From de66bf010d13ee2d63580376368b1f34d1110bb3 Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Tue, 23 Mar 2021 15:09:44 -0700 Subject: [PATCH] Revert "Merge branch 'master' of github.com:airbytehq/airbyte" This reverts commit a3a27f2fa26ac9e137be6993b0be1f3da94a7b5b, reversing changes made to 5917feee28877aa5ebfada7be32394056ed44530. --- .../new_python_source.md | 1 - .github/workflows/gradle.yml | 1 + .github/workflows/publish-command.yml | 1 - .github/workflows/test-command.yml | 5 +- airbyte-api/src/main/openapi/config.yaml | 23 - .../main/java/io/airbyte/commons/io/IOs.java | 8 - .../java/io/airbyte/commons/yaml/Yamls.java | 42 +- .../java/io/airbyte/commons/io/IOsTest.java | 7 - .../io/airbyte/commons/yaml/YamlsTest.java | 31 +- .../36c891d9-4bd9-43ac-bad2-10e12756272c.json | 7 - .../57eb1576-8f52-463d-beb6-2e107cdf571d.json | 7 + .../5e6175e5-68e1-4c17-bff9-56103bbb0d80.json | 7 - .../resources/seed/source_definitions.yaml | 11 +- .../cypress/integration/destination.spec.js | 2 +- .../cypress/integration/source.spec.js | 2 +- .../bases/base-normalization/setup.py | 2 +- .../source/StandardSourceTest.java | 4 +- .../source-appsflyer-singer/.dockerignore | 6 - .../source-appsflyer-singer/.gitignore | 1 - .../source-appsflyer-singer/Dockerfile | 16 - .../source-appsflyer-singer/README.md | 111 - .../source-appsflyer-singer/build.gradle | 32 - .../source-appsflyer-singer/main_dev.py | 32 - .../sample_files/catalog.json | 516 - .../sample_files/configured_catalog.json | 522 - .../singer_rendered_catalog.json | 871 -- .../source_appsflyer_singer/__init__.py | 27 - .../source_appsflyer_singer/source.py | 85 - .../source_appsflyer_singer/spec.json | 20 - .../unit_tests/unit_test.py | 27 - .../integration_tests}/__init__.py | 4 +- .../standard_source_test.py} | 8 +- .../connectors/source-file/setup.py | 2 +- .../source-gitlab-singer/.dockerignore | 6 - .../source-gitlab-singer/.gitignore | 1 - .../source-gitlab-singer/Dockerfile | 16 - .../source-gitlab-singer/main_dev.py | 32 - .../source-gitlab-singer/requirements.txt | 6 - .../sample_files/configured_catalog.json | 1493 --- .../sample_files/sample_config.json | 10 - .../connectors/source-gitlab-singer/setup.py | 41 - .../source_gitlab_singer/source.py | 47 - .../source_gitlab_singer/spec.json | 69 - .../unit_tests/unit_test.py | 27 - .../source-hubspot-singer/.dockerignore | 1 + .../source-hubspot-singer/Dockerfile | 20 + .../README.md | 28 +- .../build.gradle | 5 +- .../main_dev.py | 4 +- .../requirements.txt | 1 - .../sample_files/configured_catalog.json | 8861 +++++++++++++++++ .../setup.py | 12 +- .../source_hubspot_singer}/__init__.py | 4 +- .../source_hubspot_singer/source.py | 106 + .../source_hubspot_singer/spec.json | 67 + .../connectors/source-hubspot/.dockerignore | 6 - .../connectors/source-hubspot/.gitignore | 1 - .../connectors/source-hubspot/Dockerfile | 16 - .../connectors/source-hubspot/README.md | 64 - .../connectors/source-hubspot/build.gradle | 37 - .../integration_tests/client_test.py | 51 - .../integration_tests/incremental_test.py | 107 - .../source-hubspot/requirements.txt | 5 - .../sample_files/configured_catalog.json | 6055 ----------- .../sample_files/sample_config.json | 6 - .../sample_files/sample_state.json | 8 - .../connectors/source-hubspot/setup.py | 51 - .../source-hubspot/source_hubspot/api.py | 553 - .../source-hubspot/source_hubspot/client.py | 109 - .../source-hubspot/source_hubspot/errors.py | 52 - .../source_hubspot/schemas/campaigns.json | 92 - .../source_hubspot/schemas/companies.json | 33 - .../source_hubspot/schemas/contact_lists.json | 96 - .../source_hubspot/schemas/contacts.json | 2325 ----- .../schemas/deal_pipelines.json | 70 - .../source_hubspot/schemas/deals.json | 50 - .../source_hubspot/schemas/email_events.json | 118 - .../source_hubspot/schemas/engagements.json | 166 - .../source_hubspot/schemas/forms.json | 228 - .../source_hubspot/schemas/line_items.json | 24 - .../source_hubspot/schemas/owners.json | 73 - .../source_hubspot/schemas/products.json | 24 - .../source_hubspot/schemas/quotes.json | 24 - .../schemas/subscription_changes.json | 55 - .../source_hubspot/schemas/tickets.json | 24 - .../source_hubspot/schemas/workflows.json | 47 - .../source-hubspot/source_hubspot/spec.json | 30 - .../source-hubspot/unit_tests/test_client.py | 62 - .../source/jdbc/AbstractJdbcSource.java | 106 +- .../jdbc/test/JdbcSourceStandardTest.java | 8 +- .../source/mysql/MySqlJdbcStandardTest.java | 19 +- .../source/mysql/MySqlStressTest.java | 18 +- .../main/java/io/airbyte/migrate/Migrate.java | 53 +- .../io/airbyte/migrate/MigrationRunner.java | 6 +- .../migrate/migrations/MigrationV0_14_3.java | 27 +- .../scheduler/ScheduleJobPredicate.java | 9 +- .../io/airbyte/scheduler/SchedulerApp.java | 2 +- .../DefaultSynchronousSchedulerClient.java | 50 +- .../client/SynchronousJobMetadata.java | 23 +- .../scheduler/client/SynchronousResponse.java | 20 - .../worker_run/TemporalWorkerRunFactory.java | 47 +- .../scheduler/ScheduleJobPredicateTest.java | 6 +- ...DefaultSynchronousSchedulerClientTest.java | 68 +- .../TemporalWorkerRunFactoryTest.java | 9 - airbyte-server/build.gradle | 2 - .../java/io/airbyte/server/ServerApp.java | 6 +- .../airbyte/server/apis/ConfigurationApi.java | 8 +- .../server/converters/JobConverter.java | 2 +- .../server/handlers/SchedulerHandler.java | 53 +- .../server/handlers/SchedulerHandlerTest.java | 6 +- .../ArrayOfObjectsEditor.tsx | 94 - .../components/EditorHeader.tsx | 38 - .../components/EditorRow.tsx | 61 - .../components/ArrayOfObjectsEditor/index.tsx | 4 - .../components/Multiselect/Multiselect.tsx | 136 - .../src/components/Multiselect/index.tsx | 1 - .../src/components/ServiceForm/FormRoot.tsx | 2 +- .../components/ServiceForm/ServiceForm.tsx | 29 +- .../Controls/ConnectorNameControl.tsx | 36 - .../Controls/ConnectorServiceTypeControl.tsx | 77 - .../ServiceForm/components/EditControls.tsx | 6 +- .../ServiceForm/components/FormSection.tsx | 197 +- .../components/{Controls => }/Instruction.tsx | 0 .../ServiceForm/components/Property.tsx | 139 + .../components/Property/Control.tsx | 67 +- .../components/Property/GroupControls.tsx | 38 - .../ServiceForm/components/Property/Label.tsx | 19 +- .../components/Property/LabelMessage.tsx | 18 +- .../components/PropertySection.tsx | 54 - .../components/TestingConnectionSuccess.tsx | 2 +- .../ServiceForm/serviceFormContext.tsx | 30 +- .../src/components/ServiceForm/types.ts | 3 +- .../components/ServiceForm/useBuildForm.tsx | 105 +- .../src/components/base/TagInput/TagInput.tsx | 173 - .../src/components/base/TagInput/TagItem.tsx | 71 - .../src/components/base/TagInput/index.tsx | 1 - airbyte-webapp/src/components/base/index.tsx | 1 - airbyte-webapp/src/components/index.tsx | 4 - airbyte-webapp/src/core/form/types.ts | 32 +- airbyte-webapp/src/core/form/uiWidget.test.ts | 14 +- airbyte-webapp/src/core/form/uiWidget.ts | 12 +- .../core/jsonSchema/schemaToUiWidget.test.ts | 30 +- .../src/core/jsonSchema/schemaToUiWidget.ts | 35 +- .../src/core/jsonSchema/schemaToYup.test.ts | 7 - .../src/core/jsonSchema/schemaToYup.ts | 19 - airbyte-webapp/src/locales/en.json | 2 - .../components/SourceSettings.tsx | 5 + .../io/airbyte/workers/DefaultSyncWorker.java | 22 - .../java/io/airbyte/workers/WorkerUtils.java | 9 - .../process/DockerProcessBuilderFactory.java | 1 - .../workers/protocols/Destination.java | 2 - .../io/airbyte/workers/protocols/Source.java | 2 - .../airbyte/DefaultAirbyteDestination.java | 13 - .../airbyte/DefaultAirbyteSource.java | 13 - .../protocols/airbyte/EmptyAirbyteSource.java | 5 - .../workers/temporal/CancellationHandler.java | 73 - .../temporal/CheckConnectionWorkflow.java | 36 +- .../temporal/DiscoverCatalogWorkflow.java | 37 +- .../workers/temporal/SpecWorkflow.java | 37 +- .../workers/temporal/SyncWorkflow.java | 47 +- .../temporal/TemporalAttemptExecution.java | 115 +- .../workers/temporal/TemporalClient.java | 60 +- ...etadata.java => TemporalJobException.java} | 42 +- .../workers/temporal/TemporalResponse.java | 90 - .../workers/temporal/TemporalUtils.java | 2 - .../TemporalAttemptExecutionTest.java | 69 +- .../workers/temporal/TemporalClientTest.java | 179 +- docs/SUMMARY.md | 5 +- docs/api/generated-api-html/index.html | 89 - docs/architecture/basic-normalization.md | 89 +- ...te.md => founding-developer-evangelist.md} | 0 .../building-new-connector/README.md | 15 +- ...on-development.md => python-connectors.md} | 7 +- .../testing-connectors.md | 21 +- docs/integrations/connector-health.md | 15 +- docs/integrations/sources/gitlab.md | 56 - docs/integrations/sources/hubspot.md | 32 +- docs/tutorials/building-a-python-source.md | 58 +- docs/tutorials/upgrading-airbyte.md | 5 +- tools/bin/ci_credentials.sh | 3 +- tools/bin/ci_integration_test.sh | 12 +- 181 files changed, 9914 insertions(+), 17417 deletions(-) delete mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json create mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/57eb1576-8f52-463d-beb6-2e107cdf571d.json delete mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/5e6175e5-68e1-4c17-bff9-56103bbb0d80.json delete mode 100644 airbyte-integrations/connectors/source-appsflyer-singer/.dockerignore delete mode 100644 airbyte-integrations/connectors/source-appsflyer-singer/.gitignore delete mode 100644 airbyte-integrations/connectors/source-appsflyer-singer/Dockerfile delete mode 100644 airbyte-integrations/connectors/source-appsflyer-singer/README.md delete mode 100644 airbyte-integrations/connectors/source-appsflyer-singer/build.gradle delete mode 100644 airbyte-integrations/connectors/source-appsflyer-singer/main_dev.py delete mode 100644 airbyte-integrations/connectors/source-appsflyer-singer/sample_files/catalog.json delete mode 100644 airbyte-integrations/connectors/source-appsflyer-singer/sample_files/configured_catalog.json delete mode 100644 airbyte-integrations/connectors/source-appsflyer-singer/singer_rendered_catalog.json delete mode 100644 airbyte-integrations/connectors/source-appsflyer-singer/source_appsflyer_singer/__init__.py delete mode 100644 airbyte-integrations/connectors/source-appsflyer-singer/source_appsflyer_singer/source.py delete mode 100644 airbyte-integrations/connectors/source-appsflyer-singer/source_appsflyer_singer/spec.json delete mode 100644 airbyte-integrations/connectors/source-appsflyer-singer/unit_tests/unit_test.py rename airbyte-integrations/connectors/{source-hubspot/source_hubspot => source-file/integration_tests}/__init__.py (91%) rename airbyte-integrations/connectors/{source-hubspot/source_hubspot/source.py => source-file/integration_tests/standard_source_test.py} (89%) delete mode 100644 airbyte-integrations/connectors/source-gitlab-singer/.dockerignore delete mode 100644 airbyte-integrations/connectors/source-gitlab-singer/.gitignore delete mode 100644 airbyte-integrations/connectors/source-gitlab-singer/Dockerfile delete mode 100644 airbyte-integrations/connectors/source-gitlab-singer/main_dev.py delete mode 100644 airbyte-integrations/connectors/source-gitlab-singer/requirements.txt delete mode 100644 airbyte-integrations/connectors/source-gitlab-singer/sample_files/configured_catalog.json delete mode 100644 airbyte-integrations/connectors/source-gitlab-singer/sample_files/sample_config.json delete mode 100644 airbyte-integrations/connectors/source-gitlab-singer/setup.py delete mode 100644 airbyte-integrations/connectors/source-gitlab-singer/source_gitlab_singer/source.py delete mode 100644 airbyte-integrations/connectors/source-gitlab-singer/source_gitlab_singer/spec.json delete mode 100644 airbyte-integrations/connectors/source-gitlab-singer/unit_tests/unit_test.py create mode 100644 airbyte-integrations/connectors/source-hubspot-singer/.dockerignore create mode 100644 airbyte-integrations/connectors/source-hubspot-singer/Dockerfile rename airbyte-integrations/connectors/{source-gitlab-singer => source-hubspot-singer}/README.md (81%) rename airbyte-integrations/connectors/{source-gitlab-singer => source-hubspot-singer}/build.gradle (84%) rename airbyte-integrations/connectors/{source-hubspot => source-hubspot-singer}/main_dev.py (93%) rename airbyte-integrations/connectors/{source-appsflyer-singer => source-hubspot-singer}/requirements.txt (86%) create mode 100644 airbyte-integrations/connectors/source-hubspot-singer/sample_files/configured_catalog.json rename airbyte-integrations/connectors/{source-appsflyer-singer => source-hubspot-singer}/setup.py (79%) rename airbyte-integrations/connectors/{source-gitlab-singer/source_gitlab_singer => source-hubspot-singer/source_hubspot_singer}/__init__.py (93%) create mode 100644 airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/source.py create mode 100644 airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/spec.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/.dockerignore delete mode 100644 airbyte-integrations/connectors/source-hubspot/.gitignore delete mode 100644 airbyte-integrations/connectors/source-hubspot/Dockerfile delete mode 100644 airbyte-integrations/connectors/source-hubspot/README.md delete mode 100644 airbyte-integrations/connectors/source-hubspot/build.gradle delete mode 100644 airbyte-integrations/connectors/source-hubspot/integration_tests/client_test.py delete mode 100644 airbyte-integrations/connectors/source-hubspot/integration_tests/incremental_test.py delete mode 100644 airbyte-integrations/connectors/source-hubspot/requirements.txt delete mode 100644 airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/sample_files/sample_config.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/sample_files/sample_state.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/setup.py delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deal_pipelines.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/email_events.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/line_items.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/products.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/quotes.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/subscription_changes.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/tickets.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.json delete mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py delete mode 100644 airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx delete mode 100644 airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx delete mode 100644 airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.tsx delete mode 100644 airbyte-webapp/src/components/ArrayOfObjectsEditor/index.tsx delete mode 100644 airbyte-webapp/src/components/Multiselect/Multiselect.tsx delete mode 100644 airbyte-webapp/src/components/Multiselect/index.tsx delete mode 100644 airbyte-webapp/src/components/ServiceForm/components/Controls/ConnectorNameControl.tsx delete mode 100644 airbyte-webapp/src/components/ServiceForm/components/Controls/ConnectorServiceTypeControl.tsx rename airbyte-webapp/src/components/ServiceForm/components/{Controls => }/Instruction.tsx (100%) create mode 100644 airbyte-webapp/src/components/ServiceForm/components/Property.tsx delete mode 100644 airbyte-webapp/src/components/ServiceForm/components/Property/GroupControls.tsx delete mode 100644 airbyte-webapp/src/components/ServiceForm/components/PropertySection.tsx delete mode 100644 airbyte-webapp/src/components/base/TagInput/TagInput.tsx delete mode 100644 airbyte-webapp/src/components/base/TagInput/TagItem.tsx delete mode 100644 airbyte-webapp/src/components/base/TagInput/index.tsx delete mode 100644 airbyte-webapp/src/components/base/index.tsx delete mode 100644 airbyte-workers/src/main/java/io/airbyte/workers/temporal/CancellationHandler.java rename airbyte-workers/src/main/java/io/airbyte/workers/temporal/{JobMetadata.java => TemporalJobException.java} (62%) delete mode 100644 airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalResponse.java rename docs/career-and-open-positions/{founding-developer-advocate.md => founding-developer-evangelist.md} (100%) rename docs/contributing-to-airbyte/building-new-connector/{monorepo-python-development.md => python-connectors.md} (91%) delete mode 100644 docs/integrations/sources/gitlab.md diff --git a/.github/PULL_REQUEST_TEMPLATE/new_python_source.md b/.github/PULL_REQUEST_TEMPLATE/new_python_source.md index 848ac69a799..994c8f54c33 100644 --- a/.github/PULL_REQUEST_TEMPLATE/new_python_source.md +++ b/.github/PULL_REQUEST_TEMPLATE/new_python_source.md @@ -2,7 +2,6 @@ Thanks for contributing to Airbyte! Please complete the following items in order so we can review your PR. - [ ] Followed all the instructions in the locally generated checklist and your connector is functional & ready for review - [ ] Ran the standard test suite locally via `./gradlew :airbyte-integrations:connectors:source-:standardSourceTestPython` and pasted the summarized output as a comment in this PR -- [ ] Added the connector to the [connector health page](https://docs.airbyte.io/integrations/connector-health) ## Reviewer Pre-merge Checklist - [ ] Finished iterating with the PR author on the code* diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 48334a25ba2..a93f9e8279d 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -102,6 +102,7 @@ jobs: run: ./tools/bin/acceptance_test.sh - name: Run End-to-End Frontend Tests + if: success() && github.ref == 'refs/heads/master' run: ./tools/bin/e2e_test.sh - name: Push Core Docker Images diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 2194c0c7297..1f23fc95052 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -53,7 +53,6 @@ jobs: FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS }} FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS }} FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }} - GITLAB_INTEGRATION_TEST_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_CREDS }} GH_INTEGRATION_TEST_CREDS: ${{ secrets.GH_INTEGRATION_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_TRACKING_ID: ${{ secrets.GOOGLE_ANALYTICS_TEST_TRACKING_ID }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 762d9e0ad10..a7fa5da9b22 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -13,11 +13,11 @@ jobs: integration_test: runs-on: ubuntu-latest steps: - - name: Search for valid connector name format + - name: Search for valid integration name format id: regex uses: AsasInnab/regex-action@v1 with: - regex_pattern: '^((connectors|bases)/)?[a-zA-Z0-9-_]+$' + regex_pattern: '^[a-zA-Z0-9-_]+$' regex_flags: 'i' # required to be set for this plugin search_string: ${{ github.event.inputs.connector }} - name: Validate input workflow format @@ -53,7 +53,6 @@ jobs: FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS }} FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS: ${{ secrets.FACEBOOK_MARKETING_API_TEST_INTEGRATION_CREDS }} FRESHDESK_TEST_CREDS: ${{ secrets.FRESHDESK_TEST_CREDS }} - GITLAB_INTEGRATION_TEST_CREDS: ${{ secrets.GITLAB_INTEGRATION_TEST_CREDS }} GH_INTEGRATION_TEST_CREDS: ${{ secrets.GH_INTEGRATION_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_CREDS: ${{ secrets.GOOGLE_ANALYTICS_TEST_CREDS }} GOOGLE_ANALYTICS_TEST_TRACKING_ID: ${{ secrets.GOOGLE_ANALYTICS_TEST_TRACKING_ID }} diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 31463c45be6..8bbe96e2511 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -1101,29 +1101,6 @@ paths: description: Job not found "422": $ref: "#/components/responses/InvalidInput" - /v1/jobs/cancel: - post: - tags: - - jobs - summary: Cancels a job - operationId: cancelJob - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/JobIdRequestBody" - required: true - responses: - "200": - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/JobInfoRead" - "404": - description: Job not found - "422": - $ref: "#/components/responses/InvalidInput" /v1/health: get: tags: diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java b/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java index 21cc8a7dce6..ab05b04c4c6 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java @@ -91,14 +91,6 @@ public class IOs { } } - public static InputStream inputStream(final Path path) { - try { - return Files.newInputStream(path); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - public static void silentClose(final Closeable closeable) { try { closeable.close(); diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java b/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java index 48947323b21..6b98c52d9bf 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java @@ -25,27 +25,19 @@ package io.airbyte.commons.yaml; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SequenceWriter; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.fasterxml.jackson.dataformat.yaml.YAMLParser; -import com.google.common.collect.AbstractIterator; import io.airbyte.commons.lang.CloseableConsumer; import io.airbyte.commons.lang.Exceptions; -import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.commons.util.AutoCloseableIterators; import java.io.IOException; -import java.io.InputStream; import java.io.Writer; -import java.util.Iterator; public class Yamls { - public static final YAMLFactory YAML_FACTORY = new YAMLFactory(); // Object Mapper is thread-safe - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(YAML_FACTORY); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(new YAMLFactory()); public static String serialize(T object) { try { @@ -71,38 +63,6 @@ public class Yamls { } } - public static AutoCloseableIterator deserializeArray(final InputStream stream) { - try { - YAMLParser parser = YAML_FACTORY.createParser(stream); - - // Check the first token - if (parser.nextToken() != JsonToken.START_ARRAY) { - throw new IllegalStateException("Expected content to be an array"); - } - - Iterator iterator = new AbstractIterator<>() { - - @Override - protected JsonNode computeNext() { - try { - while (parser.nextToken() != JsonToken.END_ARRAY) { - return parser.readValueAsTree(); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - return endOfData(); - } - - }; - - return AutoCloseableIterators.fromIterator(iterator, parser::close); - - } catch (IOException e) { - throw new RuntimeException(e); - } - } - // todo (cgardens) - share this with Jsons if ever needed. /** diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java index 44928121b58..84d486f9ccf 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java @@ -89,13 +89,6 @@ class IOsTest { assertEquals(expectedTail, tail); } - @Test - void testInputStream() { - assertThrows(RuntimeException.class, () -> { - IOs.inputStream(Path.of("idontexist")); - }); - } - @Test void testSilentClose() throws IOException { Closeable closeable = Mockito.mock(Closeable.class); diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java b/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java index c1360aa35b9..cbd590a079f 100644 --- a/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java +++ b/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java @@ -24,26 +24,18 @@ package io.airbyte.commons.yaml; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.fail; +import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.CloseableConsumer; -import io.airbyte.commons.stream.MoreStreams; -import io.airbyte.commons.util.AutoCloseableIterator; -import java.io.ByteArrayInputStream; -import java.io.IOException; import java.io.StringWriter; -import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; import org.junit.jupiter.api.Test; class YamlsTest { @@ -129,27 +121,6 @@ class YamlsTest { assertEquals(values, deserialize); } - @Test - void testStreamRead() throws IOException { - List classes = Lists.newArrayList( - new ToClass("1", 1, 1), - new ToClass("2", 2, 2), - new ToClass("3", 3, 3)); - ByteArrayInputStream input = spy(new ByteArrayInputStream(Yamls.serialize(classes).getBytes(StandardCharsets.UTF_8))); - - try (AutoCloseableIterator iterator = Yamls.deserializeArray(input)) { - assertEquals( - classes, - MoreStreams.toStream(iterator) - .map(e -> Jsons.object(e, ToClass.class)) - .collect(Collectors.toList())); - } catch (Exception e) { - fail(); - } - - verify(input).close(); - } - private static class ToClass { @JsonProperty("str") diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json deleted file mode 100644 index d09e3152a47..00000000000 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c", - "name": "Hubspot", - "dockerRepository": "airbyte/source-hubspot", - "dockerImageTag": "0.1.1", - "documentationUrl": "https://https://docs.airbyte.io/integrations/sources/hubspot" -} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/57eb1576-8f52-463d-beb6-2e107cdf571d.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/57eb1576-8f52-463d-beb6-2e107cdf571d.json new file mode 100644 index 00000000000..bf5fbf68969 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/57eb1576-8f52-463d-beb6-2e107cdf571d.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "57eb1576-8f52-463d-beb6-2e107cdf571d", + "name": "Hubspot", + "dockerRepository": "airbyte/source-hubspot-singer", + "dockerImageTag": "0.2.0", + "documentationUrl": "https://https://docs.airbyte.io/integrations/sources/hubspot" +} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/5e6175e5-68e1-4c17-bff9-56103bbb0d80.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/5e6175e5-68e1-4c17-bff9-56103bbb0d80.json deleted file mode 100644 index adcd7d0f810..00000000000 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/5e6175e5-68e1-4c17-bff9-56103bbb0d80.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "sourceDefinitionId": "5e6175e5-68e1-4c17-bff9-56103bbb0d80", - "name": "Gitlab", - "dockerRepository": "airbyte/source-gitlab-singer", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://hub.docker.com/r/airbyte/source-gitlab-singer" -} diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 129cab2c6c7..de50afd13f2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -78,10 +78,10 @@ dockerRepository: airbyte/source-facebook-marketing dockerImageTag: 0.2.1 documentationUrl: https://hub.docker.com/r/airbyte/source-facebook-marketing -- sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c +- sourceDefinitionId: 57eb1576-8f52-463d-beb6-2e107cdf571d name: Hubspot - dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.1.1 + dockerRepository: airbyte/source-hubspot-singer + dockerImageTag: 0.2.0 documentationUrl: https://https://docs.airbyte.io/integrations/sources/hubspot - sourceDefinitionId: b1892b11-788d-44bd-b9ec-3a436f7b54ce name: Shopify @@ -188,8 +188,3 @@ dockerRepository: airbyte/source-instagram dockerImageTag: 0.1.1 documentationUrl: https://hub.docker.com/r/airbyte/source-instagram -- sourceDefinitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80 - name: Gitlab - dockerRepository: airbyte/source-gitlab-singer - dockerImageTag: 0.1.0 - documentationUrl: https://hub.docker.com/r/airbyte/source-gitlab-singer diff --git a/airbyte-e2e-testing/cypress/integration/destination.spec.js b/airbyte-e2e-testing/cypress/integration/destination.spec.js index bcbb32c9d46..d4d57b3720e 100644 --- a/airbyte-e2e-testing/cypress/integration/destination.spec.js +++ b/airbyte-e2e-testing/cypress/integration/destination.spec.js @@ -9,7 +9,7 @@ describe("Destination main actions", () => { cy.createTestDestination("Test destination cypress for update"); cy.updateDestination("Test destination cypress for update", "connectionConfiguration.destination_path", "/local/my-json"); - cy.get("div[data-id='success-result']").should("exist"); + cy.get("span[data-id='success-result']").should("exist"); cy.get("input[value='/local/my-json']").should("exist"); }); diff --git a/airbyte-e2e-testing/cypress/integration/source.spec.js b/airbyte-e2e-testing/cypress/integration/source.spec.js index 60f13c35bde..644de08629e 100644 --- a/airbyte-e2e-testing/cypress/integration/source.spec.js +++ b/airbyte-e2e-testing/cypress/integration/source.spec.js @@ -9,7 +9,7 @@ describe("Source main actions", () => { cy.createTestSource("Test source cypress for update"); cy.updateSource("Test source cypress for update", "connectionConfiguration.start_date", "2020-11-11"); - cy.get("div[data-id='success-result']").should("exist"); + cy.get("span[data-id='success-result']").should("exist"); cy.get("input[value='2020-11-11']").should("exist"); }); diff --git a/airbyte-integrations/bases/base-normalization/setup.py b/airbyte-integrations/bases/base-normalization/setup.py index 5bd0b636f90..b2bccf30340 100644 --- a/airbyte-integrations/bases/base-normalization/setup.py +++ b/airbyte-integrations/bases/base-normalization/setup.py @@ -31,7 +31,7 @@ setuptools.setup( author_email="contact@airbyte.io", url="https://github.com/airbytehq/airbyte", packages=setuptools.find_packages(), - install_requires=["airbyte-protocol", "dbt==0.18.2rc1", "pyyaml"], + install_requires=["airbyte-protocol", "dbt==0.18.1", "pyyaml"], package_data={"": ["*.yml"]}, setup_requires=["pytest-runner"], entry_points={ diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/StandardSourceTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/StandardSourceTest.java index a5ce8512435..682f0e107f2 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/StandardSourceTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/StandardSourceTest.java @@ -97,7 +97,7 @@ public abstract class StandardSourceTest { private Set IMAGES_TO_SKIP_SECOND_INCREMENTAL_READ = Sets.newHashSet( "airbyte/source-intercom-singer", "airbyte/source-exchangeratesapi-singer", - "airbyte/source-hubspot", + "airbyte/source-hubspot-singer", "airbyte/source-marketo-singer", "airbyte/source-twilio-singer", "airbyte/source-mixpanel-singer", @@ -106,7 +106,7 @@ public abstract class StandardSourceTest { "airbyte/source-salesforce-singer", "airbyte/source-stripe-singer", "airbyte/source-github-singer", - "airbyte/source-gitlab-singer"); + "airbyte/source-hubspot-singer"); /** * Name of the docker image that the tests will run against. diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/.dockerignore b/airbyte-integrations/connectors/source-appsflyer-singer/.dockerignore deleted file mode 100644 index 593d2826b43..00000000000 --- a/airbyte-integrations/connectors/source-appsflyer-singer/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!Dockerfile.test -!source_appsflyer_singer -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/.gitignore b/airbyte-integrations/connectors/source-appsflyer-singer/.gitignore deleted file mode 100644 index 29fffc6a50c..00000000000 --- a/airbyte-integrations/connectors/source-appsflyer-singer/.gitignore +++ /dev/null @@ -1 +0,0 @@ -NEW_SOURCE_CHECKLIST.md diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/Dockerfile b/airbyte-integrations/connectors/source-appsflyer-singer/Dockerfile deleted file mode 100644 index e165269240d..00000000000 --- a/airbyte-integrations/connectors/source-appsflyer-singer/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM airbyte/integration-base-singer:0.1.0 - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash git gcc && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="source_appsflyer_singer" -ENV AIRBYTE_IMPL_MODULE="source_appsflyer_singer" -ENV AIRBYTE_IMPL_PATH="SourceAppsflyerSinger" - -WORKDIR /airbyte/integration_code -COPY $CODE_PATH ./$CODE_PATH -COPY setup.py ./ -RUN pip install . - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-appsflyer-singer diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/README.md b/airbyte-integrations/connectors/source-appsflyer-singer/README.md deleted file mode 100644 index 035db1cf2d0..00000000000 --- a/airbyte-integrations/connectors/source-appsflyer-singer/README.md +++ /dev/null @@ -1,111 +0,0 @@ -# Source Appsflyer Singer - -This is the repository for the Appsflyer source connector, based on a Singer tap. -For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/sources/appsflyer). - -## Local development - -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-appsflyer:build -``` - -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/appsflyer) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_appsflyer_singer/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. - -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source appsflyer test creds` -and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector root run: -``` -pytest unit_tests -``` - -### Locally running the connector -``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - -### Locally running the connector docker image - -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/appsflyer-singer:dev -``` - -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-appsflyer:airbyteDocker -``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-appsflyer-singer:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-appsflyer-singer:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-appsflyer-singer:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-appsflyer-singer:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json -``` - -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-appsflyer-singer:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, create a directory `integration_tests` which contain your tests and run them with `pytest integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. - -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. - -### Publishing a new version of the connector -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/build.gradle b/airbyte-integrations/connectors/source-appsflyer-singer/build.gradle deleted file mode 100644 index 699f0f85c97..00000000000 --- a/airbyte-integrations/connectors/source-appsflyer-singer/build.gradle +++ /dev/null @@ -1,32 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-standard-source-test-file' -} - -airbytePython { - moduleDirectory 'source_appsflyer_singer' -} - -airbyteStandardSourceTestFile { - // For more information on standard source tests, see https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/testing-connectors - - // All these input paths must live inside this connector's directory (or subdirectories) - // TODO update the spec JSON file - specPath = "source_appsflyer_singer/spec.json" - - // configPath points to a config file which matches the spec.json supplied above. secrets/ is gitignored by default, so place your config file - // there (in case it contains any credentials) - // TODO update the config file to contain actual credentials - configPath = "secrets/config.json" - // TODO update the sample configured_catalog JSON for use in testing - // Note: If your source supports incremental syncing, then make sure that the catalog that is returned in the get_catalog method is configured - // for incremental syncing (e.g. include cursor fields, etc). - configuredCatalogPath = "sample_files/configured_catalog.json" -} - - -dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) - implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) -} diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/main_dev.py b/airbyte-integrations/connectors/source-appsflyer-singer/main_dev.py deleted file mode 100644 index a307814ce60..00000000000 --- a/airbyte-integrations/connectors/source-appsflyer-singer/main_dev.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import sys - -from base_python.entrypoint import launch -from source_appsflyer_singer import SourceAppsflyerSinger - -if __name__ == "__main__": - source = SourceAppsflyerSinger() - launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/sample_files/catalog.json b/airbyte-integrations/connectors/source-appsflyer-singer/sample_files/catalog.json deleted file mode 100644 index 23203ab0822..00000000000 --- a/airbyte-integrations/connectors/source-appsflyer-singer/sample_files/catalog.json +++ /dev/null @@ -1,516 +0,0 @@ -{ - "streams": [ - { - "name": "in_app_events", - "supported_sync_modes": ["incremental"], - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "attributed_touch_type": { - "type": ["null", "string"] - }, - "attributed_touch_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "install_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "af_prt": { - "type": ["null", "string"] - }, - "media_source": { - "type": ["null", "string"] - }, - "is_retargeting": { - "type": ["null", "boolean"] - }, - "retargeting_conversion_type": { - "type": ["null", "string"] - }, - "af_channel": { - "type": ["null", "string"] - }, - "af_keywords": { - "type": ["null", "string"] - }, - "campaign": { - "type": ["null", "string"] - }, - "af_c_id": { - "type": ["null", "string"] - }, - "af_adset": { - "type": ["null", "string"] - }, - "af_adset_id": { - "type": ["null", "string"] - }, - "af_ad": { - "type": ["null", "string"] - }, - "af_ad_id": { - "type": ["null", "string"] - }, - "af_ad_type": { - "type": ["null", "string"] - }, - "af_siteid": { - "type": ["null", "string"] - }, - "af_sub1": { - "type": ["null", "string"] - }, - "af_sub2": { - "type": ["null", "string"] - }, - "af_sub3": { - "type": ["null", "string"] - }, - "af_sub4": { - "type": ["null", "string"] - }, - "af_sub5": { - "type": ["null", "string"] - }, - "http_referrer": { - "type": ["null", "string"] - }, - "original_url": { - "type": ["null", "string"] - }, - "user_agent": { - "type": ["null", "string"] - }, - "af_cost_model": { - "type": ["null", "string"] - }, - "af_cost_value": { - "type": ["null", "string"] - }, - "af_cost_currency": { - "type": ["null", "string"] - }, - "contributor1_af_prt": { - "type": ["null", "string"] - }, - "contributor1_media_source": { - "type": ["null", "string"] - }, - "contributor1_campaign": { - "type": ["null", "string"] - }, - "contributor1_touch_type": { - "type": ["null", "string"] - }, - "contributor1_touch_time": { - "type": ["null", "string"] - }, - "contributor2_af_prt": { - "type": ["null", "string"] - }, - "contributor2_media_source": { - "type": ["null", "string"] - }, - "contributor2_campaign": { - "type": ["null", "string"] - }, - "contributor2_touch_type": { - "type": ["null", "string"] - }, - "contributor2_touch_time": { - "type": ["null", "string"] - }, - "contributor3_af_prt": { - "type": ["null", "string"] - }, - "contributor3_media_source": { - "type": ["null", "string"] - }, - "contributor3_campaign": { - "type": ["null", "string"] - }, - "contributor3_touch_type": { - "type": ["null", "string"] - }, - "contributor3_touch_time": { - "type": ["null", "string"] - }, - "country_code": { - "type": ["null", "string"] - }, - "ip": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "wifi": { - "type": ["null", "boolean"] - }, - "operator": { - "type": ["null", "string"] - }, - "carrier": { - "type": ["null", "string"] - }, - "language": { - "type": ["null", "string"] - }, - "appsflyer_id": { - "type": ["null", "string"] - }, - "advertising_id": { - "type": ["null", "string"] - }, - "idfa": { - "type": ["null", "string"] - }, - "android_id": { - "type": ["null", "string"] - }, - "customer_user_id": { - "type": ["null", "string"] - }, - "imei": { - "type": ["null", "string"] - }, - "idfv": { - "type": ["null", "string"] - }, - "platform": { - "type": ["null", "string"] - }, - "device_type": { - "type": ["null", "string"] - }, - "os_version": { - "type": ["null", "string"] - }, - "app_version": { - "type": ["null", "string"] - }, - "sdk_version": { - "type": ["null", "string"] - }, - "app_id": { - "type": ["null", "string"] - }, - "app_name": { - "type": ["null", "string"] - }, - "bundle_id": { - "type": ["null", "string"] - }, - "af_attribution_lookback": { - "type": ["null", "string"] - }, - "event_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "event_name": { - "type": ["null", "string"] - }, - "event_value": { - "type": ["null", "string"] - }, - "event_revenue": { - "type": ["null", "string"] - }, - "event_revenue_usd": { - "type": ["null", "string"] - }, - "event_revenue_currency": { - "type": ["null", "string"] - }, - "is_receipt_validated": { - "type": ["null", "boolean"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "dma": { - "type": ["null", "string"] - }, - "af_reengagement_window": { - "type": ["null", "string"] - }, - "event_source": { - "type": ["null", "string"] - }, - "is_primary_attribution": { - "type": ["null", "boolean"] - }, - "af_sub_siteid": { - "type": ["null", "string"] - } - } - } - }, - { - "name": "installations", - "supported_sync_modes": ["incremental"], - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "attributed_touch_type": { - "type": ["null", "string"] - }, - "attributed_touch_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "install_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "af_prt": { - "type": ["null", "string"] - }, - "media_source": { - "type": ["null", "string"] - }, - "is_retargeting": { - "type": ["null", "boolean"] - }, - "retargeting_conversion_type": { - "type": ["null", "string"] - }, - "af_channel": { - "type": ["null", "string"] - }, - "af_keywords": { - "type": ["null", "string"] - }, - "campaign": { - "type": ["null", "string"] - }, - "af_c_id": { - "type": ["null", "string"] - }, - "af_adset": { - "type": ["null", "string"] - }, - "af_adset_id": { - "type": ["null", "string"] - }, - "af_ad": { - "type": ["null", "string"] - }, - "af_ad_id": { - "type": ["null", "string"] - }, - "af_ad_type": { - "type": ["null", "string"] - }, - "af_siteid": { - "type": ["null", "string"] - }, - "af_sub1": { - "type": ["null", "string"] - }, - "af_sub2": { - "type": ["null", "string"] - }, - "af_sub3": { - "type": ["null", "string"] - }, - "af_sub4": { - "type": ["null", "string"] - }, - "af_sub5": { - "type": ["null", "string"] - }, - "http_referrer": { - "type": ["null", "string"] - }, - "original_url": { - "type": ["null", "string"] - }, - "user_agent": { - "type": ["null", "string"] - }, - "af_cost_model": { - "type": ["null", "string"] - }, - "af_cost_value": { - "type": ["null", "string"] - }, - "af_cost_currency": { - "type": ["null", "string"] - }, - "contributor1_af_prt": { - "type": ["null", "string"] - }, - "contributor1_media_source": { - "type": ["null", "string"] - }, - "contributor1_campaign": { - "type": ["null", "string"] - }, - "contributor1_touch_type": { - "type": ["null", "string"] - }, - "contributor1_touch_time": { - "type": ["null", "string"] - }, - "contributor2_af_prt": { - "type": ["null", "string"] - }, - "contributor2_media_source": { - "type": ["null", "string"] - }, - "contributor2_campaign": { - "type": ["null", "string"] - }, - "contributor2_touch_type": { - "type": ["null", "string"] - }, - "contributor2_touch_time": { - "type": ["null", "string"] - }, - "contributor3_af_prt": { - "type": ["null", "string"] - }, - "contributor3_media_source": { - "type": ["null", "string"] - }, - "contributor3_campaign": { - "type": ["null", "string"] - }, - "contributor3_touch_type": { - "type": ["null", "string"] - }, - "contributor3_touch_time": { - "type": ["null", "string"] - }, - "country_code": { - "type": ["null", "string"] - }, - "ip": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "wifi": { - "type": ["null", "boolean"] - }, - "operator": { - "type": ["null", "string"] - }, - "carrier": { - "type": ["null", "string"] - }, - "language": { - "type": ["null", "string"] - }, - "appsflyer_id": { - "type": ["null", "string"] - }, - "advertising_id": { - "type": ["null", "string"] - }, - "idfa": { - "type": ["null", "string"] - }, - "android_id": { - "type": ["null", "string"] - }, - "customer_user_id": { - "type": ["null", "string"] - }, - "imei": { - "type": ["null", "string"] - }, - "idfv": { - "type": ["null", "string"] - }, - "platform": { - "type": ["null", "string"] - }, - "device_type": { - "type": ["null", "string"] - }, - "os_version": { - "type": ["null", "string"] - }, - "app_version": { - "type": ["null", "string"] - }, - "sdk_version": { - "type": ["null", "string"] - }, - "app_id": { - "type": ["null", "string"] - }, - "app_name": { - "type": ["null", "string"] - }, - "bundle_id": { - "type": ["null", "string"] - }, - "af_attribution_lookback": { - "type": ["null", "string"] - }, - "event_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "event_name": { - "type": ["null", "string"] - }, - "event_value": { - "type": ["null", "string"] - }, - "event_revenue": { - "type": ["null", "string"] - }, - "event_revenue_usd": { - "type": ["null", "string"] - }, - "event_revenue_currency": { - "type": ["null", "string"] - }, - "is_receipt_validated": { - "type": ["null", "boolean"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "dma": { - "type": ["null", "string"] - }, - "af_reengagement_window": { - "type": ["null", "string"] - }, - "event_source": { - "type": ["null", "string"] - }, - "is_primary_attribution": { - "type": ["null", "boolean"] - }, - "af_sub_siteid": { - "type": ["null", "string"] - } - } - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-appsflyer-singer/sample_files/configured_catalog.json deleted file mode 100644 index 9b476b5c8dd..00000000000 --- a/airbyte-integrations/connectors/source-appsflyer-singer/sample_files/configured_catalog.json +++ /dev/null @@ -1,522 +0,0 @@ -{ - "streams": [ - { - "sync_mode": "incremental", - "stream": { - "name": "in_app_events", - "supported_sync_modes": ["incremental"], - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "attributed_touch_type": { - "type": ["null", "string"] - }, - "attributed_touch_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "install_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "af_prt": { - "type": ["null", "string"] - }, - "media_source": { - "type": ["null", "string"] - }, - "is_retargeting": { - "type": ["null", "boolean"] - }, - "retargeting_conversion_type": { - "type": ["null", "string"] - }, - "af_channel": { - "type": ["null", "string"] - }, - "af_keywords": { - "type": ["null", "string"] - }, - "campaign": { - "type": ["null", "string"] - }, - "af_c_id": { - "type": ["null", "string"] - }, - "af_adset": { - "type": ["null", "string"] - }, - "af_adset_id": { - "type": ["null", "string"] - }, - "af_ad": { - "type": ["null", "string"] - }, - "af_ad_id": { - "type": ["null", "string"] - }, - "af_ad_type": { - "type": ["null", "string"] - }, - "af_siteid": { - "type": ["null", "string"] - }, - "af_sub1": { - "type": ["null", "string"] - }, - "af_sub2": { - "type": ["null", "string"] - }, - "af_sub3": { - "type": ["null", "string"] - }, - "af_sub4": { - "type": ["null", "string"] - }, - "af_sub5": { - "type": ["null", "string"] - }, - "http_referrer": { - "type": ["null", "string"] - }, - "original_url": { - "type": ["null", "string"] - }, - "user_agent": { - "type": ["null", "string"] - }, - "af_cost_model": { - "type": ["null", "string"] - }, - "af_cost_value": { - "type": ["null", "string"] - }, - "af_cost_currency": { - "type": ["null", "string"] - }, - "contributor1_af_prt": { - "type": ["null", "string"] - }, - "contributor1_media_source": { - "type": ["null", "string"] - }, - "contributor1_campaign": { - "type": ["null", "string"] - }, - "contributor1_touch_type": { - "type": ["null", "string"] - }, - "contributor1_touch_time": { - "type": ["null", "string"] - }, - "contributor2_af_prt": { - "type": ["null", "string"] - }, - "contributor2_media_source": { - "type": ["null", "string"] - }, - "contributor2_campaign": { - "type": ["null", "string"] - }, - "contributor2_touch_type": { - "type": ["null", "string"] - }, - "contributor2_touch_time": { - "type": ["null", "string"] - }, - "contributor3_af_prt": { - "type": ["null", "string"] - }, - "contributor3_media_source": { - "type": ["null", "string"] - }, - "contributor3_campaign": { - "type": ["null", "string"] - }, - "contributor3_touch_type": { - "type": ["null", "string"] - }, - "contributor3_touch_time": { - "type": ["null", "string"] - }, - "country_code": { - "type": ["null", "string"] - }, - "ip": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "wifi": { - "type": ["null", "boolean"] - }, - "operator": { - "type": ["null", "string"] - }, - "carrier": { - "type": ["null", "string"] - }, - "language": { - "type": ["null", "string"] - }, - "appsflyer_id": { - "type": ["null", "string"] - }, - "advertising_id": { - "type": ["null", "string"] - }, - "idfa": { - "type": ["null", "string"] - }, - "android_id": { - "type": ["null", "string"] - }, - "customer_user_id": { - "type": ["null", "string"] - }, - "imei": { - "type": ["null", "string"] - }, - "idfv": { - "type": ["null", "string"] - }, - "platform": { - "type": ["null", "string"] - }, - "device_type": { - "type": ["null", "string"] - }, - "os_version": { - "type": ["null", "string"] - }, - "app_version": { - "type": ["null", "string"] - }, - "sdk_version": { - "type": ["null", "string"] - }, - "app_id": { - "type": ["null", "string"] - }, - "app_name": { - "type": ["null", "string"] - }, - "bundle_id": { - "type": ["null", "string"] - }, - "af_attribution_lookback": { - "type": ["null", "string"] - }, - "event_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "event_name": { - "type": ["null", "string"] - }, - "event_value": { - "type": ["null", "string"] - }, - "event_revenue": { - "type": ["null", "string"] - }, - "event_revenue_usd": { - "type": ["null", "string"] - }, - "event_revenue_currency": { - "type": ["null", "string"] - }, - "is_receipt_validated": { - "type": ["null", "boolean"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "dma": { - "type": ["null", "string"] - }, - "af_reengagement_window": { - "type": ["null", "string"] - }, - "event_source": { - "type": ["null", "string"] - }, - "is_primary_attribution": { - "type": ["null", "boolean"] - }, - "af_sub_siteid": { - "type": ["null", "string"] - } - } - } - } - }, - { - "sync_mode": "incremental", - "stream": { - "name": "installations", - "supported_sync_modes": ["incremental"], - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "attributed_touch_type": { - "type": ["null", "string"] - }, - "attributed_touch_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "install_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "af_prt": { - "type": ["null", "string"] - }, - "media_source": { - "type": ["null", "string"] - }, - "is_retargeting": { - "type": ["null", "boolean"] - }, - "retargeting_conversion_type": { - "type": ["null", "string"] - }, - "af_channel": { - "type": ["null", "string"] - }, - "af_keywords": { - "type": ["null", "string"] - }, - "campaign": { - "type": ["null", "string"] - }, - "af_c_id": { - "type": ["null", "string"] - }, - "af_adset": { - "type": ["null", "string"] - }, - "af_adset_id": { - "type": ["null", "string"] - }, - "af_ad": { - "type": ["null", "string"] - }, - "af_ad_id": { - "type": ["null", "string"] - }, - "af_ad_type": { - "type": ["null", "string"] - }, - "af_siteid": { - "type": ["null", "string"] - }, - "af_sub1": { - "type": ["null", "string"] - }, - "af_sub2": { - "type": ["null", "string"] - }, - "af_sub3": { - "type": ["null", "string"] - }, - "af_sub4": { - "type": ["null", "string"] - }, - "af_sub5": { - "type": ["null", "string"] - }, - "http_referrer": { - "type": ["null", "string"] - }, - "original_url": { - "type": ["null", "string"] - }, - "user_agent": { - "type": ["null", "string"] - }, - "af_cost_model": { - "type": ["null", "string"] - }, - "af_cost_value": { - "type": ["null", "string"] - }, - "af_cost_currency": { - "type": ["null", "string"] - }, - "contributor1_af_prt": { - "type": ["null", "string"] - }, - "contributor1_media_source": { - "type": ["null", "string"] - }, - "contributor1_campaign": { - "type": ["null", "string"] - }, - "contributor1_touch_type": { - "type": ["null", "string"] - }, - "contributor1_touch_time": { - "type": ["null", "string"] - }, - "contributor2_af_prt": { - "type": ["null", "string"] - }, - "contributor2_media_source": { - "type": ["null", "string"] - }, - "contributor2_campaign": { - "type": ["null", "string"] - }, - "contributor2_touch_type": { - "type": ["null", "string"] - }, - "contributor2_touch_time": { - "type": ["null", "string"] - }, - "contributor3_af_prt": { - "type": ["null", "string"] - }, - "contributor3_media_source": { - "type": ["null", "string"] - }, - "contributor3_campaign": { - "type": ["null", "string"] - }, - "contributor3_touch_type": { - "type": ["null", "string"] - }, - "contributor3_touch_time": { - "type": ["null", "string"] - }, - "country_code": { - "type": ["null", "string"] - }, - "ip": { - "type": ["null", "string"] - }, - "region": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "wifi": { - "type": ["null", "boolean"] - }, - "operator": { - "type": ["null", "string"] - }, - "carrier": { - "type": ["null", "string"] - }, - "language": { - "type": ["null", "string"] - }, - "appsflyer_id": { - "type": ["null", "string"] - }, - "advertising_id": { - "type": ["null", "string"] - }, - "idfa": { - "type": ["null", "string"] - }, - "android_id": { - "type": ["null", "string"] - }, - "customer_user_id": { - "type": ["null", "string"] - }, - "imei": { - "type": ["null", "string"] - }, - "idfv": { - "type": ["null", "string"] - }, - "platform": { - "type": ["null", "string"] - }, - "device_type": { - "type": ["null", "string"] - }, - "os_version": { - "type": ["null", "string"] - }, - "app_version": { - "type": ["null", "string"] - }, - "sdk_version": { - "type": ["null", "string"] - }, - "app_id": { - "type": ["null", "string"] - }, - "app_name": { - "type": ["null", "string"] - }, - "bundle_id": { - "type": ["null", "string"] - }, - "af_attribution_lookback": { - "type": ["null", "string"] - }, - "event_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "event_name": { - "type": ["null", "string"] - }, - "event_value": { - "type": ["null", "string"] - }, - "event_revenue": { - "type": ["null", "string"] - }, - "event_revenue_usd": { - "type": ["null", "string"] - }, - "event_revenue_currency": { - "type": ["null", "string"] - }, - "is_receipt_validated": { - "type": ["null", "boolean"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "dma": { - "type": ["null", "string"] - }, - "af_reengagement_window": { - "type": ["null", "string"] - }, - "event_source": { - "type": ["null", "string"] - }, - "is_primary_attribution": { - "type": ["null", "boolean"] - }, - "af_sub_siteid": { - "type": ["null", "string"] - } - } - } - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/singer_rendered_catalog.json b/airbyte-integrations/connectors/source-appsflyer-singer/singer_rendered_catalog.json deleted file mode 100644 index b4697b490b4..00000000000 --- a/airbyte-integrations/connectors/source-appsflyer-singer/singer_rendered_catalog.json +++ /dev/null @@ -1,871 +0,0 @@ -{ - "streams": [ - { - "tap_stream_id": "installs", - "schema": { - "properties": { - "attributed_touch_type": { "type": ["null", "string"] }, - "attributed_touch_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "install_time": { "format": "date-time", "type": ["null", "string"] }, - "af_prt": { "type": ["null", "string"] }, - "media_source": { "type": ["null", "string"] }, - "is_retargeting": { "type": ["null", "boolean"] }, - "retargeting_conversion_type": { "type": ["null", "string"] }, - "af_channel": { "type": ["null", "string"] }, - "af_keywords": { "type": ["null", "string"] }, - "campaign": { "type": ["null", "string"] }, - "af_c_id": { "type": ["null", "string"] }, - "af_adset": { "type": ["null", "string"] }, - "af_adset_id": { "type": ["null", "string"] }, - "af_ad": { "type": ["null", "string"] }, - "af_ad_id": { "type": ["null", "string"] }, - "af_ad_type": { "type": ["null", "string"] }, - "af_siteid": { "type": ["null", "string"] }, - "af_sub1": { "type": ["null", "string"] }, - "af_sub2": { "type": ["null", "string"] }, - "af_sub3": { "type": ["null", "string"] }, - "af_sub4": { "type": ["null", "string"] }, - "af_sub5": { "type": ["null", "string"] }, - "http_referrer": { "type": ["null", "string"] }, - "original_url": { "type": ["null", "string"] }, - "user_agent": { "type": ["null", "string"] }, - "af_cost_model": { "type": ["null", "string"] }, - "af_cost_value": { "type": ["null", "string"] }, - "af_cost_currency": { "type": ["null", "string"] }, - "contributor1_af_prt": { "type": ["null", "string"] }, - "contributor1_media_source": { "type": ["null", "string"] }, - "contributor1_campaign": { "type": ["null", "string"] }, - "contributor1_touch_type": { "type": ["null", "string"] }, - "contributor1_touch_time": { "type": ["null", "string"] }, - "contributor2_af_prt": { "type": ["null", "string"] }, - "contributor2_media_source": { "type": ["null", "string"] }, - "contributor2_campaign": { "type": ["null", "string"] }, - "contributor2_touch_type": { "type": ["null", "string"] }, - "contributor2_touch_time": { "type": ["null", "string"] }, - "contributor3_af_prt": { "type": ["null", "string"] }, - "contributor3_media_source": { "type": ["null", "string"] }, - "contributor3_campaign": { "type": ["null", "string"] }, - "contributor3_touch_type": { "type": ["null", "string"] }, - "contributor3_touch_time": { "type": ["null", "string"] }, - "country_code": { "type": ["null", "string"] }, - "ip": { "type": ["null", "string"] }, - "region": { "type": ["null", "string"] }, - "state": { "type": ["null", "string"] }, - "city": { "type": ["null", "string"] }, - "wifi": { "type": ["null", "boolean"] }, - "operator": { "type": ["null", "string"] }, - "carrier": { "type": ["null", "string"] }, - "language": { "type": ["null", "string"] }, - "appsflyer_id": { "type": ["null", "string"] }, - "advertising_id": { "type": ["null", "string"] }, - "idfa": { "type": ["null", "string"] }, - "android_id": { "type": ["null", "string"] }, - "customer_user_id": { "type": ["null", "string"] }, - "imei": { "type": ["null", "string"] }, - "idfv": { "type": ["null", "string"] }, - "platform": { "type": ["null", "string"] }, - "device_type": { "type": ["null", "string"] }, - "os_version": { "type": ["null", "string"] }, - "app_version": { "type": ["null", "string"] }, - "sdk_version": { "type": ["null", "string"] }, - "app_id": { "type": ["null", "string"] }, - "app_name": { "type": ["null", "string"] }, - "bundle_id": { "type": ["null", "string"] }, - "af_attribution_lookback": { "type": ["null", "string"] }, - "event_time": { "format": "date-time", "type": ["null", "string"] }, - "event_name": { "type": ["null", "string"] }, - "event_value": { "type": ["null", "string"] }, - "event_revenue": { "type": ["null", "string"] }, - "event_revenue_usd": { "type": ["null", "string"] }, - "event_revenue_currency": { "type": ["null", "string"] }, - "is_receipt_validated": { "type": ["null", "boolean"] }, - "postal_code": { "type": ["null", "string"] }, - "dma": { "type": ["null", "string"] }, - "af_reengagement_window": { "type": ["null", "string"] }, - "event_source": { "type": ["null", "string"] }, - "is_primary_attribution": { "type": ["null", "boolean"] }, - "af_sub_siteid": { "type": ["null", "string"] } - }, - "type": "object" - }, - "stream": "installs", - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "event_time", - "event_name", - "appsflyer_id" - ], - "forced-replication-method": "INCREMENTAL", - "inclusion": "available", - "valid-replication-keys": ["event_time"] - } - }, - { - "breadcrumb": ["properties", "attributed_touch_type"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "attributed_touch_time"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "install_time"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_prt"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "media_source"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "is_retargeting"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "retargeting_conversion_type"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_channel"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_keywords"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "campaign"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_c_id"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_adset"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_adset_id"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_ad"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_ad_id"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_ad_type"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_siteid"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_sub1"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_sub2"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_sub3"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_sub4"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_sub5"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "http_referrer"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "original_url"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "user_agent"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_cost_model"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_cost_value"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_cost_currency"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor1_af_prt"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor1_media_source"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor1_campaign"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor1_touch_type"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor1_touch_time"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor2_af_prt"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor2_media_source"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor2_campaign"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor2_touch_type"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor2_touch_time"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor3_af_prt"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor3_media_source"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor3_campaign"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor3_touch_type"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "contributor3_touch_time"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "country_code"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "ip"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "region"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "state"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "city"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "wifi"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "operator"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "carrier"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "language"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "appsflyer_id"], - "metadata": { "inclusion": "automatic" } - }, - { - "breadcrumb": ["properties", "advertising_id"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "idfa"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "android_id"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "customer_user_id"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "imei"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "idfv"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "platform"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "device_type"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "os_version"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "app_version"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "sdk_version"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "app_id"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "app_name"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "bundle_id"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_attribution_lookback"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "event_time"], - "metadata": { "inclusion": "automatic" } - }, - { - "breadcrumb": ["properties", "event_name"], - "metadata": { "inclusion": "automatic" } - }, - { - "breadcrumb": ["properties", "event_value"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "event_revenue"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "event_revenue_usd"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "event_revenue_currency"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "is_receipt_validated"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "postal_code"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "dma"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_reengagement_window"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "event_source"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "is_primary_attribution"], - "metadata": { "inclusion": "available" } - }, - { - "breadcrumb": ["properties", "af_sub_siteid"], - "metadata": { "inclusion": "available" } - } - ] - }, - { - "tap_stream_id": "in_app_events", - "schema": { - "properties": { - "attributed_touch_type": { "type": ["null", "string"] }, - "attributed_touch_time": { - "format": "date-time", - "type": ["null", "string"] - }, - "install_time": { "format": "date-time", "type": ["null", "string"] }, - "af_prt": { "type": ["null", "string"] }, - "media_source": { "type": ["null", "string"] }, - "is_retargeting": { "type": ["null", "boolean"] }, - "retargeting_conversion_type": { "type": ["null", "string"] }, - "af_channel": { "type": ["null", "string"] }, - "af_keywords": { "type": ["null", "string"] }, - "campaign": { "type": ["null", "string"] }, - "af_c_id": { "type": ["null", "string"] }, - "af_adset": { "type": ["null", "string"] }, - "af_adset_id": { "type": ["null", "string"] }, - "af_ad": { "type": ["null", "string"] }, - "af_ad_id": { "type": ["null", "string"] }, - "af_ad_type": { "type": ["null", "string"] }, - "af_siteid": { "type": ["null", "string"] }, - "af_sub1": { "type": ["null", "string"] }, - "af_sub2": { "type": ["null", "string"] }, - "af_sub3": { "type": ["null", "string"] }, - "af_sub4": { "type": ["null", "string"] }, - "af_sub5": { "type": ["null", "string"] }, - "http_referrer": { "type": ["null", "string"] }, - "original_url": { "type": ["null", "string"] }, - "user_agent": { "type": ["null", "string"] }, - "af_cost_model": { "type": ["null", "string"] }, - "af_cost_value": { "type": ["null", "string"] }, - "af_cost_currency": { "type": ["null", "string"] }, - "contributor1_af_prt": { "type": ["null", "string"] }, - "contributor1_media_source": { "type": ["null", "string"] }, - "contributor1_campaign": { "type": ["null", "string"] }, - "contributor1_touch_type": { "type": ["null", "string"] }, - "contributor1_touch_time": { "type": ["null", "string"] }, - "contributor2_af_prt": { "type": ["null", "string"] }, - "contributor2_media_source": { "type": ["null", "string"] }, - "contributor2_campaign": { "type": ["null", "string"] }, - "contributor2_touch_type": { "type": ["null", "string"] }, - "contributor2_touch_time": { "type": ["null", "string"] }, - "contributor3_af_prt": { "type": ["null", "string"] }, - "contributor3_media_source": { "type": ["null", "string"] }, - "contributor3_campaign": { "type": ["null", "string"] }, - "contributor3_touch_type": { "type": ["null", "string"] }, - "contributor3_touch_time": { "type": ["null", "string"] }, - "country_code": { "type": ["null", "string"] }, - "ip": { "type": ["null", "string"] }, - "region": { "type": ["null", "string"] }, - "state": { "type": ["null", "string"] }, - "city": { "type": ["null", "string"] }, - "wifi": { "type": ["null", "boolean"] }, - "operator": { "type": ["null", "string"] }, - "carrier": { "type": ["null", "string"] }, - "language": { "type": ["null", "string"] }, - "appsflyer_id": { "type": ["null", "string"] }, - "advertising_id": { "type": ["null", "string"] }, - "idfa": { "type": ["null", "string"] }, - "android_id": { "type": ["null", "string"] }, - "customer_user_id": { "type": ["null", "string"] }, - "imei": { "type": ["null", "string"] }, - "idfv": { "type": ["null", "string"] }, - "platform": { "type": ["null", "string"] }, - "device_type": { "type": ["null", "string"] }, - "os_version": { "type": ["null", "string"] }, - "app_version": { "type": ["null", "string"] }, - "sdk_version": { "type": ["null", "string"] }, - "app_id": { "type": ["null", "string"] }, - "app_name": { "type": ["null", "string"] }, - "bundle_id": { "type": ["null", "string"] }, - "af_attribution_lookback": { "type": ["null", "string"] }, - "event_time": { "format": "date-time", "type": ["null", "string"] }, - "event_name": { "type": ["null", "string"] }, - "event_value": { "type": ["null", "string"] }, - "event_revenue": { "type": ["null", "string"] }, - "event_revenue_usd": { "type": ["null", "string"] }, - "event_revenue_currency": { "type": ["null", "string"] }, - "is_receipt_validated": { "type": ["null", "boolean"] }, - "postal_code": { "type": ["null", "string"] }, - "dma": { "type": ["null", "string"] }, - "af_reengagement_window": { "type": ["null", "string"] }, - "event_source": { "type": ["null", "string"] }, - "is_primary_attribution": { "type": ["null", "boolean"] }, - "af_sub_siteid": { "type": ["null", "string"] } - }, - "type": "object", - "selected": true - }, - "stream": "in_app_events", - "metadata": [ - { - "breadcrumb": [], - "metadata": { - "table-key-properties": [ - "event_time", - "event_name", - "appsflyer_id" - ], - "forced-replication-method": "INCREMENTAL", - "inclusion": "available", - "valid-replication-keys": ["event_time"], - "selected": true, - "replication-method": "INCREMENTAL" - } - }, - { - "breadcrumb": ["properties", "attributed_touch_type"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "attributed_touch_time"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "install_time"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_prt"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "media_source"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "is_retargeting"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "retargeting_conversion_type"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_channel"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_keywords"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "campaign"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_c_id"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_adset"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_adset_id"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_ad"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_ad_id"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_ad_type"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_siteid"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_sub1"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_sub2"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_sub3"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_sub4"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_sub5"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "http_referrer"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "original_url"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "user_agent"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_cost_model"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_cost_value"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_cost_currency"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor1_af_prt"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor1_media_source"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor1_campaign"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor1_touch_type"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor1_touch_time"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor2_af_prt"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor2_media_source"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor2_campaign"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor2_touch_type"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor2_touch_time"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor3_af_prt"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor3_media_source"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor3_campaign"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor3_touch_type"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "contributor3_touch_time"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "country_code"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "ip"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "region"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "state"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "city"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "wifi"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "operator"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "carrier"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "language"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "appsflyer_id"], - "metadata": { "inclusion": "automatic", "selected": true } - }, - { - "breadcrumb": ["properties", "advertising_id"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "idfa"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "android_id"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "customer_user_id"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "imei"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "idfv"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "platform"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "device_type"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "os_version"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "app_version"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "sdk_version"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "app_id"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "app_name"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "bundle_id"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_attribution_lookback"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "event_time"], - "metadata": { "inclusion": "automatic", "selected": true } - }, - { - "breadcrumb": ["properties", "event_name"], - "metadata": { "inclusion": "automatic", "selected": true } - }, - { - "breadcrumb": ["properties", "event_value"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "event_revenue"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "event_revenue_usd"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "event_revenue_currency"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "is_receipt_validated"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "postal_code"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "dma"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_reengagement_window"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "event_source"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "is_primary_attribution"], - "metadata": { "inclusion": "available", "selected": true } - }, - { - "breadcrumb": ["properties", "af_sub_siteid"], - "metadata": { "inclusion": "available", "selected": true } - } - ] - } - ] -} diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/source_appsflyer_singer/__init__.py b/airbyte-integrations/connectors/source-appsflyer-singer/source_appsflyer_singer/__init__.py deleted file mode 100644 index 254859504ca..00000000000 --- a/airbyte-integrations/connectors/source-appsflyer-singer/source_appsflyer_singer/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .source import SourceAppsflyerSinger - -__all__ = ["SourceAppsflyerSinger"] diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/source_appsflyer_singer/source.py b/airbyte-integrations/connectors/source-appsflyer-singer/source_appsflyer_singer/source.py deleted file mode 100644 index df60fc06a83..00000000000 --- a/airbyte-integrations/connectors/source-appsflyer-singer/source_appsflyer_singer/source.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import json -from datetime import date, timedelta - -import requests -from airbyte_protocol import AirbyteConnectionStatus -from base_python import AirbyteLogger -from base_singer import SingerSource, Status - - -class SourceAppsflyerSinger(SingerSource): - TAP_CMD = "tap-appsflyer" - - def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus: - """ - Tests if the input configuration can be used to successfully connect to the integration - e.g: if a provided Stripe API token can be used to connect to the Stripe API. - - :param logger: Logging object to display debug/info/error to the logs - (logs will not be accessible via airbyte UI if they are not passed to this logger) - :param config_path: Path to the file containing the configuration json config - :param config: Json object containing the configuration of this source, content of this json is as specified in - the properties of the spec.json file - - :return: AirbyteConnectionStatus indicating a Success or Failure - """ - try: - test_date = (date.today() - timedelta(days=2)).strftime("%Y-%m-%d %H:%M") - params = {"from": test_date, "to": test_date, "api_token": config["api_token"]} - - base_url = "https://hq.appsflyer.com" - test_endpoint = "/export/{}/installs_report/v5".format(config["app_id"]) - - url = base_url + test_endpoint - - logger.info("GET {}".format(url)) - resp = requests.get(url, params=params) - - if resp.status_code == 200: - return AirbyteConnectionStatus(status=Status.SUCCEEDED) - else: - return AirbyteConnectionStatus( - status=Status.FAILED, - message=f"An exception occurred: Status Code: {0}, content: {1}".format(resp.status_code, resp.content), - ) - except Exception as e: - return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {str(e)}") - - def discover_cmd(self, logger: AirbyteLogger, config_path: str) -> str: - """ - Return the string commands to invoke the tap with the --discover flag and the right configuration options - """ - return f"{self.TAP_CMD} -c {config_path} --discover" - - def read_cmd(self, logger: AirbyteLogger, config_path: str, catalog_path: str, state_path: str = None) -> str: - """ - Return the string commands to invoke the tap with the right configuration options to read data from the source - """ - config_option = f"--config {config_path}" - properties_option = f"--properties {catalog_path}" - state_option = f"--state {state_path}" if state_path else "" - return f"{self.TAP_CMD} {config_option} {properties_option} {state_option}" diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/source_appsflyer_singer/spec.json b/airbyte-integrations/connectors/source-appsflyer-singer/source_appsflyer_singer/spec.json deleted file mode 100644 index 0de7ad93dfa..00000000000 --- a/airbyte-integrations/connectors/source-appsflyer-singer/source_appsflyer_singer/spec.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/appflyer", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Appsflyer Singer Spec", - "type": "object", - "required": ["app_id", "api_token"], - "additionalProperties": false, - "properties": { - "app_id": { - "type": "string", - "description": "app_id is an app identifier found in AppsFlyer." - }, - "api_token": { - "type": "string", - "description": "api_token is the Pull API token for authentication." - } - } - } -} diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-appsflyer-singer/unit_tests/unit_test.py deleted file mode 100644 index d685046c40b..00000000000 --- a/airbyte-integrations/connectors/source-appsflyer-singer/unit_tests/unit_test.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - - -def test_example_method(): - assert True diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/__init__.py b/airbyte-integrations/connectors/source-file/integration_tests/__init__.py similarity index 91% rename from airbyte-integrations/connectors/source-hubspot/source_hubspot/__init__.py rename to airbyte-integrations/connectors/source-file/integration_tests/__init__.py index 2964c78d202..e4383c2b7b0 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/__init__.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/__init__.py @@ -22,6 +22,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from .source import SourceHubspot +from .standard_source_test import SourceFileStandardTest -__all__ = ["SourceHubspot"] +__all__ = ["SourceFileStandardTest"] diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-file/integration_tests/standard_source_test.py similarity index 89% rename from airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py rename to airbyte-integrations/connectors/source-file/integration_tests/standard_source_test.py index bc4c895fec8..c52bfba53ce 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/standard_source_test.py @@ -22,10 +22,8 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from base_python import BaseSource - -from .client import Client +from base_python_test import DefaultStandardSourceTest -class SourceHubspot(BaseSource): - client_class = Client +class SourceFileStandardTest(DefaultStandardSourceTest): + pass diff --git a/airbyte-integrations/connectors/source-file/setup.py b/airbyte-integrations/connectors/source-file/setup.py index 74db06f13f4..f5181fb3f0c 100644 --- a/airbyte-integrations/connectors/source-file/setup.py +++ b/airbyte-integrations/connectors/source-file/setup.py @@ -34,7 +34,7 @@ MAIN_REQUIREMENTS = [ "paramiko==2.7.2", "s3fs==0.5.2", "smart-open[all]==4.1.2", - "lxml==4.6.3", + "lxml==4.6.2", "html5lib==1.1", "beautifulsoup4==4.9.3", "pyarrow==3.0.0", diff --git a/airbyte-integrations/connectors/source-gitlab-singer/.dockerignore b/airbyte-integrations/connectors/source-gitlab-singer/.dockerignore deleted file mode 100644 index a1f7ef84d0f..00000000000 --- a/airbyte-integrations/connectors/source-gitlab-singer/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!Dockerfile.test -!source_gitlab_singer -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-gitlab-singer/.gitignore b/airbyte-integrations/connectors/source-gitlab-singer/.gitignore deleted file mode 100644 index 29fffc6a50c..00000000000 --- a/airbyte-integrations/connectors/source-gitlab-singer/.gitignore +++ /dev/null @@ -1 +0,0 @@ -NEW_SOURCE_CHECKLIST.md diff --git a/airbyte-integrations/connectors/source-gitlab-singer/Dockerfile b/airbyte-integrations/connectors/source-gitlab-singer/Dockerfile deleted file mode 100644 index c69afc09438..00000000000 --- a/airbyte-integrations/connectors/source-gitlab-singer/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM airbyte/integration-base-singer:0.1.0 - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash -y gcc && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="source_gitlab_singer" -ENV AIRBYTE_IMPL_MODULE="source_gitlab_singer" -ENV AIRBYTE_IMPL_PATH="SourceGitlabSinger" - -WORKDIR /airbyte/integration_code -COPY $CODE_PATH ./$CODE_PATH -COPY setup.py ./ -RUN pip install . - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-gitlab-singer diff --git a/airbyte-integrations/connectors/source-gitlab-singer/main_dev.py b/airbyte-integrations/connectors/source-gitlab-singer/main_dev.py deleted file mode 100644 index 11e76547034..00000000000 --- a/airbyte-integrations/connectors/source-gitlab-singer/main_dev.py +++ /dev/null @@ -1,32 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import sys - -from base_python.entrypoint import launch -from source_gitlab_singer import SourceGitlabSinger - -if __name__ == "__main__": - source = SourceGitlabSinger() - launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gitlab-singer/requirements.txt b/airbyte-integrations/connectors/source-gitlab-singer/requirements.txt deleted file mode 100644 index 484253df1df..00000000000 --- a/airbyte-integrations/connectors/source-gitlab-singer/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/airbyte-protocol --e ../../bases/base-singer --e ../../bases/base-python --e ../../bases/base-python-test --e . diff --git a/airbyte-integrations/connectors/source-gitlab-singer/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-gitlab-singer/sample_files/configured_catalog.json deleted file mode 100644 index 327a49ff707..00000000000 --- a/airbyte-integrations/connectors/source-gitlab-singer/sample_files/configured_catalog.json +++ /dev/null @@ -1,1493 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "projects", - "json_schema": { - "properties": { - "approvals_before_merge": { - "type": ["null", "integer"] - }, - "archived": { - "type": ["null", "boolean"] - }, - "avatar_url": { - "type": ["null", "string"] - }, - "builds_enabled": { - "type": ["null", "boolean"] - }, - "container_registry_enabled": { - "type": ["null", "boolean"] - }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "creator_id": { - "type": ["null", "integer"] - }, - "default_branch": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "forks_count": { - "type": ["null", "integer"] - }, - "http_url_to_repo": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "issues_enabled": { - "type": ["null", "boolean"] - }, - "last_activity_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "lfs_enabled": { - "type": ["null", "boolean"] - }, - "merge_requests_enabled": { - "type": ["null", "boolean"] - }, - "merge_method": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "name_with_namespace": { - "type": ["null", "string"] - }, - "namespace": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "kind": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "path": { - "type": ["null", "string"] - }, - "full_path": { - "type": ["null", "string"] - }, - "parent_id": { - "type": ["null", "integer"] - } - }, - "type": "object" - }, - "only_allow_merge_if_all_discussions_are_resolved": { - "type": ["null", "boolean"] - }, - "only_allow_merge_if_build_succeeds": { - "type": ["null", "boolean"] - }, - "open_issues_count": { - "type": ["null", "integer"] - }, - "owner_id": { - "type": ["null", "integer"] - }, - "path": { - "type": ["null", "string"] - }, - "path_with_namespace": { - "type": ["null", "string"] - }, - "permissions": { - "properties": { - "group_access": { - "anyOf": [ - { - "type": "object", - "properties": { - "access_level": { - "type": ["null", "integer"] - }, - "notification_level": { - "type": ["null", "integer"] - } - } - }, - { - "type": "null" - } - ] - }, - "project_access": { - "anyOf": [ - { - "type": "object", - "properties": { - "access_level": { - "type": ["null", "integer"] - }, - "notification_level": { - "type": ["null", "integer"] - } - } - }, - { - "type": "null" - } - ] - } - }, - "type": "object" - }, - "public": { - "type": ["null", "boolean"] - }, - "public_builds": { - "type": ["null", "boolean"] - }, - "request_access_enabled": { - "type": ["null", "boolean"] - }, - "shared_runners_enabled": { - "type": ["null", "boolean"] - }, - "shared_with_groups": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "object", - "properties": { - "group_id": { - "type": ["null", "integer"] - }, - "group_name": { - "type": ["null", "string"] - }, - "group_access_level": { - "type": ["null", "integer"] - } - } - } - }, - { - "type": "null" - } - ] - }, - "snippets_enabled": { - "type": ["null", "boolean"] - }, - "ssh_url_to_repo": { - "type": ["null", "string"] - }, - "star_count": { - "type": ["null", "integer"] - }, - "statistics": { - "properties": { - "commit_count": { - "type": ["null", "integer"] - }, - "storage_size": { - "type": ["null", "integer"] - }, - "repository_size": { - "type": ["null", "integer"] - }, - "lfs_objects_size": { - "type": ["null", "integer"] - }, - "job_artifacts_size": { - "type": ["null", "integer"] - } - }, - "type": "object" - }, - "tag_list": { - "anyOf": [ - { - "type": "array", - "items": { - "type": ["null", "string"] - } - }, - { - "type": "null" - } - ] - }, - "visibility_level": { - "type": ["null", "integer"] - }, - "visibility": { - "type": ["null", "string"] - }, - "web_url": { - "type": ["null", "string"] - }, - "wiki_enabled": { - "type": ["null", "boolean"] - } - }, - "type": "object" - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["last_activity_at"] - }, - "sync_mode": "incremental", - "cursor_field": ["last_activity_at"] - }, - { - "stream": { - "name": "branches", - "json_schema": { - "properties": { - "project_id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "merged": { - "type": ["null", "boolean"] - }, - "protected": { - "type": ["null", "boolean"] - }, - "default": { - "type": ["null", "boolean"] - }, - "developers_can_push": { - "type": ["null", "boolean"] - }, - "developers_can_merge": { - "type": ["null", "boolean"] - }, - "can_push": { - "type": ["null", "boolean"] - }, - "commit_id": { - "type": ["null", "string"] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - }, - { - "stream": { - "name": "commits", - "json_schema": { - "properties": { - "id": { - "type": ["null", "string"] - }, - "project_id": { - "type": ["null", "integer"] - }, - "short_id": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "author_name": { - "type": ["null", "string"] - }, - "author_email": { - "type": ["null", "string"] - }, - "authored_date": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "committer_name": { - "type": ["null", "string"] - }, - "committer_email": { - "type": ["null", "string"] - }, - "committed_date": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "message": { - "type": ["null", "string"] - }, - "allow_failure": { - "type": ["null", "boolean"] - }, - "parent_ids": { - "anyOf": [ - { - "type": "array", - "items": { - "type": ["null", "string"] - } - }, - { - "type": "null" - } - ] - }, - "stats": { - "properties": { - "additions": { - "type": ["null", "integer"] - }, - "deletions": { - "type": ["null", "integer"] - }, - "total": { - "type": ["null", "integer"] - } - }, - "type": "object" - } - }, - "type": "object" - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created_at"] - }, - "sync_mode": "incremental", - "cursor_field": ["created_at"] - }, - { - "stream": { - "name": "issues", - "json_schema": { - "properties": { - "project_id": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "iid": { - "type": ["null", "integer"] - }, - "milestone_id": { - "type": ["null", "integer"] - }, - "author_id": { - "type": ["null", "integer"] - }, - "assignee_id": { - "type": ["null", "integer"] - }, - "assignees": { - "items": { - "type": ["null", "integer"] - }, - "type": "array" - }, - "closed_by_id": { - "type": ["null", "integer"] - }, - "title": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "labels": { - "items": { - "type": ["null", "string"] - }, - "type": "array" - }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "updated_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "closed_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "subscribed": { - "type": ["null", "boolean"] - }, - "upvotes": { - "type": ["null", "integer"] - }, - "downvotes": { - "type": ["null", "integer"] - }, - "merge_requests_count": { - "type": ["null", "integer"] - }, - "user_notes_count": { - "type": ["null", "integer"] - }, - "due_date": { - "type": ["null", "string"] - }, - "weight": { - "type": ["null", "integer"] - }, - "web_url": { - "type": ["null", "string"] - }, - "confidential": { - "type": ["null", "boolean"] - }, - "discussion_locked": { - "type": ["null", "boolean"] - }, - "has_tasks": { - "type": ["null", "boolean"] - }, - "task_status": { - "type": ["null", "string"] - }, - "time_estimate": { - "type": ["null", "integer"] - }, - "total_time_spent": { - "type": ["null", "integer"] - }, - "human_time_estimate": { - "type": ["null", "string"] - }, - "human_total_time_spent": { - "type": ["null", "string"] - } - }, - "type": "object" - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_at"] - }, - "sync_mode": "incremental", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "jobs", - "json_schema": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "project_id": { - "type": ["null", "integer"] - }, - "user_id": { - "type": ["null", "integer"] - }, - "commit_id": { - "type": ["null", "string"] - }, - "pipeline_id": { - "type": ["null", "integer"] - }, - "runner_id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "stage": { - "type": ["null", "string"] - }, - "ref": { - "type": ["null", "string"] - }, - "status": { - "anyOf": [ - { - "type": "string", - "enum": [ - "created", - "waiting_for_resource", - "pending", - "running", - "failed", - "success", - "canceled", - "skipped", - "manual" - ] - }, - { - "type": "null" - } - ] - }, - "duration": { - "anyOf": [ - { - "type": "number", - "format": "float" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number", - "format": "float" - }, - { - "type": "null" - } - ] - }, - "allow_failure": { - "type": ["null", "boolean"] - }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "started_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "finished_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "artifacts_expire_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - }, - { - "stream": { - "name": "merge_requests", - "json_schema": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "iid": { - "type": ["null", "integer"] - }, - "project_id": { - "type": ["null", "integer"] - }, - "milestone_id": { - "type": ["null", "integer"] - }, - "author_id": { - "type": ["null", "integer"] - }, - "assignee_id": { - "type": ["null", "integer"] - }, - "assignees": { - "items": { - "type": ["null", "integer"] - }, - "type": "array" - }, - "merged_by_id": { - "type": ["null", "integer"] - }, - "closed_by_id": { - "type": ["null", "integer"] - }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "updated_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "merged_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "closed_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "title": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "target_project_id": { - "type": ["null", "integer"] - }, - "target_branch": { - "type": ["null", "string"] - }, - "source_project_id": { - "type": ["null", "integer"] - }, - "source_branch": { - "type": ["null", "string"] - }, - "upvotes": { - "type": ["null", "integer"] - }, - "downvotes": { - "type": ["null", "integer"] - }, - "labels": { - "items": { - "type": ["null", "string"] - }, - "type": "array" - }, - "work_in_progress": { - "type": ["null", "boolean"] - }, - "merge_when_pipeline_succeeds": { - "type": ["null", "boolean"] - }, - "merge_status": { - "type": ["null", "string"] - }, - "sha": { - "type": ["null", "string"] - }, - "user_notes_count": { - "type": ["null", "integer"] - }, - "should_remove_source_branch": { - "type": ["null", "boolean"] - }, - "force_remove_source_branch": { - "type": ["null", "boolean"] - }, - "allow_collaboration": { - "type": ["null", "boolean"] - }, - "allow_maintainer_to_push": { - "type": ["null", "boolean"] - }, - "web_url": { - "type": ["null", "string"] - }, - "squash": { - "type": ["null", "boolean"] - }, - "time_estimate": { - "type": ["null", "integer"] - }, - "total_time_spent": { - "type": ["null", "integer"] - }, - "human_time_estimate": { - "type": ["null", "string"] - }, - "human_total_time_spent": { - "type": ["null", "string"] - } - }, - "type": "object" - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_at"] - }, - "sync_mode": "incremental", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "merge_request_commits", - "json_schema": { - "properties": { - "project_id": { - "type": ["null", "integer"] - }, - "merge_request_iid": { - "type": ["null", "integer"] - }, - "commit_id": { - "type": ["null", "string"] - }, - "commit_short_id": { - "type": ["null", "string"] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - }, - { - "stream": { - "name": "project_milestones", - "json_schema": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "iid": { - "type": ["null", "integer"] - }, - "project_id": { - "type": ["null", "integer"] - }, - "group_id": { - "type": ["null", "integer"] - }, - "title": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "start_date": { - "type": ["null", "string"] - }, - "due_date": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "updated_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - }, - { - "stream": { - "name": "group_milestones", - "json_schema": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "iid": { - "type": ["null", "integer"] - }, - "project_id": { - "type": ["null", "integer"] - }, - "group_id": { - "type": ["null", "integer"] - }, - "title": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "start_date": { - "type": ["null", "string"] - }, - "due_date": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "updated_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - }, - { - "stream": { - "name": "users", - "json_schema": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "username": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "avatar_url": { - "type": ["null", "string"] - }, - "web_url": { - "type": ["null", "string"] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - }, - { - "stream": { - "name": "groups", - "json_schema": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "path": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "visibility_level": { - "type": ["null", "integer"] - }, - "lfs_enabled": { - "type": ["null", "boolean"] - }, - "avatar_url": { - "type": ["null", "string"] - }, - "web_url": { - "type": ["null", "string"] - }, - "request_access_enabled": { - "type": ["null", "boolean"] - }, - "full_name": { - "type": ["null", "string"] - }, - "full_path": { - "type": ["null", "string"] - }, - "projects": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - } - } - } - }, - { - "type": "null" - } - ] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - }, - { - "stream": { - "name": "project_members", - "json_schema": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "project_id": { - "type": ["null", "integer"] - }, - "user_id": { - "type": ["null", "integer"] - }, - "access_level": { - "type": ["null", "integer"] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - }, - { - "stream": { - "name": "group_members", - "json_schema": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "group_id": { - "type": ["null", "integer"] - }, - "user_id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "username": { - "type": ["null", "string"] - }, - "access_level": { - "type": ["null", "integer"] - }, - "expires_at": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - }, - { - "stream": { - "name": "releases", - "json_schema": { - "properties": { - "tag_name": { - "type": ["null", "string"] - }, - "author_id": { - "type": ["null", "integer"] - }, - "commit_id": { - "type": ["null", "string"] - }, - "project_id": { - "type": ["null", "integer"] - }, - "description": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "released_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - }, - { - "stream": { - "name": "tags", - "json_schema": { - "properties": { - "commit_id": { - "type": ["null", "string"] - }, - "project_id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "target": { - "type": ["null", "string"] - }, - "message": { - "type": ["null", "string"] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - }, - { - "stream": { - "name": "project_labels", - "json_schema": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "project_id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "color": { - "type": ["null", "string"] - }, - "text_color": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "open_issues_count": { - "type": ["null", "integer"] - }, - "closed_issues_count": { - "type": ["null", "integer"] - }, - "open_merge_requests_count": { - "type": ["null", "integer"] - }, - "subscribed": { - "type": ["null", "boolean"] - }, - "priority": { - "type": ["null", "integer"] - }, - "is_project_label": { - "type": ["null", "boolean"] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - }, - { - "stream": { - "name": "group_labels", - "json_schema": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "group_id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "color": { - "type": ["null", "string"] - }, - "text_color": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "open_issues_count": { - "type": ["null", "integer"] - }, - "closed_issues_count": { - "type": ["null", "integer"] - }, - "open_merge_requests_count": { - "type": ["null", "integer"] - }, - "subscribed": { - "type": ["null", "boolean"] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - }, - { - "stream": { - "name": "pipelines", - "json_schema": { - "properties": { - "id": { - "type": ["null", "integer"] - }, - "status": { - "type": ["null", "string"] - }, - "ref": { - "type": ["null", "string"] - }, - "sha": { - "type": ["null", "string"] - }, - "web_url": { - "type": ["null", "string"] - }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "updated_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - } - }, - "type": "object" - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_at"] - }, - "sync_mode": "incremental", - "cursor_field": ["updated_at"] - }, - { - "stream": { - "name": "pipelines_extended", - "json_schema": { - "properties": { - "project_id": { - "type": ["null", "integer"] - }, - "id": { - "type": ["null", "integer"] - }, - "status": { - "type": ["null", "string"] - }, - "ref": { - "type": ["null", "string"] - }, - "sha": { - "type": ["null", "string"] - }, - "before_sha": { - "type": ["null", "string"] - }, - "tag": { - "type": ["null", "boolean"] - }, - "yaml_errors": { - "type": ["null", "string"] - }, - "user": { - "properties": { - "name": { - "type": "string" - }, - "username": { - "type": "string" - }, - "id": { - "type": "integer" - }, - "state": { - "type": "string" - } - }, - "type": "object" - }, - "created_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "updated_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "started_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "finished_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "committed_at": { - "anyOf": [ - { - "type": "string", - "format": "date-time" - }, - { - "type": "null" - } - ] - }, - "duration": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ] - }, - "coverage": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ] - }, - "web_url": { - "type": ["null", "string"] - } - }, - "type": "object" - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-gitlab-singer/sample_files/sample_config.json b/airbyte-integrations/connectors/source-gitlab-singer/sample_files/sample_config.json deleted file mode 100644 index 67b9ea9c91e..00000000000 --- a/airbyte-integrations/connectors/source-gitlab-singer/sample_files/sample_config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "api_url": "gitlab.com", - "private_token": ">", - "groups": " ", - "projects": " ", - "start_date": "2021-01-01T00:00:00Z", - "ultimate_license": false, - "fetch_merge_request_commits": false, - "fetch_pipelines_extended": false -} diff --git a/airbyte-integrations/connectors/source-gitlab-singer/setup.py b/airbyte-integrations/connectors/source-gitlab-singer/setup.py deleted file mode 100644 index 22347644f04..00000000000 --- a/airbyte-integrations/connectors/source-gitlab-singer/setup.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from setuptools import find_packages, setup - -setup( - name="source_gitlab_singer", - description="Source implementation for Gitlab, built on the Singer tap implementation.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=[ - "tap-gitlab @ https://gitlab.com/meltano/tap-gitlab/-/archive/v0.9.14/tap-gitlab-v0.9.14.tar", - "airbyte-protocol", - "base-singer", - "base-python", - "pytest==6.1.2", - ], - package_data={"": ["*.json"]}, -) diff --git a/airbyte-integrations/connectors/source-gitlab-singer/source_gitlab_singer/source.py b/airbyte-integrations/connectors/source-gitlab-singer/source_gitlab_singer/source.py deleted file mode 100644 index 40e15c299ee..00000000000 --- a/airbyte-integrations/connectors/source-gitlab-singer/source_gitlab_singer/source.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import requests -from base_python import AirbyteLogger -from base_singer import BaseSingerSource -from requests import HTTPError - - -class SourceGitlabSinger(BaseSingerSource): - """ - Gitlab API Reference: https://docs.gitlab.com/ee/api/README.html - """ - - tap_cmd = "tap-gitlab" - tap_name = "Gitlab API" - api_error = HTTPError, Exception - - def transform_config(self, raw_config): - return {**raw_config, "api_url": f'https://{raw_config["api_url"]}/'} - - def try_connect(self, logger: AirbyteLogger, config: dict): - if not config["projects"] and not config["groups"]: - raise Exception("Either groups or projects need to be provided for connect to Gitlab API") - response = requests.get(f"{config['api_url']}api/v4/projects", params={"private_token": config["private_token"]}) - response.raise_for_status() diff --git a/airbyte-integrations/connectors/source-gitlab-singer/source_gitlab_singer/spec.json b/airbyte-integrations/connectors/source-gitlab-singer/source_gitlab_singer/spec.json deleted file mode 100644 index 073460cff3a..00000000000 --- a/airbyte-integrations/connectors/source-gitlab-singer/source_gitlab_singer/spec.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/gitlab", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Gitlab Singer Spec", - "type": "object", - "required": [ - "api_url", - "private_token", - "start_date", - "ultimate_license", - "fetch_merge_request_commits", - "fetch_pipelines_extended" - ], - "additionalProperties": false, - "properties": { - "api_url": { - "type": "string", - "examples": ["gitlab.com"], - "description": "Please enter your basic URL from Gitlab instance", - "pattern": ["^gitlab[a-zA-Z0-9._-]*\\.com$"] - }, - "private_token": { - "type": "string", - "description": "Log into your Gitlab account and then generate a personal Access Token.", - "airbyte_secret": true - }, - "groups": { - "type": "string", - "examples": ["airbyte.io"], - "description": "Space-delimited list of groups. e.g. airbyte.io" - }, - "projects": { - "type": "string", - "examples": ["airbyte.io/documentation"], - "description": "Space-delimited list of projects. e.g. airbyte.io/documentation meltano/tap-gitlab" - }, - "start_date": { - "type": "string", - "description": "The date from which you'd like to replicate data for Gitlab API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "examples": ["2021-03-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - }, - "ultimate_license": { - "type": "boolean", - "description": "If this parameter is set to True, then the GitLab account used has access to the GitLab Ultimate or GitLab.com Gold features. It will enable fetching Epics and Epic Issues.", - "default": false - }, - "fetch_merge_request_commits": { - "type": "boolean", - "description": "If this parameter is set to True, then for each Merge Request, also fetch the MR's commits and create the join table merge_request_commits with the Merge Request and related Commit IDs.", - "default": false - }, - "fetch_pipelines_extended": { - "type": "boolean", - "description": "If you enable this option, then for every Pipeline fetched with sync_pipelines (which returns N pages containing all pipelines per project), also fetch extended details of each of these pipelines with sync_pipelines_extended.", - "default": false - } - }, - "anyOf": [ - { - "required": ["groups"] - }, - { - "required": ["projects"] - } - ] - } -} diff --git a/airbyte-integrations/connectors/source-gitlab-singer/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-gitlab-singer/unit_tests/unit_test.py deleted file mode 100644 index d685046c40b..00000000000 --- a/airbyte-integrations/connectors/source-gitlab-singer/unit_tests/unit_test.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - - -def test_example_method(): - assert True diff --git a/airbyte-integrations/connectors/source-hubspot-singer/.dockerignore b/airbyte-integrations/connectors/source-hubspot-singer/.dockerignore new file mode 100644 index 00000000000..378eac25d31 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/.dockerignore @@ -0,0 +1 @@ +build diff --git a/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile b/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile new file mode 100644 index 00000000000..09f648919cd --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile @@ -0,0 +1,20 @@ +FROM airbyte/integration-base-singer:0.1.0 + +RUN apt-get update && apt-get install -y \ + bash \ + && rm -rf /var/lib/apt/lists/* + +ENV CODE_PATH="source_hubspot_singer" +ENV AIRBYTE_IMPL_MODULE="source_hubspot_singer" +ENV AIRBYTE_IMPL_PATH="SourceHubspotSinger" + +LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.name=airbyte/source-hubspot-singer + +WORKDIR /airbyte/integration_code +COPY $CODE_PATH ./$CODE_PATH +COPY setup.py ./ +RUN pip install . + +WORKDIR /airbyte + diff --git a/airbyte-integrations/connectors/source-gitlab-singer/README.md b/airbyte-integrations/connectors/source-hubspot-singer/README.md similarity index 81% rename from airbyte-integrations/connectors/source-gitlab-singer/README.md rename to airbyte-integrations/connectors/source-hubspot-singer/README.md index 06149d60537..7544b8ca7c2 100644 --- a/airbyte-integrations/connectors/source-gitlab-singer/README.md +++ b/airbyte-integrations/connectors/source-hubspot-singer/README.md @@ -1,7 +1,7 @@ -# Source Gitlab Singer +# Source Hubspot Singer -This is the repository for the Gitlab source connector, based on a Singer tap. -For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/sources/gitlab). +This is the repository for the Hubspot source connector, based on a Singer tap. +For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/sources/hubspot). ## Local development @@ -30,16 +30,16 @@ should work as you expect. #### Building via Gradle From the Airbyte repository root, run: ``` -./gradlew :airbyte-integrations:connectors:source-gitlab:build +./gradlew :airbyte-integrations:connectors:source-hubspot:build ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/gitlab) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_gitlab_singer/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/hubspot) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hubspot_singer/spec.json` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source gitlab test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source hubspot test creds` and place them into `secrets/config.json`. ### Locally running the connector @@ -75,12 +75,12 @@ python -m pytest unit_tests #### Build First, make sure you build the latest Docker image: ``` -docker build . -t airbyte/gitlab-singer:dev +docker build . -t airbyte/hubspot-singer:dev ``` You can also build the connector image via Gradle: ``` -./gradlew :airbyte-integrations:connectors:source-gitlab:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-hubspot:airbyteDocker ``` When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in the Dockerfile. @@ -88,14 +88,14 @@ the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-gitlab-singer:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gitlab-singer:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gitlab-singer:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-gitlab-singer:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm airbyte/source-hubspot-singer:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubspot-singer:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubspot-singer:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-hubspot-singer:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json ``` ### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-gitlab-singer:integrationTest` to run the standard integration test suite. +1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-hubspot-singer:integrationTest` to run the standard integration test suite. 1. To run additional integration tests, create a directory `integration_tests` which contain your tests and run them with `pytest integration_tests`. Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. diff --git a/airbyte-integrations/connectors/source-gitlab-singer/build.gradle b/airbyte-integrations/connectors/source-hubspot-singer/build.gradle similarity index 84% rename from airbyte-integrations/connectors/source-gitlab-singer/build.gradle rename to airbyte-integrations/connectors/source-hubspot-singer/build.gradle index 039e6082498..ae533e155bc 100644 --- a/airbyte-integrations/connectors/source-gitlab-singer/build.gradle +++ b/airbyte-integrations/connectors/source-hubspot-singer/build.gradle @@ -5,16 +5,17 @@ plugins { } airbytePython { - moduleDirectory 'source_gitlab_singer' + moduleDirectory 'source_hubspot_singer' } airbyteStandardSourceTestFile { - specPath = "source_gitlab_singer/spec.json" + specPath = "source_hubspot_singer/spec.json" configPath = "secrets/config.json" configuredCatalogPath = "sample_files/configured_catalog.json" } + dependencies { implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) diff --git a/airbyte-integrations/connectors/source-hubspot/main_dev.py b/airbyte-integrations/connectors/source-hubspot-singer/main_dev.py similarity index 93% rename from airbyte-integrations/connectors/source-hubspot/main_dev.py rename to airbyte-integrations/connectors/source-hubspot-singer/main_dev.py index c45ce67037a..bf981e765e2 100644 --- a/airbyte-integrations/connectors/source-hubspot/main_dev.py +++ b/airbyte-integrations/connectors/source-hubspot-singer/main_dev.py @@ -25,8 +25,8 @@ SOFTWARE. import sys from base_python.entrypoint import launch -from source_hubspot import SourceHubspot +from source_hubspot_singer import SourceHubspotSinger if __name__ == "__main__": - source = SourceHubspot() + source = SourceHubspotSinger() launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/requirements.txt b/airbyte-integrations/connectors/source-hubspot-singer/requirements.txt similarity index 86% rename from airbyte-integrations/connectors/source-appsflyer-singer/requirements.txt rename to airbyte-integrations/connectors/source-hubspot-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-appsflyer-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-hubspot-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-hubspot-singer/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hubspot-singer/sample_files/configured_catalog.json new file mode 100644 index 00000000000..57a28099d70 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/sample_files/configured_catalog.json @@ -0,0 +1,8861 @@ +{ + "streams": [ + { + "stream": { + "name": "companies", + "json_schema": { + "type": "object", + "properties": { + "portalId": { + "type": ["null", "integer"] + }, + "companyId": { + "type": ["null", "integer"] + }, + "properties": { + "type": "object", + "properties": { + "about_us": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "closedate_timestamp_earliest_value_a2a17e6e": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "facebookfans": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "first_contact_createdate_timestamp_earliest_value_78b50eea": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "first_conversion_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "first_conversion_date_timestamp_earliest_value_61f58f2c": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "first_conversion_event_name": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "first_conversion_event_name_timestamp_earliest_value_68ddae0a": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "first_deal_created_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "founded_year": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_additional_domains": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_first_timestamp": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_first_touch_converting_campaign": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_first_visit_timestamp": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_last_timestamp": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_last_touch_converting_campaign": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_last_visit_timestamp": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_num_page_views": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_num_page_views_cardinality_sum_e46e85b0": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_num_visits": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_num_visits_cardinality_sum_53d952a6": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_source": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_source_data_1": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_source_data_2": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_source_timestamp_earliest_value_25a3a52c": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_avatar_filemanager_key": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_created_by_user_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_createdate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_ideal_customer_profile": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_is_target_account": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_last_booked_meeting_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_last_logged_call_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_last_open_task_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_last_sales_activity_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_last_sales_activity_timestamp": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_lastmodifieddate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_merged_object_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_num_blockers": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_num_contacts_with_buying_roles": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_num_decision_makers": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_num_open_deals": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_object_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_predictivecontactscore_v2": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_target_account": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_target_account_probability": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_target_account_recommendation_snooze_time": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_target_account_recommendation_state": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_total_deal_value": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_updated_by_user_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_user_ids_of_all_owners": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hubspot_owner_assigneddate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "is_public": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "num_associated_contacts": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "num_associated_deals": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "num_conversion_events": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "num_conversion_events_cardinality_sum_d095f14b": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "recent_conversion_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "recent_conversion_date_timestamp_latest_value_72856da1": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "recent_conversion_event_name": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "recent_conversion_event_name_timestamp_latest_value_66c820bf": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "recent_deal_amount": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "recent_deal_close_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "timezone": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "total_money_raised": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "total_revenue": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "name": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "twitterhandle": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "phone": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "twitterbio": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "twitterfollowers": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "address": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "address2": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "facebook_company_page": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "city": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "linkedin_company_page": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "linkedinbio": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "state": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "googleplus_page": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "engagements_last_meeting_booked": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "engagements_last_meeting_booked_campaign": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "engagements_last_meeting_booked_medium": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "engagements_last_meeting_booked_source": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_latest_meeting_activity": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_sales_email_last_replied": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hubspot_owner_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "notes_last_contacted": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "notes_last_updated": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "notes_next_activity_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "num_contacted_notes": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "num_notes": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "zip": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "country": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hubspot_team_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_all_owner_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "website": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "domain": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_all_team_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_all_accessible_team_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "numberofemployees": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "industry": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "annualrevenue": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "lifecyclestage": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_lead_status": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_parent_company_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "type": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "description": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_num_child_companies": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hubspotscore": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "createdate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "closedate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "first_contact_createdate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "days_to_close": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "web_technologies": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + } + } + }, + "property_about_us": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_closedate_timestamp_earliest_value_a2a17e6e": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_facebookfans": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_first_contact_createdate_timestamp_earliest_value_78b50eea": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_first_conversion_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_first_conversion_date_timestamp_earliest_value_61f58f2c": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_first_conversion_event_name": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_first_conversion_event_name_timestamp_earliest_value_68ddae0a": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_first_deal_created_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_founded_year": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_additional_domains": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_first_timestamp": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_first_touch_converting_campaign": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_first_visit_timestamp": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_last_timestamp": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_last_touch_converting_campaign": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_last_visit_timestamp": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_num_page_views": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_num_page_views_cardinality_sum_e46e85b0": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_num_visits": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_num_visits_cardinality_sum_53d952a6": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_source": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_source_data_1": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_source_data_2": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_source_timestamp_earliest_value_25a3a52c": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_avatar_filemanager_key": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_created_by_user_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_createdate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_ideal_customer_profile": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_is_target_account": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_last_booked_meeting_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_last_logged_call_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_last_open_task_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_last_sales_activity_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_last_sales_activity_timestamp": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_lastmodifieddate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_merged_object_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_num_blockers": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_num_contacts_with_buying_roles": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_num_decision_makers": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_num_open_deals": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_object_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_predictivecontactscore_v2": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_target_account": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_target_account_probability": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_target_account_recommendation_snooze_time": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_target_account_recommendation_state": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_total_deal_value": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_updated_by_user_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_user_ids_of_all_owners": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hubspot_owner_assigneddate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_is_public": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_num_associated_contacts": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_num_associated_deals": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_num_conversion_events": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_num_conversion_events_cardinality_sum_d095f14b": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_recent_conversion_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_recent_conversion_date_timestamp_latest_value_72856da1": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_recent_conversion_event_name": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_recent_conversion_event_name_timestamp_latest_value_66c820bf": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_recent_deal_amount": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_recent_deal_close_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_timezone": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_total_money_raised": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_total_revenue": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_name": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_twitterhandle": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_phone": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_twitterbio": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_twitterfollowers": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_address": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_address2": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_facebook_company_page": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_city": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_linkedin_company_page": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_linkedinbio": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_state": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_googleplus_page": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_engagements_last_meeting_booked": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_engagements_last_meeting_booked_campaign": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_engagements_last_meeting_booked_medium": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_engagements_last_meeting_booked_source": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_latest_meeting_activity": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_sales_email_last_replied": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hubspot_owner_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_notes_last_contacted": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_notes_last_updated": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_notes_next_activity_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_num_contacted_notes": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_num_notes": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_zip": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_country": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hubspot_team_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_all_owner_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_website": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_domain": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_all_team_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_all_accessible_team_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_numberofemployees": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_industry": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_annualrevenue": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_lifecyclestage": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_lead_status": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_parent_company_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_type": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_description": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_num_child_companies": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hubspotscore": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_createdate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_closedate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_first_contact_createdate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_days_to_close": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_web_technologies": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "properties_versions": { + "type": "array", + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + }, + "sourceVid": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["hs_lastmodifieddate"] + }, + "sync_mode": "incremental", + "cursor_field": ["hs_lastmodifieddate"] + }, + { + "stream": { + "name": "contacts_by_company", + "json_schema": { + "type": "object", + "properties": { + "contact-id": { + "type": ["integer"] + }, + "company-id": { + "type": ["integer"] + } + }, + "additionalProperties": false + } + } + }, + { + "stream": { + "name": "contact_lists", + "json_schema": { + "type": "object", + "properties": { + "parentId": { + "type": ["null", "integer"] + }, + "metaData": { + "type": "object", + "properties": { + "processing": { + "type": ["null", "string"] + }, + "size": { + "type": ["null", "integer"] + }, + "error": { + "type": ["null", "string"] + }, + "lastProcessingStateChangeAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "lastSizeChangeAt": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "dynamic": { + "type": ["null", "boolean"] + }, + "name": { + "type": ["null", "string"] + }, + "filters": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "filterFamily": { + "type": ["null", "string"] + }, + "withinTimeMode": { + "type": ["null", "string"] + }, + "checkPastVersions": { + "type": ["null", "boolean"] + }, + "type": { + "type": ["null", "string"] + }, + "property": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "operator": { + "type": ["null", "string"] + } + } + } + } + }, + "portalId": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "listId": { + "type": ["null", "integer"] + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "internalListId": { + "type": ["null", "integer"] + }, + "readOnly": { + "type": ["null", "boolean"] + }, + "deleteable": { + "type": ["null", "boolean"] + }, + "listType": { + "type": ["null", "string"] + }, + "archived": { + "type": ["null", "boolean"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updatedAt"] + }, + "sync_mode": "incremental", + "cursor_field": ["updatedAt"] + }, + { + "stream": { + "name": "subscription_changes", + "json_schema": { + "type": "object", + "properties": { + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "portalId": { + "type": ["null", "integer"] + }, + "recipient": { + "type": ["null", "string"] + }, + "changes": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "change": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "portalId": { + "type": ["null", "integer"] + }, + "subscriptionId": { + "type": ["null", "integer"] + }, + "changeType": { + "type": ["null", "string"] + }, + "causedByEvent": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + } + } + } + } + } + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["startTimestamp"] + }, + "sync_mode": "incremental", + "cursor_field": ["startTimestamp"] + }, + { + "stream": { + "name": "email_events", + "json_schema": { + "type": "object", + "properties": { + "appId": { + "type": ["null", "integer"] + }, + "appName": { + "type": ["null", "string"] + }, + "browser": { + "type": ["null", "object"], + "properties": { + "family": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "producer": { + "type": ["null", "string"] + }, + "producerUrl": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "deviceType": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "integer"] + }, + "emailCampaignId": { + "type": ["null", "integer"] + }, + "emailCampaignGroupId": { + "type": ["null", "integer"] + }, + "filteredEvent": { + "type": ["null", "boolean"] + }, + "from": { + "type": ["null", "string"] + }, + "hmid": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "ipAddress": { + "type": ["null", "string"] + }, + "linkId": { + "type": ["null", "integer"] + }, + "location": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + } + } + }, + "portalId": { + "type": ["null", "integer"] + }, + "recipient": { + "type": ["null", "string"] + }, + "response": { + "type": ["null", "string"] + }, + "sentBy": { + "type": ["null", "object"], + "properties": { + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "string"] + } + } + }, + "smtpId": { + "type": ["null", "string"] + }, + "subject": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "userAgent": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["startTimestamp"] + }, + "sync_mode": "incremental", + "cursor_field": ["startTimestamp"] + }, + { + "stream": { + "name": "engagements", + "json_schema": { + "type": "object", + "properties": { + "engagement_id": { + "type": "integer" + }, + "engagement": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "portalId": { + "type": "integer" + }, + "active": { + "type": "boolean" + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "lastUpdated": { + "type": ["null", "string"], + "format": "date-time" + }, + "ownerId": { + "type": "integer" + }, + "type": { + "type": "string" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "associations": { + "type": ["null", "object"], + "properties": { + "contactIds": { + "type": ["null", "array"], + "items": { + "type": "integer" + } + }, + "companyIds": { + "type": ["null", "array"], + "items": { + "type": "integer" + } + }, + "dealIds": { + "type": ["null", "array"], + "items": { + "type": "integer" + } + } + } + }, + "attachments": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + } + } + } + }, + "metadata": { + "type": ["null", "object"], + "properties": { + "body": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "object"], + "properties": { + "email": { + "type": "string" + }, + "firstName": { + "type": "string" + }, + "lastName": { + "type": "string" + } + } + }, + "to": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "email": { + "type": "string" + } + } + } + }, + "cc": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "email": { + "type": "string" + } + } + } + }, + "bcc": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "email": { + "type": "string" + } + } + } + }, + "subject": { + "type": ["null", "string"] + }, + "html": { + "type": ["null", "string"] + }, + "text": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "forObjectType": { + "type": ["null", "string"] + }, + "startTime": { + "type": ["null", "integer"] + }, + "endTime": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "toNumber": { + "type": ["null", "string"] + }, + "fromNumber": { + "type": ["null", "string"] + }, + "externalId": { + "type": ["null", "string"] + }, + "durationMilliseconds": { + "type": ["null", "integer"] + }, + "externalAccountId": { + "type": ["null", "string"] + }, + "recordingUrl": { + "type": ["null", "string"], + "format": "uri" + }, + "disposition": { + "type": ["null", "string"] + } + } + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["lastUpdated"] + }, + "sync_mode": "incremental", + "cursor_field": ["lastUpdated"] + }, + { + "stream": { + "name": "deal_pipelines", + "json_schema": { + "type": "object", + "properties": { + "pipelineId": { + "type": ["null", "string"] + }, + "stages": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "stageId": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "probability": { + "type": ["null", "number"] + }, + "active": { + "type": ["null", "boolean"] + }, + "displayOrder": { + "type": ["null", "integer"] + }, + "closedWon": { + "type": ["null", "boolean"] + } + } + } + }, + "label": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "displayOrder": { + "type": ["null", "integer"] + }, + "staticDefault": { + "type": ["null", "boolean"] + } + } + } + } + }, + { + "stream": { + "name": "deals", + "json_schema": { + "type": "object", + "properties": { + "portalId": { + "type": ["null", "integer"] + }, + "dealId": { + "type": ["null", "integer"] + }, + "isDeleted": { + "type": ["null", "boolean"] + }, + "associations": { + "type": ["null", "object"], + "properties": { + "associatedVids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "associatedCompanyIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "associatedDealIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + } + } + }, + "properties": { + "type": "object", + "properties": { + "amount_in_home_currency": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "days_to_close": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_acv": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_source": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_source_data_1": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_analytics_source_data_2": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_arr": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_campaign": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_closed_amount": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_closed_amount_in_home_currency": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_created_by_user_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_entered_appointmentscheduled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_entered_closedlost": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_entered_closedwon": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_entered_contractsent": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_entered_decisionmakerboughtin": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_entered_presentationscheduled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_entered_qualifiedtobuy": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_exited_appointmentscheduled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_exited_closedlost": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_exited_closedwon": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_exited_contractsent": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_exited_decisionmakerboughtin": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_exited_presentationscheduled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_date_exited_qualifiedtobuy": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_deal_amount_calculation_preference": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_deal_stage_probability": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_forecast_amount": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_forecast_probability": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_is_closed": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_lastmodifieddate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_likelihood_to_close": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_line_item_global_term_hs_discount_percentage": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_line_item_global_term_hs_discount_percentage_enabled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_line_item_global_term_hs_recurring_billing_period": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_line_item_global_term_hs_recurring_billing_period_enabled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_line_item_global_term_hs_recurring_billing_start_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_line_item_global_term_recurringbillingfrequency": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_line_item_global_term_recurringbillingfrequency_enabled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_manual_forecast_category": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_merged_object_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_mrr": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_next_step": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_object_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_predicted_amount": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_predicted_amount_in_home_currency": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_projected_amount": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_projected_amount_in_home_currency": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_tcv": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_time_in_appointmentscheduled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_time_in_closedlost": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_time_in_closedwon": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_time_in_contractsent": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_time_in_decisionmakerboughtin": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_time_in_presentationscheduled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_time_in_qualifiedtobuy": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_updated_by_user_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_user_ids_of_all_owners": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hubspot_owner_assigneddate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "dealname": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "amount": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "dealstage": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "pipeline": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "closedate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "createdate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "engagements_last_meeting_booked": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "engagements_last_meeting_booked_campaign": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "engagements_last_meeting_booked_medium": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "engagements_last_meeting_booked_source": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_latest_meeting_activity": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_sales_email_last_replied": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hubspot_owner_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "notes_last_contacted": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "notes_last_updated": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "notes_next_activity_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "num_contacted_notes": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "num_notes": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_createdate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hubspot_team_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "dealtype": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_all_owner_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "description": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_all_team_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "hs_all_accessible_team_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "num_associated_contacts": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "closed_lost_reason": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "closed_won_reason": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + } + } + }, + "property_amount_in_home_currency": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_days_to_close": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_acv": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_source": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_source_data_1": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_analytics_source_data_2": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_arr": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_campaign": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_closed_amount": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_closed_amount_in_home_currency": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_created_by_user_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_entered_appointmentscheduled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_entered_closedlost": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_entered_closedwon": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_entered_contractsent": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_entered_decisionmakerboughtin": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_entered_presentationscheduled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_entered_qualifiedtobuy": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_exited_appointmentscheduled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_exited_closedlost": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_exited_closedwon": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_exited_contractsent": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_exited_decisionmakerboughtin": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_exited_presentationscheduled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_date_exited_qualifiedtobuy": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_deal_amount_calculation_preference": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_deal_stage_probability": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_forecast_amount": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_forecast_probability": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_is_closed": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_lastmodifieddate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_likelihood_to_close": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_line_item_global_term_hs_discount_percentage": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_line_item_global_term_hs_discount_percentage_enabled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_line_item_global_term_hs_recurring_billing_period": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_line_item_global_term_hs_recurring_billing_period_enabled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_line_item_global_term_hs_recurring_billing_start_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_line_item_global_term_recurringbillingfrequency": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_line_item_global_term_recurringbillingfrequency_enabled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_manual_forecast_category": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_merged_object_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_mrr": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_next_step": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_object_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_predicted_amount": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_predicted_amount_in_home_currency": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_projected_amount": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_projected_amount_in_home_currency": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_tcv": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_time_in_appointmentscheduled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_time_in_closedlost": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_time_in_closedwon": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_time_in_contractsent": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_time_in_decisionmakerboughtin": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_time_in_presentationscheduled": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_time_in_qualifiedtobuy": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_updated_by_user_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_user_ids_of_all_owners": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hubspot_owner_assigneddate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_dealname": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_amount": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_dealstage": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_pipeline": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_closedate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_createdate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_engagements_last_meeting_booked": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_engagements_last_meeting_booked_campaign": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_engagements_last_meeting_booked_medium": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_engagements_last_meeting_booked_source": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_latest_meeting_activity": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_sales_email_last_replied": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hubspot_owner_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_notes_last_contacted": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_notes_last_updated": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_notes_next_activity_date": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_num_contacted_notes": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_num_notes": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_createdate": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"], + "format": "date-time" + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hubspot_team_id": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_dealtype": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_all_owner_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_description": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_all_team_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_hs_all_accessible_team_ids": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_num_associated_contacts": { + "type": "object", + "properties": { + "value": { + "type": ["null", "number", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_closed_lost_reason": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "property_closed_won_reason": { + "type": "object", + "properties": { + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + } + } + }, + "properties_versions": { + "type": "array", + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + }, + "sourceVid": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["hs_lastmodifieddate"] + }, + "sync_mode": "incremental", + "cursor_field": ["hs_lastmodifieddate"] + }, + { + "stream": { + "name": "owners", + "json_schema": { + "type": "object", + "properties": { + "portalId": { + "type": ["null", "integer"] + }, + "ownerId": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "signature": { + "type": ["null", "string"] + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "hasContactsAccess": { + "type": ["null", "boolean"] + }, + "isActive": { + "type": ["null", "boolean"] + }, + "activeUserId": { + "type": ["null", "integer"] + }, + "userIdIncludingInactive": { + "type": ["null", "integer"] + }, + "remoteList": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "portalId": { + "type": ["null", "integer"] + }, + "ownerId": { + "type": ["null", "integer"] + }, + "remoteId": { + "type": ["null", "string"] + }, + "remoteType": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + } + } + } + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updatedAt"] + }, + "sync_mode": "incremental", + "cursor_field": ["updatedAt"] + }, + { + "stream": { + "name": "workflows", + "json_schema": { + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "enabled": { + "type": ["null", "boolean"] + }, + "insertedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "personaTagIds": { + "type": "array", + "items": { + "type": "integer" + } + }, + "contactListIds": { + "type": "object", + "properties": { + "enrolled": { + "type": ["null", "integer"] + }, + "active": { + "type": ["null", "integer"] + }, + "steps": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updatedAt"] + }, + "sync_mode": "incremental", + "cursor_field": ["updatedAt"] + }, + { + "stream": { + "name": "forms", + "json_schema": { + "type": "object", + "properties": { + "deletedAt": { + "type": ["null", "integer"] + }, + "portalId": { + "type": ["null", "integer"] + }, + "guid": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "action": { + "type": ["null", "string"] + }, + "method": { + "type": ["null", "string"] + }, + "cssClass": { + "type": ["null", "string"] + }, + "redirect": { + "type": ["null", "string"] + }, + "submitText": { + "type": ["null", "string"] + }, + "followUpId": { + "type": ["null", "string"] + }, + "notifyRecipients": { + "type": ["null", "string"] + }, + "leadNurturingCampaignId": { + "type": ["null", "string"] + }, + "formFieldGroups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "fields": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "fieldType": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "groupName": { + "type": ["null", "string"] + }, + "displayOrder": { + "type": ["null", "integer"] + }, + "required": { + "type": ["null", "boolean"] + }, + "validation": { + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "data": { + "type": ["null", "string"] + }, + "useDefaultBlockList": { + "type": ["null", "boolean"] + }, + "blockedEmailAddresses": { + "type": "array", + "items": { + "type": ["null", "string"] + } + } + } + }, + "enabled": { + "type": ["null", "boolean"] + }, + "hidden": { + "type": ["null", "boolean"] + }, + "defaultValue": { + "type": ["null", "string"] + }, + "isSmartField": { + "type": ["null", "boolean"] + }, + "unselectedLabel": { + "type": ["null", "string"] + }, + "placeholder": { + "type": ["null", "string"] + }, + "labelHidden": { + "type": ["null", "boolean"] + }, + "options": { + "type": "array", + "items": { + "type": "object", + "properties": { + "description": { + "type": ["null", "string"] + }, + "displayOrder": { + "type": ["null", "integer"] + }, + "doubleData": { + "type": ["null", "number"] + }, + "hidden": { + "type": ["null", "boolean"] + }, + "label": { + "type": ["null", "string"] + }, + "readOnly": { + "type": ["null", "boolean"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "selectedOptions": { + "type": "array", + "items": { + "type": ["null", "string"] + } + } + } + } + }, + "default": { + "type": ["null", "boolean"] + }, + "isSmartGroup": { + "type": ["null", "boolean"] + }, + "richText": { + "type": "object", + "properties": { + "content": { + "type": ["null", "string"] + } + } + } + } + } + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "performableHtml": { + "type": ["null", "string"] + }, + "migratedFrom": { + "type": ["null", "string"] + }, + "ignoreCurrentValues": { + "type": ["null", "boolean"] + }, + "deletable": { + "type": ["null", "boolean"] + }, + "inlineMessage": { + "type": ["null", "string"] + }, + "tmsId": { + "type": ["null", "string"] + }, + "captchaEnabled": { + "type": ["null", "boolean"] + }, + "campaignGuid": { + "type": ["null", "string"] + }, + "cloneable": { + "type": ["null", "boolean"] + }, + "editable": { + "type": ["null", "boolean"] + }, + "formType": { + "type": ["null", "string"] + }, + "metaData": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updatedAt"] + }, + "sync_mode": "incremental", + "cursor_field": ["updatedAt"] + }, + { + "stream": { + "name": "campaigns", + "json_schema": { + "type": "object", + "properties": { + "appId": { + "type": ["null", "integer"] + }, + "appName": { + "type": ["null", "string"] + }, + "contentId": { + "type": ["null", "integer"] + }, + "counters": { + "type": ["null", "object"], + "properties": { + "delievered": { + "type": ["null", "integer"] + }, + "open": { + "type": ["null", "integer"] + }, + "processed": { + "type": ["null", "integer"] + }, + "sent": { + "type": ["null", "integer"] + }, + "deferred": { + "type": ["null", "integer"] + }, + "unsubscribed": { + "type": ["null", "integer"] + }, + "statuschange": { + "type": ["null", "integer"] + }, + "bounce": { + "type": ["null", "integer"] + }, + "mta_dropped": { + "type": ["null", "integer"] + }, + "dropped": { + "type": ["null", "integer"] + }, + "suppressed": { + "type": ["null", "integer"] + }, + "click": { + "type": ["null", "integer"] + }, + "delivered": { + "type": ["null", "integer"] + }, + "forward": { + "type": ["null", "integer"] + }, + "print": { + "type": ["null", "integer"] + }, + "reply": { + "type": ["null", "integer"] + }, + "spamreport": { + "type": ["null", "integer"] + } + } + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "numIncluded": { + "type": ["null", "integer"] + }, + "numQueued": { + "type": ["null", "integer"] + }, + "subType": { + "type": ["null", "string"] + }, + "subject": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-appsflyer-singer/setup.py b/airbyte-integrations/connectors/source-hubspot-singer/setup.py similarity index 79% rename from airbyte-integrations/connectors/source-appsflyer-singer/setup.py rename to airbyte-integrations/connectors/source-hubspot-singer/setup.py index d516ccdbd19..f46a326087c 100644 --- a/airbyte-integrations/connectors/source-appsflyer-singer/setup.py +++ b/airbyte-integrations/connectors/source-hubspot-singer/setup.py @@ -25,17 +25,11 @@ SOFTWARE. from setuptools import find_packages, setup setup( - name="source_appsflyer_singer", - description="Source implementation for Appsflyer, built on the Singer tap implementation.", + name="source_github_singer", + description="Source implementation for Github.", author="Airbyte", author_email="contact@airbyte.io", + install_requires=["tap-hubspot==2.8.1", "requests", "airbyte-protocol", "base-singer"], packages=find_packages(), - install_requires=[ - "tap-appsflyer @ git+https://github.com/Muriloo/tap-appsflyer", - "airbyte-protocol", - "base-singer", - "base-python", - "pytest==6.1.2", - ], package_data={"": ["*.json"]}, ) diff --git a/airbyte-integrations/connectors/source-gitlab-singer/source_gitlab_singer/__init__.py b/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/__init__.py similarity index 93% rename from airbyte-integrations/connectors/source-gitlab-singer/source_gitlab_singer/__init__.py rename to airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/__init__.py index 2e72df3ac97..72edd0735fd 100644 --- a/airbyte-integrations/connectors/source-gitlab-singer/source_gitlab_singer/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/__init__.py @@ -22,6 +22,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from .source import SourceGitlabSinger +from .source import SourceHubspotSinger -__all__ = ["SourceGitlabSinger"] +__all__ = ["SourceHubspotSinger"] diff --git a/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/source.py b/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/source.py new file mode 100644 index 00000000000..7120d3ab1a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/source.py @@ -0,0 +1,106 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import json + +import requests +from airbyte_protocol import AirbyteConnectionStatus, Status +from base_singer import AirbyteLogger, SingerSource +from requests.status_codes import codes as status_codes + + +class SourceHubspotSinger(SingerSource): + def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus: + error_msg = "Unable to connect with the provided credentials. Error: {}" + try: + api_key = config.get("hapikey") + if api_key: + logger.info("checking with api key") + r = requests.get(f"https://api.hubapi.com/contacts/v1/lists/all/contacts/all?hapikey={api_key}") + else: + logger.info("checking with oauth") + # borrowing from tap-hubspot + # https://github.com/singer-io/tap-hubspot/blob/master/tap_hubspot/__init__.py#L208-L229 + payload = { + "grant_type": "refresh_token", + "redirect_uri": config["redirect_uri"], + "refresh_token": config["refresh_token"], + "client_id": config["client_id"], + "client_secret": config["client_secret"], + } + resp = requests.post("https://api.hubapi.com/oauth/v1/token", data=payload) + + if resp.status_code == status_codes.FORBIDDEN: + return AirbyteConnectionStatus(status=Status.FAILED, message=error_msg.format(resp.text)) + + try: + resp.raise_for_status() + except Exception as e: + logger.error(str(e)) + return AirbyteConnectionStatus(status=Status.FAILED, message=error_msg.format(e)) + + auth = resp.json() + headers = {"Authorization": "Bearer {}".format(auth["access_token"])} + + r = requests.get("https://api.hubapi.com/contacts/v1/lists/all/contacts/all", headers=headers) + + if r.status_code == status_codes.OK: + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + else: + return AirbyteConnectionStatus(status=Status.FAILED, message=error_msg.format(r.text)) + except Exception as e: + logger.error(str(e)) + return AirbyteConnectionStatus(status=Status.FAILED, message=error_msg.format(e)) + + def discover_cmd(self, logger, config_path) -> str: + return f"tap-hubspot --config {config_path} --discover" + + def read_cmd(self, logger, config_path, catalog_path, state_path=None) -> str: + config_option = f"--config {config_path}" + properties_option = f"--properties {catalog_path}" + state_option = f"--state {state_path}" if state_path else "" + return f"tap-hubspot {config_option} {properties_option} {state_option}" + + def transform_config(self, raw_config): + rendered_config = dict(raw_config) + singer_config = dict() + + # the singer integration requires if the hapikey field is passed that all + # of the oauth fields get passed as well. it just checks existence. we + # do not bother our users to provide the extra oauth creds if they are + # using an api key. so we fill them in with dummy values for them. + if "api_key" in rendered_config["credentials"]: + singer_config["hapikey"] = rendered_config["credentials"]["api_key"] + singer_config["redirect_uri"] = "placeholder" + singer_config["client_id"] = "placeholder" + singer_config["client_secret"] = "placeholder" + singer_config["refresh_token"] = "placeholder" + else: + singer_config = rendered_config["credentials"] + + singer_config["start_date"] = rendered_config["start_date"] + # always turn off singer tracking. + singer_config["disable_collection"] = True + + return singer_config diff --git a/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/spec.json b/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/spec.json new file mode 100644 index 00000000000..16bd50127db --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/spec.json @@ -0,0 +1,67 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Hubspot Source Spec", + "type": "object", + "required": ["start_date", "credentials"], + "additionalProperties": false, + "properties": { + "start_date": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", + "examples": ["2017-01-25T00:00:00Z"] + }, + "credentials": { + "type": "object", + "oneOf": [ + { + "title": "api key", + "required": ["api_key"], + "properties": { + "api_key": { + "description": "Hubspot API Key. See our docs if you need help finding this key.", + "type": "string", + "airbyte_secret": true + } + } + }, + { + "title": "oauth", + "required": [ + "redirect_uri", + "client_id", + "client_secret", + "refresh_token" + ], + "properties": { + "redirect_uri": { + "description": "Hubspot API Key. See our docs if you need help finding this key.", + "type": "string", + "examples": ["https://api.hubspot.com/"] + }, + "client_id": { + "description": "Hubspot client_id. See our docs if you need help finding this id.", + "type": "string", + "examples": ["123456789000"] + }, + "client_secret": { + "description": "Hubspot client_secret. See our docs if you need help finding this secret.", + "type": "string", + "examples": ["secret"], + "airbyte_secret": true + }, + "refresh_token": { + "description": "Hubspot refresh_token. See our docs if you need help generating the token.", + "type": "string", + "examples": ["refresh_token"], + "airbyte_secret": true + } + } + } + ] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/.dockerignore b/airbyte-integrations/connectors/source-hubspot/.dockerignore deleted file mode 100644 index 0113a1636f9..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!Dockerfile.test -!source_hubspot -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-hubspot/.gitignore b/airbyte-integrations/connectors/source-hubspot/.gitignore deleted file mode 100644 index 29fffc6a50c..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/.gitignore +++ /dev/null @@ -1 +0,0 @@ -NEW_SOURCE_CHECKLIST.md diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile deleted file mode 100644 index 10ee0569532..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM airbyte/integration-base-python:dev - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="source_hubspot" -ENV AIRBYTE_IMPL_MODULE="source_hubspot" -ENV AIRBYTE_IMPL_PATH="SourceHubspot" - -WORKDIR /airbyte/integration_code -COPY $CODE_PATH ./$CODE_PATH -COPY setup.py ./ -RUN pip install ".[main]" - -LABEL io.airbyte.version=0.1.1 -LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/README.md b/airbyte-integrations/connectors/source-hubspot/README.md deleted file mode 100644 index f8a2c49191c..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# Hubspot Source - -This is the repository for the Hubspot source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/hubspot). - -## Local development - -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Build & Activate Virtual Environment -First, build the module by running the following from the `airbyte` project root directory: -``` -./gradlew :airbyte-integrations:connectors:source-hubspot:build -``` - -This will generate a virtualenv for this module in `source-hubspot/.venv`. Make sure this venv is active in your -development environment of choice. To activate the venv from the terminal, run: -``` -cd airbyte-integrations/connectors/source-hubspot # cd into the connector directory -source .venv/bin/activate -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/hubspot) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hubspot/spec.json` file. -See `sample_files/sample_config.json` for a sample config file. - -**If you are an Airbyte core member**, copy the credentials in RPass under the secret name `source-hubspot-integration-test-config` -and place them into `secrets/config.json`. - - -### Locally running the connector -``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -pytest unit_tests -``` - -### Locally running the connector docker image -``` -# in airbyte root directory -./gradlew :airbyte-integrations:connectors:source-hubspot:airbyteDocker -docker run --rm airbyte/source-hubspot:dev spec -docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-hubspot/secrets:/secrets airbyte/source-hubspot:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-hubspot/secrets:/secrets airbyte/source-hubspot:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-hubspot/secrets:/secrets -v $(pwd)/airbyte-integrations/connectors/source-hubspot/sample_files:/sample_files airbyte/source-hubspot:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json -``` - -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-hubspot:standardSourceTestPython` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `pytest integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. - -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. diff --git a/airbyte-integrations/connectors/source-hubspot/build.gradle b/airbyte-integrations/connectors/source-hubspot/build.gradle deleted file mode 100644 index 5e625a30dc6..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/build.gradle +++ /dev/null @@ -1,37 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-standard-source-test-file' -} - -airbytePython { - moduleDirectory 'source_hubspot' -} - -airbyteStandardSourceTestFile { - // For more information on standard source tests, see https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/testing-connectors - - // All these input paths must live inside this connector's directory (or subdirectories) - // TODO update the spec JSON file - specPath = "source_hubspot/spec.json" - - // configPath points to a config file which matches the spec.json supplied above. secrets/ is gitignored by default, so place your config file - // there (in case it contains any credentials) - // TODO update the config file to contain actual credentials - configPath = "secrets/config.json" - // TODO update the sample configured_catalog JSON for use in testing - // Note: If your source supports incremental syncing, then make sure that the catalog that is returned in the get_catalog method is configured - // for incremental syncing (e.g. include cursor fields, etc). - configuredCatalogPath = "sample_files/configured_catalog.json" -} - -dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) - implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) -} - -task("pythonIntegrationTests", type: PythonTask, dependsOn: installTestReqs) { - module = "pytest" - command = "-s integration_tests" -} -integrationTest.dependsOn("pythonIntegrationTests") diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/client_test.py b/airbyte-integrations/connectors/source-hubspot/integration_tests/client_test.py deleted file mode 100644 index 8ec93f82685..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/client_test.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import pytest -from source_hubspot.client import Client -from source_hubspot.errors import HubspotInvalidAuth - - -@pytest.fixture(name="wrong_credentials") -def wrong_credentials_fixture(): - return {"api_key": "wrongkey-key1-key2-key3-wrongkey1234"} - - -def test__health_check_with_wrong_token(wrong_credentials): - client = Client(start_date="2021-02-01T00:00:00Z", credentials=wrong_credentials) - alive, error = client.health_check() - - assert not alive - assert ( - error - == "HubspotInvalidAuth('The API key provided is invalid. View or manage your API key here: https://app.hubspot.com/l/api-key/')" - ) - - -def test__stream_iterator_with_wrong_token(wrong_credentials): - client = Client(start_date="2021-02-01T00:00:00Z", credentials=wrong_credentials) - with pytest.raises( - HubspotInvalidAuth, match="The API key provided is invalid. View or manage your API key here: https://app.hubspot.com/l/api-key/" - ): - _ = list(client.streams) diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/incremental_test.py b/airbyte-integrations/connectors/source-hubspot/integration_tests/incremental_test.py deleted file mode 100644 index 6ff6069cb4c..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/incremental_test.py +++ /dev/null @@ -1,107 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import json -from pathlib import Path -from typing import Any, Iterable, List, Mapping, MutableMapping, Tuple - -import pytest -from airbyte_protocol import ConfiguredAirbyteCatalog, SyncMode, Type -from base_python import AirbyteLogger -from source_hubspot.source import SourceHubspot - -HERE = Path(__file__).parent.absolute() - - -@pytest.fixture(scope="session", name="config") -def config_fixture() -> Mapping[str, Any]: - config_filename = HERE.parent / "secrets" / "config.json" - - if not config_filename.exists(): - raise RuntimeError(f"Please provide config in {config_filename}") - - with open(str(config_filename)) as json_file: - return json.load(json_file) - - -@pytest.fixture -def configured_catalog() -> ConfiguredAirbyteCatalog: - catalog_filename = HERE.parent / "sample_files" / "configured_catalog.json" - if not catalog_filename.exists(): - raise RuntimeError(f"Please provide configured catalog in {catalog_filename}") - - return ConfiguredAirbyteCatalog.parse_file(catalog_filename) - - -@pytest.fixture -def configured_catalog_with_incremental(configured_catalog) -> ConfiguredAirbyteCatalog: - streams = [] - for stream in configured_catalog.streams: - if SyncMode.incremental in stream.stream.supported_sync_modes: - stream.sync_mode = SyncMode.incremental - streams.append(stream) - - configured_catalog.streams = streams - return configured_catalog - - -def read_stream( - source: SourceHubspot, config: Mapping, catalog: ConfiguredAirbyteCatalog, state: MutableMapping = None -) -> Tuple[Mapping, List]: - records = {} - states = [] - for message in source.read(AirbyteLogger(), config, catalog, state): - if message.type == Type.RECORD: - records.setdefault(message.record.stream, []) - records[message.record.stream].append(message.record) - elif message.type == Type.STATE: - states.append(message.state) - - return records, states - - -def records_older(records: Iterable, than: int, cursor_field: str) -> Iterable: - for record in records: - if record.data.get(cursor_field) < than: - yield record - - -class TestIncrementalSync: - def test_sync_with_latest_state(self, config, configured_catalog_with_incremental): - """Sync first time, save the state and sync second time with saved state from previous sync""" - streams = {stream.stream.name: stream for stream in configured_catalog_with_incremental.streams} - records1, states1 = read_stream(SourceHubspot(), config, configured_catalog_with_incremental) - - assert states1, "should have at least one state emitted" - assert records1, "should have at least few records emitted" - - records2, states2 = read_stream(SourceHubspot(), config, configured_catalog_with_incremental, states1[-1].data) - - assert states1[-1] == states2[-1], "final states should be the same" - for stream_name, state in states2[-1].data.items(): - cursor_field = streams[stream_name].cursor_field[0] - old_records1 = records_older(records1[stream_name], than=records2[stream_name][0].data[cursor_field], cursor_field=cursor_field) - old_records2 = records_older(records2[stream_name], than=records2[stream_name][0].data[cursor_field], cursor_field=cursor_field) - assert list(old_records1), "should have older records from the first read" - assert not list(old_records2), "should not have older records from the second read" diff --git a/airbyte-integrations/connectors/source-hubspot/requirements.txt b/airbyte-integrations/connectors/source-hubspot/requirements.txt deleted file mode 100644 index e0281021aed..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. --e ../../bases/airbyte-protocol --e ../../bases/base-python --e ../../bases/base-python-test --e . diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json deleted file mode 100644 index 964214e1fa9..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json +++ /dev/null @@ -1,6055 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "campaigns", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "appId": { - "type": ["null", "integer"] - }, - "appName": { - "type": ["null", "string"] - }, - "contentId": { - "type": ["null", "integer"] - }, - "counters": { - "type": ["null", "object"], - "properties": { - "open": { - "type": ["null", "integer"] - }, - "processed": { - "type": ["null", "integer"] - }, - "sent": { - "type": ["null", "integer"] - }, - "deferred": { - "type": ["null", "integer"] - }, - "unsubscribed": { - "type": ["null", "integer"] - }, - "statuschange": { - "type": ["null", "integer"] - }, - "bounce": { - "type": ["null", "integer"] - }, - "mta_dropped": { - "type": ["null", "integer"] - }, - "dropped": { - "type": ["null", "integer"] - }, - "suppressed": { - "type": ["null", "integer"] - }, - "click": { - "type": ["null", "integer"] - }, - "delivered": { - "type": ["null", "integer"] - }, - "forward": { - "type": ["null", "integer"] - }, - "print": { - "type": ["null", "integer"] - }, - "reply": { - "type": ["null", "integer"] - }, - "spamreport": { - "type": ["null", "integer"] - } - } - }, - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "numIncluded": { - "type": ["null", "integer"] - }, - "numQueued": { - "type": ["null", "integer"] - }, - "subType": { - "type": ["null", "string"] - }, - "subject": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "lastUpdatedTime": { - "type": "string" - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - }, - { - "stream": { - "name": "companies", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "portalId": { - "type": ["null", "integer"] - }, - "companyId": { - "type": "integer" - }, - "isDeleted": { - "type": ["null", "boolean"] - }, - "additionalDomains": { - "type": ["null", "array"] - }, - "stateChanges": { - "type": ["null", "array"] - }, - "mergeAudits": { - "type": ["null", "array"] - }, - "contacts": { - "type": ["null", "array"] - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "properties": { - "type": "object", - "properties": { - "about_us": { - "type": "string" - }, - "closedate_timestamp_earliest_value_a2a17e6e": { - "type": "datetime" - }, - "facebookfans": { - "type": "number" - }, - "first_contact_createdate_timestamp_earliest_value_78b50eea": { - "type": "datetime" - }, - "first_conversion_date": { - "type": "datetime" - }, - "first_conversion_date_timestamp_earliest_value_61f58f2c": { - "type": "datetime" - }, - "first_conversion_event_name": { - "type": "string" - }, - "first_conversion_event_name_timestamp_earliest_value_68ddae0a": { - "type": "datetime" - }, - "first_deal_created_date": { - "type": "datetime" - }, - "founded_year": { - "type": "string" - }, - "hs_additional_domains": { - "type": "string" - }, - "hs_all_assigned_business_unit_ids": { - "type": "string" - }, - "hs_analytics_first_timestamp": { - "type": "datetime" - }, - "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": { - "type": "datetime" - }, - "hs_analytics_first_touch_converting_campaign": { - "type": "string" - }, - "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": { - "type": "datetime" - }, - "hs_analytics_first_visit_timestamp": { - "type": "datetime" - }, - "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": { - "type": "datetime" - }, - "hs_analytics_last_timestamp": { - "type": "datetime" - }, - "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": { - "type": "datetime" - }, - "hs_analytics_last_touch_converting_campaign": { - "type": "string" - }, - "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": { - "type": "datetime" - }, - "hs_analytics_last_visit_timestamp": { - "type": "datetime" - }, - "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": { - "type": "datetime" - }, - "hs_analytics_num_page_views": { - "type": "number" - }, - "hs_analytics_num_page_views_cardinality_sum_e46e85b0": { - "type": "number" - }, - "hs_analytics_num_visits": { - "type": "number" - }, - "hs_analytics_num_visits_cardinality_sum_53d952a6": { - "type": "number" - }, - "hs_analytics_source": { - "type": "string" - }, - "hs_analytics_source_data_1": { - "type": "string" - }, - "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": { - "type": "datetime" - }, - "hs_analytics_source_data_2": { - "type": "string" - }, - "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": { - "type": "datetime" - }, - "hs_analytics_source_timestamp_earliest_value_25a3a52c": { - "type": "datetime" - }, - "hs_avatar_filemanager_key": { - "type": "string" - }, - "hs_created_by_user_id": { - "type": "number" - }, - "hs_createdate": { - "type": "datetime" - }, - "hs_ideal_customer_profile": { - "type": "string" - }, - "hs_is_target_account": { - "type": "bool" - }, - "hs_last_booked_meeting_date": { - "type": "datetime" - }, - "hs_last_logged_call_date": { - "type": "datetime" - }, - "hs_last_open_task_date": { - "type": "datetime" - }, - "hs_last_sales_activity_date": { - "type": "datetime" - }, - "hs_last_sales_activity_timestamp": { - "type": "datetime" - }, - "hs_lastmodifieddate": { - "type": "datetime" - }, - "hs_merged_object_ids": { - "type": "string" - }, - "hs_num_blockers": { - "type": "number" - }, - "hs_num_contacts_with_buying_roles": { - "type": "number" - }, - "hs_num_decision_makers": { - "type": "number" - }, - "hs_num_open_deals": { - "type": "number" - }, - "hs_object_id": { - "type": "number" - }, - "hs_predictivecontactscore_v2": { - "type": "number" - }, - "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": { - "type": "number" - }, - "hs_target_account": { - "type": "string" - }, - "hs_target_account_probability": { - "type": "number" - }, - "hs_target_account_recommendation_snooze_time": { - "type": "datetime" - }, - "hs_target_account_recommendation_state": { - "type": "string" - }, - "hs_total_deal_value": { - "type": "number" - }, - "hs_updated_by_user_id": { - "type": "number" - }, - "hs_user_ids_of_all_owners": { - "type": "string" - }, - "hubspot_owner_assigneddate": { - "type": "datetime" - }, - "is_public": { - "type": "bool" - }, - "num_associated_contacts": { - "type": "number" - }, - "num_associated_deals": { - "type": "number" - }, - "num_conversion_events": { - "type": "number" - }, - "num_conversion_events_cardinality_sum_d095f14b": { - "type": "number" - }, - "recent_conversion_date": { - "type": "datetime" - }, - "recent_conversion_date_timestamp_latest_value_72856da1": { - "type": "datetime" - }, - "recent_conversion_event_name": { - "type": "string" - }, - "recent_conversion_event_name_timestamp_latest_value_66c820bf": { - "type": "datetime" - }, - "recent_deal_amount": { - "type": "number" - }, - "recent_deal_close_date": { - "type": "datetime" - }, - "timezone": { - "type": "string" - }, - "total_money_raised": { - "type": "string" - }, - "total_revenue": { - "type": "number" - }, - "name": { - "type": "string" - }, - "twitterhandle": { - "type": "string" - }, - "phone": { - "type": "string" - }, - "twitterbio": { - "type": "string" - }, - "twitterfollowers": { - "type": "number" - }, - "address": { - "type": "string" - }, - "address2": { - "type": "string" - }, - "facebook_company_page": { - "type": "string" - }, - "city": { - "type": "string" - }, - "linkedin_company_page": { - "type": "string" - }, - "linkedinbio": { - "type": "string" - }, - "state": { - "type": "string" - }, - "googleplus_page": { - "type": "string" - }, - "engagements_last_meeting_booked": { - "type": "datetime" - }, - "engagements_last_meeting_booked_campaign": { - "type": "string" - }, - "engagements_last_meeting_booked_medium": { - "type": "string" - }, - "engagements_last_meeting_booked_source": { - "type": "string" - }, - "hs_latest_meeting_activity": { - "type": "datetime" - }, - "hs_sales_email_last_replied": { - "type": "datetime" - }, - "hubspot_owner_id": { - "type": "string" - }, - "notes_last_contacted": { - "type": "datetime" - }, - "notes_last_updated": { - "type": "datetime" - }, - "notes_next_activity_date": { - "type": "datetime" - }, - "num_contacted_notes": { - "type": "number" - }, - "num_notes": { - "type": "number" - }, - "zip": { - "type": "string" - }, - "country": { - "type": "string" - }, - "hubspot_team_id": { - "type": "string" - }, - "hs_all_owner_ids": { - "type": "string" - }, - "website": { - "type": "string" - }, - "domain": { - "type": "string" - }, - "hs_all_team_ids": { - "type": "string" - }, - "hs_all_accessible_team_ids": { - "type": "string" - }, - "numberofemployees": { - "type": "number" - }, - "industry": { - "type": "string" - }, - "annualrevenue": { - "type": "number" - }, - "lifecyclestage": { - "type": "string" - }, - "hs_lead_status": { - "type": "string" - }, - "hs_parent_company_id": { - "type": "number" - }, - "type": { - "type": "string" - }, - "description": { - "type": "string" - }, - "hs_num_child_companies": { - "type": "number" - }, - "hubspotscore": { - "type": "number" - }, - "createdate": { - "type": "datetime" - }, - "closedate": { - "type": "datetime" - }, - "first_contact_createdate": { - "type": "datetime" - }, - "days_to_close": { - "type": "number" - }, - "web_technologies": { - "type": "string" - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - }, - { - "stream": { - "name": "contact_lists", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "parentId": { - "type": ["null", "integer"] - }, - "metaData": { - "type": "object", - "properties": { - "processing": { - "type": ["null", "string"] - }, - "size": { - "type": ["null", "integer"] - }, - "error": { - "type": ["null", "string"] - }, - "lastProcessingStateChangeAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "lastSizeChangeAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "dynamic": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "filters": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "object", - "properties": { - "filterFamily": { - "type": ["null", "string"] - }, - "withinTimeMode": { - "type": ["null", "string"] - }, - "checkPastVersions": { - "type": ["null", "boolean"] - }, - "type": { - "type": ["null", "string"] - }, - "property": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "operator": { - "type": ["null", "string"] - } - } - } - } - }, - "portalId": { - "type": ["null", "integer"] - }, - "createdAt": { - "type": "integer" - }, - "listId": { - "type": ["null", "integer"] - }, - "updatedAt": { - "type": "integer" - }, - "internalListId": { - "type": ["null", "integer"] - }, - "readOnly": { - "type": ["null", "boolean"] - }, - "deleteable": { - "type": ["null", "boolean"] - }, - "listType": { - "type": ["null", "string"] - }, - "archived": { - "type": ["null", "boolean"] - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - }, - { - "stream": { - "name": "contacts", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "vid": { - "type": ["null", "integer"] - }, - "canonical-vid": { - "type": ["null", "integer"] - }, - "merged-vids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "portal-id": { - "type": ["null", "integer"] - }, - "is-contact": { - "type": ["null", "boolean"] - }, - "profile-token": { - "type": ["null", "string"] - }, - "profile-url": { - "type": ["null", "string"] - }, - "associated-company": { - "type": "object", - "properties": { - "properties": { - "type": "object", - "properties": { - "about_us": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "closedate_timestamp_earliest_value_a2a17e6e": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "facebookfans": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_contact_createdate_timestamp_earliest_value_78b50eea": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_conversion_date_timestamp_earliest_value_61f58f2c": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_conversion_event_name_timestamp_earliest_value_68ddae0a": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_deal_created_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "founded_year": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_additional_domains": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_timestamp": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_touch_converting_campaign": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_visit_timestamp": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_timestamp": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_touch_converting_campaign": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_visit_timestamp": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_num_page_views": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_num_page_views_cardinality_sum_e46e85b0": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_num_visits": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_num_visits_cardinality_sum_53d952a6": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_data_1": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_data_2": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_timestamp_earliest_value_25a3a52c": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_avatar_filemanager_key": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_created_by_user_id": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_createdate": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_ideal_customer_profile": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_is_target_account": { - "type": "object", - "properties": { - "value": { - "type": ["null", "boolean"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_booked_meeting_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_logged_call_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_open_task_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_sales_activity_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_sales_activity_timestamp": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_lastmodifieddate": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_merged_object_ids": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_blockers": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_contacts_with_buying_roles": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_decision_makers": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_open_deals": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_object_id": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_target_account": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_target_account_probability": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_target_account_recommendation_snooze_time": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_target_account_recommendation_state": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_total_deal_value": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_updated_by_user_id": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_user_ids_of_all_owners": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hubspot_owner_assigneddate": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "is_public": { - "type": "object", - "properties": { - "value": { - "type": ["null", "boolean"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_associated_contacts": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_associated_deals": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_conversion_events_cardinality_sum_d095f14b": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "recent_conversion_date_timestamp_latest_value_72856da1": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "recent_conversion_event_name_timestamp_latest_value_66c820bf": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "recent_deal_amount": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "recent_deal_close_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "timezone": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "total_money_raised": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "total_revenue": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "twitterhandle": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "phone": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "twitterbio": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "twitterfollowers": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "address": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "address2": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "facebook_company_page": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "city": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "linkedin_company_page": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "linkedinbio": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "state": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "googleplus_page": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "engagements_last_meeting_booked": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "engagements_last_meeting_booked_campaign": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "engagements_last_meeting_booked_medium": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "engagements_last_meeting_booked_source": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_latest_meeting_activity": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_sales_email_last_replied": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hubspot_owner_id": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "notes_last_contacted": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "notes_last_updated": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "notes_next_activity_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_contacted_notes": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_notes": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "zip": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "country": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hubspot_team_id": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_all_owner_ids": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "website": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "domain": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_all_team_ids": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_all_accessible_team_ids": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "numberofemployees": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "industry": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "annualrevenue": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "lifecyclestage": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_lead_status": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_parent_company_id": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "type": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "description": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_child_companies": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "createdate": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "closedate": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_contact_createdate": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "days_to_close": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "web_technologies": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - } - } - }, - "company-id": { - "type": ["null", "integer"] - }, - "portal-id": { - "type": ["null", "integer"] - } - } - }, - "identity-profiles": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "deleted-changed-timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "saved-at-timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "vid": { - "type": ["null", "integer"] - }, - "identities": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "type": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "list-memberships": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "internal-list-id": { - "type": ["null", "integer"] - }, - "is-member": { - "type": ["null", "boolean"] - }, - "static-list-id": { - "type": ["null", "integer"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "vid": { - "type": ["null", "integer"] - } - } - } - }, - "form-submissions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "conversion-id": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "form-id": { - "type": ["null", "string"] - }, - "portal-id": { - "type": ["null", "integer"] - }, - "page-url": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - } - } - } - }, - "merge-audits": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "canonical-vid": { - "type": ["null", "integer"] - }, - "vid-to-merge": { - "type": ["null", "integer"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "user-id": { - "type": ["null", "integer"] - }, - "num-properties-moved": { - "type": ["null", "integer"] - }, - "merged_from_email": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "source-type": { - "type": ["null", "string"] - }, - "source-id": { - "type": ["null", "string"] - }, - "source-label": { - "type": ["null", "string"] - }, - "source-vids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "timestamp": { - "type": ["null", "integer"] - }, - "selected": { - "type": ["null", "boolean"] - } - } - }, - "merged_to_email": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "source-type": { - "type": ["null", "string"] - }, - "source-id": { - "type": ["null", "string"] - }, - "source-label": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "integer"] - }, - "selected": { - "type": ["null", "boolean"] - } - } - } - } - } - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "properties": { - "type": "object", - "properties": { - "company_size": { - "type": "string" - }, - "date_of_birth": { - "type": "string" - }, - "days_to_close": { - "type": "number" - }, - "degree": { - "type": "string" - }, - "field_of_study": { - "type": "string" - }, - "first_conversion_date": { - "type": "datetime" - }, - "first_conversion_event_name": { - "type": "string" - }, - "first_deal_created_date": { - "type": "datetime" - }, - "gender": { - "type": "string" - }, - "graduation_date": { - "type": "string" - }, - "hs_additional_emails": { - "type": "string" - }, - "hs_all_assigned_business_unit_ids": { - "type": "string" - }, - "hs_all_contact_vids": { - "type": "string" - }, - "hs_analytics_first_touch_converting_campaign": { - "type": "string" - }, - "hs_analytics_last_touch_converting_campaign": { - "type": "string" - }, - "hs_avatar_filemanager_key": { - "type": "string" - }, - "hs_buying_role": { - "type": "string" - }, - "hs_calculated_form_submissions": { - "type": "string" - }, - "hs_calculated_merged_vids": { - "type": "string" - }, - "hs_calculated_mobile_number": { - "type": "string" - }, - "hs_calculated_phone_number": { - "type": "string" - }, - "hs_calculated_phone_number_area_code": { - "type": "string" - }, - "hs_calculated_phone_number_country_code": { - "type": "string" - }, - "hs_calculated_phone_number_region_code": { - "type": "string" - }, - "hs_content_membership_email_confirmed": { - "type": "bool" - }, - "hs_content_membership_notes": { - "type": "string" - }, - "hs_content_membership_registered_at": { - "type": "datetime" - }, - "hs_content_membership_registration_domain_sent_to": { - "type": "string" - }, - "hs_content_membership_registration_email_sent_at": { - "type": "datetime" - }, - "hs_content_membership_status": { - "type": "string" - }, - "hs_conversations_visitor_email": { - "type": "string" - }, - "hs_count_is_unworked": { - "type": "number" - }, - "hs_count_is_worked": { - "type": "number" - }, - "hs_created_by_conversations": { - "type": "bool" - }, - "hs_created_by_user_id": { - "type": "number" - }, - "hs_createdate": { - "type": "datetime" - }, - "hs_document_last_revisited": { - "type": "datetime" - }, - "hs_email_bad_address": { - "type": "bool" - }, - "hs_email_customer_quarantined_reason": { - "type": "string" - }, - "hs_email_domain": { - "type": "string" - }, - "hs_email_hard_bounce_reason": { - "type": "string" - }, - "hs_email_hard_bounce_reason_enum": { - "type": "string" - }, - "hs_email_quarantined": { - "type": "bool" - }, - "hs_email_quarantined_reason": { - "type": "string" - }, - "hs_email_recipient_fatigue_recovery_time": { - "type": "datetime" - }, - "hs_email_sends_since_last_engagement": { - "type": "number" - }, - "hs_emailconfirmationstatus": { - "type": "string" - }, - "hs_facebook_ad_clicked": { - "type": "bool" - }, - "hs_facebook_click_id": { - "type": "string" - }, - "hs_feedback_last_nps_follow_up": { - "type": "string" - }, - "hs_feedback_last_nps_rating": { - "type": "string" - }, - "hs_feedback_last_survey_date": { - "type": "datetime" - }, - "hs_feedback_show_nps_web_survey": { - "type": "bool" - }, - "hs_first_engagement_object_id": { - "type": "number" - }, - "hs_google_click_id": { - "type": "string" - }, - "hs_ip_timezone": { - "type": "string" - }, - "hs_is_contact": { - "type": "bool" - }, - "hs_is_unworked": { - "type": "bool" - }, - "hs_last_sales_activity_date": { - "type": "datetime" - }, - "hs_last_sales_activity_timestamp": { - "type": "datetime" - }, - "hs_lastmodifieddate": { - "type": "datetime" - }, - "hs_lead_status": { - "type": "string" - }, - "hs_legal_basis": { - "type": "string" - }, - "hs_merged_object_ids": { - "type": "string" - }, - "hs_object_id": { - "type": "number" - }, - "hs_predictivecontactscore_v2": { - "type": "number" - }, - "hs_predictivescoringtier": { - "type": "string" - }, - "hs_sa_first_engagement_date": { - "type": "datetime" - }, - "hs_sa_first_engagement_descr": { - "type": "string" - }, - "hs_sa_first_engagement_object_type": { - "type": "string" - }, - "hs_sales_email_last_clicked": { - "type": "datetime" - }, - "hs_sales_email_last_opened": { - "type": "datetime" - }, - "hs_searchable_calculated_international_mobile_number": { - "type": "phone_number" - }, - "hs_searchable_calculated_international_phone_number": { - "type": "phone_number" - }, - "hs_searchable_calculated_mobile_number": { - "type": "phone_number" - }, - "hs_searchable_calculated_phone_number": { - "type": "phone_number" - }, - "hs_sequences_is_enrolled": { - "type": "bool" - }, - "hs_testpurge": { - "type": "string" - }, - "hs_testrollback": { - "type": "string" - }, - "hs_time_to_first_engagement": { - "type": "number" - }, - "hs_updated_by_user_id": { - "type": "number" - }, - "hs_user_ids_of_all_owners": { - "type": "string" - }, - "hubspot_owner_assigneddate": { - "type": "datetime" - }, - "ip_city": { - "type": "string" - }, - "ip_country": { - "type": "string" - }, - "ip_country_code": { - "type": "string" - }, - "ip_latlon": { - "type": "string" - }, - "ip_state": { - "type": "string" - }, - "ip_state_code": { - "type": "string" - }, - "ip_zipcode": { - "type": "string" - }, - "job_function": { - "type": "string" - }, - "lastmodifieddate": { - "type": "datetime" - }, - "marital_status": { - "type": "string" - }, - "military_status": { - "type": "string" - }, - "num_associated_deals": { - "type": "number" - }, - "num_conversion_events": { - "type": "number" - }, - "num_unique_conversion_events": { - "type": "number" - }, - "recent_conversion_date": { - "type": "datetime" - }, - "recent_conversion_event_name": { - "type": "string" - }, - "recent_deal_amount": { - "type": "number" - }, - "recent_deal_close_date": { - "type": "datetime" - }, - "relationship_status": { - "type": "string" - }, - "school": { - "type": "string" - }, - "seniority": { - "type": "string" - }, - "start_date": { - "type": "string" - }, - "total_revenue": { - "type": "number" - }, - "work_email": { - "type": "string" - }, - "firstname": { - "type": "string" - }, - "hs_analytics_first_url": { - "type": "string" - }, - "hs_email_delivered": { - "type": "number" - }, - "hs_email_optout_10798197": { - "type": "string" - }, - "hs_email_optout_11890603": { - "type": "string" - }, - "hs_email_optout_11890831": { - "type": "string" - }, - "twitterhandle": { - "type": "string" - }, - "currentlyinworkflow": { - "type": "string" - }, - "hs_analytics_last_url": { - "type": "string" - }, - "hs_email_open": { - "type": "number" - }, - "lastname": { - "type": "string" - }, - "hs_analytics_num_page_views": { - "type": "number" - }, - "hs_email_click": { - "type": "number" - }, - "salutation": { - "type": "string" - }, - "email": { - "type": "string" - }, - "hs_analytics_num_visits": { - "type": "number" - }, - "hs_email_bounce": { - "type": "number" - }, - "hs_persona": { - "type": "string" - }, - "hs_analytics_num_event_completions": { - "type": "number" - }, - "hs_email_optout": { - "type": "bool" - }, - "mobilephone": { - "type": "string" - }, - "phone": { - "type": "string" - }, - "fax": { - "type": "string" - }, - "hs_analytics_first_timestamp": { - "type": "datetime" - }, - "hs_email_last_email_name": { - "type": "string" - }, - "hs_email_last_send_date": { - "type": "datetime" - }, - "address": { - "type": "string" - }, - "engagements_last_meeting_booked": { - "type": "datetime" - }, - "engagements_last_meeting_booked_campaign": { - "type": "string" - }, - "engagements_last_meeting_booked_medium": { - "type": "string" - }, - "engagements_last_meeting_booked_source": { - "type": "string" - }, - "hs_analytics_first_visit_timestamp": { - "type": "datetime" - }, - "hs_email_last_open_date": { - "type": "datetime" - }, - "hs_latest_meeting_activity": { - "type": "datetime" - }, - "hs_sales_email_last_replied": { - "type": "datetime" - }, - "hubspot_owner_id": { - "type": "string" - }, - "notes_last_contacted": { - "type": "datetime" - }, - "notes_last_updated": { - "type": "datetime" - }, - "notes_next_activity_date": { - "type": "datetime" - }, - "num_contacted_notes": { - "type": "number" - }, - "num_notes": { - "type": "number" - }, - "surveymonkeyeventlastupdated": { - "type": "number" - }, - "webinareventlastupdated": { - "type": "number" - }, - "city": { - "type": "string" - }, - "hs_analytics_last_timestamp": { - "type": "datetime" - }, - "hs_email_last_click_date": { - "type": "datetime" - }, - "hubspot_team_id": { - "type": "string" - }, - "hs_all_owner_ids": { - "type": "string" - }, - "hs_analytics_last_visit_timestamp": { - "type": "datetime" - }, - "hs_email_first_send_date": { - "type": "datetime" - }, - "state": { - "type": "string" - }, - "hs_all_team_ids": { - "type": "string" - }, - "hs_analytics_source": { - "type": "string" - }, - "hs_email_first_open_date": { - "type": "datetime" - }, - "zip": { - "type": "string" - }, - "country": { - "type": "string" - }, - "hs_all_accessible_team_ids": { - "type": "string" - }, - "hs_analytics_source_data_1": { - "type": "string" - }, - "hs_email_first_click_date": { - "type": "datetime" - }, - "hs_analytics_source_data_2": { - "type": "string" - }, - "hs_email_is_ineligible": { - "type": "bool" - }, - "hs_language": { - "type": "string" - }, - "hs_analytics_first_referrer": { - "type": "string" - }, - "jobtitle": { - "type": "string" - }, - "hs_analytics_last_referrer": { - "type": "string" - }, - "message": { - "type": "string" - }, - "closedate": { - "type": "datetime" - }, - "hs_analytics_average_page_views": { - "type": "number" - }, - "hs_analytics_revenue": { - "type": "number" - }, - "hs_lifecyclestage_lead_date": { - "type": "datetime" - }, - "hs_lifecyclestage_marketingqualifiedlead_date": { - "type": "datetime" - }, - "hs_lifecyclestage_opportunity_date": { - "type": "datetime" - }, - "lifecyclestage": { - "type": "string" - }, - "hs_lifecyclestage_salesqualifiedlead_date": { - "type": "datetime" - }, - "createdate": { - "type": "datetime" - }, - "hs_lifecyclestage_evangelist_date": { - "type": "datetime" - }, - "hs_lifecyclestage_customer_date": { - "type": "datetime" - }, - "hubspotscore": { - "type": "number" - }, - "company": { - "type": "string" - }, - "hs_lifecyclestage_subscriber_date": { - "type": "datetime" - }, - "hs_lifecyclestage_other_date": { - "type": "datetime" - }, - "website": { - "type": "string" - }, - "numemployees": { - "type": "string" - }, - "annualrevenue": { - "type": "string" - }, - "industry": { - "type": "string" - }, - "associatedcompanyid": { - "type": "number" - }, - "associatedcompanylastupdated": { - "type": "number" - }, - "hs_predictivecontactscorebucket": { - "type": "string" - }, - "hs_predictivecontactscore": { - "type": "number" - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - }, - { - "stream": { - "name": "deal_pipelines", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "label": { - "type": "string" - }, - "displayOrder": { - "type": "integer" - }, - "active": { - "type": "boolean" - }, - "stages": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { - "type": "string" - }, - "displayOrder": { - "type": "integer" - }, - "metadata": { - "type": "object", - "properties": { - "isClosed": { - "type": "string" - }, - "probability": { - "type": "string" - } - } - }, - "stageId": { - "type": "string" - }, - "createdAt": { - "type": ["null", "integer"] - }, - "updatedAt": { - "type": ["null", "integer"] - }, - "active": { - "type": "boolean" - } - } - } - }, - "objectType": { - "type": "string" - }, - "objectTypeId": { - "type": "string" - }, - "pipelineId": { - "type": "string" - }, - "createdAt": { - "type": "integer" - }, - "updatedAt": { - "type": ["null", "integer"] - }, - "default": { - "type": "boolean" - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - }, - { - "stream": { - "name": "deals", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "portalId": { - "type": ["null", "integer"] - }, - "dealId": { - "type": ["null", "integer"] - }, - "isDeleted": { - "type": ["null", "boolean"] - }, - "associations": { - "type": ["null", "object"], - "properties": { - "associatedVids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "associatedCompanyIds": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "associatedDealIds": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - } - } - }, - "imports": { - "type": ["null", "array"] - }, - "stateChanges": { - "type": ["null", "array"] - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "properties": { - "type": "object", - "properties": { - "amount_in_home_currency": { - "type": "number" - }, - "days_to_close": { - "type": "number" - }, - "hs_acv": { - "type": "number" - }, - "hs_all_assigned_business_unit_ids": { - "type": "string" - }, - "hs_analytics_source": { - "type": "string" - }, - "hs_analytics_source_data_1": { - "type": "string" - }, - "hs_analytics_source_data_2": { - "type": "string" - }, - "hs_arr": { - "type": "number" - }, - "hs_campaign": { - "type": "string" - }, - "hs_closed_amount": { - "type": "number" - }, - "hs_closed_amount_in_home_currency": { - "type": "number" - }, - "hs_created_by_user_id": { - "type": "number" - }, - "hs_date_entered_9567448": { - "type": "datetime" - }, - "hs_date_entered_9567449": { - "type": "datetime" - }, - "hs_date_entered_appointmentscheduled": { - "type": "datetime" - }, - "hs_date_entered_closedlost": { - "type": "datetime" - }, - "hs_date_entered_closedwon": { - "type": "datetime" - }, - "hs_date_entered_contractsent": { - "type": "datetime" - }, - "hs_date_entered_customclosedwonstage": { - "type": "datetime" - }, - "hs_date_entered_decisionmakerboughtin": { - "type": "datetime" - }, - "hs_date_entered_presentationscheduled": { - "type": "datetime" - }, - "hs_date_entered_qualifiedtobuy": { - "type": "datetime" - }, - "hs_date_exited_9567448": { - "type": "datetime" - }, - "hs_date_exited_9567449": { - "type": "datetime" - }, - "hs_date_exited_appointmentscheduled": { - "type": "datetime" - }, - "hs_date_exited_closedlost": { - "type": "datetime" - }, - "hs_date_exited_closedwon": { - "type": "datetime" - }, - "hs_date_exited_contractsent": { - "type": "datetime" - }, - "hs_date_exited_customclosedwonstage": { - "type": "datetime" - }, - "hs_date_exited_decisionmakerboughtin": { - "type": "datetime" - }, - "hs_date_exited_presentationscheduled": { - "type": "datetime" - }, - "hs_date_exited_qualifiedtobuy": { - "type": "datetime" - }, - "hs_deal_amount_calculation_preference": { - "type": "string" - }, - "hs_deal_stage_probability": { - "type": "number" - }, - "hs_forecast_amount": { - "type": "number" - }, - "hs_forecast_probability": { - "type": "number" - }, - "hs_is_closed": { - "type": "bool" - }, - "hs_lastmodifieddate": { - "type": "datetime" - }, - "hs_likelihood_to_close": { - "type": "number" - }, - "hs_line_item_global_term_hs_discount_percentage": { - "type": "string" - }, - "hs_line_item_global_term_hs_discount_percentage_enabled": { - "type": "bool" - }, - "hs_line_item_global_term_hs_recurring_billing_period": { - "type": "string" - }, - "hs_line_item_global_term_hs_recurring_billing_period_enabled": { - "type": "bool" - }, - "hs_line_item_global_term_hs_recurring_billing_start_date": { - "type": "string" - }, - "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": { - "type": "bool" - }, - "hs_line_item_global_term_recurringbillingfrequency": { - "type": "string" - }, - "hs_line_item_global_term_recurringbillingfrequency_enabled": { - "type": "bool" - }, - "hs_manual_forecast_category": { - "type": "string" - }, - "hs_merged_object_ids": { - "type": "string" - }, - "hs_mrr": { - "type": "number" - }, - "hs_next_step": { - "type": "string" - }, - "hs_object_id": { - "type": "number" - }, - "hs_predicted_amount": { - "type": "number" - }, - "hs_predicted_amount_in_home_currency": { - "type": "number" - }, - "hs_projected_amount": { - "type": "number" - }, - "hs_projected_amount_in_home_currency": { - "type": "number" - }, - "hs_tcv": { - "type": "number" - }, - "hs_time_in_9567448": { - "type": "number" - }, - "hs_time_in_9567449": { - "type": "number" - }, - "hs_time_in_appointmentscheduled": { - "type": "number" - }, - "hs_time_in_closedlost": { - "type": "number" - }, - "hs_time_in_closedwon": { - "type": "number" - }, - "hs_time_in_contractsent": { - "type": "number" - }, - "hs_time_in_customclosedwonstage": { - "type": "number" - }, - "hs_time_in_decisionmakerboughtin": { - "type": "number" - }, - "hs_time_in_presentationscheduled": { - "type": "number" - }, - "hs_time_in_qualifiedtobuy": { - "type": "number" - }, - "hs_updated_by_user_id": { - "type": "number" - }, - "hs_user_ids_of_all_owners": { - "type": "string" - }, - "hubspot_owner_assigneddate": { - "type": "datetime" - }, - "dealname": { - "type": "string" - }, - "amount": { - "type": "number" - }, - "dealstage": { - "type": "string" - }, - "pipeline": { - "type": "string" - }, - "closedate": { - "type": "datetime" - }, - "createdate": { - "type": "datetime" - }, - "engagements_last_meeting_booked": { - "type": "datetime" - }, - "engagements_last_meeting_booked_campaign": { - "type": "string" - }, - "engagements_last_meeting_booked_medium": { - "type": "string" - }, - "engagements_last_meeting_booked_source": { - "type": "string" - }, - "hs_latest_meeting_activity": { - "type": "datetime" - }, - "hs_sales_email_last_replied": { - "type": "datetime" - }, - "hubspot_owner_id": { - "type": "string" - }, - "notes_last_contacted": { - "type": "datetime" - }, - "notes_last_updated": { - "type": "datetime" - }, - "notes_next_activity_date": { - "type": "datetime" - }, - "num_contacted_notes": { - "type": "number" - }, - "num_notes": { - "type": "number" - }, - "hs_createdate": { - "type": "datetime" - }, - "hubspot_team_id": { - "type": "string" - }, - "dealtype": { - "type": "string" - }, - "hs_all_owner_ids": { - "type": "string" - }, - "description": { - "type": "string" - }, - "hs_all_team_ids": { - "type": "string" - }, - "hs_all_accessible_team_ids": { - "type": "string" - }, - "num_associated_contacts": { - "type": "number" - }, - "closed_lost_reason": { - "type": "string" - }, - "closed_won_reason": { - "type": "string" - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - }, - { - "stream": { - "name": "email_events", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "appId": { - "type": ["null", "integer"] - }, - "appName": { - "type": ["null", "string"] - }, - "browser": { - "type": ["null", "object"], - "properties": { - "family": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "producer": { - "type": ["null", "string"] - }, - "producerUrl": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - }, - "created": { - "type": "integer" - }, - "deviceType": { - "type": ["null", "string"] - }, - "duration": { - "type": ["null", "integer"] - }, - "emailCampaignId": { - "type": ["null", "integer"] - }, - "emailCampaignGroupId": { - "type": ["null", "integer"] - }, - "filteredEvent": { - "type": ["null", "boolean"] - }, - "from": { - "type": ["null", "string"] - }, - "hmid": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "ipAddress": { - "type": ["null", "string"] - }, - "linkId": { - "type": ["null", "integer"] - }, - "location": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } - }, - "portalId": { - "type": ["null", "integer"] - }, - "recipient": { - "type": ["null", "string"] - }, - "response": { - "type": ["null", "string"] - }, - "sentBy": { - "type": ["null", "object"], - "properties": { - "created": { - "type": ["null", "string"], - "format": "date-time" - }, - "id": { - "type": ["null", "string"] - } - } - }, - "smtpId": { - "type": ["null", "string"] - }, - "subject": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "userAgent": { - "type": ["null", "string"] - } - } - }, - "supported_sync_modes": ["incremental", "full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": ["created"] - }, - "sync_mode": "incremental", - "cursor_field": ["created"] - }, - { - "stream": { - "name": "engagements", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "portalId": { - "type": "integer" - }, - "active": { - "type": "boolean" - }, - "createdAt": { - "type": "integer" - }, - "lastUpdated": { - "type": "integer" - }, - "ownerId": { - "type": "integer" - }, - "type": { - "type": "string" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "associations": { - "type": ["null", "object"], - "properties": { - "contactIds": { - "type": ["null", "array"], - "items": { - "type": "integer" - } - }, - "companyIds": { - "type": ["null", "array"], - "items": { - "type": "integer" - } - }, - "dealIds": { - "type": ["null", "array"], - "items": { - "type": "integer" - } - } - } - }, - "attachments": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "id": { - "type": "integer" - } - } - } - }, - "metadata": { - "type": ["null", "object"], - "properties": { - "body": { - "type": ["null", "string"] - }, - "from": { - "type": ["null", "object"], - "properties": { - "email": { - "type": "string" - }, - "firstName": { - "type": "string" - }, - "lastName": { - "type": "string" - } - } - }, - "to": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "email": { - "type": "string" - } - } - } - }, - "cc": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "email": { - "type": "string" - } - } - } - }, - "bcc": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "email": { - "type": "string" - } - } - } - }, - "subject": { - "type": ["null", "string"] - }, - "html": { - "type": ["null", "string"] - }, - "text": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "forObjectType": { - "type": ["null", "string"] - }, - "startTime": { - "type": ["null", "integer"] - }, - "endTime": { - "type": ["null", "integer"] - }, - "title": { - "type": ["null", "string"] - }, - "toNumber": { - "type": ["null", "string"] - }, - "fromNumber": { - "type": ["null", "string"] - }, - "externalId": { - "type": ["null", "string"] - }, - "durationMilliseconds": { - "type": ["null", "integer"] - }, - "externalAccountId": { - "type": ["null", "string"] - }, - "recordingUrl": { - "type": ["null", "string"], - "format": "uri" - }, - "disposition": { - "type": ["null", "string"] - } - } - }, - "properties": { - "type": "object", - "properties": { - "hs_activity_type": { - "type": "string" - }, - "hs_all_assigned_business_unit_ids": { - "type": "string" - }, - "hs_at_mentioned_owner_ids": { - "type": "string" - }, - "hs_attachment_ids": { - "type": "string" - }, - "hs_body_preview": { - "type": "string" - }, - "hs_body_preview_html": { - "type": "string" - }, - "hs_body_preview_is_truncated": { - "type": "bool" - }, - "hs_call_app_id": { - "type": "number" - }, - "hs_call_authed_url_provider": { - "type": "string" - }, - "hs_call_body": { - "type": "string" - }, - "hs_call_callee_object_id": { - "type": "number" - }, - "hs_call_callee_object_type": { - "type": "string" - }, - "hs_call_disposition": { - "type": "string" - }, - "hs_call_duration": { - "type": "number" - }, - "hs_call_external_account_id": { - "type": "string" - }, - "hs_call_external_id": { - "type": "string" - }, - "hs_call_from_number": { - "type": "string" - }, - "hs_call_has_transcript": { - "type": "bool" - }, - "hs_call_recording_url": { - "type": "string" - }, - "hs_call_source": { - "type": "string" - }, - "hs_call_status": { - "type": "string" - }, - "hs_call_title": { - "type": "string" - }, - "hs_call_to_number": { - "type": "string" - }, - "hs_call_transcription_id": { - "type": "number" - }, - "hs_call_video_recording_url": { - "type": "string" - }, - "hs_calls_service_call_id": { - "type": "number" - }, - "hs_conversation_session_agent_join_time": { - "type": "datetime" - }, - "hs_conversation_session_agent_response_time": { - "type": "number" - }, - "hs_conversation_session_duration": { - "type": "number" - }, - "hs_conversation_session_is_bot": { - "type": "bool" - }, - "hs_conversation_session_num_agent_messages": { - "type": "number" - }, - "hs_conversation_session_num_visitor_messages": { - "type": "number" - }, - "hs_conversation_session_online": { - "type": "bool" - }, - "hs_conversation_session_session_closed_at": { - "type": "datetime" - }, - "hs_conversation_session_source": { - "type": "string" - }, - "hs_conversation_session_thread_id": { - "type": "number" - }, - "hs_conversation_session_url": { - "type": "string" - }, - "hs_conversation_session_visitor_end_time": { - "type": "datetime" - }, - "hs_conversation_session_visitor_start_time": { - "type": "datetime" - }, - "hs_conversation_session_visitor_wait_time": { - "type": "number" - }, - "hs_created_by": { - "type": "string" - }, - "hs_created_by_user_id": { - "type": "number" - }, - "hs_createdate": { - "type": "datetime" - }, - "hs_direction_and_unique_id": { - "type": "string" - }, - "hs_email_attached_video_id": { - "type": "string" - }, - "hs_email_attached_video_name": { - "type": "string" - }, - "hs_email_attached_video_opened": { - "type": "bool" - }, - "hs_email_attached_video_watched": { - "type": "bool" - }, - "hs_email_bcc_email": { - "type": "string" - }, - "hs_email_bcc_firstname": { - "type": "string" - }, - "hs_email_bcc_lastname": { - "type": "string" - }, - "hs_email_bcc_raw": { - "type": "string" - }, - "hs_email_cc_email": { - "type": "string" - }, - "hs_email_cc_firstname": { - "type": "string" - }, - "hs_email_cc_lastname": { - "type": "string" - }, - "hs_email_cc_raw": { - "type": "string" - }, - "hs_email_direction": { - "type": "string" - }, - "hs_email_encoded_email_associations_request": { - "type": "string" - }, - "hs_email_error_message": { - "type": "string" - }, - "hs_email_facsimile_send_id": { - "type": "string" - }, - "hs_email_from_email": { - "type": "string" - }, - "hs_email_from_firstname": { - "type": "string" - }, - "hs_email_from_lastname": { - "type": "string" - }, - "hs_email_from_raw": { - "type": "string" - }, - "hs_email_headers": { - "type": "json" - }, - "hs_email_html": { - "type": "string" - }, - "hs_email_logged_from": { - "type": "string" - }, - "hs_email_media_processing_status": { - "type": "string" - }, - "hs_email_member_of_forwarded_subthread": { - "type": "bool" - }, - "hs_email_message_id": { - "type": "string" - }, - "hs_email_post_send_status": { - "type": "string" - }, - "hs_email_recipient_drop_reasons": { - "type": "json" - }, - "hs_email_send_event_id": { - "type": "string" - }, - "hs_email_send_event_id_created": { - "type": "datetime" - }, - "hs_email_sender_email": { - "type": "string" - }, - "hs_email_sender_firstname": { - "type": "string" - }, - "hs_email_sender_lastname": { - "type": "string" - }, - "hs_email_sender_raw": { - "type": "string" - }, - "hs_email_sent_via": { - "type": "string" - }, - "hs_email_status": { - "type": "string" - }, - "hs_email_subject": { - "type": "string" - }, - "hs_email_text": { - "type": "string" - }, - "hs_email_thread_id": { - "type": "string" - }, - "hs_email_to_email": { - "type": "string" - }, - "hs_email_to_firstname": { - "type": "string" - }, - "hs_email_to_lastname": { - "type": "string" - }, - "hs_email_to_raw": { - "type": "string" - }, - "hs_email_tracker_key": { - "type": "string" - }, - "hs_email_validation_skipped": { - "type": "string" - }, - "hs_engagement_source": { - "type": "string" - }, - "hs_engagement_source_id": { - "type": "string" - }, - "hs_engagement_type": { - "type": "string" - }, - "hs_follow_up_action": { - "type": "string" - }, - "hs_gdpr_deleted": { - "type": "bool" - }, - "hs_internal_meeting_notes": { - "type": "string" - }, - "hs_lastmodifieddate": { - "type": "datetime" - }, - "hs_meeting_body": { - "type": "string" - }, - "hs_meeting_created_from_link_id": { - "type": "string" - }, - "hs_meeting_end_time": { - "type": "datetime" - }, - "hs_meeting_external_url": { - "type": "string" - }, - "hs_meeting_location": { - "type": "string" - }, - "hs_meeting_outcome": { - "type": "string" - }, - "hs_meeting_pre_meeting_prospect_reminders": { - "type": "string" - }, - "hs_meeting_source": { - "type": "string" - }, - "hs_meeting_source_id": { - "type": "string" - }, - "hs_meeting_start_time": { - "type": "datetime" - }, - "hs_meeting_title": { - "type": "string" - }, - "hs_meeting_web_conference_meeting_id": { - "type": "string" - }, - "hs_merged_object_ids": { - "type": "string" - }, - "hs_modified_by": { - "type": "string" - }, - "hs_note_body": { - "type": "string" - }, - "hs_num_associated_companies": { - "type": "number" - }, - "hs_num_associated_contacts": { - "type": "number" - }, - "hs_num_associated_deals": { - "type": "number" - }, - "hs_num_associated_queue_objects": { - "type": "number" - }, - "hs_num_associated_tickets": { - "type": "number" - }, - "hs_object_id": { - "type": "number" - }, - "hs_product_name": { - "type": "string" - }, - "hs_publishing_task_body": { - "type": "string" - }, - "hs_publishing_task_campaign_guid": { - "type": "string" - }, - "hs_publishing_task_category": { - "type": "string" - }, - "hs_publishing_task_category_id": { - "type": "number" - }, - "hs_publishing_task_content_id": { - "type": "string" - }, - "hs_publishing_task_name": { - "type": "string" - }, - "hs_publishing_task_state": { - "type": "string" - }, - "hs_queue_membership_ids": { - "type": "string" - }, - "hs_scheduled_tasks": { - "type": "json" - }, - "hs_task_body": { - "type": "string" - }, - "hs_task_completion_date": { - "type": "datetime" - }, - "hs_task_for_object_type": { - "type": "string" - }, - "hs_task_is_all_day": { - "type": "bool" - }, - "hs_task_last_contact_outreach": { - "type": "datetime" - }, - "hs_task_last_sales_activity_timestamp": { - "type": "datetime" - }, - "hs_task_priority": { - "type": "string" - }, - "hs_task_probability_to_complete": { - "type": "number" - }, - "hs_task_reminders": { - "type": "string" - }, - "hs_task_repeat_interval": { - "type": "string" - }, - "hs_task_send_default_reminder": { - "type": "bool" - }, - "hs_task_sequence_enrollment_active": { - "type": "bool" - }, - "hs_task_sequence_step_enrollment_id": { - "type": "string" - }, - "hs_task_sequence_step_order": { - "type": "number" - }, - "hs_task_status": { - "type": "string" - }, - "hs_task_subject": { - "type": "string" - }, - "hs_task_template_id": { - "type": "number" - }, - "hs_task_type": { - "type": "string" - }, - "hs_timestamp": { - "type": "datetime" - }, - "hs_unique_id": { - "type": "string" - }, - "hs_unknown_visitor_conversation": { - "type": "bool" - }, - "hs_updated_by_user_id": { - "type": "number" - }, - "hs_user_ids_of_all_owners": { - "type": "string" - }, - "hubspot_owner_assigneddate": { - "type": "datetime" - }, - "hubspot_owner_id": { - "type": "string" - }, - "hubspot_team_id": { - "type": "string" - }, - "hs_all_owner_ids": { - "type": "string" - }, - "hs_all_team_ids": { - "type": "string" - }, - "hs_all_accessible_team_ids": { - "type": "string" - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - }, - { - "stream": { - "name": "forms", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "deletedAt": { - "type": ["null", "integer"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "guid": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "action": { - "type": ["null", "string"] - }, - "method": { - "type": ["null", "string"] - }, - "cssClass": { - "type": ["null", "string"] - }, - "redirect": { - "type": ["null", "string"] - }, - "submitText": { - "type": ["null", "string"] - }, - "followUpId": { - "type": ["null", "string"] - }, - "notifyRecipients": { - "type": ["null", "string"] - }, - "leadNurturingCampaignId": { - "type": ["null", "string"] - }, - "formFieldGroups": { - "type": "array", - "items": { - "type": "object", - "properties": { - "fields": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "fieldType": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "groupName": { - "type": ["null", "string"] - }, - "displayOrder": { - "type": ["null", "integer"] - }, - "required": { - "type": ["null", "boolean"] - }, - "validation": { - "type": "object", - "properties": { - "name": { - "type": ["null", "string"] - }, - "message": { - "type": ["null", "string"] - }, - "data": { - "type": ["null", "string"] - }, - "useDefaultBlockList": { - "type": ["null", "boolean"] - }, - "blockedEmailAddresses": { - "type": "array", - "items": { - "type": ["null", "string"] - } - } - } - }, - "enabled": { - "type": ["null", "boolean"] - }, - "hidden": { - "type": ["null", "boolean"] - }, - "defaultValue": { - "type": ["null", "string"] - }, - "isSmartField": { - "type": ["null", "boolean"] - }, - "unselectedLabel": { - "type": ["null", "string"] - }, - "placeholder": { - "type": ["null", "string"] - }, - "labelHidden": { - "type": ["null", "boolean"] - }, - "options": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": ["null", "string"] - }, - "displayOrder": { - "type": ["null", "integer"] - }, - "doubleData": { - "type": ["null", "number"] - }, - "hidden": { - "type": ["null", "boolean"] - }, - "label": { - "type": ["null", "string"] - }, - "readOnly": { - "type": ["null", "boolean"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "selectedOptions": { - "type": "array", - "items": { - "type": ["null", "string"] - } - } - } - } - }, - "default": { - "type": ["null", "boolean"] - }, - "isSmartGroup": { - "type": ["null", "boolean"] - }, - "richText": { - "type": "object", - "properties": { - "content": { - "type": ["null", "string"] - } - } - } - } - } - }, - "createdAt": { - "type": "integer" - }, - "updatedAt": { - "type": "integer" - }, - "performableHtml": { - "type": ["null", "string"] - }, - "migratedFrom": { - "type": ["null", "string"] - }, - "ignoreCurrentValues": { - "type": ["null", "boolean"] - }, - "deletable": { - "type": ["null", "boolean"] - }, - "inlineMessage": { - "type": ["null", "string"] - }, - "tmsId": { - "type": ["null", "string"] - }, - "captchaEnabled": { - "type": ["null", "boolean"] - }, - "campaignGuid": { - "type": ["null", "string"] - }, - "cloneable": { - "type": ["null", "boolean"] - }, - "editable": { - "type": ["null", "boolean"] - }, - "formType": { - "type": ["null", "string"] - }, - "metaData": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "properties": { - "type": "object", - "properties": { - "hs_all_accessible_team_ids": { - "type": "string" - }, - "hs_all_assigned_business_unit_ids": { - "type": "string" - }, - "hs_asset_subtype": { - "type": "string" - }, - "hs_created_at": { - "type": "datetime" - }, - "hs_created_by_user_id": { - "type": "number" - }, - "hs_createdate": { - "type": "datetime" - }, - "hs_lastmodifieddate": { - "type": "datetime" - }, - "hs_merged_object_ids": { - "type": "string" - }, - "hs_name": { - "type": "string" - }, - "hs_object_id": { - "type": "number" - }, - "hs_origin_asset_id": { - "type": "string" - }, - "hs_publish_status": { - "type": "string" - }, - "hs_published_at": { - "type": "datetime" - }, - "hs_updated_by_user_id": { - "type": "number" - }, - "hs_user_ids_of_all_owners": { - "type": "string" - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - }, - { - "stream": { - "name": "line_items", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "objectType": { - "type": ["null", "string"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "objectId": { - "type": ["null", "integer"] - }, - "isDeleted": { - "type": ["null", "boolean"] - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "properties": { - "type": "object", - "properties": { - "amount": { - "type": "number" - }, - "createdate": { - "type": "datetime" - }, - "description": { - "type": "string" - }, - "discount": { - "type": "number" - }, - "hs_acv": { - "type": "number" - }, - "hs_all_accessible_team_ids": { - "type": "string" - }, - "hs_all_assigned_business_unit_ids": { - "type": "string" - }, - "hs_arr": { - "type": "number" - }, - "hs_cost_of_goods_sold": { - "type": "number" - }, - "hs_created_by_user_id": { - "type": "number" - }, - "hs_createdate": { - "type": "datetime" - }, - "hs_discount_percentage": { - "type": "number" - }, - "hs_images": { - "type": "string" - }, - "hs_lastmodifieddate": { - "type": "datetime" - }, - "hs_line_item_currency_code": { - "type": "string" - }, - "hs_margin": { - "type": "number" - }, - "hs_margin_acv": { - "type": "number" - }, - "hs_margin_arr": { - "type": "number" - }, - "hs_margin_mrr": { - "type": "number" - }, - "hs_margin_tcv": { - "type": "number" - }, - "hs_merged_object_ids": { - "type": "string" - }, - "hs_mrr": { - "type": "number" - }, - "hs_object_id": { - "type": "number" - }, - "hs_position_on_quote": { - "type": "number" - }, - "hs_product_id": { - "type": "number" - }, - "hs_recurring_billing_end_date": { - "type": "string" - }, - "hs_recurring_billing_period": { - "type": "string" - }, - "hs_recurring_billing_start_date": { - "type": "string" - }, - "hs_sku": { - "type": "string" - }, - "hs_sync_amount": { - "type": "number" - }, - "hs_tcv": { - "type": "number" - }, - "hs_term_in_months": { - "type": "number" - }, - "hs_updated_by_user_id": { - "type": "number" - }, - "hs_url": { - "type": "string" - }, - "hs_user_ids_of_all_owners": { - "type": "string" - }, - "name": { - "type": "string" - }, - "price": { - "type": "number" - }, - "quantity": { - "type": "number" - }, - "recurringbillingfrequency": { - "type": "string" - }, - "tax": { - "type": "number" - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - }, - { - "stream": { - "name": "owners", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "portalId": { - "type": ["null", "integer"] - }, - "ownerId": { - "type": ["null", "integer"] - }, - "type": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "signature": { - "type": ["null", "string"] - }, - "updatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "hasContactsAccess": { - "type": ["null", "boolean"] - }, - "isActive": { - "type": ["null", "boolean"] - }, - "activeUserId": { - "type": ["null", "integer"] - }, - "userIdIncludingInactive": { - "type": ["null", "integer"] - }, - "remoteList": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "ownerId": { - "type": ["null", "integer"] - }, - "remoteId": { - "type": ["null", "string"] - }, - "remoteType": { - "type": ["null", "string"] - }, - "active": { - "type": ["null", "boolean"] - } - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - }, - { - "stream": { - "name": "products", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "objectType": { - "type": ["null", "string"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "objectId": { - "type": ["null", "integer"] - }, - "isDeleted": { - "type": ["null", "boolean"] - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "properties": { - "type": "object", - "properties": { - "amount": { - "type": "number" - }, - "createdate": { - "type": "datetime" - }, - "description": { - "type": "string" - }, - "discount": { - "type": "number" - }, - "hs_all_accessible_team_ids": { - "type": "string" - }, - "hs_all_assigned_business_unit_ids": { - "type": "string" - }, - "hs_avatar_filemanager_key": { - "type": "string" - }, - "hs_cost_of_goods_sold": { - "type": "number" - }, - "hs_created_by_user_id": { - "type": "number" - }, - "hs_createdate": { - "type": "datetime" - }, - "hs_discount_percentage": { - "type": "number" - }, - "hs_folder_id": { - "type": "number" - }, - "hs_images": { - "type": "string" - }, - "hs_lastmodifieddate": { - "type": "datetime" - }, - "hs_merged_object_ids": { - "type": "string" - }, - "hs_object_id": { - "type": "number" - }, - "hs_recurring_billing_period": { - "type": "string" - }, - "hs_recurring_billing_start_date": { - "type": "string" - }, - "hs_sku": { - "type": "string" - }, - "hs_updated_by_user_id": { - "type": "number" - }, - "hs_url": { - "type": "string" - }, - "hs_user_ids_of_all_owners": { - "type": "string" - }, - "name": { - "type": "string" - }, - "price": { - "type": "number" - }, - "quantity": { - "type": "number" - }, - "recurringbillingfrequency": { - "type": "string" - }, - "tax": { - "type": "number" - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - }, - { - "stream": { - "name": "quotes", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "createdAt": { - "type": ["null", "string"] - }, - "updatedAt": { - "type": ["null", "string"] - }, - "archived": { - "type": ["null", "boolean"] - }, - "archivedAt": { - "type": ["null", "string"] - }, - "associations": { - "type": "object" - }, - "properties": { - "type": "object", - "properties": { - "hs_all_assigned_business_unit_ids": { - "type": "string" - }, - "hs_approver_id": { - "type": "string" - }, - "hs_created_by_user_id": { - "type": "number" - }, - "hs_createdate": { - "type": "datetime" - }, - "hs_esign_date": { - "type": "datetime" - }, - "hs_feedback": { - "type": "string" - }, - "hs_language": { - "type": "string" - }, - "hs_lastmodifieddate": { - "type": "datetime" - }, - "hs_line_item_global_term_hs_discount_percentage": { - "type": "string" - }, - "hs_line_item_global_term_hs_discount_percentage_enabled": { - "type": "bool" - }, - "hs_line_item_global_term_hs_recurring_billing_period": { - "type": "string" - }, - "hs_line_item_global_term_hs_recurring_billing_period_enabled": { - "type": "bool" - }, - "hs_line_item_global_term_hs_recurring_billing_start_date": { - "type": "string" - }, - "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": { - "type": "bool" - }, - "hs_line_item_global_term_recurringbillingfrequency": { - "type": "string" - }, - "hs_line_item_global_term_recurringbillingfrequency_enabled": { - "type": "bool" - }, - "hs_locale": { - "type": "string" - }, - "hs_locked": { - "type": "bool" - }, - "hs_merged_object_ids": { - "type": "string" - }, - "hs_object_id": { - "type": "number" - }, - "hs_payment_date": { - "type": "datetime" - }, - "hs_pdf_download_link": { - "type": "string" - }, - "hs_public_url_key": { - "type": "string" - }, - "hs_sender_image_url": { - "type": "string" - }, - "hs_template_type": { - "type": "string" - }, - "hs_updated_by_user_id": { - "type": "number" - }, - "hs_user_ids_of_all_owners": { - "type": "string" - }, - "hubspot_owner_assigneddate": { - "type": "datetime" - }, - "hs_sender_company_name": { - "type": "string" - }, - "hs_sender_firstname": { - "type": "string" - }, - "hs_title": { - "type": "string" - }, - "hs_expiration_date": { - "type": "datetime" - }, - "hs_sender_company_domain": { - "type": "string" - }, - "hs_sender_lastname": { - "type": "string" - }, - "hs_comments": { - "type": "string" - }, - "hs_sender_company_address": { - "type": "string" - }, - "hs_sender_email": { - "type": "string" - }, - "hs_sender_company_address2": { - "type": "string" - }, - "hs_sender_phone": { - "type": "string" - }, - "hs_terms": { - "type": "string" - }, - "hs_logo_url": { - "type": "string" - }, - "hs_sender_company_city": { - "type": "string" - }, - "hs_sender_jobtitle": { - "type": "string" - }, - "hs_sender_company_state": { - "type": "string" - }, - "hs_show_signature_box": { - "type": "bool" - }, - "hs_primary_color": { - "type": "string" - }, - "hs_sales_email_last_replied": { - "type": "datetime" - }, - "hs_sender_company_zip": { - "type": "string" - }, - "hs_show_countersignature_box": { - "type": "bool" - }, - "hubspot_owner_id": { - "type": "string" - }, - "notes_last_contacted": { - "type": "datetime" - }, - "notes_last_updated": { - "type": "datetime" - }, - "notes_next_activity_date": { - "type": "datetime" - }, - "num_contacted_notes": { - "type": "number" - }, - "num_notes": { - "type": "number" - }, - "hs_currency": { - "type": "string" - }, - "hs_sender_company_country": { - "type": "string" - }, - "hubspot_team_id": { - "type": "string" - }, - "hs_all_owner_ids": { - "type": "string" - }, - "hs_sender_company_image_url": { - "type": "string" - }, - "hs_timezone": { - "type": "string" - }, - "hs_all_team_ids": { - "type": "string" - }, - "hs_payment_enabled": { - "type": "bool" - }, - "hs_all_accessible_team_ids": { - "type": "string" - }, - "hs_esign_enabled": { - "type": "bool" - }, - "hs_num_associated_deals": { - "type": "number" - }, - "hs_esign_num_signers_required": { - "type": "number" - }, - "hs_template": { - "type": "string" - }, - "hs_esign_num_signers_completed": { - "type": "number" - }, - "hs_quote_amount": { - "type": "number" - }, - "hs_status": { - "type": "string" - }, - "hs_quote_number": { - "type": "string" - }, - "hs_collect_shipping_address": { - "type": "bool" - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - }, - { - "stream": { - "name": "subscription_changes", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "portalId": { - "type": ["null", "integer"] - }, - "recipient": { - "type": ["null", "string"] - }, - "changes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "change": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "subscriptionId": { - "type": ["null", "integer"] - }, - "changeType": { - "type": ["null", "string"] - }, - "causedByEvent": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "created": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - } - } - } - } - }, - "supported_sync_modes": ["incremental", "full_refresh"], - "source_defined_cursor": true, - "default_cursor_field": ["timestamp"] - }, - "sync_mode": "incremental", - "cursor_field": ["timestamp"] - }, - { - "stream": { - "name": "tickets", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "objectType": { - "type": ["null", "string"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "objectId": { - "type": ["null", "integer"] - }, - "isDeleted": { - "type": ["null", "boolean"] - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - }, - "properties": { - "type": "object", - "properties": { - "closed_date": { - "type": "datetime" - }, - "created_by": { - "type": "number" - }, - "createdate": { - "type": "datetime" - }, - "first_agent_reply_date": { - "type": "datetime" - }, - "hs_all_assigned_business_unit_ids": { - "type": "string" - }, - "hs_conversations_originating_message_id": { - "type": "string" - }, - "hs_conversations_originating_thread_id": { - "type": "number" - }, - "hs_created_by_user_id": { - "type": "number" - }, - "hs_createdate": { - "type": "datetime" - }, - "hs_custom_inbox": { - "type": "number" - }, - "hs_date_entered_1": { - "type": "datetime" - }, - "hs_date_entered_2": { - "type": "datetime" - }, - "hs_date_entered_3": { - "type": "datetime" - }, - "hs_date_entered_4": { - "type": "datetime" - }, - "hs_date_exited_1": { - "type": "datetime" - }, - "hs_date_exited_2": { - "type": "datetime" - }, - "hs_date_exited_3": { - "type": "datetime" - }, - "hs_date_exited_4": { - "type": "datetime" - }, - "hs_external_object_ids": { - "type": "string" - }, - "hs_feedback_last_ces_follow_up": { - "type": "string" - }, - "hs_feedback_last_ces_rating": { - "type": "string" - }, - "hs_feedback_last_survey_date": { - "type": "datetime" - }, - "hs_file_upload": { - "type": "string" - }, - "hs_last_email_activity": { - "type": "string" - }, - "hs_last_email_date": { - "type": "datetime" - }, - "hs_lastactivitydate": { - "type": "datetime" - }, - "hs_lastcontacted": { - "type": "datetime" - }, - "hs_lastmodifieddate": { - "type": "datetime" - }, - "hs_merged_object_ids": { - "type": "string" - }, - "hs_nextactivitydate": { - "type": "datetime" - }, - "hs_num_associated_companies": { - "type": "number" - }, - "hs_num_times_contacted": { - "type": "number" - }, - "hs_object_id": { - "type": "number" - }, - "hs_originating_email_engagement_id": { - "type": "number" - }, - "hs_pipeline": { - "type": "string" - }, - "hs_pipeline_stage": { - "type": "string" - }, - "hs_resolution": { - "type": "string" - }, - "hs_ticket_category": { - "type": "string" - }, - "hs_ticket_id": { - "type": "number" - }, - "hs_ticket_priority": { - "type": "string" - }, - "hs_time_in_1": { - "type": "number" - }, - "hs_time_in_2": { - "type": "number" - }, - "hs_time_in_3": { - "type": "number" - }, - "hs_time_in_4": { - "type": "number" - }, - "hs_updated_by_user_id": { - "type": "number" - }, - "hs_user_ids_of_all_owners": { - "type": "string" - }, - "hubspot_owner_assigneddate": { - "type": "datetime" - }, - "last_engagement_date": { - "type": "datetime" - }, - "last_reply_date": { - "type": "datetime" - }, - "nps_follow_up_answer": { - "type": "string" - }, - "nps_follow_up_question_version": { - "type": "number" - }, - "nps_score": { - "type": "string" - }, - "source_thread_id": { - "type": "string" - }, - "time_to_close": { - "type": "number" - }, - "time_to_first_agent_reply": { - "type": "number" - }, - "subject": { - "type": "string" - }, - "content": { - "type": "string" - }, - "source_type": { - "type": "string" - }, - "source_ref": { - "type": "string" - }, - "tags": { - "type": "string" - }, - "hs_sales_email_last_replied": { - "type": "datetime" - }, - "hubspot_owner_id": { - "type": "string" - }, - "notes_last_contacted": { - "type": "datetime" - }, - "notes_last_updated": { - "type": "datetime" - }, - "notes_next_activity_date": { - "type": "datetime" - }, - "num_contacted_notes": { - "type": "number" - }, - "num_notes": { - "type": "number" - }, - "hubspot_team_id": { - "type": "string" - }, - "hs_all_owner_ids": { - "type": "string" - }, - "hs_all_team_ids": { - "type": "string" - }, - "hs_all_accessible_team_ids": { - "type": "string" - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - }, - { - "stream": { - "name": "workflows", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "name": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "type": { - "type": ["null", "string"] - }, - "enabled": { - "type": ["null", "boolean"] - }, - "insertedAt": { - "type": "integer" - }, - "updatedAt": { - "type": "integer" - }, - "personaTagIds": { - "type": "array", - "items": { - "type": "integer" - } - }, - "contactListIds": { - "type": "object", - "properties": { - "enrolled": { - "type": ["null", "integer"] - }, - "active": { - "type": ["null", "integer"] - }, - "steps": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "default_cursor_field": null - }, - "sync_mode": "full_refresh", - "cursor_field": null - } - ] -} diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/sample_config.json b/airbyte-integrations/connectors/source-hubspot/sample_files/sample_config.json deleted file mode 100644 index 470938cb24c..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/sample_config.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "start_date": "2020-01-01T00:00:00Z", - "credentials": { - "api_key": "demo" - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/sample_state.json b/airbyte-integrations/connectors/source-hubspot/sample_files/sample_state.json deleted file mode 100644 index b56de1fc71e..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/sample_state.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "subscription_changes": { - "timestamp": "2021-02-23T00:00:00Z" - }, - "email_events": { - "timestamp": "2021-02-23T00:00:00Z" - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py deleted file mode 100644 index e6099aae261..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = [ - "airbyte-protocol", - "base-python", - "backoff==1.10.0", - "pendulum==1.2.0", - "requests==2.25.1", -] - -TEST_REQUIREMENTS = [ - "pytest", - "requests_mock==1.8.0", -] - -setup( - name="source_hubspot", - description="Source implementation for Hubspot.", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py deleted file mode 100644 index 4c1611d7292..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py +++ /dev/null @@ -1,553 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import sys -import time -from abc import ABC, abstractmethod -from datetime import datetime, timedelta -from functools import partial -from http import HTTPStatus -from typing import Any, Callable, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Union - -import backoff -import pendulum as pendulum -import requests -from base_python.entrypoint import logger -from source_hubspot.errors import HubspotAccessDenied, HubspotInvalidAuth, HubspotRateLimited, HubspotTimeout - -# we got this when provided API Token has incorrect format -CLOUDFLARE_ORIGIN_DNS_ERROR = 530 - - -def retry_connection_handler(**kwargs): - """Retry helper, log each attempt""" - - def log_retry_attempt(details): - _, exc, _ = sys.exc_info() - logger.info(str(exc)) - logger.info(f"Caught retryable error after {details['tries']} tries. Waiting {details['wait']} more seconds then retrying...") - - def giveup_handler(exc): - if isinstance(exc, (HubspotInvalidAuth, HubspotAccessDenied)): - return True - return exc.response is not None and HTTPStatus.BAD_REQUEST <= exc.response.status_code < HTTPStatus.INTERNAL_SERVER_ERROR - - return backoff.on_exception( - backoff.expo, - requests.exceptions.RequestException, - jitter=None, - on_backoff=log_retry_attempt, - giveup=giveup_handler, - **kwargs, - ) - - -def retry_after_handler(**kwargs): - """Retry helper when we hit the call limit, sleeps for specific duration""" - - def sleep_on_ratelimit(_details): - _, exc, _ = sys.exc_info() - if isinstance(exc, HubspotRateLimited) and exc.response.headers.get("Retry-After"): - retry_after = int(exc.response.headers["Retry-After"]) - logger.info(f"Rate limit reached. Sleeping for {retry_after} seconds") - time.sleep(retry_after + 1) # extra second to cover any fractions of second - - def log_giveup(_details): - logger.error("Max retry limit reached") - - return backoff.on_exception( - backoff.constant, - HubspotRateLimited, - jitter=None, - on_backoff=sleep_on_ratelimit, - on_giveup=log_giveup, - interval=0, # skip waiting part, we will wait in on_backoff handler - **kwargs, - ) - - -class API: - """Hubspot API interface, authorize, retrieve and post, supports backoff logic""" - - BASE_URL = "https://api.hubapi.com" - USER_AGENT = "Airbyte" - - def __init__(self, credentials: Mapping[str, Any]): - self._credentials = {**credentials} - self._session = requests.Session() - self._session.headers = { - "Content-Type": "application/json", - "User-Agent": self.USER_AGENT, - } - - def _acquire_access_token_from_refresh_token(self): - payload = { - "grant_type": "refresh_token", - "redirect_uri": self._credentials["redirect_uri"], - "refresh_token": self._credentials["refresh_token"], - "client_id": self._credentials["client_id"], - "client_secret": self._credentials["client_secret"], - } - - resp = requests.post(self.BASE_URL + "/oauth/v1/token", data=payload) - if resp.status_code == HTTPStatus.FORBIDDEN: - raise HubspotInvalidAuth(resp.content, response=resp) - - resp.raise_for_status() - auth = resp.json() - self._credentials["access_token"] = auth["access_token"] - self._credentials["refresh_token"] = auth["refresh_token"] - self._credentials["token_expires"] = datetime.utcnow() + timedelta(seconds=auth["expires_in"] - 600) - logger.info("Token refreshed. Expires at %s", self._credentials["token_expires"]) - - @property - def api_key(self) -> Optional[str]: - """Get API Key if set""" - return self._credentials.get("api_key") - - @property - def access_token(self) -> Optional[str]: - """Get Access Token if set, refreshes token if needed""" - if not self._credentials.get("access_token"): - return None - - if self._credentials["token_expires"] is None or self._credentials["token_expires"] < datetime.utcnow(): - self._acquire_access_token_from_refresh_token() - return self._credentials.get("access_token") - - def _add_auth(self, params: Mapping[str, Any] = None) -> Mapping[str, Any]: - """Add auth info to request params/header""" - params = params or {} - - if self.access_token: - self._session.headers["Authorization"] = f"Bearer {self.access_token}" - else: - params["hapikey"] = self.api_key - - return params - - @staticmethod - def _parse_and_handle_errors(response) -> Union[MutableMapping[str, Any], List[MutableMapping[str, Any]]]: - """Handle response""" - message = "Unknown error" - if response.headers.get("content-type") == "application/json;charset=utf-8" and response.status_code != HTTPStatus.OK: - message = response.json().get("message") - - if response.status_code == HTTPStatus.FORBIDDEN: - raise HubspotAccessDenied(message, response=response) - elif response.status_code in (HTTPStatus.UNAUTHORIZED, CLOUDFLARE_ORIGIN_DNS_ERROR): - raise HubspotInvalidAuth(message, response=response) - elif response.status_code == HTTPStatus.TOO_MANY_REQUESTS: - retry_after = response.headers.get("Retry-After") - raise HubspotRateLimited( - f"429 Rate Limit Exceeded: API rate-limit has been reached until {retry_after} seconds." - " See https://developers.hubspot.com/docs/api/usage-details", - response=response, - ) - elif response.status_code in (HTTPStatus.BAD_GATEWAY, HTTPStatus.SERVICE_UNAVAILABLE): - raise HubspotTimeout(message, response=response) - else: - response.raise_for_status() - - return response.json() - - @retry_connection_handler(max_tries=5, factor=5) - @retry_after_handler(max_tries=3) - def get(self, url: str, params=None) -> Union[MutableMapping[str, Any], List[MutableMapping[str, Any]]]: - response = self._session.get(self.BASE_URL + url, params=self._add_auth(params)) - return self._parse_and_handle_errors(response) - - def post(self, url: str, data: Mapping[str, Any], params=None) -> Union[Mapping[str, Any], List[Mapping[str, Any]]]: - response = self._session.post(self.BASE_URL + url, params=self._add_auth(params), json=data) - return self._parse_and_handle_errors(response) - - -class Stream(ABC): - """Base class for all streams. Responsible for data fetching and pagination""" - - entity: str = None - updated_at_field: str = None - created_at_field: str = None - - more_key: str = None - data_field = "results" - - page_filter = "offset" - page_field = "offset" - limit_field = "limit" - limit = 100 - - @property - @abstractmethod - def url(self): - """Default URL to read from""" - - def __init__(self, api: API, start_date: str = None, **kwargs): - self._api: API = api - self._start_date = pendulum.parse(start_date) - - @property - def name(self) -> str: - stream_name = self.__class__.__name__ - if stream_name.endswith("Stream"): - stream_name = stream_name[: -len("Stream")] - return stream_name - - def list(self, fields) -> Iterable: - yield from self.read(partial(self._api.get, url=self.url)) - - def _filter_dynamic_fields(self, records: Iterable) -> Iterable: - """Skip certain fields because they are too dynamic and change every call (timers, etc), - see https://github.com/airbytehq/airbyte/issues/2397 - """ - for record in records: - if isinstance(record, Mapping) and "properties" in record: - for key in list(record["properties"].keys()): - if key.startswith("hs_time_in"): - record["properties"].pop(key) - yield record - - def _transform(self, records: Iterable) -> Iterable: - """Preprocess record before emitting""" - for record in records: - if self.created_at_field and self.updated_at_field and record.get(self.updated_at_field) is None: - record[self.updated_at_field] = record[self.created_at_field] - yield record - - @staticmethod - def _field_to_datetime(value: Union[int, str]) -> pendulum.datetime: - if isinstance(value, int): - value = pendulum.from_timestamp(value / 1000.0) - elif isinstance(value, str): - value = pendulum.parse(value) - else: - raise ValueError(f"Unsupported type of datetime field {type(value)}") - return value - - def _filter_old_records(self, records: Iterable) -> Iterable: - """Skip records that was updated before our start_date""" - for record in records: - updated_at = record[self.updated_at_field] - if updated_at: - updated_at = self._field_to_datetime(updated_at) - if updated_at < self._start_date: - continue - yield record - - def _read(self, getter: Callable, params: MutableMapping[str, Any] = None) -> Iterator: - while True: - response = getter(params=params) - if isinstance(response, Mapping): - if response.get(self.data_field) is None: - raise RuntimeError("Unexpected API response: {} not in {}".format(self.data_field, response.keys())) - - yield from response[self.data_field] - - # pagination - if "paging" in response: # APIv3 pagination - if "next" in response["paging"]: - params["after"] = response["paging"]["next"]["after"] - else: - break - else: - if not response.get(self.more_key, False): - break - if self.page_field in response: - params[self.page_filter] = response[self.page_field] - else: - response = list(response) - yield from response - - # pagination - if len(response) < self.limit: - break - else: - params[self.page_filter] = params.get(self.page_filter, 0) + self.limit - - def read(self, getter: Callable, params: Mapping[str, Any] = None) -> Iterator: - default_params = {self.limit_field: self.limit, "properties": ",".join(self.properties.keys())} - params = {**default_params, **params} if params else {**default_params} - - yield from self._filter_dynamic_fields(self._filter_old_records(self._transform(self._read(getter, params)))) - - @property - def properties(self) -> Mapping[str, Any]: - """Some entities has dynamic set of properties, so we trying to resolve those at runtime""" - if not self.entity: - return {} - - props = {} - data = self._api.get(f"/properties/v2/{self.entity}/properties") - for row in data: - field_type = row["type"] - if field_type in ["enumeration", "date", "date-time"]: - field_type = "string" - props[row["name"]] = {"type": field_type} - - return props - - -class IncrementalStream(Stream, ABC): - """Stream that supports state and incremental read""" - - state_pk = "timestamp" - limit = 1000 - - @property - @abstractmethod - def updated_at_field(self): - """Name of the field associated with the state""" - - @property - def state(self) -> Optional[Mapping[str, Any]]: - """Current state, if wasn't set return None""" - if self._state: - return {self.state_pk: str(self._state)} - return None - - @state.setter - def state(self, value): - self._state = pendulum.parse(value[self.state_pk]) - self._start_date = max(self._state, self._start_date) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._state = None - - def read(self, getter: Callable, params: Mapping[str, Any] = None) -> Iterator: - """Apply state filter to set of records, update cursor(state) if necessary in the end""" - latest_cursor = None - # to track state, there is no guarantee that returned records sorted in ascending order. Having exact - # boundary we could always ensure we don't miss records between states. In the future, if we would - # like to save the state more often we can do this every batch - for record in self.read_chunked(getter, params): - yield record - cursor = self._field_to_datetime(record[self.updated_at_field]) - latest_cursor = max(cursor, latest_cursor) if latest_cursor else cursor - - if latest_cursor: - new_state = max(latest_cursor, self._state) if self._state else latest_cursor - if new_state != self._state: - logger.info(f"Advancing bookmark for {self.name} stream from {self._state} to {latest_cursor}") - self._state = new_state - self._start_date = self._state - - def read_chunked( - self, getter: Callable, params: Mapping[str, Any] = None, chunk_size: pendulum.Interval = pendulum.interval(days=1) - ) -> Iterator: - params = {**params} if params else {} - now_ts = int(pendulum.now().timestamp() * 1000) - start_ts = int(self._start_date.timestamp() * 1000) - chunk_size = int(chunk_size.total_seconds() * 1000) - - for ts in range(start_ts, now_ts, chunk_size): - end_ts = ts + chunk_size - params["startTimestamp"] = ts - params["endTimestamp"] = end_ts - logger.info( - f"Reading chunk from stream {self.name} between {pendulum.from_timestamp(ts / 1000)} and {pendulum.from_timestamp(end_ts / 1000)}" - ) - yield from super().read(getter, params) - - -class CRMObjectStream(Stream): - """Unified stream interface for CRM objects. - You need to provide `entity` parameter to read concrete stream, possible values are: - company, contact, deal, line_item, owner, product, ticket, quote - You can also include associated records (IDs), provide associations parameter - a list of entity names: - contacts, tickets, deals, engagements - see https://developers.hubspot.com/docs/api/crm/understanding-the-crm for more details - """ - - entity: Optional[str] = None - associations: List[str] = [] - updated_at_field = "updatedAt" - created_at_field = "createdAt" - - @property - def url(self): - """Entity URL""" - return f"/crm/v3/objects/{self.entity}" - - def __init__(self, entity: str = None, associations: List[str] = None, include_archived_only: bool = False, **kwargs): - super().__init__(**kwargs) - self.entity = entity or self.entity - self.associations = associations or self.associations - self._include_archived_only = include_archived_only - - if not self.entity: - raise ValueError("Entity must be set either on class or instance level") - - def list(self, fields) -> Iterable: - params = { - "archived": str(self._include_archived_only).lower(), - "associations": self.associations, - } - generator = self.read(partial(self._api.get, url=self.url), params) - yield from self._flat_associations(generator) - - def _flat_associations(self, records: Iterable[MutableMapping]) -> Iterable[MutableMapping]: - """When result has associations we prefer to have it flat, so we transform this: - - "associations": { - "contacts": { - "results": [{"id": "201", "type": "company_to_contact"}, {"id": "251", "type": "company_to_contact"}]} - } - } - - to this: - - "contacts": [201, 251] - """ - for record in records: - if "associations" in record: - associations = record.pop("associations") - for name, association in associations.items(): - record[name] = [row["id"] for row in association.get("results", [])] - yield record - - -class CampaignStream(Stream): - """Email campaigns, API v1 - There is some confusion between emails and campaigns in docs, this endpoint returns actual emails - Docs: https://legacydocs.hubspot.com/docs/methods/email/get_campaign_data - """ - - url = "/email/public/v1/campaigns" - more_key = "hasMore" - data_field = "campaigns" - limit = 500 - updated_at_field = "lastUpdatedTime" - - def list(self, fields) -> Iterable: - for row in self.read(getter=partial(self._api.get, url=self.url)): - record = self._api.get(f"/email/public/v1/campaigns/{row['id']}") - yield {**row, **record} - - -class ContactListStream(Stream): - """Contact lists, API v1 - Docs: https://legacydocs.hubspot.com/docs/methods/lists/get_lists - """ - - url = "/contacts/v1/lists" - data_field = "lists" - more_key = "has-more" - updated_at_field = "updatedAt" - created_at_field = "createdAt" - limit_field = "count" - - -class DealPipelineStream(Stream): - """Deal pipelines, API v1, - This endpoint requires the contacts scope the tickets scope. - Docs: https://legacydocs.hubspot.com/docs/methods/pipelines/get_pipelines_for_object_type - """ - - url = "/crm-pipelines/v1/pipelines/deals" - updated_at_field = "updatedAt" - created_at_field = "createdAt" - - -class TicketPipelineStream(Stream): - """Ticket pipelines, API v1 - This endpoint requires the tickets scope. - Docs: https://legacydocs.hubspot.com/docs/methods/pipelines/get_pipelines_for_object_type - """ - - url = "/crm-pipelines/v1/pipelines/tickets" - updated_at_field = "updatedAt" - created_at_field = "createdAt" - - -class EmailEventStream(IncrementalStream): - """Email events, API v1 - Docs: https://legacydocs.hubspot.com/docs/methods/email/get_events - """ - - url = "/email/public/v1/events" - data_field = "events" - more_key = "hasMore" - updated_at_field = "created" - created_at_field = "created" - - -class EngagementStream(Stream): - """Engagements, API v1 - Docs: https://legacydocs.hubspot.com/docs/methods/engagements/get-all-engagements - """ - - entity = "engagement" - url = "/engagements/v1/engagements/paged" - more_key = "hasMore" - limit = 250 - updated_at_field = "lastUpdated" - created_at_field = "createdAt" - - def _transform(self, records: Iterable) -> Iterable: - yield from super()._transform({**record.pop("engagement"), **record} for record in records) - - -class FormStream(Stream): - """Marketing Forms, API v2 - by default non-marketing forms are filtered out of this endpoint - Docs: https://developers.hubspot.com/docs/api/marketing/forms - """ - - entity = "form" - url = "/forms/v2/forms" - updated_at_field = "updatedAt" - created_at_field = "createdAt" - - -class OwnerStream(Stream): - """Owners, API v3 - Docs: https://legacydocs.hubspot.com/docs/methods/owners/get_owners - """ - - url = "/crm/v3/owners" - updated_at_field = "updatedAt" - created_at_field = "createdAt" - - -class SubscriptionChangeStream(IncrementalStream): - """Subscriptions timeline for a portal, API v1 - Docs: https://legacydocs.hubspot.com/docs/methods/email/get_subscriptions_timeline - """ - - url = "/email/public/v1/subscriptions/timeline" - data_field = "timeline" - more_key = "hasMore" - updated_at_field = "timestamp" - - -class WorkflowStream(Stream): - """Workflows, API v3 - Docs: https://legacydocs.hubspot.com/docs/methods/workflows/v3/get_workflows - """ - - url = "/automation/v3/workflows" - data_field = "workflows" - updated_at_field = "updatedAt" - created_at_field = "insertedAt" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py deleted file mode 100644 index 53bc472f303..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/client.py +++ /dev/null @@ -1,109 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from typing import Any, Iterator, Mapping, Tuple - -from airbyte_protocol import AirbyteStream -from base_python import BaseClient -from requests import HTTPError -from source_hubspot.api import ( - API, - CampaignStream, - ContactListStream, - CRMObjectStream, - DealPipelineStream, - EmailEventStream, - EngagementStream, - FormStream, - OwnerStream, - SubscriptionChangeStream, - WorkflowStream, -) - - -class Client(BaseClient): - """Hubspot client, provides methods to discover and read streams""" - - def __init__(self, start_date, credentials, **kwargs): - self._start_date = start_date - self._api = API(credentials=credentials) - - common_params = dict(api=self._api, start_date=self._start_date) - self._apis = { - "campaigns": CampaignStream(**common_params), - "companies": CRMObjectStream(entity="company", associations=["contacts"], **common_params), - "contact_lists": ContactListStream(**common_params), - "contacts": CRMObjectStream(entity="contact", **common_params), - "deal_pipelines": DealPipelineStream(**common_params), - "deals": CRMObjectStream(entity="deal", **common_params), - "email_events": EmailEventStream(**common_params), - "engagements": EngagementStream(**common_params), - "forms": FormStream(**common_params), - "line_items": CRMObjectStream(entity="line_item", **common_params), - "owners": OwnerStream(**common_params), - "products": CRMObjectStream(entity="product", **common_params), - "quotes": CRMObjectStream(entity="quote", **common_params), - "subscription_changes": SubscriptionChangeStream(**common_params), - "tickets": CRMObjectStream(entity="ticket", **common_params), - "workflows": WorkflowStream(**common_params), - } - - super().__init__(**kwargs) - - def _enumerate_methods(self) -> Mapping[str, callable]: - return {name: api.list for name, api in self._apis.items()} - - @property - def streams(self) -> Iterator[AirbyteStream]: - """List of available streams, patch streams to append properties dynamically""" - for stream in super().streams: - properties = self._apis[stream.name].properties - if properties: - stream.json_schema["properties"]["properties"] = {"type": "object", "properties": properties} - stream.default_cursor_field = [self._apis[stream.name].updated_at_field] - yield stream - - def stream_has_state(self, name: str) -> bool: - """Tell if stream supports incremental sync""" - return hasattr(self._apis[name], "state") - - def get_stream_state(self, name: str) -> Any: - """Get state of stream with corresponding name""" - return self._apis[name].state - - def set_stream_state(self, name: str, state: Any): - """Set state of stream with corresponding name""" - self._apis[name].state = state - - def health_check(self) -> Tuple[bool, str]: - alive = True - error_msg = None - - try: - _ = self._apis["contacts"].properties - except HTTPError as error: - alive = False - error_msg = repr(error) - - return alive, error_msg diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py deleted file mode 100644 index 6c0bc45f450..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/errors.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from requests import HTTPError - - -class HubspotError(HTTPError): - """ - Base error class. - Subclassing HTTPError to avoid breaking existing code that expects only HTTPErrors. - """ - - -class HubspotTimeout(HubspotError): - """502/504 HubSpot has processing limits in place to prevent a single client from causing degraded performance, - and these responses indicate that those limits have been hit. You'll normally only see these timeout responses - when making a large number of requests over a sustained period. If you get one of these responses, - you should pause your requests for a few seconds, then retry. - """ - - -class HubspotInvalidAuth(HubspotError): - """401 Unauthorized""" - - -class HubspotAccessDenied(HubspotError): - """403 Forbidden""" - - -class HubspotRateLimited(HubspotError): - """429 Rate Limit Reached""" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json deleted file mode 100644 index 185bc429c4e..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "appId": { - "type": ["null", "integer"] - }, - "appName": { - "type": ["null", "string"] - }, - "contentId": { - "type": ["null", "integer"] - }, - "counters": { - "type": ["null", "object"], - "properties": { - "open": { - "type": ["null", "integer"] - }, - "processed": { - "type": ["null", "integer"] - }, - "sent": { - "type": ["null", "integer"] - }, - "deferred": { - "type": ["null", "integer"] - }, - "unsubscribed": { - "type": ["null", "integer"] - }, - "statuschange": { - "type": ["null", "integer"] - }, - "bounce": { - "type": ["null", "integer"] - }, - "mta_dropped": { - "type": ["null", "integer"] - }, - "dropped": { - "type": ["null", "integer"] - }, - "suppressed": { - "type": ["null", "integer"] - }, - "click": { - "type": ["null", "integer"] - }, - "delivered": { - "type": ["null", "integer"] - }, - "forward": { - "type": ["null", "integer"] - }, - "print": { - "type": ["null", "integer"] - }, - "reply": { - "type": ["null", "integer"] - }, - "spamreport": { - "type": ["null", "integer"] - } - } - }, - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "numIncluded": { - "type": ["null", "integer"] - }, - "numQueued": { - "type": ["null", "integer"] - }, - "subType": { - "type": ["null", "string"] - }, - "subject": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "lastUpdatedTime": { - "type": "string" - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies.json deleted file mode 100644 index 31448368c7c..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "portalId": { - "type": ["null", "integer"] - }, - "companyId": { - "type": "integer" - }, - "isDeleted": { - "type": ["null", "boolean"] - }, - "additionalDomains": { - "type": ["null", "array"] - }, - "stateChanges": { - "type": ["null", "array"] - }, - "mergeAudits": { - "type": ["null", "array"] - }, - "contacts": { - "type": ["null", "array"] - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json deleted file mode 100644 index 3b2a9b118fa..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "parentId": { - "type": ["null", "integer"] - }, - "metaData": { - "type": "object", - "properties": { - "processing": { - "type": ["null", "string"] - }, - "size": { - "type": ["null", "integer"] - }, - "error": { - "type": ["null", "string"] - }, - "lastProcessingStateChangeAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "lastSizeChangeAt": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "dynamic": { - "type": ["null", "boolean"] - }, - "name": { - "type": ["null", "string"] - }, - "filters": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "object", - "properties": { - "filterFamily": { - "type": ["null", "string"] - }, - "withinTimeMode": { - "type": ["null", "string"] - }, - "checkPastVersions": { - "type": ["null", "boolean"] - }, - "type": { - "type": ["null", "string"] - }, - "property": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "operator": { - "type": ["null", "string"] - } - } - } - } - }, - "portalId": { - "type": ["null", "integer"] - }, - "createdAt": { - "type": "integer" - }, - "listId": { - "type": ["null", "integer"] - }, - "updatedAt": { - "type": "integer" - }, - "internalListId": { - "type": ["null", "integer"] - }, - "readOnly": { - "type": ["null", "boolean"] - }, - "deleteable": { - "type": ["null", "boolean"] - }, - "listType": { - "type": ["null", "string"] - }, - "archived": { - "type": ["null", "boolean"] - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts.json deleted file mode 100644 index e1f5c9f758e..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts.json +++ /dev/null @@ -1,2325 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "vid": { - "type": ["null", "integer"] - }, - "canonical-vid": { - "type": ["null", "integer"] - }, - "merged-vids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "portal-id": { - "type": ["null", "integer"] - }, - "is-contact": { - "type": ["null", "boolean"] - }, - "profile-token": { - "type": ["null", "string"] - }, - "profile-url": { - "type": ["null", "string"] - }, - "associated-company": { - "type": "object", - "properties": { - "properties": { - "type": "object", - "properties": { - "about_us": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "closedate_timestamp_earliest_value_a2a17e6e": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "facebookfans": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_contact_createdate_timestamp_earliest_value_78b50eea": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_conversion_date_timestamp_earliest_value_61f58f2c": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_conversion_event_name_timestamp_earliest_value_68ddae0a": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_deal_created_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "founded_year": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_additional_domains": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_timestamp": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_touch_converting_campaign": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_visit_timestamp": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_timestamp": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_touch_converting_campaign": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_visit_timestamp": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_num_page_views": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_num_page_views_cardinality_sum_e46e85b0": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_num_visits": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_num_visits_cardinality_sum_53d952a6": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_data_1": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_data_2": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_analytics_source_timestamp_earliest_value_25a3a52c": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_avatar_filemanager_key": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_created_by_user_id": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_createdate": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_ideal_customer_profile": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_is_target_account": { - "type": "object", - "properties": { - "value": { - "type": ["null", "boolean"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_booked_meeting_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_logged_call_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_open_task_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_sales_activity_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_last_sales_activity_timestamp": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_lastmodifieddate": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_merged_object_ids": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_blockers": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_contacts_with_buying_roles": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_decision_makers": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_open_deals": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_object_id": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_target_account": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_target_account_probability": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_target_account_recommendation_snooze_time": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_target_account_recommendation_state": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_total_deal_value": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_updated_by_user_id": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_user_ids_of_all_owners": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hubspot_owner_assigneddate": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "is_public": { - "type": "object", - "properties": { - "value": { - "type": ["null", "boolean"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_associated_contacts": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_associated_deals": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_conversion_events_cardinality_sum_d095f14b": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "recent_conversion_date_timestamp_latest_value_72856da1": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "recent_conversion_event_name_timestamp_latest_value_66c820bf": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "recent_deal_amount": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "recent_deal_close_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "timezone": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "total_money_raised": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "total_revenue": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "name": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "twitterhandle": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "phone": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "twitterbio": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "twitterfollowers": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "address": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "address2": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "facebook_company_page": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "city": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "linkedin_company_page": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "linkedinbio": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "state": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "googleplus_page": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "engagements_last_meeting_booked": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "engagements_last_meeting_booked_campaign": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "engagements_last_meeting_booked_medium": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "engagements_last_meeting_booked_source": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_latest_meeting_activity": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_sales_email_last_replied": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hubspot_owner_id": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "notes_last_contacted": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "notes_last_updated": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "notes_next_activity_date": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_contacted_notes": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "num_notes": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "zip": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "country": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hubspot_team_id": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_all_owner_ids": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "website": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "domain": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_all_team_ids": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_all_accessible_team_ids": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "numberofemployees": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "industry": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "annualrevenue": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "lifecyclestage": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_lead_status": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_parent_company_id": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "type": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "description": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "hs_num_child_companies": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "createdate": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "closedate": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "first_contact_createdate": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"], - "format": "date-time" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "days_to_close": { - "type": "object", - "properties": { - "value": { - "type": ["null", "number", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - }, - "web_technologies": { - "type": "object", - "properties": { - "value": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "sourceId": { - "type": ["null", "string"] - } - } - } - } - }, - "company-id": { - "type": ["null", "integer"] - }, - "portal-id": { - "type": ["null", "integer"] - } - } - }, - "identity-profiles": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "deleted-changed-timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "saved-at-timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "vid": { - "type": ["null", "integer"] - }, - "identities": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "type": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "list-memberships": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "internal-list-id": { - "type": ["null", "integer"] - }, - "is-member": { - "type": ["null", "boolean"] - }, - "static-list-id": { - "type": ["null", "integer"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "vid": { - "type": ["null", "integer"] - } - } - } - }, - "form-submissions": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "conversion-id": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "form-id": { - "type": ["null", "string"] - }, - "portal-id": { - "type": ["null", "integer"] - }, - "page-url": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - } - } - } - }, - "merge-audits": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "canonical-vid": { - "type": ["null", "integer"] - }, - "vid-to-merge": { - "type": ["null", "integer"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "user-id": { - "type": ["null", "integer"] - }, - "num-properties-moved": { - "type": ["null", "integer"] - }, - "merged_from_email": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "source-type": { - "type": ["null", "string"] - }, - "source-id": { - "type": ["null", "string"] - }, - "source-label": { - "type": ["null", "string"] - }, - "source-vids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "timestamp": { - "type": ["null", "integer"] - }, - "selected": { - "type": ["null", "boolean"] - } - } - }, - "merged_to_email": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "source-type": { - "type": ["null", "string"] - }, - "source-id": { - "type": ["null", "string"] - }, - "source-label": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "integer"] - }, - "selected": { - "type": ["null", "boolean"] - } - } - } - } - } - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deal_pipelines.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deal_pipelines.json deleted file mode 100644 index f73b83ad54c..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deal_pipelines.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "label": { - "type": "string" - }, - "displayOrder": { - "type": "integer" - }, - "active": { - "type": "boolean" - }, - "stages": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { - "type": "string" - }, - "displayOrder": { - "type": "integer" - }, - "metadata": { - "type": "object", - "properties": { - "isClosed": { - "type": "string" - }, - "probability": { - "type": "string" - } - } - }, - "stageId": { - "type": "string" - }, - "createdAt": { - "type": ["null", "integer"] - }, - "updatedAt": { - "type": ["null", "integer"] - }, - "active": { - "type": "boolean" - } - } - } - }, - "objectType": { - "type": "string" - }, - "objectTypeId": { - "type": "string" - }, - "pipelineId": { - "type": "string" - }, - "createdAt": { - "type": "integer" - }, - "updatedAt": { - "type": ["null", "integer"] - }, - "default": { - "type": "boolean" - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json deleted file mode 100644 index b05ff1d9253..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "portalId": { - "type": ["null", "integer"] - }, - "dealId": { - "type": ["null", "integer"] - }, - "isDeleted": { - "type": ["null", "boolean"] - }, - "associations": { - "type": ["null", "object"], - "properties": { - "associatedVids": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "associatedCompanyIds": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - }, - "associatedDealIds": { - "type": ["null", "array"], - "items": { - "type": ["null", "integer"] - } - } - } - }, - "imports": { - "type": ["null", "array"] - }, - "stateChanges": { - "type": ["null", "array"] - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/email_events.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/email_events.json deleted file mode 100644 index 7334325e95e..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/email_events.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "appId": { - "type": ["null", "integer"] - }, - "appName": { - "type": ["null", "string"] - }, - "browser": { - "type": ["null", "object"], - "properties": { - "family": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "producer": { - "type": ["null", "string"] - }, - "producerUrl": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - }, - "created": { - "type": "integer" - }, - "deviceType": { - "type": ["null", "string"] - }, - "duration": { - "type": ["null", "integer"] - }, - "emailCampaignId": { - "type": ["null", "integer"] - }, - "emailCampaignGroupId": { - "type": ["null", "integer"] - }, - "filteredEvent": { - "type": ["null", "boolean"] - }, - "from": { - "type": ["null", "string"] - }, - "hmid": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "ipAddress": { - "type": ["null", "string"] - }, - "linkId": { - "type": ["null", "integer"] - }, - "location": { - "type": ["null", "object"], - "properties": { - "city": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - } - } - }, - "portalId": { - "type": ["null", "integer"] - }, - "recipient": { - "type": ["null", "string"] - }, - "response": { - "type": ["null", "string"] - }, - "sentBy": { - "type": ["null", "object"], - "properties": { - "created": { - "type": ["null", "string"], - "format": "date-time" - }, - "id": { - "type": ["null", "string"] - } - } - }, - "smtpId": { - "type": ["null", "string"] - }, - "subject": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "userAgent": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json deleted file mode 100644 index 7ceed014919..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "portalId": { - "type": "integer" - }, - "active": { - "type": "boolean" - }, - "createdAt": { - "type": "integer" - }, - "lastUpdated": { - "type": "integer" - }, - "ownerId": { - "type": "integer" - }, - "type": { - "type": "string" - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "associations": { - "type": ["null", "object"], - "properties": { - "contactIds": { - "type": ["null", "array"], - "items": { - "type": "integer" - } - }, - "companyIds": { - "type": ["null", "array"], - "items": { - "type": "integer" - } - }, - "dealIds": { - "type": ["null", "array"], - "items": { - "type": "integer" - } - } - } - }, - "attachments": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "id": { - "type": "integer" - } - } - } - }, - "metadata": { - "type": ["null", "object"], - "properties": { - "body": { - "type": ["null", "string"] - }, - "from": { - "type": ["null", "object"], - "properties": { - "email": { - "type": "string" - }, - "firstName": { - "type": "string" - }, - "lastName": { - "type": "string" - } - } - }, - "to": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "email": { - "type": "string" - } - } - } - }, - "cc": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "email": { - "type": "string" - } - } - } - }, - "bcc": { - "type": ["null", "array"], - "items": { - "type": "object", - "properties": { - "email": { - "type": "string" - } - } - } - }, - "subject": { - "type": ["null", "string"] - }, - "html": { - "type": ["null", "string"] - }, - "text": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "forObjectType": { - "type": ["null", "string"] - }, - "startTime": { - "type": ["null", "integer"] - }, - "endTime": { - "type": ["null", "integer"] - }, - "title": { - "type": ["null", "string"] - }, - "toNumber": { - "type": ["null", "string"] - }, - "fromNumber": { - "type": ["null", "string"] - }, - "externalId": { - "type": ["null", "string"] - }, - "durationMilliseconds": { - "type": ["null", "integer"] - }, - "externalAccountId": { - "type": ["null", "string"] - }, - "recordingUrl": { - "type": ["null", "string"], - "format": "uri" - }, - "disposition": { - "type": ["null", "string"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json deleted file mode 100644 index 2ad839cabc5..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json +++ /dev/null @@ -1,228 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "deletedAt": { - "type": ["null", "integer"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "guid": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "action": { - "type": ["null", "string"] - }, - "method": { - "type": ["null", "string"] - }, - "cssClass": { - "type": ["null", "string"] - }, - "redirect": { - "type": ["null", "string"] - }, - "submitText": { - "type": ["null", "string"] - }, - "followUpId": { - "type": ["null", "string"] - }, - "notifyRecipients": { - "type": ["null", "string"] - }, - "leadNurturingCampaignId": { - "type": ["null", "string"] - }, - "formFieldGroups": { - "type": "array", - "items": { - "type": "object", - "properties": { - "fields": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "fieldType": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "groupName": { - "type": ["null", "string"] - }, - "displayOrder": { - "type": ["null", "integer"] - }, - "required": { - "type": ["null", "boolean"] - }, - "validation": { - "type": "object", - "properties": { - "name": { - "type": ["null", "string"] - }, - "message": { - "type": ["null", "string"] - }, - "data": { - "type": ["null", "string"] - }, - "useDefaultBlockList": { - "type": ["null", "boolean"] - }, - "blockedEmailAddresses": { - "type": "array", - "items": { - "type": ["null", "string"] - } - } - } - }, - "enabled": { - "type": ["null", "boolean"] - }, - "hidden": { - "type": ["null", "boolean"] - }, - "defaultValue": { - "type": ["null", "string"] - }, - "isSmartField": { - "type": ["null", "boolean"] - }, - "unselectedLabel": { - "type": ["null", "string"] - }, - "placeholder": { - "type": ["null", "string"] - }, - "labelHidden": { - "type": ["null", "boolean"] - }, - "options": { - "type": "array", - "items": { - "type": "object", - "properties": { - "description": { - "type": ["null", "string"] - }, - "displayOrder": { - "type": ["null", "integer"] - }, - "doubleData": { - "type": ["null", "number"] - }, - "hidden": { - "type": ["null", "boolean"] - }, - "label": { - "type": ["null", "string"] - }, - "readOnly": { - "type": ["null", "boolean"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "selectedOptions": { - "type": "array", - "items": { - "type": ["null", "string"] - } - } - } - } - }, - "default": { - "type": ["null", "boolean"] - }, - "isSmartGroup": { - "type": ["null", "boolean"] - }, - "richText": { - "type": "object", - "properties": { - "content": { - "type": ["null", "string"] - } - } - } - } - } - }, - "createdAt": { - "type": "integer" - }, - "updatedAt": { - "type": "integer" - }, - "performableHtml": { - "type": ["null", "string"] - }, - "migratedFrom": { - "type": ["null", "string"] - }, - "ignoreCurrentValues": { - "type": ["null", "boolean"] - }, - "deletable": { - "type": ["null", "boolean"] - }, - "inlineMessage": { - "type": ["null", "string"] - }, - "tmsId": { - "type": ["null", "string"] - }, - "captchaEnabled": { - "type": ["null", "boolean"] - }, - "campaignGuid": { - "type": ["null", "string"] - }, - "cloneable": { - "type": ["null", "boolean"] - }, - "editable": { - "type": ["null", "boolean"] - }, - "formType": { - "type": ["null", "string"] - }, - "metaData": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/line_items.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/line_items.json deleted file mode 100644 index 6fc90652f1c..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/line_items.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "objectType": { - "type": ["null", "string"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "objectId": { - "type": ["null", "integer"] - }, - "isDeleted": { - "type": ["null", "boolean"] - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners.json deleted file mode 100644 index 8efc00d8e3e..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/owners.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "portalId": { - "type": ["null", "integer"] - }, - "ownerId": { - "type": ["null", "integer"] - }, - "type": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "createdAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "signature": { - "type": ["null", "string"] - }, - "updatedAt": { - "type": ["null", "string"], - "format": "date-time" - }, - "hasContactsAccess": { - "type": ["null", "boolean"] - }, - "isActive": { - "type": ["null", "boolean"] - }, - "activeUserId": { - "type": ["null", "integer"] - }, - "userIdIncludingInactive": { - "type": ["null", "integer"] - }, - "remoteList": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "ownerId": { - "type": ["null", "integer"] - }, - "remoteId": { - "type": ["null", "string"] - }, - "remoteType": { - "type": ["null", "string"] - }, - "active": { - "type": ["null", "boolean"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/products.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/products.json deleted file mode 100644 index 6fc90652f1c..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/products.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "objectType": { - "type": ["null", "string"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "objectId": { - "type": ["null", "integer"] - }, - "isDeleted": { - "type": ["null", "boolean"] - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/quotes.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/quotes.json deleted file mode 100644 index 53a9e1ae69c..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/quotes.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "createdAt": { - "type": ["null", "string"] - }, - "updatedAt": { - "type": ["null", "string"] - }, - "archived": { - "type": ["null", "boolean"] - }, - "archivedAt": { - "type": ["null", "string"] - }, - "associations": { - "type": "object" - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/subscription_changes.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/subscription_changes.json deleted file mode 100644 index f45f72663ff..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/subscription_changes.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "portalId": { - "type": ["null", "integer"] - }, - "recipient": { - "type": ["null", "string"] - }, - "changes": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "change": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"], - "format": "date-time" - }, - "source": { - "type": ["null", "string"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "subscriptionId": { - "type": ["null", "integer"] - }, - "changeType": { - "type": ["null", "string"] - }, - "causedByEvent": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "created": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/tickets.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/tickets.json deleted file mode 100644 index 6fc90652f1c..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/tickets.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "objectType": { - "type": ["null", "string"] - }, - "portalId": { - "type": ["null", "integer"] - }, - "objectId": { - "type": ["null", "integer"] - }, - "isDeleted": { - "type": ["null", "boolean"] - }, - "createdAt": { - "type": "string" - }, - "updatedAt": { - "type": "string" - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json deleted file mode 100644 index 06cbe775039..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "name": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "type": { - "type": ["null", "string"] - }, - "enabled": { - "type": ["null", "boolean"] - }, - "insertedAt": { - "type": "integer" - }, - "updatedAt": { - "type": "integer" - }, - "personaTagIds": { - "type": "array", - "items": { - "type": "integer" - } - }, - "contactListIds": { - "type": "object", - "properties": { - "enrolled": { - "type": ["null", "integer"] - }, - "active": { - "type": ["null", "integer"] - }, - "steps": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.json deleted file mode 100644 index 77d0bb325c1..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Hubspot Source Spec", - "type": "object", - "required": ["start_date", "credentials"], - "additionalProperties": false, - "properties": { - "start_date": { - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2017-01-25T00:00:00Z"] - }, - "credentials": { - "type": "object", - "title": "api key", - "required": ["api_key"], - "properties": { - "api_key": { - "description": "Hubspot API Key. See our docs if you need help finding this key.", - "type": "string", - "airbyte_secret": true - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py deleted file mode 100644 index 8da70f47308..00000000000 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_client.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import pytest -from source_hubspot.client import Client - - -@pytest.fixture(name="some_credentials") -def some_credentials_fixture(): - return {"api_key": "wrong_key"} - - -def test_client_backoff_on_limit_reached(requests_mock, some_credentials): - """Error once, check that we retry and not fail""" - responses = [ - {"json": {"error": "limit reached"}, "status_code": 429, "headers": {"Retry-After": "0"}}, - {"json": [], "status_code": 200}, - ] - - requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) - client = Client(start_date="2021-02-01T00:00:00Z", credentials=some_credentials) - - alive, error = client.health_check() - - assert alive - assert not error - - -def test_client_backoff_on_server_error(requests_mock, some_credentials): - """Error once, check that we retry and not fail""" - responses = [ - {"json": {"error": "something bad"}, "status_code": 500}, - {"json": [], "status_code": 200}, - ] - requests_mock.register_uri("GET", "/properties/v2/contact/properties", responses) - client = Client(start_date="2021-02-01T00:00:00Z", credentials=some_credentials) - - alive, error = client.health_check() - - assert alive - assert not error diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java index 3eb82be18d9..ba4f620d5b4 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java @@ -55,10 +55,8 @@ import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.SQLException; import java.time.Instant; -import java.util.AbstractMap.SimpleImmutableEntry; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -139,11 +137,8 @@ public abstract class AbstractJdbcSource extends BaseConnector implements Source .stream() .map(t -> CatalogHelpers.createAirbyteStream(t.getName(), t.getFields()) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(t.getPrimaryKeys() - .stream() - .filter(Objects::nonNull) - .map(Collections::singletonList) - .collect(Collectors.toList()))) + .withSourceDefinedPrimaryKey( + t.getPrimaryKeys().stream().filter(Objects::nonNull).map(Collections::singletonList).collect(Collectors.toList()))) .collect(Collectors.toList())); } } @@ -285,9 +280,8 @@ public abstract class AbstractJdbcSource extends BaseConnector implements Source final Optional databaseOptional, final Optional schemaOptional) throws Exception { - final List tableInfos = discoverInternal(database, databaseOptional, schemaOptional); - final Map> tablePrimaryKeys = discoverPrimaryKeys(database, databaseOptional, schemaOptional, tableInfos); - return tableInfos.stream() + + return discoverInternal(database, databaseOptional, schemaOptional).stream() .map(t -> { // some databases return multiple copies of the same record for a column (e.g. redshift) because // they have at least once delivery guarantees. we want to dedupe these, but first we check that the @@ -298,77 +292,12 @@ public abstract class AbstractJdbcSource extends BaseConnector implements Source .map(f -> Field.of(f.getColumnName(), JdbcUtils.getType(f.getColumnType()))) .distinct() .collect(Collectors.toList()); - final String streamName = JdbcUtils.getFullyQualifiedTableName(t.getSchemaName(), t.getName()); - final List primaryKeys = tablePrimaryKeys.getOrDefault(streamName, Collections.emptyList()); - return new TableInfo(streamName, fields, primaryKeys); + + return new TableInfo(JdbcUtils.getFullyQualifiedTableName(t.getSchemaName(), t.getName()), fields, t.getPrimaryKeys()); }) .collect(Collectors.toList()); } - /** - * Discover Primary keys for each table and @return a map of schema.table name to their associated - * list of primary key fields. - * - * When invoking the conn.getMetaData().getPrimaryKeys() function without a table name, it may fail - * on some databases (for example MySql) but works on others (for instance Postgres). To avoid - * making repeated queries to the DB, we try to get all primary keys without specifying a table - * first, if it doesn't work, we retry one table at a time. - */ - private Map> discoverPrimaryKeys(JdbcDatabase database, - Optional databaseOptional, - Optional schemaOptional, - List tableInfos) { - try { - // Get all primary keys without specifying a table name - final Map> tablePrimaryKeys = aggregatePrimateKeys(database.bufferedResultSetQuery( - conn -> conn.getMetaData().getPrimaryKeys(databaseOptional.orElse(null), schemaOptional.orElse(null), null), - r -> { - final String schemaName = - r.getObject(JDBC_COLUMN_SCHEMA_NAME) != null ? r.getString(JDBC_COLUMN_SCHEMA_NAME) : r.getString(JDBC_COLUMN_DATABASE_NAME); - final String streamName = JdbcUtils.getFullyQualifiedTableName(schemaName, r.getString(JDBC_COLUMN_TABLE_NAME)); - final String primaryKey = r.getString(JDBC_COLUMN_COLUMN_NAME); - return new SimpleImmutableEntry<>(streamName, primaryKey); - })); - if (!tablePrimaryKeys.isEmpty()) { - return tablePrimaryKeys; - } - } catch (SQLException e) { - LOGGER.debug(String.format("Could not retrieve primary keys without a table name (%s), retrying", e)); - } - // Get primary keys one table at a time - return tableInfos.stream() - .collect(Collectors.toMap( - tableInfo -> JdbcUtils.getFullyQualifiedTableName(tableInfo.getSchemaName(), tableInfo.getName()), - tableInfo -> { - final String streamName = JdbcUtils.getFullyQualifiedTableName(tableInfo.getSchemaName(), tableInfo.getName()); - try { - final Map> primaryKeys = aggregatePrimateKeys(database.bufferedResultSetQuery( - conn -> conn.getMetaData().getPrimaryKeys(databaseOptional.orElse(null), tableInfo.getSchemaName(), tableInfo.getName()), - r -> new SimpleImmutableEntry<>(streamName, r.getString(JDBC_COLUMN_COLUMN_NAME)))); - return primaryKeys.getOrDefault(streamName, Collections.emptyList()); - } catch (SQLException e) { - LOGGER.error(String.format("Could not retrieve primary keys for %s: %s", streamName, e)); - return Collections.emptyList(); - } - })); - } - - /** - * Aggregate list of @param entries of StreamName and PrimaryKey and - * - * @return a map by StreamName to associated list of primary keys - */ - private static Map> aggregatePrimateKeys(List> entries) { - final Map> result = new HashMap<>(); - entries.forEach(entry -> { - if (!result.containsKey(entry.getKey())) { - result.put(entry.getKey(), new ArrayList<>()); - } - result.get(entry.getKey()).add(entry.getValue()); - }); - return result; - } - private static void assertColumnsWithSameNameAreSame(String schemaName, String tableName, List columns) { columns.stream() .collect(Collectors.groupingBy(ColumnInfo::getColumnName)) @@ -391,7 +320,7 @@ public abstract class AbstractJdbcSource extends BaseConnector implements Source final Optional schemaOptional) throws Exception { final Set internalSchemas = new HashSet<>(getExcludedInternalSchemas()); - return database.bufferedResultSetQuery( + final List result = database.bufferedResultSetQuery( conn -> conn.getMetaData().getColumns(databaseOptional.orElse(null), schemaOptional.orElse(null), null, null), resultSet -> Jsons.jsonNode(ImmutableMap.builder() // we always want a namespace, if we cannot get a schema, use db name. @@ -429,6 +358,17 @@ public abstract class AbstractJdbcSource extends BaseConnector implements Source }) .collect(Collectors.toList()))) .collect(Collectors.toList()); + result.forEach(t -> { + try { + final List primaryKeys = database.bufferedResultSetQuery( + conn -> conn.getMetaData().getPrimaryKeys(databaseOptional.orElse(null), t.getSchemaName(), t.getName()), + resultSet -> resultSet.getString(JDBC_COLUMN_COLUMN_NAME)); + t.addPrimaryKeys(primaryKeys); + } catch (SQLException e) { + LOGGER.warn(String.format("Could not find primary keys for %s.%s: %s", t.getSchemaName(), t.getName(), e)); + } + }); + return result; } private static AutoCloseableIterator getMessageIterator(AutoCloseableIterator recordIterator, @@ -541,11 +481,13 @@ public abstract class AbstractJdbcSource extends BaseConnector implements Source private final String schemaName; private final String name; private final List fields; + private final List primaryKeys; public TableInfoInternal(String schemaName, String tableName, List fields) { this.schemaName = schemaName; this.name = tableName; this.fields = fields; + this.primaryKeys = new ArrayList<>(); } public String getSchemaName() { @@ -560,6 +502,14 @@ public abstract class AbstractJdbcSource extends BaseConnector implements Source return fields; } + public void addPrimaryKeys(List primaryKeys) { + this.primaryKeys.addAll(primaryKeys); + } + + public List getPrimaryKeys() { + return primaryKeys; + } + } protected static class ColumnInfo { diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceStandardTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceStandardTest.java index 17b17fbcf9a..81752a9bb79 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceStandardTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceStandardTest.java @@ -89,7 +89,7 @@ public abstract class JdbcSourceStandardTest { private static final String TABLE_NAME = "id_and_name"; private static final String TABLE_NAME_WITHOUT_PK = "id_and_name_without_pk"; - private static final String TABLE_NAME_COMPOSITE_PK = "full_name_composite_pk"; + private static final String TABLE_NAME_FULL_NAMES = "full_names"; private JsonNode config; private JdbcDatabase database; @@ -164,11 +164,11 @@ public abstract class JdbcSourceStandardTest { connection.createStatement() .execute( String.format("CREATE TABLE %s(first_name VARCHAR(200), last_name VARCHAR(200), updated_at DATE, PRIMARY KEY (first_name, last_name));", - getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK))); + getFullyQualifiedTableName(TABLE_NAME_FULL_NAMES))); connection.createStatement().execute( String.format( "INSERT INTO %s(first_name, last_name, updated_at) VALUES ('first' ,'picard', '2004-10-19'), ('second', 'crusher', '2005-10-19'), ('third', 'vash', '2006-10-19');", - getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK))); + getFullyQualifiedTableName(TABLE_NAME_FULL_NAMES))); }); } @@ -627,7 +627,7 @@ public abstract class JdbcSourceStandardTest { .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(Collections.emptyList()), CatalogHelpers.createAirbyteStream( - defaultNamespace + "." + TABLE_NAME_COMPOSITE_PK, + defaultNamespace + "." + TABLE_NAME_FULL_NAMES, Field.of("first_name", JsonSchemaPrimitive.STRING), Field.of("last_name", JsonSchemaPrimitive.STRING), Field.of("updated_at", JsonSchemaPrimitive.STRING)) diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcStandardTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcStandardTest.java index 92436d1c949..7060fa4c3c4 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcStandardTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcStandardTest.java @@ -27,13 +27,11 @@ package io.airbyte.integrations.source.mysql; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceStandardTest; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; import org.apache.commons.lang3.RandomStringUtils; import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterAll; @@ -52,15 +50,13 @@ class MySqlJdbcStandardTest extends JdbcSourceStandardTest { private Database database; @BeforeAll - static void init() throws SQLException { - container = new MySQLContainer<>("mysql:8.0") - .withUsername(TEST_USER) - .withPassword(TEST_PASSWORD) - .withEnv("MYSQL_ROOT_HOST", "%") - .withEnv("MYSQL_ROOT_PASSWORD", TEST_PASSWORD); + static void init() { + // test containers withInitScript only accepts scripts that are mounted as resources. + MoreResources.writeResource("init.sql", + "CREATE USER '" + TEST_USER + "'@'%' IDENTIFIED BY '" + TEST_PASSWORD + "';\n" + + "GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n"); + container = new MySQLContainer<>("mysql:8.0").withInitScript("init.sql").withUsername("root").withPassword(""); container.start(); - Connection connection = DriverManager.getConnection(container.getJdbcUrl(), "root", TEST_PASSWORD); - connection.createStatement().execute("GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n"); } @BeforeEach @@ -80,7 +76,6 @@ class MySqlJdbcStandardTest extends JdbcSourceStandardTest { config.get("host").asText(), config.get("port").asText()), MySqlSource.DRIVER_CLASS, - SQLDialect.MYSQL); database.query(ctx -> { diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java index 187c03471cc..ca324c363c6 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java @@ -27,13 +27,11 @@ package io.airbyte.integrations.source.mysql; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.Database; import io.airbyte.db.Databases; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; import java.util.Optional; import org.apache.commons.lang3.RandomStringUtils; import org.jooq.SQLDialect; @@ -55,15 +53,13 @@ class MySqlStressTest extends JdbcStressTest { private Database database; @BeforeAll - static void init() throws SQLException { - container = new MySQLContainer<>("mysql:8.0") - .withUsername(TEST_USER) - .withPassword(TEST_PASSWORD) - .withEnv("MYSQL_ROOT_HOST", "%") - .withEnv("MYSQL_ROOT_PASSWORD", TEST_PASSWORD); + static void init() { + // test containers withInitScript only accepts scripts that are mounted as resources. + MoreResources.writeResource("init.sql", + "CREATE USER '" + TEST_USER + "'@'%' IDENTIFIED BY '" + TEST_PASSWORD + "';\n" + + "GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n"); + container = new MySQLContainer<>("mysql:8.0").withInitScript("init.sql").withUsername("root").withPassword(""); container.start(); - Connection connection = DriverManager.getConnection(container.getJdbcUrl(), "root", TEST_PASSWORD); - connection.createStatement().execute("GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n"); } @BeforeEach diff --git a/airbyte-migration/src/main/java/io/airbyte/migrate/Migrate.java b/airbyte-migration/src/main/java/io/airbyte/migrate/Migrate.java index f3f92825e5f..f10a9ff8252 100644 --- a/airbyte-migration/src/main/java/io/airbyte/migrate/Migrate.java +++ b/airbyte-migration/src/main/java/io/airbyte/migrate/Migrate.java @@ -32,7 +32,6 @@ import io.airbyte.commons.lang.Exceptions; import io.airbyte.commons.map.MoreMaps; import io.airbyte.commons.set.MoreSets; import io.airbyte.commons.stream.MoreStreams; -import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.version.AirbyteVersion; import io.airbyte.commons.yaml.Yamls; import io.airbyte.validation.json.JsonSchemaValidator; @@ -47,10 +46,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Consumer; +import java.util.stream.BaseStream; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.io.FileUtils; -import org.apache.logging.log4j.util.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -83,7 +82,7 @@ public class Migrate { // detect desired version. final String targetVersion = migrateConfig.getTargetVersion(); Preconditions.checkArgument(!currentVersion.equals("dev"), "Cannot migrate data with version dev."); - Preconditions.checkArgument(targetVersion == null || !targetVersion.equals("dev"), "Cannot migrate data to version dev."); + Preconditions.checkArgument(!targetVersion.equals("dev"), "Cannot migrate data to version dev."); LOGGER.info("Starting migrations. Current version: {}, Target version: {}", currentVersion, targetVersion); @@ -91,13 +90,7 @@ public class Migrate { final List migrationVersions = migrations.stream().map(m -> new AirbyteVersion(m.getVersion())).collect(Collectors.toList()); final int currentVersionIndex = getPreviousMigration(migrationVersions, new AirbyteVersion(currentVersion)); Preconditions.checkArgument(currentVersionIndex >= 0, "No migration found for current version: " + currentVersion); - final int targetVersionIndex; - if (Strings.isNotEmpty(targetVersion)) { - targetVersionIndex = migrations.stream().map(Migration::getVersion).collect(Collectors.toList()).indexOf(targetVersion); - } else { - targetVersionIndex = migrations.size() - 1; - } - + final int targetVersionIndex = migrations.stream().map(Migration::getVersion).collect(Collectors.toList()).indexOf(targetVersion); Preconditions.checkArgument(targetVersionIndex >= 0, "No migration found for target version: " + targetVersion); Preconditions.checkArgument(currentVersionIndex < targetVersionIndex, String.format( "Target version is not greater than the current version. current version: %s, target version: %s. Note migration order is determined by membership in migrations list, not any canonical sorting of the version string itself.", @@ -127,18 +120,7 @@ public class Migrate { final Path tmpOutputDir = Files.createDirectories(migrateRoot.resolve(migration.getVersion())); // create a map of each input resource path to the input stream. - final Map> inputData = createInputStreams(migration, migrationInputRoot); - final Map> inputDataStreams = inputData.entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> MoreStreams.toStream(entry.getValue()) - .peek(r -> { - try { - jsonSchemaValidator.ensure(migration.getInputSchema().get(entry.getKey()), r); - } catch (JsonValidationException e) { - throw new IllegalArgumentException("Input data schema does not match declared input schema.", e); - } - }))); + final Map> inputData = createInputStreams(migration, migrationInputRoot); final Map outputStreams = createOutputStreams(migration, tmpOutputDir); // make the java compiler happy (it can't resolve that RecordConsumer is, in fact, a @@ -146,19 +128,19 @@ public class Migrate { final Map> outputDataWithGenericType = MigrationUtils.mapRecordConsumerToConsumer(outputStreams); // do the migration. - new MigrateWithMetadata(migration).migrate(inputDataStreams, outputDataWithGenericType); + new MigrateWithMetadata(migration).migrate(inputData, outputDataWithGenericType); // clean up. - inputData.values().forEach(v -> Exceptions.toRuntime(v::close)); + inputData.values().forEach(BaseStream::close); outputStreams.values().forEach(v -> Exceptions.toRuntime(v::close)); return tmpOutputDir; } - private Map> createInputStreams(Migration migration, Path migrationInputRoot) { - final Map> resourceIdToInputStreams = MoreMaps.merge( - createInputStreamsForResourceType(migrationInputRoot, ResourceType.CONFIG), - createInputStreamsForResourceType(migrationInputRoot, ResourceType.JOB)); + private Map> createInputStreams(Migration migration, Path migrationInputRoot) { + final Map> resourceIdToInputStreams = MoreMaps.merge( + createInputStreamsForResourceType(migration, migrationInputRoot, ResourceType.CONFIG), + createInputStreamsForResourceType(migration, migrationInputRoot, ResourceType.JOB)); try { MoreSets.assertEqualsVerbose(migration.getInputSchema().keySet(), resourceIdToInputStreams.keySet()); @@ -168,16 +150,25 @@ public class Migrate { return resourceIdToInputStreams; } - private Map> createInputStreamsForResourceType(Path migrationInputRoot, ResourceType resourceType) { + private Map> createInputStreamsForResourceType(Migration migration, + Path migrationInputRoot, + ResourceType resourceType) { final List inputFilePaths = FileUtils.listFiles(migrationInputRoot.resolve(resourceType.getDirectoryName()).toFile(), null, false) .stream() .map(File::toPath) .collect(Collectors.toList()); - final Map> inputData = new HashMap<>(); + final Map> inputData = new HashMap<>(); for (final Path absolutePath : inputFilePaths) { final ResourceId resourceId = ResourceId.fromRecordFilePath(resourceType, absolutePath); - AutoCloseableIterator recordInputStream = Yamls.deserializeArray(IOs.inputStream(absolutePath)); + final Stream recordInputStream = MoreStreams.toStream(Yamls.deserialize(IOs.readFile(absolutePath)).elements()) + .peek(r -> { + try { + jsonSchemaValidator.ensure(migration.getInputSchema().get(resourceId), r); + } catch (JsonValidationException e) { + throw new IllegalArgumentException("Input data schema does not match declared input schema.", e); + } + }); inputData.put(resourceId, recordInputStream); } diff --git a/airbyte-migration/src/main/java/io/airbyte/migrate/MigrationRunner.java b/airbyte-migration/src/main/java/io/airbyte/migrate/MigrationRunner.java index 5fd59b9cca6..8576ac26932 100644 --- a/airbyte-migration/src/main/java/io/airbyte/migrate/MigrationRunner.java +++ b/airbyte-migration/src/main/java/io/airbyte/migrate/MigrationRunner.java @@ -46,7 +46,7 @@ public class MigrationRunner { MigrateConfig migrateConfig = parse(args); - if (migrateConfig.getInputPath().toString().endsWith(".gz")) { + if (migrateConfig.getInputPath().toString().endsWith("gz") || migrateConfig.getInputPath().toString().endsWith(".tar.gz")) { LOGGER.info("Unpacking tarball"); final Path uncompressedInputPath = Files.createDirectories(workspaceRoot.resolve("uncompressed")); Archives.extractArchive(migrateConfig.getInputPath(), uncompressedInputPath); @@ -89,8 +89,8 @@ public class MigrationRunner { .help("Full path of the output tarball. By convention should end with .tar.gz"); parser.addArgument("--target-version") - .required(false) - .help("Version to upgrade the data to (default to latest migration available if left empty)"); + .required(true) + .help("Version to upgrade the data to"); try { final Namespace parsed = parser.parseArgs(args); diff --git a/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_14_3.java b/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_14_3.java index a92cdd319ab..d757d58f9ee 100644 --- a/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_14_3.java +++ b/airbyte-migration/src/main/java/io/airbyte/migrate/migrations/MigrationV0_14_3.java @@ -27,7 +27,6 @@ package io.airbyte.migrate.migrations; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.util.MoreIterators; @@ -102,19 +101,21 @@ public class MigrationV0_14_3 extends BaseMigration implements Migration { // sync mode enum is identical in Schema and ConfiguredCatalog. .map(JsonNode::asText) .collect(Collectors.toList()); - // catalog fields - final Map airbyteStream = Maps.newHashMap(); - airbyteStream.put("name", stream.get("name")); - airbyteStream.put("supported_sync_modes", Jsons.jsonNode(supportedSyncModes)); - airbyteStream.put("json_schema", fieldsToJsonSchema(stream.get("fields"))); - airbyteStream.put("source_defined_cursor", stream.get("sourceDefinedCursor")); - airbyteStream.put("default_cursor_field", stream.get("defaultCursorField")); + final Map airbyteStream = ImmutableMap.builder() + // catalog fields + .put("name", stream.get("name")) + .put("supported_sync_modes", Jsons.jsonNode(supportedSyncModes)) + .put("json_schema", fieldsToJsonSchema(stream.get("fields"))) + .put("source_defined_cursor", stream.get("sourceDefinedCursor")) + .put("default_cursor_field", stream.get("defaultCursorField")) + .build(); // configured catalog fields - final Map catalog = Maps.newHashMap(); - catalog.put("stream", Jsons.jsonNode(airbyteStream)); - catalog.put("sync_mode", Jsons.jsonNode(stream.get("syncMode").asText())); - catalog.put("cursor_field", stream.get("cursorField")); - return catalog; + return (Map) ImmutableMap.builder() + .put("stream", Jsons.jsonNode(airbyteStream)) + // sync mode enum is identical in Schema and ConfiguredCatalog. + .put("sync_mode", Jsons.jsonNode(stream.get("syncMode").asText())) + .put("cursor_field", stream.get("cursorField")) + .build(); }) .collect(Collectors.toList()); diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/ScheduleJobPredicate.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/ScheduleJobPredicate.java index a2552a4312e..bd7289cb1e6 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/ScheduleJobPredicate.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/ScheduleJobPredicate.java @@ -72,14 +72,7 @@ public class ScheduleJobPredicate implements BiPredicate, Standard } final Job previousJob = previousJobOptional.get(); - - // if there is an active job, do not start a new one. - if (!JobStatus.TERMINAL_STATUSES.contains(previousJob.getStatus())) { - return false; - } - - long prevRunStart = previousJob.getStartedAtInSecond().orElse(previousJob.getCreatedAtInSecond()); - long nextRunStart = prevRunStart + ScheduleHelpers.getIntervalInSecond(standardSyncSchedule.getSchedule()); + long nextRunStart = previousJob.getUpdatedAtInSecond() + ScheduleHelpers.getIntervalInSecond(standardSyncSchedule.getSchedule()); return nextRunStart < timeSupplier.get().getEpochSecond(); } diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/SchedulerApp.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/SchedulerApp.java index eeb735b83ee..a31433c4881 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/SchedulerApp.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/SchedulerApp.java @@ -99,7 +99,7 @@ public class SchedulerApp { final ExecutorService workerThreadPool = Executors.newFixedThreadPool(MAX_WORKERS, THREAD_FACTORY); final ScheduledExecutorService scheduledPool = Executors.newSingleThreadScheduledExecutor(); - final TemporalWorkerRunFactory temporalWorkerRunFactory = new TemporalWorkerRunFactory(TemporalClient.production(workspaceRoot), workspaceRoot); + final TemporalWorkerRunFactory temporalWorkerRunFactory = new TemporalWorkerRunFactory(TemporalClient.production(), workspaceRoot); final JobRetrier jobRetrier = new JobRetrier(jobPersistence, Instant::now); final JobScheduler jobScheduler = new JobScheduler(jobPersistence, configRepository); final JobSubmitter jobSubmitter = new JobSubmitter( diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClient.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClient.java index ead1b7793de..80fba1edaf0 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClient.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClient.java @@ -25,6 +25,7 @@ package io.airbyte.scheduler.client; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.config.DestinationConnection; import io.airbyte.config.JobCheckConnectionConfig; import io.airbyte.config.JobConfig.ConfigType; @@ -37,11 +38,10 @@ import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.scheduler.JobTracker; import io.airbyte.scheduler.JobTracker.JobState; import io.airbyte.workers.temporal.TemporalClient; -import io.airbyte.workers.temporal.TemporalResponse; +import io.airbyte.workers.temporal.TemporalJobException; import java.io.IOException; import java.time.Instant; import java.util.UUID; -import java.util.function.Function; public class DefaultSynchronousSchedulerClient implements SynchronousSchedulerClient { @@ -54,7 +54,8 @@ public class DefaultSynchronousSchedulerClient implements SynchronousSchedulerCl } @Override - public SynchronousResponse createSourceCheckConnectionJob(final SourceConnection source, final String dockerImage) { + public SynchronousResponse createSourceCheckConnectionJob(final SourceConnection source, final String dockerImage) + throws IOException { final JobCheckConnectionConfig jobCheckConnectionConfig = new JobCheckConnectionConfig() .withConnectionConfiguration(source.getConfiguration()) .withDockerImage(dockerImage); @@ -68,7 +69,8 @@ public class DefaultSynchronousSchedulerClient implements SynchronousSchedulerCl @Override public SynchronousResponse createDestinationCheckConnectionJob(final DestinationConnection destination, - final String dockerImage) { + final String dockerImage) + throws IOException { final JobCheckConnectionConfig jobCheckConnectionConfig = new JobCheckConnectionConfig() .withConnectionConfiguration(destination.getConfiguration()) .withDockerImage(dockerImage); @@ -81,7 +83,7 @@ public class DefaultSynchronousSchedulerClient implements SynchronousSchedulerCl } @Override - public SynchronousResponse createDiscoverSchemaJob(final SourceConnection source, final String dockerImage) { + public SynchronousResponse createDiscoverSchemaJob(final SourceConnection source, final String dockerImage) throws IOException { final JobDiscoverCatalogConfig jobDiscoverCatalogConfig = new JobDiscoverCatalogConfig() .withConnectionConfiguration(source.getConfiguration()) .withDockerImage(dockerImage); @@ -108,29 +110,35 @@ public class DefaultSynchronousSchedulerClient implements SynchronousSchedulerCl @VisibleForTesting SynchronousResponse execute(ConfigType configType, UUID configId, - Function> executor, + CheckedFunction executor, UUID jobTrackerId) { final long createdAt = Instant.now().toEpochMilli(); + T operationOutput = null; + TemporalJobException exception = null; final UUID jobId = UUID.randomUUID(); try { track(jobId, configType, jobTrackerId, JobState.STARTED, null); - final TemporalResponse operationOutput = executor.apply(jobId); - JobState outputState = operationOutput.getMetadata().isSucceeded() ? JobState.SUCCEEDED : JobState.FAILED; - track(jobId, configType, jobTrackerId, outputState, operationOutput.getOutput().orElse(null)); - final long endedAt = Instant.now().toEpochMilli(); - - return SynchronousResponse.fromTemporalResponse( - operationOutput, - jobId, - configType, - configId, - createdAt, - endedAt); - } catch (RuntimeException e) { - // todo handle null. - track(jobId, configType, jobTrackerId, JobState.FAILED, null); + operationOutput = executor.apply(jobId); + track(jobId, configType, jobTrackerId, JobState.SUCCEEDED, operationOutput); + } catch (TemporalJobException e) { + exception = e; + track(jobId, configType, jobTrackerId, JobState.FAILED, operationOutput); + } catch (Exception e) { + track(jobId, configType, jobTrackerId, JobState.FAILED, operationOutput); throw e; } + final long endedAt = Instant.now().toEpochMilli(); + + final SynchronousJobMetadata metadata = new SynchronousJobMetadata( + jobId, + configType, + configId, + createdAt, + endedAt, + exception == null, + exception != null ? exception.getLogPath() : null); + + return new SynchronousResponse<>(operationOutput, metadata); } private void track(UUID jobId, ConfigType configType, UUID jobTrackerId, JobState jobState, T value) { diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/client/SynchronousJobMetadata.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/client/SynchronousJobMetadata.java index f0931dcdd14..61e024e9b35 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/client/SynchronousJobMetadata.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/client/SynchronousJobMetadata.java @@ -25,7 +25,6 @@ package io.airbyte.scheduler.client; import io.airbyte.config.JobConfig.ConfigType; -import io.airbyte.workers.temporal.JobMetadata; import java.nio.file.Path; import java.util.Objects; import java.util.Optional; @@ -43,22 +42,6 @@ public class SynchronousJobMetadata { private final Path logPath; - public static SynchronousJobMetadata fromJobMetadata(JobMetadata jobMetadata, - UUID id, - ConfigType configType, - UUID configId, - long createdAt, - long endedAt) { - return new SynchronousJobMetadata( - id, - configType, - configId, - createdAt, - endedAt, - jobMetadata.isSucceeded(), - jobMetadata.getLogPath()); - } - public SynchronousJobMetadata(final UUID id, final ConfigType configType, final UUID configId, @@ -99,8 +82,10 @@ public class SynchronousJobMetadata { return succeeded; } - public Path getLogPath() { - return logPath; + // todo (cgardens) - this should always be present. + // only present if there was an error. + public Optional getLogPath() { + return Optional.ofNullable(logPath); } @Override diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/client/SynchronousResponse.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/client/SynchronousResponse.java index e0bb32bb2a1..ef8179c070d 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/client/SynchronousResponse.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/client/SynchronousResponse.java @@ -24,10 +24,7 @@ package io.airbyte.scheduler.client; -import io.airbyte.config.JobConfig.ConfigType; -import io.airbyte.workers.temporal.TemporalResponse; import java.util.Objects; -import java.util.UUID; public class SynchronousResponse { @@ -42,23 +39,6 @@ public class SynchronousResponse { return new SynchronousResponse<>(output, metadata); } - public static SynchronousResponse fromTemporalResponse(TemporalResponse temporalResponse, - UUID id, - ConfigType configType, - UUID configId, - long createdAt, - long endedAt) { - - final SynchronousJobMetadata metadata = SynchronousJobMetadata.fromJobMetadata( - temporalResponse.getMetadata(), - id, - configType, - configId, - createdAt, - endedAt); - return new SynchronousResponse<>(temporalResponse.getOutput().orElse(null), metadata); - } - public SynchronousResponse(final T output, final SynchronousJobMetadata metadata) { this.output = output; this.metadata = metadata; diff --git a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/worker_run/TemporalWorkerRunFactory.java b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/worker_run/TemporalWorkerRunFactory.java index 4ca9f94207f..7afe801c021 100644 --- a/airbyte-scheduler/src/main/java/io/airbyte/scheduler/worker_run/TemporalWorkerRunFactory.java +++ b/airbyte-scheduler/src/main/java/io/airbyte/scheduler/worker_run/TemporalWorkerRunFactory.java @@ -36,8 +36,9 @@ import io.airbyte.workers.JobStatus; import io.airbyte.workers.OutputAndStatus; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.temporal.TemporalClient; +import io.airbyte.workers.temporal.TemporalJobException; import io.airbyte.workers.temporal.TemporalJobType; -import io.airbyte.workers.temporal.TemporalResponse; +import io.temporal.failure.TemporalException; import java.nio.file.Path; public class TemporalWorkerRunFactory { @@ -55,25 +56,32 @@ public class TemporalWorkerRunFactory { return WorkerRun.create(workspaceRoot, job.getId(), attemptId, createSupplier(job, attemptId)); } + // suppress "CodeBlock2Expr" because in the lambda syntax without a return statement the formatting + // makes the switch statement very hard to read. + @SuppressWarnings({"CodeBlock2Expr"}) public CheckedSupplier, Exception> createSupplier(Job job, int attemptId) { final TemporalJobType temporalJobType = toTemporalJobType(job.getConfigType()); return switch (job.getConfigType()) { case SYNC -> () -> { - final TemporalResponse output = temporalClient.submitSync(job.getId(), attemptId, job.getConfig().getSync()); - return toOutputAndStatus(output); + return toOutputAndStatus(() -> { + final StandardSyncOutput output = temporalClient.submitSync(job.getId(), attemptId, job.getConfig().getSync()); + return new JobOutput().withSync(output); + }); }; case RESET_CONNECTION -> () -> { - final JobResetConnectionConfig resetConnection = job.getConfig().getResetConnection(); - final JobSyncConfig config = new JobSyncConfig() - .withPrefix(resetConnection.getPrefix()) - .withSourceDockerImage(WorkerConstants.RESET_JOB_SOURCE_DOCKER_IMAGE_STUB) - .withDestinationDockerImage(resetConnection.getDestinationDockerImage()) - .withSourceConfiguration(Jsons.emptyObject()) - .withDestinationConfiguration(resetConnection.getDestinationConfiguration()) - .withConfiguredAirbyteCatalog(resetConnection.getConfiguredAirbyteCatalog()); + return toOutputAndStatus(() -> { + final JobResetConnectionConfig resetConnection = job.getConfig().getResetConnection(); + final JobSyncConfig config = new JobSyncConfig() + .withPrefix(resetConnection.getPrefix()) + .withSourceDockerImage(WorkerConstants.RESET_JOB_SOURCE_DOCKER_IMAGE_STUB) + .withDestinationDockerImage(resetConnection.getDestinationDockerImage()) + .withSourceConfiguration(Jsons.emptyObject()) + .withDestinationConfiguration(resetConnection.getDestinationConfiguration()) + .withConfiguredAirbyteCatalog(resetConnection.getConfiguredAirbyteCatalog()); - final TemporalResponse output = temporalClient.submitSync(job.getId(), attemptId, config); - return toOutputAndStatus(output); + final StandardSyncOutput output = temporalClient.submitSync(job.getId(), attemptId, config); + return new JobOutput().withSync(output); + }); }; default -> throw new IllegalArgumentException("Does not support job type: " + temporalJobType); }; @@ -88,9 +96,16 @@ public class TemporalWorkerRunFactory { }; } - private OutputAndStatus toOutputAndStatus(TemporalResponse response) { - final JobStatus status = response.isSuccess() ? JobStatus.SUCCEEDED : JobStatus.FAILED; - return new OutputAndStatus<>(status, new JobOutput().withSync(response.getOutput().orElse(null))); + private OutputAndStatus toOutputAndStatus(CheckedSupplier supplier) { + try { + return new OutputAndStatus<>(JobStatus.SUCCEEDED, supplier.get()); + } catch (TemporalJobException | TemporalException e) { + // while from within the temporal activity we throw TemporalJobException, Temporal wraps any + // exception thrown by an activity in an ApplicationFailure exception, which gets wrapped in + // ActivityFailure exception which get wrapped in a WorkflowFailedException. We will need to unwrap + // these later, but for now we just catch the parent. + return new OutputAndStatus<>(JobStatus.FAILED); + } } } diff --git a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/ScheduleJobPredicateTest.java b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/ScheduleJobPredicateTest.java index 696b9dd0dcb..8ce56c553fa 100644 --- a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/ScheduleJobPredicateTest.java +++ b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/ScheduleJobPredicateTest.java @@ -78,7 +78,7 @@ class ScheduleJobPredicateTest { @Test public void testScheduleNotReady() { when(job.getStatus()).thenReturn(JobStatus.SUCCEEDED); - when(job.getStartedAtInSecond()).thenReturn(Optional.of(now.minus(Duration.ofDays(1)).getEpochSecond())); + when(job.getUpdatedAtInSecond()).thenReturn(now.minus(Duration.ofDays(1)).getEpochSecond()); assertFalse(scheduleJobPredicate.test(Optional.of(job), SCHEDULE)); } @@ -91,7 +91,7 @@ class ScheduleJobPredicateTest { names = {"PENDING", "RUNNING", "INCOMPLETE"}) public void testShouldScheduleBasedOnPreviousJobStatus(JobStatus status) { when(job.getStatus()).thenReturn(status); - when(job.getStartedAtInSecond()).thenReturn(Optional.of(now.minus(Duration.ofDays(2)).getEpochSecond())); + when(job.getUpdatedAtInSecond()).thenReturn(now.minus(Duration.ofDays(2)).getEpochSecond()); assertTrue(scheduleJobPredicate.test(Optional.of(job), SCHEDULE), "job status: " + status.toString()); } @@ -102,7 +102,7 @@ class ScheduleJobPredicateTest { names = {"FAILED", "SUCCEEDED", "CANCELLED"}) public void testScheduleShouldNotScheduleBasedOnPreviousJobStatus(JobStatus status) { when(job.getStatus()).thenReturn(status); - when(job.getStartedAtInSecond()).thenReturn(Optional.of(now.minus(Duration.ofDays(2)).getEpochSecond())); + when(job.getUpdatedAtInSecond()).thenReturn(now.minus(Duration.ofDays(2)).getEpochSecond()); assertFalse(scheduleJobPredicate.test(Optional.of(job), SCHEDULE), "job status: " + status.toString()); } diff --git a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClientTest.java b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClientTest.java index a599dfa1d48..a2a7c366523 100644 --- a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClientTest.java +++ b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/client/DefaultSynchronousSchedulerClientTest.java @@ -38,6 +38,7 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; import io.airbyte.config.DestinationConnection; import io.airbyte.config.JobCheckConnectionConfig; @@ -50,13 +51,11 @@ import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.scheduler.JobTracker; import io.airbyte.scheduler.JobTracker.JobState; -import io.airbyte.workers.temporal.JobMetadata; import io.airbyte.workers.temporal.TemporalClient; -import io.airbyte.workers.temporal.TemporalResponse; +import io.airbyte.workers.temporal.TemporalJobException; import java.io.IOException; import java.nio.file.Path; import java.util.UUID; -import java.util.function.Function; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; @@ -67,7 +66,6 @@ import org.junit.jupiter.api.Test; // execution exception cases again. class DefaultSynchronousSchedulerClientTest { - private static final Path LOG_PATH = Path.of("/tmp"); private static final String DOCKER_IMAGE = "foo/bar"; private static final UUID UUID1 = UUID.randomUUID(); private static final UUID UUID2 = UUID.randomUUID(); @@ -95,25 +93,19 @@ class DefaultSynchronousSchedulerClientTest { schedulerClient = new DefaultSynchronousSchedulerClient(temporalClient, jobTracker); } - private static JobMetadata createMetadata(boolean succeeded) { - return new JobMetadata( - succeeded, - LOG_PATH); - } - @Nested @DisplayName("Test execute method.") class ExecuteSynchronousJob { @SuppressWarnings("unchecked") @Test - void testExecuteJobSuccess() { + void testExecute() throws TemporalJobException { final UUID configId = UUID.randomUUID(); final UUID jobTrackingId = UUID.randomUUID(); - final Function> function = mock(Function.class); - when(function.apply(any(UUID.class))).thenReturn(new TemporalResponse<>("hello", createMetadata(true))); + final CheckedFunction checkedFunction = mock(CheckedFunction.class); + when(checkedFunction.apply(any(UUID.class))).thenReturn("hello"); - final SynchronousResponse response = schedulerClient.execute(ConfigType.DISCOVER_SCHEMA, configId, function, jobTrackingId); + final SynchronousResponse response = schedulerClient.execute(ConfigType.DISCOVER_SCHEMA, configId, checkedFunction, jobTrackingId); assertNotNull(response); assertEquals("hello", response.getOutput()); @@ -121,7 +113,7 @@ class DefaultSynchronousSchedulerClientTest { assertTrue(response.getMetadata().getConfigId().isPresent()); assertEquals(configId, response.getMetadata().getConfigId().get()); assertTrue(response.getMetadata().isSucceeded()); - assertEquals(LOG_PATH, response.getMetadata().getLogPath()); + assertTrue(response.getMetadata().getLogPath().isEmpty()); verify(jobTracker).trackDiscover(any(UUID.class), eq(jobTrackingId), eq(JobState.STARTED)); verify(jobTracker).trackDiscover(any(UUID.class), eq(jobTrackingId), eq(JobState.SUCCEEDED)); @@ -129,13 +121,13 @@ class DefaultSynchronousSchedulerClientTest { @SuppressWarnings("unchecked") @Test - void testExecuteJobFailure() { + void testExecuteTemporalJobException() throws TemporalJobException { final UUID configId = UUID.randomUUID(); final UUID jobTrackingId = UUID.randomUUID(); - final Function> function = mock(Function.class); - when(function.apply(any(UUID.class))).thenReturn(new TemporalResponse<>(null, createMetadata(false))); + final CheckedFunction checkedFunction = mock(CheckedFunction.class); + when(checkedFunction.apply(any(UUID.class))).thenThrow(new TemporalJobException(Path.of("/tmp"))); - final SynchronousResponse response = schedulerClient.execute(ConfigType.DISCOVER_SCHEMA, configId, function, jobTrackingId); + final SynchronousResponse response = schedulerClient.execute(ConfigType.DISCOVER_SCHEMA, configId, checkedFunction, jobTrackingId); assertNotNull(response); assertNull(response.getOutput()); @@ -143,7 +135,7 @@ class DefaultSynchronousSchedulerClientTest { assertTrue(response.getMetadata().getConfigId().isPresent()); assertEquals(configId, response.getMetadata().getConfigId().get()); assertFalse(response.getMetadata().isSucceeded()); - assertEquals(LOG_PATH, response.getMetadata().getLogPath()); + assertTrue(response.getMetadata().getLogPath().isPresent()); verify(jobTracker).trackDiscover(any(UUID.class), eq(jobTrackingId), eq(JobState.STARTED)); verify(jobTracker).trackDiscover(any(UUID.class), eq(jobTrackingId), eq(JobState.FAILED)); @@ -151,13 +143,13 @@ class DefaultSynchronousSchedulerClientTest { @SuppressWarnings("unchecked") @Test - void testExecuteRuntimeException() { + void testExecuteRuntimeException() throws TemporalJobException { final UUID configId = UUID.randomUUID(); final UUID jobTrackingId = UUID.randomUUID(); - final Function> function = mock(Function.class); - when(function.apply(any(UUID.class))).thenThrow(new RuntimeException()); + final CheckedFunction checkedFunction = mock(CheckedFunction.class); + when(checkedFunction.apply(any(UUID.class))).thenThrow(new RuntimeException()); - assertThrows(RuntimeException.class, () -> schedulerClient.execute(ConfigType.DISCOVER_SCHEMA, configId, function, jobTrackingId)); + assertThrows(RuntimeException.class, () -> schedulerClient.execute(ConfigType.DISCOVER_SCHEMA, configId, checkedFunction, jobTrackingId)); verify(jobTracker).trackDiscover(any(UUID.class), eq(jobTrackingId), eq(JobState.STARTED)); verify(jobTracker).trackDiscover(any(UUID.class), eq(jobTrackingId), eq(JobState.FAILED)); @@ -170,53 +162,49 @@ class DefaultSynchronousSchedulerClientTest { class TestJobCreation { @Test - void testCreateSourceCheckConnectionJob() { + void testCreateSourceCheckConnectionJob() throws IOException, TemporalJobException { final JobCheckConnectionConfig jobCheckConnectionConfig = new JobCheckConnectionConfig() .withConnectionConfiguration(SOURCE_CONNECTION.getConfiguration()) .withDockerImage(DOCKER_IMAGE); - final StandardCheckConnectionOutput mockOutput = mock(StandardCheckConnectionOutput.class); - when(temporalClient.submitCheckConnection(any(UUID.class), eq(0), eq(jobCheckConnectionConfig))) - .thenReturn(new TemporalResponse<>(mockOutput, createMetadata(true))); + StandardCheckConnectionOutput mockOutput = mock(StandardCheckConnectionOutput.class); + when(temporalClient.submitCheckConnection(any(UUID.class), eq(0), eq(jobCheckConnectionConfig))).thenReturn(mockOutput); final SynchronousResponse response = schedulerClient.createSourceCheckConnectionJob(SOURCE_CONNECTION, DOCKER_IMAGE); assertEquals(mockOutput, response.getOutput()); } @Test - void testCreateDestinationCheckConnectionJob() { + void testCreateDestinationCheckConnectionJob() throws IOException, TemporalJobException { final JobCheckConnectionConfig jobCheckConnectionConfig = new JobCheckConnectionConfig() .withConnectionConfiguration(DESTINATION_CONNECTION.getConfiguration()) .withDockerImage(DOCKER_IMAGE); - final StandardCheckConnectionOutput mockOutput = mock(StandardCheckConnectionOutput.class); - when(temporalClient.submitCheckConnection(any(UUID.class), eq(0), eq(jobCheckConnectionConfig))) - .thenReturn(new TemporalResponse<>(mockOutput, createMetadata(true))); + StandardCheckConnectionOutput mockOutput = mock(StandardCheckConnectionOutput.class); + when(temporalClient.submitCheckConnection(any(UUID.class), eq(0), eq(jobCheckConnectionConfig))).thenReturn(mockOutput); final SynchronousResponse response = schedulerClient.createDestinationCheckConnectionJob(DESTINATION_CONNECTION, DOCKER_IMAGE); assertEquals(mockOutput, response.getOutput()); } @Test - void testCreateDiscoverSchemaJob() { + void testCreateDiscoverSchemaJob() throws IOException, TemporalJobException { final JobDiscoverCatalogConfig jobDiscoverCatalogConfig = new JobDiscoverCatalogConfig() .withConnectionConfiguration(SOURCE_CONNECTION.getConfiguration()) .withDockerImage(DOCKER_IMAGE); - final AirbyteCatalog mockOutput = mock(AirbyteCatalog.class); - when(temporalClient.submitDiscoverSchema(any(UUID.class), eq(0), eq(jobDiscoverCatalogConfig))) - .thenReturn(new TemporalResponse<>(mockOutput, createMetadata(true))); + AirbyteCatalog mockOutput = mock(AirbyteCatalog.class); + when(temporalClient.submitDiscoverSchema(any(UUID.class), eq(0), eq(jobDiscoverCatalogConfig))).thenReturn(mockOutput); final SynchronousResponse response = schedulerClient.createDiscoverSchemaJob(SOURCE_CONNECTION, DOCKER_IMAGE); assertEquals(mockOutput, response.getOutput()); } @Test - void testCreateGetSpecJob() throws IOException { + void testCreateGetSpecJob() throws IOException, TemporalJobException { final JobGetSpecConfig jobSpecConfig = new JobGetSpecConfig().withDockerImage(DOCKER_IMAGE); - final ConnectorSpecification mockOutput = mock(ConnectorSpecification.class); - when(temporalClient.submitGetSpec(any(UUID.class), eq(0), eq(jobSpecConfig))) - .thenReturn(new TemporalResponse<>(mockOutput, createMetadata(true))); + ConnectorSpecification mockOutput = mock(ConnectorSpecification.class); + when(temporalClient.submitGetSpec(any(UUID.class), eq(0), eq(jobSpecConfig))).thenReturn(mockOutput); final SynchronousResponse response = schedulerClient.createGetSpecJob(DOCKER_IMAGE); assertEquals(mockOutput, response.getOutput()); } diff --git a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/temporal/TemporalWorkerRunFactoryTest.java b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/temporal/TemporalWorkerRunFactoryTest.java index bb488ee9db5..d4cab9f773f 100644 --- a/airbyte-scheduler/src/test/java/io/airbyte/scheduler/temporal/TemporalWorkerRunFactoryTest.java +++ b/airbyte-scheduler/src/test/java/io/airbyte/scheduler/temporal/TemporalWorkerRunFactoryTest.java @@ -36,14 +36,12 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.config.JobConfig.ConfigType; import io.airbyte.config.JobResetConnectionConfig; import io.airbyte.config.JobSyncConfig; -import io.airbyte.config.StandardSyncOutput; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.scheduler.Job; import io.airbyte.scheduler.worker_run.TemporalWorkerRunFactory; import io.airbyte.scheduler.worker_run.WorkerRun; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.temporal.TemporalClient; -import io.airbyte.workers.temporal.TemporalResponse; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -72,20 +70,15 @@ class TemporalWorkerRunFactoryTest { when(job.getAttemptsCount()).thenReturn(ATTEMPT_ID); } - @SuppressWarnings("unchecked") @Test void testSync() throws Exception { when(job.getConfigType()).thenReturn(ConfigType.SYNC); - final TemporalResponse mockResponse = mock(TemporalResponse.class); - when(temporalClient.submitSync(JOB_ID, ATTEMPT_ID, job.getConfig().getSync())).thenReturn(mockResponse); - final WorkerRun workerRun = workerRunFactory.create(job); workerRun.call(); verify(temporalClient).submitSync(JOB_ID, ATTEMPT_ID, job.getConfig().getSync()); assertEquals(jobRoot, workerRun.getJobRoot()); } - @SuppressWarnings("unchecked") @Test void testResetConnection() throws Exception { final JobResetConnectionConfig resetConfig = new JobResetConnectionConfig() @@ -100,8 +93,6 @@ class TemporalWorkerRunFactoryTest { .withConfiguredAirbyteCatalog(resetConfig.getConfiguredAirbyteCatalog()); when(job.getConfigType()).thenReturn(ConfigType.RESET_CONNECTION); when(job.getConfig().getResetConnection()).thenReturn(resetConfig); - final TemporalResponse mockResponse = mock(TemporalResponse.class); - when(temporalClient.submitSync(JOB_ID, ATTEMPT_ID, syncConfig)).thenReturn(mockResponse); final WorkerRun workerRun = workerRunFactory.create(job); workerRun.call(); diff --git a/airbyte-server/build.gradle b/airbyte-server/build.gradle index 3d0bf2f747c..b9b5fdf851e 100644 --- a/airbyte-server/build.gradle +++ b/airbyte-server/build.gradle @@ -3,8 +3,6 @@ plugins { } dependencies { - implementation 'io.temporal:temporal-sdk:1.0.4' - implementation 'org.apache.cxf:cxf-core:3.4.2' implementation 'org.eclipse.jetty:jetty-server:9.4.31.v20200723' diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 9a662167f65..3f50069d4a0 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -99,10 +99,8 @@ public class ServerApp { ConfigurationApiFactory.setSchedulerJobClient(new DefaultSchedulerJobClient(jobPersistence, new DefaultJobCreator(jobPersistence))); final JobTracker jobTracker = new JobTracker(configRepository); - final TemporalClient temporalClient = TemporalClient.production(configs.getWorkspaceRoot()); - - ConfigurationApiFactory - .setSynchronousSchedulerClient(new SpecCachingSynchronousSchedulerClient(new DefaultSynchronousSchedulerClient(temporalClient, jobTracker))); + ConfigurationApiFactory.setSynchronousSchedulerClient( + new SpecCachingSynchronousSchedulerClient(new DefaultSynchronousSchedulerClient(TemporalClient.production(), jobTracker))); ConfigurationApiFactory.setConfigRepository(configRepository); ConfigurationApiFactory.setJobPersistence(jobPersistence); ConfigurationApiFactory.setConfigs(configs); diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 357dcbb38b2..b5c99bd9bdb 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -133,8 +133,7 @@ public class ConfigurationApi implements io.airbyte.api.V1Api { final FileTtlManager archiveTtlManager) { final SpecFetcher specFetcher = new SpecFetcher(synchronousSchedulerClient); final JsonSchemaValidator schemaValidator = new JsonSchemaValidator(); - schedulerHandler = - new SchedulerHandler(configRepository, schedulerJobClient, synchronousSchedulerClient, jobPersistence, configs.getWorkspaceRoot()); + schedulerHandler = new SchedulerHandler(configRepository, schedulerJobClient, synchronousSchedulerClient); final DockerImageValidator dockerImageValidator = new DockerImageValidator(synchronousSchedulerClient); sourceDefinitionsHandler = new SourceDefinitionsHandler(configRepository, dockerImageValidator, synchronousSchedulerClient); connectionsHandler = new ConnectionsHandler(configRepository); @@ -397,11 +396,6 @@ public class ConfigurationApi implements io.airbyte.api.V1Api { return execute(() -> schedulerHandler.discoverSchemaForSourceFromSourceCreate(sourceCreate)); } - @Override - public JobInfoRead cancelJob(@Valid JobIdRequestBody jobIdRequestBody) { - return execute(() -> schedulerHandler.cancelJob(jobIdRequestBody)); - } - // JOB HISTORY @Override diff --git a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java index cb6d649d91f..05bb0b84e54 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java +++ b/airbyte-server/src/main/java/io/airbyte/server/converters/JobConverter.java @@ -119,7 +119,7 @@ public class JobConverter { .createdAt(metadata.getCreatedAt()) .endedAt(metadata.getEndedAt()) .succeeded(metadata.isSucceeded()) - .logs(JobConverter.getLogRead(metadata.getLogPath())); + .logs(metadata.getLogPath().map(JobConverter::getLogRead).orElse(null)); } } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java index 79d661dbc1f..9d9455703a9 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/SchedulerHandler.java @@ -33,7 +33,6 @@ import io.airbyte.api.model.DestinationDefinitionIdRequestBody; import io.airbyte.api.model.DestinationDefinitionSpecificationRead; import io.airbyte.api.model.DestinationIdRequestBody; import io.airbyte.api.model.DestinationUpdate; -import io.airbyte.api.model.JobIdRequestBody; import io.airbyte.api.model.JobInfoRead; import io.airbyte.api.model.SourceCoreConfig; import io.airbyte.api.model.SourceDefinitionIdRequestBody; @@ -43,7 +42,6 @@ import io.airbyte.api.model.SourceIdRequestBody; import io.airbyte.api.model.SourceUpdate; import io.airbyte.commons.docker.DockerUtils; import io.airbyte.commons.enums.Enums; -import io.airbyte.commons.io.IOs; import io.airbyte.config.DestinationConnection; import io.airbyte.config.SourceConnection; import io.airbyte.config.StandardCheckConnectionOutput; @@ -58,20 +56,13 @@ import io.airbyte.scheduler.Job; import io.airbyte.scheduler.client.SchedulerJobClient; import io.airbyte.scheduler.client.SynchronousResponse; import io.airbyte.scheduler.client.SynchronousSchedulerClient; -import io.airbyte.scheduler.persistence.JobPersistence; import io.airbyte.server.converters.CatalogConverter; import io.airbyte.server.converters.ConfigurationUpdate; import io.airbyte.server.converters.JobConverter; import io.airbyte.server.converters.SpecFetcher; import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; -import io.airbyte.workers.WorkerUtils; -import io.airbyte.workers.temporal.TemporalAttemptExecution; -import io.airbyte.workers.temporal.TemporalUtils; -import io.temporal.api.common.v1.WorkflowExecution; -import io.temporal.api.workflowservice.v1.RequestCancelWorkflowExecutionRequest; import java.io.IOException; -import java.nio.file.Path; import java.util.UUID; public class SchedulerHandler { @@ -82,23 +73,17 @@ public class SchedulerHandler { private final SpecFetcher specFetcher; private final ConfigurationUpdate configurationUpdate; private final JsonSchemaValidator jsonSchemaValidator; - private final JobPersistence jobPersistence; - private final Path workspaceRoot; public SchedulerHandler(ConfigRepository configRepository, SchedulerJobClient schedulerJobClient, - SynchronousSchedulerClient synchronousSchedulerClient, - JobPersistence jobPersistence, - Path workspaceRoot) { + SynchronousSchedulerClient synchronousSchedulerClient) { this( configRepository, schedulerJobClient, synchronousSchedulerClient, new ConfigurationUpdate(configRepository, new SpecFetcher(synchronousSchedulerClient)), new JsonSchemaValidator(), - new SpecFetcher(synchronousSchedulerClient), - jobPersistence, - workspaceRoot); + new SpecFetcher(synchronousSchedulerClient)); } @VisibleForTesting @@ -107,17 +92,13 @@ public class SchedulerHandler { SynchronousSchedulerClient synchronousSchedulerClient, ConfigurationUpdate configurationUpdate, JsonSchemaValidator jsonSchemaValidator, - SpecFetcher specFetcher, - JobPersistence jobPersistence, - Path workspaceRoot) { + SpecFetcher specFetcher) { this.configRepository = configRepository; this.schedulerJobClient = schedulerJobClient; this.synchronousSchedulerClient = synchronousSchedulerClient; this.configurationUpdate = configurationUpdate; this.jsonSchemaValidator = jsonSchemaValidator; this.specFetcher = specFetcher; - this.jobPersistence = jobPersistence; - this.workspaceRoot = workspaceRoot; } public CheckConnectionRead checkSourceConnectionFromSourceId(SourceIdRequestBody sourceIdRequestBody) @@ -295,30 +276,6 @@ public class SchedulerHandler { return JobConverter.getJobInfoRead(job); } - public JobInfoRead cancelJob(JobIdRequestBody jobIdRequestBody) throws IOException { - final long jobId = jobIdRequestBody.getId(); - - // first prevent this job from being scheduled again - jobPersistence.cancelJob(jobId); - - // second cancel the temporal execution - // TODO: this is hacky, resolve https://github.com/airbytehq/airbyte/issues/2564 to avoid this - // behavior - final Path attemptParentDir = WorkerUtils.getJobRoot(workspaceRoot, String.valueOf(jobId), 0L).getParent(); - final String workflowId = IOs.readFile(attemptParentDir, TemporalAttemptExecution.WORKFLOW_ID_FILENAME); - final WorkflowExecution workflowExecution = WorkflowExecution.newBuilder() - .setWorkflowId(workflowId) - .build(); - final RequestCancelWorkflowExecutionRequest cancelRequest = RequestCancelWorkflowExecutionRequest.newBuilder() - .setWorkflowExecution(workflowExecution) - .setNamespace(TemporalUtils.DEFAULT_NAMESPACE) - .build(); - - TemporalUtils.TEMPORAL_SERVICE.blockingStub().requestCancelWorkflowExecution(cancelRequest); - - return JobConverter.getJobInfoRead(jobPersistence.getJob(jobId)); - } - private CheckConnectionRead reportConnectionStatus(final SynchronousResponse response) { final CheckConnectionRead checkConnectionRead = new CheckConnectionRead() .jobInfo(JobConverter.getSynchronousJobRead(response)); @@ -327,10 +284,6 @@ public class SchedulerHandler { checkConnectionRead .status(Enums.convertTo(response.getOutput().getStatus(), StatusEnum.class)) .message(response.getOutput().getMessage()); - } else { - checkConnectionRead - .status(StatusEnum.FAILED) - .message("Check Connection Failed!"); } return checkConnectionRead; diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java index ca5cb4baf59..8809ce8c448 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/SchedulerHandlerTest.java @@ -72,7 +72,6 @@ import io.airbyte.scheduler.client.SchedulerJobClient; import io.airbyte.scheduler.client.SynchronousJobMetadata; import io.airbyte.scheduler.client.SynchronousResponse; import io.airbyte.scheduler.client.SynchronousSchedulerClient; -import io.airbyte.scheduler.persistence.JobPersistence; import io.airbyte.server.converters.ConfigurationUpdate; import io.airbyte.server.converters.SpecFetcher; import io.airbyte.server.helpers.ConnectionHelpers; @@ -82,7 +81,6 @@ import io.airbyte.validation.json.JsonSchemaValidator; import io.airbyte.validation.json.JsonValidationException; import java.io.IOException; import java.net.URI; -import java.nio.file.Path; import java.util.HashMap; import java.util.Optional; import java.util.UUID; @@ -130,7 +128,6 @@ class SchedulerHandlerTest { private ConfigurationUpdate configurationUpdate; private JsonSchemaValidator jsonSchemaValidator; private SpecFetcher specFetcher; - private JobPersistence jobPersistence; @BeforeEach void setup() { @@ -146,10 +143,9 @@ class SchedulerHandlerTest { schedulerJobClient = spy(SchedulerJobClient.class); synchronousSchedulerClient = mock(SynchronousSchedulerClient.class); configRepository = mock(ConfigRepository.class); - jobPersistence = mock(JobPersistence.class); schedulerHandler = new SchedulerHandler(configRepository, schedulerJobClient, synchronousSchedulerClient, configurationUpdate, - jsonSchemaValidator, specFetcher, jobPersistence, mock(Path.class)); + jsonSchemaValidator, specFetcher); } @Test diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx deleted file mode 100644 index ba03f8c3650..00000000000 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx +++ /dev/null @@ -1,94 +0,0 @@ -import React from "react"; -import styled from "styled-components"; - -import { Button } from "components/Button"; - -import { EditorHeader } from "./components/EditorHeader"; -import { EditorRow } from "./components/EditorRow"; -import { FormattedMessage } from "react-intl"; - -const ItemsList = styled.div` - background: ${({ theme }) => theme.greyColor0}; - border-radius: 4px; -`; - -const ButtonContainer = styled.div` - display: flex; - justify-content: flex-end; -`; - -const SmallButton = styled(Button)` - margin-left: 8px; - padding: 6px 8px 7px; -`; - -type ArrayOfObjectsEditorProps = { - items: { name: string }[]; - children?: React.ReactNode; - onStartEdit: (n: number) => void; - onCancelEdit: () => void; - onDone: () => void; - onRemove: (index: number) => void; - isEditMode: boolean; -}; - -const ArrayOfObjectsEditor: React.FC = ({ - onStartEdit, - onDone, - onRemove, - onCancelEdit, - isEditMode, - items, - children, -}) => { - const onAddItem = React.useCallback(() => onStartEdit(items.length), [ - onStartEdit, - items, - ]); - const handleRemove = React.useCallback((idx: number) => onRemove(idx), [ - onRemove, - items, - ]); - const handleEdit = React.useCallback((idx: number) => onStartEdit(idx), [ - onStartEdit, - items, - ]); - - if (isEditMode) { - return ( - <> - {typeof children === "function" ? children() : children} - - - - - - - - - - ); - } - - return ( - <> - - {items.length ? ( - - {items.map((item, key) => ( - - ))} - - ) : null} - - ); -}; - -export { ArrayOfObjectsEditor }; -export type { ArrayOfObjectsEditorProps }; diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx deleted file mode 100644 index 60159c6d352..00000000000 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import React from "react"; -import styled from "styled-components"; -import { FormattedMessage } from "react-intl"; - -import { Button } from "components/Button"; - -const Content = styled.div` - display: flex; - justify-content: space-between; - align-items: center; - flex-direction: row; - color: ${({ theme }) => theme.textColor}; - font-weight: 500; - font-size: 14px; - line-height: 17px; - margin: 5px 0; -`; - -type EditorHeaderProps = { - itemsCount: number; - onAddItem: () => void; -}; - -const EditorHeader: React.FC = ({ - itemsCount, - onAddItem, -}) => { - return ( - - - - - ); -}; - -export { EditorHeader }; diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.tsx deleted file mode 100644 index 91823a32788..00000000000 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorRow.tsx +++ /dev/null @@ -1,61 +0,0 @@ -import React from "react"; -import styled from "styled-components"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import { faTimes } from "@fortawesome/free-solid-svg-icons"; -import { FormattedMessage } from "react-intl"; - -import { Button } from "components/Button"; - -const Content = styled.div` - display: flex; - justify-content: space-between; - align-items: center; - flex-direction: row; - color: ${({ theme }) => theme.textColor}; - font-weight: 500; - font-size: 14px; - line-height: 17px; - padding: 5px 12px 6px 14px; - border-bottom: 1px solid ${({ theme }) => theme.greyColor20}; - - &:last-child { - border: none; - } -`; - -const Delete = styled(FontAwesomeIcon)` - color: ${({ theme }) => theme.greyColor55}; - font-weight: 300; - font-size: 14px; - line-height: 24px; - margin-left: 7px; - cursor: pointer; -`; - -type EditorRowProps = { - name: string; - id: number; - onEdit: (id: number) => void; - onRemove: (id: number) => void; -}; - -const EditorRow: React.FC = ({ - name, - id, - onEdit, - onRemove, -}) => { - return ( - -
{name || id}
-
- - onRemove(id)} /> -
- - ); -}; - -export { EditorRow }; diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/index.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/index.tsx deleted file mode 100644 index b4a645f0160..00000000000 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/index.tsx +++ /dev/null @@ -1,4 +0,0 @@ -import { ArrayOfObjectsEditor } from "./ArrayOfObjectsEditor"; - -export default ArrayOfObjectsEditor; -export { ArrayOfObjectsEditor }; diff --git a/airbyte-webapp/src/components/Multiselect/Multiselect.tsx b/airbyte-webapp/src/components/Multiselect/Multiselect.tsx deleted file mode 100644 index 9f5bfed8783..00000000000 --- a/airbyte-webapp/src/components/Multiselect/Multiselect.tsx +++ /dev/null @@ -1,136 +0,0 @@ -import { Multiselect as ReactMultiselect } from "react-widgets"; -import styled from "styled-components"; -import { MultiselectProps as WidgetMultiselectProps } from "react-widgets/lib/Multiselect"; - -type MultiselectProps = { - disabled?: boolean; - error?: boolean; -} & WidgetMultiselectProps; - -const Multiselect = styled(ReactMultiselect)` - box-shadow: none; - padding: 0; - margin: 0; - - & .rw-list-option.rw-state-focus, - & .rw-list-option { - color: ${({ theme }) => theme.textColor}; - border: none; - padding: 10px 16px; - font-size: 14px; - line-height: 19px; - } - - & .rw-list-option:hover { - background: ${({ theme }) => theme.primaryColor12}; - color: ${({ theme }) => theme.primaryColor}; - } - - & .rw-list-option.rw-state-selected { - background: ${({ theme }) => theme.primaryColor12}; - color: ${({ theme }) => theme.primaryColor}; - pointer-events: none; - } - - & .rw-popup { - border: 0.5px solid ${({ theme }) => theme.greyColor20}; - border-radius: 4px; - box-shadow: 0 8px 10px 0 rgba(11, 10, 26, 0.04), - 0 3px 14px 0 rgba(11, 10, 26, 0.08), 0 5px 5px 0 rgba(11, 10, 26, 0.12); - } - - & .rw-popup-container { - & .rw-select { - display: none; - } - - & .rw-list-optgroup { - width: 100%; - padding: 0; - border: none; - } - } - - & .rw-widget-container { - box-shadow: none; - outline: none; - width: 100%; - min-height: 37px; - padding: 4px 8px; - border-radius: 4px; - font-size: 14px; - line-height: 20px; - font-weight: normal; - border: 1px solid - ${(props) => - props.error ? props.theme.dangerColor : props.theme.greyColor0}; - background: ${(props) => - props.error ? props.theme.greyColor10 : props.theme.greyColor0}; - caret-color: ${({ theme }) => theme.primaryColor}; - - & > div { - vertical-align: middle; - } - - & .rw-btn-select { - color: ${({ theme }) => theme.primaryColor}; - } - - & input { - padding: 0; - height: auto; - line-height: 26px; - } - - &::placeholder { - color: ${({ theme }) => theme.greyColor40}; - } - - &:hover { - box-shadow: none; - background: ${({ theme }) => theme.greyColor20}; - border-color: ${(props) => - props.error ? props.theme.dangerColor : props.theme.greyColor20}; - } - - & .rw-multiselect-taglist { - vertical-align: top; - - & .rw-multiselect-tag { - background: ${({ theme }) => theme.mediumPrimaryColor}; - border-color: ${({ theme }) => theme.mediumPrimaryColor}; - color: ${({ theme }) => theme.whiteColor}; - border-radius: 4px; - font-weight: 500; - font-size: 12px; - line-height: 21px; - height: 23px; - margin: 0 3px 0 0; - padding: 0 4px 0 6px; - - & .rw-multiselect-tag-btn { - color: ${({ theme }) => theme.greyColor55}; - font-size: 20px; - line-height: 23px; - } - } - } - } - - &.rw-state-focus { - & .rw-widget-container { - background: ${({ theme }) => theme.primaryColor12}; - border-color: ${({ theme }) => theme.primaryColor}; - } - } - - &.rw-state-disabled { - & .rw-widget-container { - pointer-events: none; - color: ${({ theme }) => theme.greyColor55}; - } - } -`; - -export { Multiselect }; -export type { MultiselectProps }; diff --git a/airbyte-webapp/src/components/Multiselect/index.tsx b/airbyte-webapp/src/components/Multiselect/index.tsx deleted file mode 100644 index 4182b5f1b86..00000000000 --- a/airbyte-webapp/src/components/Multiselect/index.tsx +++ /dev/null @@ -1 +0,0 @@ -export * from "./Multiselect"; diff --git a/airbyte-webapp/src/components/ServiceForm/FormRoot.tsx b/airbyte-webapp/src/components/ServiceForm/FormRoot.tsx index 8cca8910aaf..ce27c242e20 100644 --- a/airbyte-webapp/src/components/ServiceForm/FormRoot.tsx +++ b/airbyte-webapp/src/components/ServiceForm/FormRoot.tsx @@ -30,7 +30,7 @@ const FormRoot: React.FC<{ additionBottomControls?: React.ReactNode; errorMessage?: React.ReactNode; successMessage?: React.ReactNode; - formFields: FormBlock; + formFields: FormBlock[]; connector?: string; onRetest?: () => void; }> = ({ diff --git a/airbyte-webapp/src/components/ServiceForm/ServiceForm.tsx b/airbyte-webapp/src/components/ServiceForm/ServiceForm.tsx index ec014bedfd4..f85e691f429 100644 --- a/airbyte-webapp/src/components/ServiceForm/ServiceForm.tsx +++ b/airbyte-webapp/src/components/ServiceForm/ServiceForm.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useMemo } from "react"; +import React, { useCallback } from "react"; import { Formik } from "formik"; import { JSONSchema7 } from "json-schema"; @@ -16,19 +16,19 @@ import { FormRoot } from "./FormRoot"; type ServiceFormProps = { formType: "source" | "destination"; + dropDownData: Array; onSubmit: (values: ServiceFormValues) => void; onRetest?: (values: ServiceFormValues) => void; specifications?: JSONSchema7; isLoading?: boolean; isEditMode?: boolean; + onDropDownSelect?: (id: string) => void; allowChangeConnector?: boolean; formValues?: Partial; hasSuccess?: boolean; additionBottomControls?: React.ReactNode; errorMessage?: React.ReactNode; successMessage?: React.ReactNode; - dropDownData: Array; - onDropDownSelect?: (id: string) => void; documentationUrl?: string; }; @@ -39,25 +39,12 @@ const FormikPatch: React.FC = () => { const ServiceForm: React.FC = (props) => { const { specifications, formValues, onSubmit, isLoading, onRetest } = props; - const jsonSchema: JSONSchema7 = useMemo( - () => ({ - type: "object", - properties: { - name: { type: "string" }, - serviceType: { type: "string" }, - ...Object.fromEntries( - Object.entries({ - connectionConfiguration: isLoading ? null : specifications, - }).filter(([, v]) => !!v) - ), - }, - required: ["name", "serviceType"], - }), - [isLoading, specifications] + const { formFields, initialValues } = useBuildForm( + isLoading, + formValues, + specifications ); - const { formFields, initialValues } = useBuildForm(jsonSchema, formValues); - const { uiWidgetsInfo, setUiWidgetsInfo } = useBuildUiWidgets( formFields, initialValues @@ -65,7 +52,7 @@ const ServiceForm: React.FC = (props) => { const validationSchema = useConstructValidationSchema( uiWidgetsInfo, - jsonSchema + specifications ); const onFormSubmit = useCallback( diff --git a/airbyte-webapp/src/components/ServiceForm/components/Controls/ConnectorNameControl.tsx b/airbyte-webapp/src/components/ServiceForm/components/Controls/ConnectorNameControl.tsx deleted file mode 100644 index 32e6724b1d2..00000000000 --- a/airbyte-webapp/src/components/ServiceForm/components/Controls/ConnectorNameControl.tsx +++ /dev/null @@ -1,36 +0,0 @@ -import React from "react"; -import { FormattedMessage, useIntl } from "react-intl"; -import { useField } from "formik"; - -import { Input, ControlLabels } from "components"; -import { FormBaseItem } from "core/form/types"; -import { useServiceForm } from "../../serviceFormContext"; - -const ConnectorNameControl: React.FC<{ property: FormBaseItem }> = ({ - property, -}) => { - const formatMessage = useIntl().formatMessage; - const [field, fieldMeta] = useField(property.path); - const { formType } = useServiceForm(); - - return ( - } - message={formatMessage({ - id: `form.${formType}Name.message`, - })} - > - - - ); -}; - -export { ConnectorNameControl }; diff --git a/airbyte-webapp/src/components/ServiceForm/components/Controls/ConnectorServiceTypeControl.tsx b/airbyte-webapp/src/components/ServiceForm/components/Controls/ConnectorServiceTypeControl.tsx deleted file mode 100644 index 3df6f795728..00000000000 --- a/airbyte-webapp/src/components/ServiceForm/components/Controls/ConnectorServiceTypeControl.tsx +++ /dev/null @@ -1,77 +0,0 @@ -import React, { useCallback, useMemo } from "react"; -import { useIntl } from "react-intl"; -import { useField } from "formik"; -import styled from "styled-components"; -import { DropDown, DropDownRow, ControlLabels } from "components"; - -import { FormBaseItem } from "core/form/types"; -import { useServiceForm } from "../../serviceFormContext"; - -const DropdownLabels = styled(ControlLabels)` - max-width: 202px; -`; - -const ConnectorServiceTypeControl: React.FC<{ property: FormBaseItem }> = ({ - property, -}) => { - const formatMessage = useIntl().formatMessage; - const [field, fieldMeta, { setValue }] = useField(property.path); - - const { - formType, - isEditMode, - allowChangeConnector, - onChangeServiceType, - dropDownData, - } = useServiceForm(); - - const sortedDropDownData = useMemo( - () => dropDownData.sort(DropDownRow.defaultDataItemSort), - [dropDownData] - ); - - const handleSelect = useCallback( - (item: DropDownRow.IDataItem) => { - setValue(item.value); - if (onChangeServiceType) { - onChangeServiceType(item.value); - } - }, - [setValue, onChangeServiceType] - ); - - return ( - <> - - - - {/*TODO: figure out when we want to include instruction*/} - {/*{field.value && includeInstruction && (*/} - {/* */} - {/*)}*/} - - ); -}; - -export { ConnectorServiceTypeControl }; diff --git a/airbyte-webapp/src/components/ServiceForm/components/EditControls.tsx b/airbyte-webapp/src/components/ServiceForm/components/EditControls.tsx index c105cf5ca67..e6875fcb079 100644 --- a/airbyte-webapp/src/components/ServiceForm/components/EditControls.tsx +++ b/airbyte-webapp/src/components/ServiceForm/components/EditControls.tsx @@ -2,7 +2,7 @@ import React from "react"; import styled from "styled-components"; import { FormattedMessage } from "react-intl"; -import { Button } from "components"; +import Button from "components/Button"; import { useServiceForm } from "../serviceFormContext"; import TestingConnectionSpinner from "./TestingConnectionSpinner"; import TestingConnectionSuccess from "./TestingConnectionSuccess"; @@ -41,7 +41,7 @@ const EditControls: React.FC = ({ successMessage, errorMessage, }) => { - const { unfinishedFlows } = useServiceForm(); + const { unfinishedSecrets } = useServiceForm(); if (isSubmitting) { return ; @@ -67,7 +67,7 @@ const EditControls: React.FC = ({ isSubmitting || !isValid || !dirty || - Object.keys(unfinishedFlows).length > 0 + Object.keys(unfinishedSecrets).length > 0 } > diff --git a/airbyte-webapp/src/components/ServiceForm/components/FormSection.tsx b/airbyte-webapp/src/components/ServiceForm/components/FormSection.tsx index d8d1b956a53..dc41f4c5d18 100644 --- a/airbyte-webapp/src/components/ServiceForm/components/FormSection.tsx +++ b/airbyte-webapp/src/components/ServiceForm/components/FormSection.tsx @@ -1,177 +1,94 @@ import React from "react"; import styled from "styled-components"; -import { DropDown, Label, ArrayOfObjectsEditor } from "components"; -import { - FormBlock, - FormConditionItem, - FormObjectArrayItem, -} from "core/form/types"; -import { PropertySection } from "./PropertySection"; +import { DropDown, Label } from "components"; +import { FormBlock, FormConditionItem } from "core/form/types"; +import { Property } from "./Property"; import { useServiceForm } from "../serviceFormContext"; -import GroupControls from "./Property/GroupControls"; -import { FieldArray, useField } from "formik"; -const SectionContainer = styled.div` +const ItemSection = styled.div` margin-bottom: 27px; `; +const FormItemGroupDropDown = styled(ItemSection)` + margin-top: -17px; + background: ${({ theme }) => theme.whiteColor}; + padding: 0 5px; + display: inline-block; + vertical-align: middle; + + & > div { + min-width: 180px; + display: inline-block; + } +`; + +const FormItemGroup = styled(ItemSection)` + border: 2px solid ${({ theme }) => theme.greyColor20}; + box-sizing: border-box; + border-radius: 8px; + padding: 0 20px; + margin-top: 41px; +`; + const GroupLabel = styled(Label)` width: auto; margin-right: 8px; display: inline-block; `; -const ConditionControls = styled.div` - padding-top: 25px; -`; - -const ConditionSection: React.FC<{ - formField: FormConditionItem; - path?: string; -}> = ({ formField }) => { +const ConditionSection: React.FC<{ formField: FormConditionItem }> = ({ + formField, +}) => { const { widgetsInfo, setUiWidgetsInfo } = useServiceForm(); - const currentlySelectedCondition = widgetsInfo[formField.path]?.selectedItem; + const currentlySelectedCondition = + widgetsInfo[formField.fieldName]?.selectedItem; const label = formField.title || formField.fieldKey; return ( - - {label ? {label}: : null} - ({ - text: dataItem, - value: dataItem, - }))} - onSelect={(selectedItem) => - setUiWidgetsInfo(formField.path, { - selectedItem: selectedItem.value, - }) - } - value={currentlySelectedCondition} - /> - - } - > - - + + {label ? {label}: : null} + ({ + text: dataItem, + value: dataItem, + }))} + onSelect={(selectedItem) => + setUiWidgetsInfo(formField.fieldName, { + selectedItem: selectedItem.value, + }) + } + value={currentlySelectedCondition} /> - - + + + ); }; -const ArraySection: React.FC<{ - formField: FormObjectArrayItem; - path: string; -}> = ({ formField, path }) => { - const { - addUnfinishedFlow, - removeUnfinishedFlow, - unfinishedFlows, - } = useServiceForm(); - const [field, , form] = useField(path); +const FormSection: React.FC<{ blocks: FormBlock[] }> = (props) => { + const { blocks } = props; - const items = field.value ?? []; - const flow = unfinishedFlows[path]; - - return ( - - - ( - - addUnfinishedFlow(path, { - id: index, - startValue: index < items.length ? items[index] : null, - }) - } - onDone={() => removeUnfinishedFlow(path)} - onCancelEdit={() => { - removeUnfinishedFlow(path); - - if (flow.startValue) { - form.setValue(flow.startValue); - } - }} - onRemove={arrayHelpers.remove} - items={items} - > - {() => ( - - )} - - )} - /> - - - ); -}; - -const FormSection: React.FC<{ - blocks: FormBlock[] | FormBlock; - path?: string; - skipAppend?: boolean; -}> = ({ blocks, path, skipAppend }) => { - const sections = Array.isArray(blocks) ? blocks : [blocks]; return ( <> - {sections.map((formField) => { - const sectionPath = path - ? skipAppend - ? path - : `${path}.${formField.fieldKey}` - : formField.fieldKey; - + {blocks.map((formField) => { if (formField._type === "formGroup") { - return ( - - ); + return ; } if (formField._type === "formCondition") { - return ( - - ); - } - - if (formField._type === "objectArray") { - return ( - - ); + return ; } return ( - - - + + + ); })} diff --git a/airbyte-webapp/src/components/ServiceForm/components/Controls/Instruction.tsx b/airbyte-webapp/src/components/ServiceForm/components/Instruction.tsx similarity index 100% rename from airbyte-webapp/src/components/ServiceForm/components/Controls/Instruction.tsx rename to airbyte-webapp/src/components/ServiceForm/components/Instruction.tsx diff --git a/airbyte-webapp/src/components/ServiceForm/components/Property.tsx b/airbyte-webapp/src/components/ServiceForm/components/Property.tsx new file mode 100644 index 00000000000..fc13deb8195 --- /dev/null +++ b/airbyte-webapp/src/components/ServiceForm/components/Property.tsx @@ -0,0 +1,139 @@ +import React, { useMemo } from "react"; + +import { useField } from "formik"; +import { FormattedMessage, useIntl } from "react-intl"; +import styled from "styled-components"; + +import { + ControlLabels, + DropDown, + DropDownRow, + Input, + LabeledToggle, + TextWithHTML, +} from "components"; +import { FormBaseItem } from "core/form/types"; +import Instruction from "./Instruction"; +import { Label } from "./Property/Label"; +import { Control } from "./Property/Control"; +import { useServiceForm } from "../serviceFormContext"; + +const DropdownLabels = styled(ControlLabels)` + max-width: 202px; +`; + +const ServiceTypeProperty: React.FC<{ property: FormBaseItem }> = ({ + property, +}) => { + const formatMessage = useIntl().formatMessage; + const { fieldName, meta } = property; + const [field, fieldMeta, form] = useField(fieldName); + + const { + formType, + isEditMode, + allowChangeConnector, + onChangeServiceType, + dropDownData, + documentationUrl, + } = useServiceForm(); + + const sortedDropDownData = useMemo( + () => dropDownData.sort(DropDownRow.defaultDataItemSort), + [dropDownData] + ); + + return ( + <> + + { + form.setValue(item.value); + if (onChangeServiceType) { + onChangeServiceType(item.value); + } + }} + /> + + {field.value && meta?.includeInstruction && ( + + )} + + ); +}; + +const Property: React.FC<{ property: FormBaseItem }> = ({ property }) => { + const formatMessage = useIntl().formatMessage; + const { fieldName, fieldKey } = property; + const [field, fieldMeta] = useField(fieldName); + const { + addUnfinishedSecret, + removeUnfinishedSecret, + unfinishedSecrets, + formType, + } = useServiceForm(); + + if (fieldKey === "name") { + return ( + } + message={formatMessage({ + id: `form.${formType}Name.message`, + })} + > + + + ); + } else if (fieldKey === "serviceType") { + return ; + } else if (property.type === "boolean") { + return ( + } + value={field.value ?? property.default} + /> + ); + } + + return ( + + ); +}; + +export { Property }; diff --git a/airbyte-webapp/src/components/ServiceForm/components/Property/Control.tsx b/airbyte-webapp/src/components/ServiceForm/components/Property/Control.tsx index b5906bf7deb..0f58b40f9ab 100644 --- a/airbyte-webapp/src/components/ServiceForm/components/Property/Control.tsx +++ b/airbyte-webapp/src/components/ServiceForm/components/Property/Control.tsx @@ -1,28 +1,27 @@ import React from "react"; import { useIntl } from "react-intl"; -import { FieldArray, useField } from "formik"; +import { useField } from "formik"; -import { DropDown, Input, Multiselect, TextArea, TagInput } from "components"; +import { DropDown, Input, TextArea } from "components"; import ConfirmationControl from "./ConfirmationControl"; import { FormBaseItem } from "core/form/types"; type IProps = { property: FormBaseItem; - name: string; - unfinishedFlows: Record; - addUnfinishedFlow: (key: string, info?: Record) => void; - removeUnfinishedFlow: (key: string) => void; + unfinishedSecrets: Record; + addUnfinishedSecret: (key: string, info?: Record) => void; + removeUnfinishedSecret: (key: string) => void; }; const Control: React.FC = ({ property, - name, - addUnfinishedFlow, - removeUnfinishedFlow, - unfinishedFlows, + addUnfinishedSecret, + removeUnfinishedSecret, + unfinishedSecrets, }) => { const formatMessage = useIntl().formatMessage; - const [field, meta, form] = useField(name); + const { fieldName } = property; + const [field, meta, form] = useField(fieldName); // TODO: think what to do with other cases let placeholder: string | undefined; @@ -41,42 +40,8 @@ const Control: React.FC = ({ break; } - if (property.type === "array" && !property.enum) { - return ( - ( - ({ - id, - value, - }))} - onEnter={(newItem) => arrayHelpers.push(newItem)} - onDelete={(item) => arrayHelpers.remove(Number.parseInt(item))} - addOnBlur - error={!!meta.error} - /> - )} - /> - ); - } - - if (property.type === "array" && property.enum) { - const data = - property.enum?.length && typeof property.enum[0] !== "object" - ? (property.enum as string[] | number[]) - : undefined; - return ( - form.setValue(dataItems)} - value={field.value} - /> - ); - } - const value = field.value ?? property.default; + if (property.enum) { return ( = ({ /> ); } else if (property.isSecret) { - const unfinishedSecret = unfinishedFlows[name]; + const unfinishedSecret = unfinishedSecrets[fieldName]; const isEditInProgress = !!unfinishedSecret; const isFormInEditMode = !!meta.initialValue; return ( @@ -130,13 +95,13 @@ const Control: React.FC = ({ } showButtons={isFormInEditMode} isEditInProgress={isEditInProgress} - onDone={() => removeUnfinishedFlow(name)} + onDone={() => removeUnfinishedSecret(fieldName)} onStart={() => { - addUnfinishedFlow(name, { startValue: field.value }); + addUnfinishedSecret(fieldName, { startValue: field.value }); form.setValue(""); }} onCancel={() => { - removeUnfinishedFlow(name); + removeUnfinishedSecret(fieldName); if ( unfinishedSecret && unfinishedSecret.hasOwnProperty("startValue") @@ -147,7 +112,7 @@ const Control: React.FC = ({ /> ); } else { - const inputType = property.type === "integer" ? "number" : "text"; + const inputType: string = property.type === "integer" ? "number" : "text"; return ( theme.whiteColor}; - padding: 0 5px; - display: inline-block; - vertical-align: middle; - - & > div { - min-width: 180px; - display: inline-block; - } -`; - -const FormGroup = styled.div` - margin: 41px 0 27px; - border: 2px solid ${({ theme }) => theme.greyColor20}; - box-sizing: border-box; - border-radius: 8px; - padding: 0 20px; -`; - -type GroupControlsProps = { - title: React.ReactNode; -}; - -const GroupControls: React.FC = ({ title, children }) => { - return ( - - {title} - {children} - - ); -}; - -export default GroupControls; diff --git a/airbyte-webapp/src/components/ServiceForm/components/Property/Label.tsx b/airbyte-webapp/src/components/ServiceForm/components/Property/Label.tsx index 77fdb84c9a1..c0e50942993 100644 --- a/airbyte-webapp/src/components/ServiceForm/components/Property/Label.tsx +++ b/airbyte-webapp/src/components/ServiceForm/components/Property/Label.tsx @@ -1,4 +1,6 @@ import React from "react"; +import { useField } from "formik"; + import { ControlLabels } from "components/LabeledControl"; import { FormBaseItem } from "core/form/types"; @@ -6,30 +8,23 @@ import { LabelMessage } from "./LabelMessage"; type LabelMessageProps = { property: FormBaseItem; - error: string | undefined; - touched: boolean; }; -const Label: React.FC = ({ - property, - error, - touched, - children, -}) => { +const Label: React.FC = ({ property, children }) => { + const [, meta] = useField(property.fieldName); + const labelText = property.title || property.fieldKey; const labelRequiredAppendix = property.isRequired ? " *" : ""; const label = `${labelText}${labelRequiredAppendix}`; - const displayError = !!error && touched; + const displayError = !!meta.error && meta.touched; return ( - } + message={} > {children} diff --git a/airbyte-webapp/src/components/ServiceForm/components/Property/LabelMessage.tsx b/airbyte-webapp/src/components/ServiceForm/components/Property/LabelMessage.tsx index e53b125da36..e71397b10ad 100644 --- a/airbyte-webapp/src/components/ServiceForm/components/Property/LabelMessage.tsx +++ b/airbyte-webapp/src/components/ServiceForm/components/Property/LabelMessage.tsx @@ -1,16 +1,17 @@ import React from "react"; +import { useField } from "formik"; import { FormattedMessage } from "react-intl"; -import { TextWithHTML } from "components"; +import TextWithHTML from "components/TextWithHTML"; import { FormBaseItem } from "core/form/types"; type IProps = { property: FormBaseItem; - error: string | undefined; - touched: boolean; }; -const LabelMessage: React.FC = ({ property, error, touched }) => { +const LabelMessage: React.FC = ({ property }) => { + const [, meta] = useField(property.fieldName); + const constructExamples = () => { if (!property.examples) { return null; @@ -25,11 +26,14 @@ const LabelMessage: React.FC = ({ property, error, touched }) => { ); }; - const displayError = !!error && touched; + const displayError = !!meta.error && meta.touched; const errorMessage = - displayError && error === "form.pattern.error" ? ( - + displayError && meta.error === "form.pattern.error" ? ( + ) : null; const message = property.description ? ( diff --git a/airbyte-webapp/src/components/ServiceForm/components/PropertySection.tsx b/airbyte-webapp/src/components/ServiceForm/components/PropertySection.tsx deleted file mode 100644 index 7e2a37b57a0..00000000000 --- a/airbyte-webapp/src/components/ServiceForm/components/PropertySection.tsx +++ /dev/null @@ -1,54 +0,0 @@ -import React from "react"; - -import { useField } from "formik"; - -import { LabeledToggle, TextWithHTML } from "components"; -import { FormBaseItem } from "core/form/types"; -import { Label } from "./Property/Label"; -import { Control } from "./Property/Control"; -import { useServiceForm } from "../serviceFormContext"; - -const PropertySection: React.FC<{ property: FormBaseItem; path?: string }> = ({ - property, - path, -}) => { - const propertyPath = path ?? property.path; - const formikBag = useField(propertyPath); - const [field, meta] = formikBag; - const { - addUnfinishedFlow, - removeUnfinishedFlow, - unfinishedFlows, - widgetsInfo, - } = useServiceForm(); - - const overriddenComponent = widgetsInfo[propertyPath]?.component; - if (widgetsInfo[propertyPath]?.component) { - return <>{overriddenComponent(property)}; - } - - if (property.type === "boolean") { - return ( - } - value={field.value ?? property.default} - /> - ); - } - - return ( - - ); -}; - -export { PropertySection }; diff --git a/airbyte-webapp/src/components/ServiceForm/components/TestingConnectionSuccess.tsx b/airbyte-webapp/src/components/ServiceForm/components/TestingConnectionSuccess.tsx index 96578c179a9..37cb1b0aab9 100644 --- a/airbyte-webapp/src/components/ServiceForm/components/TestingConnectionSuccess.tsx +++ b/airbyte-webapp/src/components/ServiceForm/components/TestingConnectionSuccess.tsx @@ -25,7 +25,7 @@ const Success = styled(StatusIcon)` const TestingConnectionSuccess: React.FC = () => { return ( - + diff --git a/airbyte-webapp/src/components/ServiceForm/serviceFormContext.tsx b/airbyte-webapp/src/components/ServiceForm/serviceFormContext.tsx index 7ebf869560f..d6c89d84e23 100644 --- a/airbyte-webapp/src/components/ServiceForm/serviceFormContext.tsx +++ b/airbyte-webapp/src/components/ServiceForm/serviceFormContext.tsx @@ -13,9 +13,9 @@ type Context = { documentationUrl?: string; widgetsInfo: WidgetConfigMap; setUiWidgetsInfo: (path: string, value: Record) => void; - unfinishedFlows: Record; - addUnfinishedFlow: (key: string, info?: Record) => void; - removeUnfinishedFlow: (key: string) => void; + unfinishedSecrets: Record; + addUnfinishedSecret: (key: string, info?: Record) => void; + removeUnfinishedSecret: (key: string) => void; resetUiFormProgress: () => void; }; @@ -23,10 +23,10 @@ const context: Context = { formType: "source", dropDownData: [], widgetsInfo: {}, + unfinishedSecrets: {}, setUiWidgetsInfo: (_path: string, _value: Record) => ({}), - unfinishedFlows: {}, - addUnfinishedFlow: (_key: string, _info?: Record) => ({}), - removeUnfinishedFlow: (_key: string) => ({}), + addUnfinishedSecret: (_key: string, _info?: Record) => ({}), + removeUnfinishedSecret: (_key: string) => ({}), resetUiFormProgress: () => ({}), }; @@ -46,7 +46,7 @@ const ServiceFormContextProvider: React.FC<{ setUiWidgetsInfo: (path: string, value: Record) => void; }> = ({ children, widgetsInfo, setUiWidgetsInfo, ...props }) => { const ctx = useMemo(() => { - const unfinishedFlows = widgetsInfo["_common.unfinishedFlows"] ?? {}; + const unfinishedSecrets = widgetsInfo["_common.unfinishedSecrets"] ?? {}; return { widgetsInfo, setUiWidgetsInfo, @@ -56,21 +56,21 @@ const ServiceFormContextProvider: React.FC<{ onChangeServiceType: props.onChangeServiceType, dropDownData: props.dropDownData, documentationUrl: props.documentationUrl, - unfinishedFlows: unfinishedFlows, - addUnfinishedFlow: (path, info) => - setUiWidgetsInfo("_common.unfinishedFlows", { - ...unfinishedFlows, + unfinishedSecrets, + addUnfinishedSecret: (path, info) => + setUiWidgetsInfo("_common.unfinishedSecrets", { + ...unfinishedSecrets, [path]: info ?? {}, }), - removeUnfinishedFlow: (path: string) => + removeUnfinishedSecret: (path: string) => setUiWidgetsInfo( - "_common.unfinishedFlows", + "_common.unfinishedSecrets", Object.fromEntries( - Object.entries(unfinishedFlows).filter(([key]) => key !== path) + Object.entries(unfinishedSecrets).filter(([key]) => key !== path) ) ), resetUiFormProgress: () => { - setUiWidgetsInfo("_common.unfinishedFlows", {}); + setUiWidgetsInfo("_common.unfinishedSecrets", {}); }, }; }, [props, widgetsInfo, setUiWidgetsInfo]); diff --git a/airbyte-webapp/src/components/ServiceForm/types.ts b/airbyte-webapp/src/components/ServiceForm/types.ts index c2e7761a3b0..8068abc0060 100644 --- a/airbyte-webapp/src/components/ServiceForm/types.ts +++ b/airbyte-webapp/src/components/ServiceForm/types.ts @@ -3,7 +3,8 @@ import { ConnectionConfiguration } from "core/domain/connection"; type ServiceFormValues = { name: string; serviceType: string; - connectionConfiguration: ConnectionConfiguration; + frequency?: string; + connectionConfiguration?: ConnectionConfiguration; }; export type { ServiceFormValues }; diff --git a/airbyte-webapp/src/components/ServiceForm/useBuildForm.tsx b/airbyte-webapp/src/components/ServiceForm/useBuildForm.tsx index 7d8be91531c..930790ddfc5 100644 --- a/airbyte-webapp/src/components/ServiceForm/useBuildForm.tsx +++ b/airbyte-webapp/src/components/ServiceForm/useBuildForm.tsx @@ -5,40 +5,51 @@ import { JSONSchema7 } from "json-schema"; import flatten from "flat"; import merge from "lodash.merge"; -import { - FormBaseItem, - FormBlock, - WidgetConfig, - WidgetConfigMap, -} from "core/form/types"; +import { FormBlock, WidgetConfig, WidgetConfigMap } from "core/form/types"; import { jsonSchemaToUiWidget } from "core/jsonSchema/schemaToUiWidget"; import { buildYupFormForJsonSchema } from "core/jsonSchema/schemaToYup"; import { buildPathInitialState } from "core/form/uiWidget"; import { ServiceFormValues } from "./types"; -import { ConnectorNameControl } from "./components/Controls/ConnectorNameControl"; -import { ConnectorServiceTypeControl } from "./components/Controls/ConnectorServiceTypeControl"; function useBuildForm( - jsonSchema: JSONSchema7, - initialValues?: Partial + isLoadingConnectionConfiguration?: boolean, + initialValues?: Partial, + jsonSchema?: JSONSchema7 ): { initialValues: ServiceFormValues; - formFields: FormBlock; + formFields: FormBlock[]; } { - const startValues = useMemo( - () => ({ + const startValues = useMemo(() => { + const values: ServiceFormValues = { name: "", serviceType: "", - connectionConfiguration: {}, ...initialValues, - }), - [initialValues] - ); + }; - const formFields = useMemo( - () => jsonSchemaToUiWidget(jsonSchema), - [jsonSchema] - ); + return values; + }, [initialValues]); + + const formFields = useMemo(() => { + return [ + { + _type: "formItem", + type: "string", + fieldKey: "name", + fieldName: "name", + isRequired: true, + }, + { + _type: "formItem", + type: "string", + fieldKey: "serviceType", + fieldName: "serviceType", + isRequired: true, + }, + ...(jsonSchema && !isLoadingConnectionConfiguration + ? [jsonSchemaToUiWidget(jsonSchema, "connectionConfiguration")] + : []), + ]; + }, [jsonSchema, isLoadingConnectionConfiguration]); return { initialValues: startValues, @@ -47,37 +58,21 @@ function useBuildForm( } const useBuildUiWidgets = ( - formFields: FormBlock[] | FormBlock, + formFields: FormBlock[], formValues: ServiceFormValues ): { uiWidgetsInfo: WidgetConfigMap; setUiWidgetsInfo: (widgetId: string, updatedValues: WidgetConfig) => void; } => { - const uiOverrides = { - name: { - component: (property: FormBaseItem) => ( - - ), - }, - serviceType: { - component: (property: FormBaseItem) => ( - - ), - }, - }; - const [overriddenWidgetState, setUiWidgetsInfo] = useState( - uiOverrides + {} ); // As schema is dynamic, it is possible, that new updated values, will differ from one stored. const mergedState = useMemo( () => merge( - buildPathInitialState( - Array.isArray(formFields) ? formFields : [formFields], - formValues - ), + buildPathInitialState(formFields, formValues), overriddenWidgetState ), [formFields, formValues, overriddenWidgetState] @@ -98,12 +93,30 @@ const useBuildUiWidgets = ( // As validation schema depends on what path of oneOf is currently selected in jsonschema const useConstructValidationSchema = ( uiWidgetsInfo: WidgetConfigMap, - jsonSchema: JSONSchema7 -): yup.Schema => - useMemo(() => buildYupFormForJsonSchema(jsonSchema, uiWidgetsInfo), [ - uiWidgetsInfo, - jsonSchema, - ]); + jsonSchema?: JSONSchema7 + // eslint-disable-next-line @typescript-eslint/no-explicit-any +): yup.ObjectSchema => { + return useMemo(() => { + let validationShape: yup.ObjectSchema = yup.object().shape({ + name: yup.string().required("form.empty.error"), + serviceType: yup.string().required("form.empty.error"), + }); + + // We have additional fields. Lets build schema for them + if (jsonSchema?.properties) { + validationShape = validationShape.shape({ + connectionConfiguration: buildYupFormForJsonSchema( + jsonSchema, + uiWidgetsInfo, + undefined, + "connectionConfiguration" + ), + }); + } + + return validationShape; + }, [uiWidgetsInfo, jsonSchema]); +}; const usePatchFormik = (): void => { const { diff --git a/airbyte-webapp/src/components/base/TagInput/TagInput.tsx b/airbyte-webapp/src/components/base/TagInput/TagInput.tsx deleted file mode 100644 index f0067e97cbe..00000000000 --- a/airbyte-webapp/src/components/base/TagInput/TagInput.tsx +++ /dev/null @@ -1,173 +0,0 @@ -import React, { useRef, useState } from "react"; -import styled from "styled-components"; - -import TagItem, { IItemProps } from "./TagItem"; - -const MainContainer = styled.div<{ error?: boolean }>` - width: 100%; - min-height: 36px; - border-radius: 4px; - padding: 6px 6px 0; - cursor: text; - max-height: 100%; - overflow: auto; - display: flex; - flex-direction: row; - flex-wrap: wrap; - align-self: stretch; - border: 1px solid - ${(props) => - props.error ? props.theme.dangerColor : props.theme.greyColor0}; - background: ${(props) => - props.error ? props.theme.greyColor10 : props.theme.greyColor0}; - caret-color: ${({ theme }) => theme.primaryColor}; - - &:hover { - background: ${({ theme }) => theme.greyColor20}; - border-color: ${(props) => - props.error ? props.theme.dangerColor : props.theme.greyColor20}; - } - - &:focus, - &:focus-within { - background: ${({ theme }) => theme.primaryColor12}; - border-color: ${({ theme }) => theme.primaryColor}; - } -`; - -const InputElement = styled.input` - margin-bottom: 4px; - border: none; - outline: none; - font-size: 14px; - line-height: 20px; - font-weight: normal; - color: ${({ theme }) => theme.textColor}; - flex: 1 1 auto; - background: rgba(0, 0, 0, 0); - - &::placeholder { - color: ${({ theme }) => theme.greyColor40}; - } -`; - -type TagInputProps = { - inputProps?: React.InputHTMLAttributes; - value: IItemProps[]; - className?: string; - validationRegex?: RegExp; - error?: boolean; - addOnBlur?: boolean; - disabled?: boolean; - - onEnter: (value?: string | number | readonly string[]) => void; - onDelete: (value: string) => void; - onError?: () => void; -}; - -const TagInput: React.FC = ({ - inputProps, - onEnter, - value, - className, - onDelete, - validationRegex, - error, - disabled, - onError, - addOnBlur, -}) => { - const inputElement = useRef(null); - const [selectedElementId, setSelectedElementId] = useState(""); - const [currentInputValue, setCurrentInputValue] = useState(""); - - const handleContainerBlur = () => setSelectedElementId(""); - const handleContainerClick = () => { - if (inputElement.current !== null) { - inputElement.current.focus(); - } - }; - - const onAddValue = () => { - if (!inputElement.current?.value) { - return; - } - - const isValid = validationRegex - ? !!inputElement.current?.value.match(validationRegex) - : true; - - if (isValid) { - onEnter(currentInputValue); - setCurrentInputValue(""); - } else if (onError) { - onError(); - } - }; - - const handleInputKeyDown = (event: React.KeyboardEvent) => { - const { keyCode } = event; - - // on ENTER click - if (keyCode === 13) { - event.stopPropagation(); - event.preventDefault(); - onAddValue(); - - // on DELETE or BACKSPACE click when input is empty (select or delete last tag in valuesList) - } else if ((keyCode === 46 || keyCode === 8) && currentInputValue === "") { - if (selectedElementId !== "") { - const nextId = value.length - 1 > 0 ? value[value.length - 2].id : ""; - onDelete(selectedElementId); - setSelectedElementId(nextId); - } else if (value.length) { - setSelectedElementId(value[value.length - 1].id); - } - } - }; - - const handleInputBlur = () => { - if (addOnBlur) { - onAddValue(); - } - }; - - const inputPlaceholder = - !value.length && inputProps?.placeholder ? inputProps.placeholder : ""; - - return ( - - {value.map((item, key) => ( - - ))} - { - setSelectedElementId(""); - setCurrentInputValue(event.target.value); - }} - /> - - ); -}; - -export { TagInput }; -export type { TagInputProps }; diff --git a/airbyte-webapp/src/components/base/TagInput/TagItem.tsx b/airbyte-webapp/src/components/base/TagInput/TagItem.tsx deleted file mode 100644 index 09e12450e85..00000000000 --- a/airbyte-webapp/src/components/base/TagInput/TagItem.tsx +++ /dev/null @@ -1,71 +0,0 @@ -import React from "react"; -import styled from "styled-components"; - -const Tag = styled.div<{ isSelected?: boolean }>` - max-width: 100%; - display: flex; - background: ${({ theme }) => theme.mediumPrimaryColor}; - color: ${({ theme }) => theme.whiteColor}; - font-size: 12px; - line-height: 20px; - font-weight: 500; - border-radius: 4px; - padding-left: 6px; - margin: 0 5px 4px 0; - border: 2px solid - ${({ theme, isSelected }) => - isSelected ? theme.primaryColor : theme.mediumPrimaryColor}; -`; - -const Text = styled.div` - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; -`; - -const Delete = styled.button` - border: none; - outline: none; - font-weight: 300; - font-size: 14px; - line-height: 24px; - cursor: pointer; - color: ${({ theme }) => theme.greyColor55}; - background: none; - text-decoration: none; - padding: 0 4px; - height: 20px; - &:hover { - color: ${({ theme }) => theme.greyColor40}; - } -`; - -type IProps = { - item: IItemProps; - isSelected?: boolean; - disabled?: boolean; - onDeleteTag: (id: string) => void; -}; - -export type IItemProps = { - value: string; - id: string; -}; - -const TagItem: React.FC = ({ - item, - onDeleteTag, - isSelected, - disabled, -}) => { - const clickOnDeleteButton = () => onDeleteTag(item.id); - - return ( - - {item.value} - - - ); -}; - -export default TagItem; diff --git a/airbyte-webapp/src/components/base/TagInput/index.tsx b/airbyte-webapp/src/components/base/TagInput/index.tsx deleted file mode 100644 index 2c0ae630fd0..00000000000 --- a/airbyte-webapp/src/components/base/TagInput/index.tsx +++ /dev/null @@ -1 +0,0 @@ -export * from "./TagInput"; diff --git a/airbyte-webapp/src/components/base/index.tsx b/airbyte-webapp/src/components/base/index.tsx deleted file mode 100644 index 2c0ae630fd0..00000000000 --- a/airbyte-webapp/src/components/base/index.tsx +++ /dev/null @@ -1 +0,0 @@ -export * from "./TagInput"; diff --git a/airbyte-webapp/src/components/index.tsx b/airbyte-webapp/src/components/index.tsx index eca9bec51f9..9a0d4e54ae5 100644 --- a/airbyte-webapp/src/components/index.tsx +++ b/airbyte-webapp/src/components/index.tsx @@ -1,10 +1,7 @@ -export * from "./base"; - export * from "./Button"; export * from "./Input"; export * from "./Spinner"; export * from "./DropDown"; -export * from "./Multiselect"; export * from "./Titles"; export * from "./StatusIcon"; export * from "./Label"; @@ -13,4 +10,3 @@ export * from "./LabeledToggle"; export * from "./Link"; export * from "./TextWithHTML"; export * from "./TextArea"; -export * from "./ArrayOfObjectsEditor"; diff --git a/airbyte-webapp/src/core/form/types.ts b/airbyte-webapp/src/core/form/types.ts index dc6003954a6..45e6ce5e30d 100644 --- a/airbyte-webapp/src/core/form/types.ts +++ b/airbyte-webapp/src/core/form/types.ts @@ -4,20 +4,22 @@ export type FormBaseItem = { _type: "formItem"; type: JSONSchema7TypeName; fieldKey: string; - path: string; + fieldName: string; isRequired: boolean; isSecret?: boolean; title?: string; multiline?: boolean; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + meta?: { [key: string]: any }; } & Partial; type FormGroupItem = { _type: "formGroup"; - path: string; + jsonSchema: JSONSchema7; + fieldName: string; fieldKey: string; isRequired: boolean; title?: string; - jsonSchema: JSONSchema7; properties: FormBlock[]; isLoading?: boolean; description?: string; @@ -27,34 +29,16 @@ type FormGroupItem = { type FormConditionItem = { _type: "formCondition"; - path: string; + fieldName: string; fieldKey: string; isRequired: boolean; - title?: string; conditions: { [key: string]: FormGroupItem | FormBaseItem }; -}; - -type FormObjectArrayItem = { - _type: "objectArray"; - path: string; - fieldKey: string; - isRequired: boolean; title?: string; - properties: FormBlock; }; -type FormBlock = - | FormGroupItem - | FormBaseItem - | FormConditionItem - | FormObjectArrayItem; +type FormBlock = FormGroupItem | FormBaseItem | FormConditionItem; -export type { - FormBlock, - FormConditionItem, - FormGroupItem, - FormObjectArrayItem, -}; +export type { FormBlock, FormConditionItem, FormGroupItem }; // eslint-disable-next-line @typescript-eslint/no-explicit-any export type WidgetConfig = { [key: string]: any }; diff --git a/airbyte-webapp/src/core/form/uiWidget.test.ts b/airbyte-webapp/src/core/form/uiWidget.test.ts index 3720854fc5f..a69e71f2a9e 100644 --- a/airbyte-webapp/src/core/form/uiWidget.test.ts +++ b/airbyte-webapp/src/core/form/uiWidget.test.ts @@ -6,7 +6,7 @@ const formItems: FormBlock[] = [ { _type: "formGroup", fieldKey: "key", - path: "key", + fieldName: "key", isRequired: true, jsonSchema: { type: "object", @@ -39,21 +39,21 @@ const formItems: FormBlock[] = [ { _type: "formItem", fieldKey: "start_date", - path: "key.start_date", + fieldName: "key.start_date", isRequired: true, type: "string", }, { _type: "formCondition", fieldKey: "credentials", - path: "key.credentials", + fieldName: "key.credentials", isRequired: true, conditions: { "api key": { title: "api key", _type: "formGroup", fieldKey: "credentials", - path: "key.credentials", + fieldName: "key.credentials", isRequired: false, jsonSchema: { title: "api key", @@ -64,7 +64,7 @@ const formItems: FormBlock[] = [ { _type: "formItem", fieldKey: "api_key", - path: "key.credentials.api_key", + fieldName: "key.credentials.api_key", isRequired: true, type: "string", }, @@ -74,7 +74,7 @@ const formItems: FormBlock[] = [ title: "oauth", _type: "formGroup", fieldKey: "credentials", - path: "key.credentials", + fieldName: "key.credentials", isRequired: false, jsonSchema: { title: "oauth", @@ -91,7 +91,7 @@ const formItems: FormBlock[] = [ _type: "formItem", examples: ["https://api.hubspot.com/"], fieldKey: "redirect_uri", - path: "key.credentials.redirect_uri", + fieldName: "key.credentials.redirect_uri", isRequired: true, type: "string", }, diff --git a/airbyte-webapp/src/core/form/uiWidget.ts b/airbyte-webapp/src/core/form/uiWidget.ts index 7d8370a28d9..168bae67dd6 100644 --- a/airbyte-webapp/src/core/form/uiWidget.ts +++ b/airbyte-webapp/src/core/form/uiWidget.ts @@ -4,7 +4,8 @@ import { buildYupFormForJsonSchema } from "core/jsonSchema/schemaToYup"; export const buildPathInitialState = ( formBlock: FormBlock[], - formValues: { [key: string]: unknown }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + formValues: { [key: string]: any }, widgetState: WidgetConfigMap = {} ): { [key: string]: WidgetConfigMap } => formBlock.reduce((widgetStateBuilder, formItem) => { @@ -16,14 +17,17 @@ export const buildPathInitialState = ( widgetStateBuilder ); case "formItem": - widgetStateBuilder[formItem.path] = {}; + widgetStateBuilder[formItem.fieldName] = {}; return widgetStateBuilder; case "formCondition": const defaultCondition = Object.entries(formItem.conditions).find( ([key, subConditionItems]) => { switch (subConditionItems._type) { case "formGroup": - const selectedValues = get(formValues, subConditionItems.path); + const selectedValues = get( + formValues, + subConditionItems.fieldName + ); const subPathSchema = buildYupFormForJsonSchema({ type: "object", @@ -45,7 +49,7 @@ export const buildPathInitialState = ( const selectedPath = defaultCondition ?? Object.keys(formItem.conditions)?.[0]; - widgetStateBuilder[formItem.path] = { + widgetStateBuilder[formItem.fieldName] = { selectedItem: selectedPath, }; diff --git a/airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.test.ts b/airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.test.ts index a5be8dcf23b..4657f510d75 100644 --- a/airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.test.ts +++ b/airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.test.ts @@ -30,7 +30,7 @@ test("should reformat jsonSchema to internal widget representation", () => { const expected = { _type: "formGroup", - path: "key", + fieldName: "key", fieldKey: "key", isRequired: false, jsonSchema: { @@ -64,7 +64,7 @@ test("should reformat jsonSchema to internal widget representation", () => { { _type: "formItem", description: "Hostname of the database.", - path: "key.host", + fieldName: "key.host", fieldKey: "host", isRequired: true, type: "string", @@ -72,7 +72,7 @@ test("should reformat jsonSchema to internal widget representation", () => { { _type: "formItem", description: "Port of the database.", - path: "key.port", + fieldName: "key.port", fieldKey: "port", isRequired: true, type: "integer", @@ -80,7 +80,7 @@ test("should reformat jsonSchema to internal widget representation", () => { { _type: "formItem", description: "Username to use to access the database.", - path: "key.user", + fieldName: "key.user", fieldKey: "user", isRequired: true, type: "string", @@ -88,7 +88,7 @@ test("should reformat jsonSchema to internal widget representation", () => { { _type: "formItem", description: "Name of the database.", - path: "key.dbname", + fieldName: "key.dbname", fieldKey: "dbname", isRequired: true, type: "string", @@ -96,7 +96,7 @@ test("should reformat jsonSchema to internal widget representation", () => { { _type: "formItem", description: "Password associated with the username.", - path: "key.password", + fieldName: "key.password", fieldKey: "password", isRequired: false, isSecret: true, @@ -125,7 +125,7 @@ test("should reformat jsonSchema to internal widget representation with parent s const expected = { _type: "formGroup", fieldKey: "key", - path: "key", + fieldName: "key", isRequired: true, jsonSchema: { properties: { @@ -143,7 +143,7 @@ test("should reformat jsonSchema to internal widget representation with parent s _type: "formItem", description: "Hostname of the database.", fieldKey: "host", - path: "key.host", + fieldName: "key.host", isRequired: true, type: "string", }, @@ -222,19 +222,19 @@ test("should reformat jsonSchema to internal widget representation when has oneO }, }, }, - path: "key", + fieldName: "key", fieldKey: "key", properties: [ { _type: "formItem", - path: "key.start_date", + fieldName: "key.start_date", fieldKey: "start_date", isRequired: true, type: "string", }, { _type: "formCondition", - path: "key.credentials", + fieldName: "key.credentials", fieldKey: "credentials", conditions: { "api key": { @@ -245,12 +245,12 @@ test("should reformat jsonSchema to internal widget representation when has oneO required: ["api_key"], properties: { api_key: { type: "string" } }, }, - path: "key.credentials", + fieldName: "key.credentials", fieldKey: "credentials", properties: [ { _type: "formItem", - path: "key.credentials.api_key", + fieldName: "key.credentials.api_key", fieldKey: "api_key", isRequired: true, type: "string", @@ -271,13 +271,13 @@ test("should reformat jsonSchema to internal widget representation when has oneO }, }, }, - path: "key.credentials", + fieldName: "key.credentials", fieldKey: "credentials", properties: [ { examples: ["https://api.hubspot.com/"], _type: "formItem", - path: "key.credentials.redirect_uri", + fieldName: "key.credentials.redirect_uri", fieldKey: "redirect_uri", isRequired: true, type: "string", diff --git a/airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.ts b/airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.ts index 274d32dbebb..5e84face8be 100644 --- a/airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.ts +++ b/airbyte-webapp/src/core/jsonSchema/schemaToUiWidget.ts @@ -15,7 +15,7 @@ import { FormBlock } from "core/form/types"; */ export const jsonSchemaToUiWidget = ( jsonSchema: JSONSchema7Definition, - key = "", + key: string, path: string = key, parentSchema?: JSONSchema7Definition ): FormBlock => { @@ -25,7 +25,7 @@ export const jsonSchemaToUiWidget = ( if (typeof jsonSchema === "boolean") { return { _type: "formItem", - path: key, + fieldName: key, fieldKey: key, type: "null", isRequired, @@ -45,41 +45,25 @@ export const jsonSchemaToUiWidget = ( return { _type: "formCondition", - path: path || key, + fieldName: path || key, fieldKey: key, conditions, isRequired, }; } - if ( - jsonSchema.type === "array" && - typeof jsonSchema.items === "object" && - !Array.isArray(jsonSchema.items) && - jsonSchema.items.type === "object" - ) { - return { - ...pickDefaultFields(jsonSchema), - _type: "objectArray", - path: path || key, - fieldKey: key, - properties: jsonSchemaToUiWidget(jsonSchema.items, key, path), - isRequired, - }; - } - if (jsonSchema.properties) { const properties = Object.entries( jsonSchema.properties ).map(([k, schema]) => - jsonSchemaToUiWidget(schema, k, path ? `${path}.${k}` : k, jsonSchema) + jsonSchemaToUiWidget(schema, k, `${path}.${k}`, jsonSchema) ); return { ...pickDefaultFields(jsonSchema), _type: "formGroup", jsonSchema, - path: path || key, + fieldName: path || key, fieldKey: key, properties, isRequired, @@ -89,7 +73,7 @@ export const jsonSchemaToUiWidget = ( return { ...pickDefaultFields(jsonSchema), _type: "formItem", - path: path || key, + fieldName: path || key, fieldKey: key, isRequired, isSecret: (jsonSchema as { airbyte_secret: boolean }).airbyte_secret, @@ -119,10 +103,5 @@ const pickDefaultFields = (schema: JSONSchema7): Partial => ({ description: schema.description, pattern: schema.pattern, title: schema.title, - enum: - typeof schema.items === "object" && - !Array.isArray(schema.items) && - schema.items.enum - ? schema.items.enum - : schema.enum, + enum: schema.enum, }); diff --git a/airbyte-webapp/src/core/jsonSchema/schemaToYup.test.ts b/airbyte-webapp/src/core/jsonSchema/schemaToYup.test.ts index 6de762858a3..7b8c6885f71 100644 --- a/airbyte-webapp/src/core/jsonSchema/schemaToYup.test.ts +++ b/airbyte-webapp/src/core/jsonSchema/schemaToYup.test.ts @@ -33,12 +33,6 @@ test("should build schema for simple case", () => { type: "string", description: "Password associated with the username.", }, - reports: { - type: "array", - items: { - type: "string", - }, - }, }, additionalProperties: false, }; @@ -52,7 +46,6 @@ test("should build schema for simple case", () => { is_field_no_default: yup.boolean().required("form.empty.error"), dbname: yup.string().trim().required("form.empty.error"), password: yup.string().trim(), - reports: yup.array().of(yup.string().trim()), }); expect(JSON.stringify(yupSchema)).toEqual(JSON.stringify(expectedSchema)); diff --git a/airbyte-webapp/src/core/jsonSchema/schemaToYup.ts b/airbyte-webapp/src/core/jsonSchema/schemaToYup.ts index 48b47ac1829..f1c718ddf6c 100644 --- a/airbyte-webapp/src/core/jsonSchema/schemaToYup.ts +++ b/airbyte-webapp/src/core/jsonSchema/schemaToYup.ts @@ -30,7 +30,6 @@ export const buildYupFormForJsonSchema = ( | yup.NumberSchema | yup.StringSchema | yup.ObjectSchema - | yup.ArraySchema | yup.BooleanSchema | null = null; @@ -79,24 +78,6 @@ export const buildYupFormForJsonSchema = ( schema = schema.max(jsonSchema?.maximum); } break; - case "array": - if ( - typeof jsonSchema.items === "object" && - !Array.isArray(jsonSchema.items) - ) { - schema = yup - .array() - .of( - buildYupFormForJsonSchema( - jsonSchema.items, - uiConfig, - jsonSchema, - propertyKey, - propertyPath ? `${propertyPath}.${propertyKey}` : propertyKey - ) - ); - } - break; case "object": schema = yup .object() diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 3693e1dc9ba..962fd66383d 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -76,8 +76,6 @@ "form.prefix": "Namespace Prefix", "form.prefix.message": "Where to replicate your data to in your destination", "form.prefix.placeholder": "/configured-setting", - "form.addItems": "Add", - "form.items": "Items", "preferences.title": "Specify your preferences", "preferences.anonymizeUsage": "Anonymize usage data collection", diff --git a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx index 9907c255173..d7704b84e5c 100644 --- a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx +++ b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx @@ -64,6 +64,11 @@ const SourceSettings: React.FC = ({ } }; + // values: { + // name: string; + // serviceType: string; + // connectionConfiguration?: ConnectionConfiguration; + // } const onRetest = async (values: { name: string; serviceType: string; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultSyncWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultSyncWorker.java index bbc48c7d263..eb17452db76 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/DefaultSyncWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/DefaultSyncWorker.java @@ -145,29 +145,7 @@ public class DefaultSyncWorker implements SyncWorker { @Override public void cancel() { - LOGGER.info("Cancelling sync worker..."); cancelled.set(true); - - LOGGER.info("Cancelling source..."); - try { - source.cancel(); - } catch (Exception e) { - e.printStackTrace(); - } - - LOGGER.info("Cancelling destination..."); - try { - destination.cancel(); - } catch (Exception e) { - e.printStackTrace(); - } - - LOGGER.info("Cancelling normalization runner..."); - try { - normalizationRunner.close(); - } catch (Exception e) { - e.printStackTrace(); - } } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerUtils.java b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerUtils.java index 297948bcb15..3762781a3be 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerUtils.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerUtils.java @@ -28,7 +28,6 @@ import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardTapConfig; import io.airbyte.config.StandardTargetConfig; import io.airbyte.scheduler.models.IntegrationLauncherConfig; -import io.airbyte.scheduler.models.JobRunConfig; import io.airbyte.workers.process.AirbyteIntegrationLauncher; import io.airbyte.workers.process.IntegrationLauncher; import io.airbyte.workers.process.ProcessBuilderFactory; @@ -125,14 +124,6 @@ public class WorkerUtils { return getJobRoot(workspaceRoot, launcherConfig.getJobId(), Math.toIntExact(launcherConfig.getAttemptId())); } - public static Path getJobRoot(Path workspaceRoot, JobRunConfig jobRunConfig) { - return getJobRoot(workspaceRoot, jobRunConfig.getJobId(), jobRunConfig.getAttemptId()); - } - - public static Path getLogPath(Path jobRoot) { - return jobRoot.resolve(WorkerConstants.LOG_FILENAME); - } - public static Path getJobRoot(Path workspaceRoot, String jobId, long attemptId) { return getJobRoot(workspaceRoot, jobId, Math.toIntExact(attemptId)); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/process/DockerProcessBuilderFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/process/DockerProcessBuilderFactory.java index 12e6349b6dc..1b45e56beb5 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/process/DockerProcessBuilderFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/process/DockerProcessBuilderFactory.java @@ -89,7 +89,6 @@ public class DockerProcessBuilderFactory implements ProcessBuilderFactory { "docker", "run", "--rm", - "--init", "-i", "-v", String.format("%s:%s", workspaceMountSource, DATA_MOUNT_DESTINATION), diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java index d8001f45324..6bbf0704df8 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Destination.java @@ -40,6 +40,4 @@ public interface Destination extends CheckedConsumer, AutoClose @Override void close() throws Exception; - void cancel() throws Exception; - } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java index 92b9a299be2..e207b5c4cd1 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/Source.java @@ -39,6 +39,4 @@ public interface Source extends AutoCloseable { @Override void close() throws Exception; - void cancel() throws Exception; - } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java index 0ec538af379..2b9bbfe43ae 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteDestination.java @@ -109,17 +109,4 @@ public class DefaultAirbyteDestination implements AirbyteDestination { } } - @Override - public void cancel() throws Exception { - LOGGER.info("Attempting to cancel destination process..."); - - if (targetProcess == null) { - LOGGER.info("Target process no longer exists, cancellation is a no-op."); - } else { - LOGGER.info("Target process exists, cancelling..."); - WorkerUtils.cancelProcess(targetProcess); - LOGGER.info("Cancelled destination process!"); - } - } - } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java index 4e80b3f5549..1dd0cde6c27 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/DefaultAirbyteSource.java @@ -113,17 +113,4 @@ public class DefaultAirbyteSource implements AirbyteSource { } } - @Override - public void cancel() throws Exception { - LOGGER.info("Attempting to cancel source process..."); - - if (tapProcess == null) { - LOGGER.info("Source process no longer exists, cancellation is a no-op."); - } else { - LOGGER.info("Source process exists, cancelling..."); - WorkerUtils.cancelProcess(tapProcess); - LOGGER.info("Cancelled source process!"); - } - } - } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java index e4bc4c4f033..c6d533dfa9c 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/protocols/airbyte/EmptyAirbyteSource.java @@ -56,9 +56,4 @@ public class EmptyAirbyteSource implements AirbyteSource { // no op. } - @Override - public void cancel() throws Exception { - // no op. - } - } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/CancellationHandler.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/CancellationHandler.java deleted file mode 100644 index 9dc292216a2..00000000000 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/CancellationHandler.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.workers.temporal; - -import io.airbyte.workers.WorkerException; -import io.temporal.activity.Activity; -import io.temporal.activity.ActivityExecutionContext; -import io.temporal.client.ActivityCompletionException; - -public interface CancellationHandler { - - void checkAndHandleCancellation(Runnable onCancellationCallback) throws WorkerException; - - class TemporalCancellationHandler implements CancellationHandler { - - final ActivityExecutionContext context; - - public TemporalCancellationHandler() { - context = Activity.getExecutionContext(); - } - - /** - * Check for a cancellation/timeout status and run any callbacks necessary to shut down underlying - * processes. This method should generally be run frequently within an activity so a change in - * cancellation status is respected. This will only be effective if the cancellation type for the - * workflow is set to - * {@link io.temporal.activity.ActivityCancellationType#WAIT_CANCELLATION_COMPLETED}; otherwise, the - * activity will be killed automatically as part of cleanup without removing underlying processes. - * - * @param onCancellationCallback a runnable that will only run when Temporal indicates the activity - * should be killed (cancellation or timeout). - * @throws WorkerException - */ - @Override - public void checkAndHandleCancellation(Runnable onCancellationCallback) throws WorkerException { - try { - // Heartbeat is somewhat misleading here. What it does is check the current Temporal activity's - // context and - // throw an exception if the sync has been cancelled or timed out. The input to this heartbeat - // function - // is available as a field in thrown ActivityCompletionExceptions, which we aren't using for now. - context.heartbeat(null); - } catch (ActivityCompletionException e) { - onCancellationCallback.run(); - throw new WorkerException("Worker cleaned up after exception", e); - } - } - - } - -} diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/CheckConnectionWorkflow.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/CheckConnectionWorkflow.java index 9c867618cc8..cdefcf3b6c9 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/CheckConnectionWorkflow.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/CheckConnectionWorkflow.java @@ -24,13 +24,11 @@ package io.airbyte.workers.temporal; -import io.airbyte.commons.functional.CheckedSupplier; import io.airbyte.config.StandardCheckConnectionInput; import io.airbyte.config.StandardCheckConnectionOutput; import io.airbyte.scheduler.models.IntegrationLauncherConfig; import io.airbyte.scheduler.models.JobRunConfig; import io.airbyte.workers.DefaultCheckConnectionWorker; -import io.airbyte.workers.Worker; import io.airbyte.workers.process.AirbyteIntegrationLauncher; import io.airbyte.workers.process.IntegrationLauncher; import io.airbyte.workers.process.ProcessBuilderFactory; @@ -42,7 +40,6 @@ import io.temporal.workflow.WorkflowInterface; import io.temporal.workflow.WorkflowMethod; import java.nio.file.Path; import java.time.Duration; -import java.util.function.Supplier; @WorkflowInterface public interface CheckConnectionWorkflow { @@ -50,7 +47,8 @@ public interface CheckConnectionWorkflow { @WorkflowMethod StandardCheckConnectionOutput run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig, - StandardCheckConnectionInput connectionConfiguration); + StandardCheckConnectionInput connectionConfiguration) + throws TemporalJobException; class WorkflowImpl implements CheckConnectionWorkflow { @@ -63,7 +61,8 @@ public interface CheckConnectionWorkflow { @Override public StandardCheckConnectionOutput run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig, - StandardCheckConnectionInput connectionConfiguration) { + StandardCheckConnectionInput connectionConfiguration) + throws TemporalJobException { return activity.run(jobRunConfig, launcherConfig, connectionConfiguration); } @@ -75,7 +74,8 @@ public interface CheckConnectionWorkflow { @ActivityMethod StandardCheckConnectionOutput run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig, - StandardCheckConnectionInput connectionConfiguration); + StandardCheckConnectionInput connectionConfiguration) + throws TemporalJobException; } @@ -91,31 +91,17 @@ public interface CheckConnectionWorkflow { public StandardCheckConnectionOutput run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig, - StandardCheckConnectionInput connectionConfiguration) { - - final Supplier inputSupplier = () -> connectionConfiguration; - - final TemporalAttemptExecution temporalAttemptExecution = - new TemporalAttemptExecution<>( - workspaceRoot, - jobRunConfig, - getWorkerFactory(launcherConfig), - inputSupplier, - new CancellationHandler.TemporalCancellationHandler()); - - return temporalAttemptExecution.get(); - } - - private CheckedSupplier, Exception> getWorkerFactory(IntegrationLauncherConfig launcherConfig) { - return () -> { + StandardCheckConnectionInput connectionConfiguration) + throws TemporalJobException { + return new TemporalAttemptExecution<>(workspaceRoot, jobRunConfig, (jobRoot) -> { final IntegrationLauncher integrationLauncher = new AirbyteIntegrationLauncher( launcherConfig.getJobId(), Math.toIntExact(launcherConfig.getAttemptId()), launcherConfig.getDockerImage(), pbf); - return new DefaultCheckConnectionWorker(integrationLauncher); - }; + return new DefaultCheckConnectionWorker(integrationLauncher).run(connectionConfiguration, jobRoot); + }).get(); } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/DiscoverCatalogWorkflow.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/DiscoverCatalogWorkflow.java index e23406f3f6b..fc49335e8e2 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/DiscoverCatalogWorkflow.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/DiscoverCatalogWorkflow.java @@ -24,13 +24,11 @@ package io.airbyte.workers.temporal; -import io.airbyte.commons.functional.CheckedSupplier; import io.airbyte.config.StandardDiscoverCatalogInput; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.scheduler.models.IntegrationLauncherConfig; import io.airbyte.scheduler.models.JobRunConfig; import io.airbyte.workers.DefaultDiscoverCatalogWorker; -import io.airbyte.workers.Worker; import io.airbyte.workers.process.AirbyteIntegrationLauncher; import io.airbyte.workers.process.IntegrationLauncher; import io.airbyte.workers.process.ProcessBuilderFactory; @@ -44,7 +42,6 @@ import io.temporal.workflow.WorkflowInterface; import io.temporal.workflow.WorkflowMethod; import java.nio.file.Path; import java.time.Duration; -import java.util.function.Supplier; @WorkflowInterface public interface DiscoverCatalogWorkflow { @@ -52,7 +49,8 @@ public interface DiscoverCatalogWorkflow { @WorkflowMethod AirbyteCatalog run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig, - StandardDiscoverCatalogInput config); + StandardDiscoverCatalogInput config) + throws TemporalJobException; class WorkflowImpl implements DiscoverCatalogWorkflow { @@ -65,7 +63,8 @@ public interface DiscoverCatalogWorkflow { @Override public AirbyteCatalog run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig, - StandardDiscoverCatalogInput config) { + StandardDiscoverCatalogInput config) + throws TemporalJobException { return activity.run(jobRunConfig, launcherConfig, config); } @@ -77,7 +76,8 @@ public interface DiscoverCatalogWorkflow { @ActivityMethod AirbyteCatalog run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig, - StandardDiscoverCatalogInput config); + StandardDiscoverCatalogInput config) + throws TemporalJobException; } @@ -93,28 +93,15 @@ public interface DiscoverCatalogWorkflow { public AirbyteCatalog run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig, - StandardDiscoverCatalogInput config) { - final Supplier inputSupplier = () -> config; - - final TemporalAttemptExecution temporalAttemptExecution = new TemporalAttemptExecution<>( - workspaceRoot, - jobRunConfig, - getWorkerFactory(launcherConfig), - inputSupplier, - new CancellationHandler.TemporalCancellationHandler()); - - return temporalAttemptExecution.get(); - } - - private CheckedSupplier, Exception> getWorkerFactory(IntegrationLauncherConfig launcherConfig) { - return () -> { + StandardDiscoverCatalogInput config) + throws TemporalJobException { + return new TemporalAttemptExecution<>(workspaceRoot, jobRunConfig, (jobRoot) -> { final IntegrationLauncher integrationLauncher = - new AirbyteIntegrationLauncher(launcherConfig.getJobId(), launcherConfig.getAttemptId().intValue(), launcherConfig.getDockerImage(), - pbf); + new AirbyteIntegrationLauncher(launcherConfig.getJobId(), launcherConfig.getAttemptId().intValue(), launcherConfig.getDockerImage(), pbf); final AirbyteStreamFactory streamFactory = new DefaultAirbyteStreamFactory(); - return new DefaultDiscoverCatalogWorker(integrationLauncher, streamFactory); - }; + return new DefaultDiscoverCatalogWorker(integrationLauncher, streamFactory).run(config, jobRoot); + }).get(); } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/SpecWorkflow.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/SpecWorkflow.java index 8e83b7d8050..90fbf4ec05f 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/SpecWorkflow.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/SpecWorkflow.java @@ -24,13 +24,11 @@ package io.airbyte.workers.temporal; -import io.airbyte.commons.functional.CheckedSupplier; import io.airbyte.config.JobGetSpecConfig; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.scheduler.models.IntegrationLauncherConfig; import io.airbyte.scheduler.models.JobRunConfig; import io.airbyte.workers.DefaultGetSpecWorker; -import io.airbyte.workers.Worker; import io.airbyte.workers.process.AirbyteIntegrationLauncher; import io.airbyte.workers.process.IntegrationLauncher; import io.airbyte.workers.process.ProcessBuilderFactory; @@ -42,13 +40,12 @@ import io.temporal.workflow.WorkflowInterface; import io.temporal.workflow.WorkflowMethod; import java.nio.file.Path; import java.time.Duration; -import java.util.function.Supplier; @WorkflowInterface public interface SpecWorkflow { @WorkflowMethod - ConnectorSpecification run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig); + ConnectorSpecification run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig) throws TemporalJobException; class WorkflowImpl implements SpecWorkflow { @@ -59,7 +56,7 @@ public interface SpecWorkflow { private final SpecActivity activity = Workflow.newActivityStub(SpecActivity.class, options); @Override - public ConnectorSpecification run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig) { + public ConnectorSpecification run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig) throws TemporalJobException { return activity.run(jobRunConfig, launcherConfig); } @@ -69,7 +66,7 @@ public interface SpecWorkflow { interface SpecActivity { @ActivityMethod - ConnectorSpecification run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig); + ConnectorSpecification run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig) throws TemporalJobException; } @@ -83,29 +80,15 @@ public interface SpecWorkflow { this.workspaceRoot = workspaceRoot; } - public ConnectorSpecification run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig) { - final Supplier inputSupplier = () -> new JobGetSpecConfig().withDockerImage(launcherConfig.getDockerImage()); + public ConnectorSpecification run(JobRunConfig jobRunConfig, IntegrationLauncherConfig launcherConfig) throws TemporalJobException { + return new TemporalAttemptExecution<>(workspaceRoot, jobRunConfig, (jobRoot) -> { - final TemporalAttemptExecution temporalAttemptExecution = new TemporalAttemptExecution<>( - workspaceRoot, - jobRunConfig, - getWorkerFactory(launcherConfig), - inputSupplier, - new CancellationHandler.TemporalCancellationHandler()); + final IntegrationLauncher integrationLauncher = + new AirbyteIntegrationLauncher(launcherConfig.getJobId(), launcherConfig.getAttemptId().intValue(), launcherConfig.getDockerImage(), pbf); - return temporalAttemptExecution.get(); - } - - private CheckedSupplier, Exception> getWorkerFactory(IntegrationLauncherConfig launcherConfig) { - return () -> { - final IntegrationLauncher integrationLauncher = new AirbyteIntegrationLauncher( - launcherConfig.getJobId(), - launcherConfig.getAttemptId().intValue(), - launcherConfig.getDockerImage(), - pbf); - - return new DefaultGetSpecWorker(integrationLauncher); - }; + final JobGetSpecConfig jobGetSpecConfig = new JobGetSpecConfig().withDockerImage(launcherConfig.getDockerImage()); + return new DefaultGetSpecWorker(integrationLauncher).run(jobGetSpecConfig, jobRoot); + }).get(); } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/SyncWorkflow.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/SyncWorkflow.java index a1f14f40fa7..9895886c88f 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/SyncWorkflow.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/SyncWorkflow.java @@ -24,13 +24,11 @@ package io.airbyte.workers.temporal; -import io.airbyte.commons.functional.CheckedSupplier; import io.airbyte.config.StandardSyncInput; import io.airbyte.config.StandardSyncOutput; import io.airbyte.scheduler.models.IntegrationLauncherConfig; import io.airbyte.scheduler.models.JobRunConfig; import io.airbyte.workers.DefaultSyncWorker; -import io.airbyte.workers.Worker; import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.normalization.NormalizationRunnerFactory; import io.airbyte.workers.process.AirbyteIntegrationLauncher; @@ -42,7 +40,6 @@ import io.airbyte.workers.protocols.airbyte.DefaultAirbyteDestination; import io.airbyte.workers.protocols.airbyte.DefaultAirbyteSource; import io.airbyte.workers.protocols.airbyte.EmptyAirbyteSource; import io.airbyte.workers.protocols.airbyte.NamespacingMapper; -import io.temporal.activity.ActivityCancellationType; import io.temporal.activity.ActivityInterface; import io.temporal.activity.ActivityMethod; import io.temporal.activity.ActivityOptions; @@ -51,9 +48,6 @@ import io.temporal.workflow.WorkflowInterface; import io.temporal.workflow.WorkflowMethod; import java.nio.file.Path; import java.time.Duration; -import java.util.function.Supplier; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @WorkflowInterface public interface SyncWorkflow { @@ -62,23 +56,23 @@ public interface SyncWorkflow { StandardSyncOutput run(JobRunConfig jobRunConfig, IntegrationLauncherConfig sourceLauncherConfig, IntegrationLauncherConfig destinationLauncherConfig, - StandardSyncInput syncInput); + StandardSyncInput syncInput) + throws TemporalJobException; class WorkflowImpl implements SyncWorkflow { - private final ActivityOptions options = ActivityOptions.newBuilder() + final ActivityOptions options = ActivityOptions.newBuilder() .setScheduleToCloseTimeout(Duration.ofDays(3)) - .setCancellationType(ActivityCancellationType.WAIT_CANCELLATION_COMPLETED) .setRetryOptions(TemporalUtils.NO_RETRY) .build(); - private final SyncActivity activity = Workflow.newActivityStub(SyncActivity.class, options); @Override public StandardSyncOutput run(JobRunConfig jobRunConfig, IntegrationLauncherConfig sourceLauncherConfig, IntegrationLauncherConfig destinationLauncherConfig, - StandardSyncInput syncInput) { + StandardSyncInput syncInput) + throws TemporalJobException { return activity.run(jobRunConfig, sourceLauncherConfig, destinationLauncherConfig, syncInput); } @@ -91,14 +85,13 @@ public interface SyncWorkflow { StandardSyncOutput run(JobRunConfig jobRunConfig, IntegrationLauncherConfig sourceLauncherConfig, IntegrationLauncherConfig destinationLauncherConfig, - StandardSyncInput syncInput); + StandardSyncInput syncInput) + throws TemporalJobException; } class SyncActivityImpl implements SyncActivity { - private static final Logger LOGGER = LoggerFactory.getLogger(SyncActivityImpl.class); - private final ProcessBuilderFactory pbf; private final Path workspaceRoot; @@ -110,26 +103,10 @@ public interface SyncWorkflow { public StandardSyncOutput run(JobRunConfig jobRunConfig, IntegrationLauncherConfig sourceLauncherConfig, IntegrationLauncherConfig destinationLauncherConfig, - StandardSyncInput syncInput) { + StandardSyncInput syncInput) + throws TemporalJobException { - final Supplier inputSupplier = () -> syncInput; - - final TemporalAttemptExecution temporalAttemptExecution = new TemporalAttemptExecution<>( - workspaceRoot, - jobRunConfig, - getWorkerFactory(sourceLauncherConfig, destinationLauncherConfig, jobRunConfig, syncInput), - inputSupplier, - new CancellationHandler.TemporalCancellationHandler()); - - return temporalAttemptExecution.get(); - } - - private CheckedSupplier, Exception> getWorkerFactory( - IntegrationLauncherConfig sourceLauncherConfig, - IntegrationLauncherConfig destinationLauncherConfig, - JobRunConfig jobRunConfig, - StandardSyncInput syncInput) { - return () -> { + return new TemporalAttemptExecution<>(workspaceRoot, jobRunConfig, (jobRoot) -> { final IntegrationLauncher sourceLauncher = new AirbyteIntegrationLauncher( sourceLauncherConfig.getJobId(), Math.toIntExact(sourceLauncherConfig.getAttemptId()), @@ -156,8 +133,8 @@ public interface SyncWorkflow { NormalizationRunnerFactory.create( destinationLauncherConfig.getDockerImage(), pbf, - syncInput.getDestinationConfiguration())); - }; + syncInput.getDestinationConfiguration())).run(syncInput, jobRoot); + }).get(); } } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java index b89c6dca747..1ea07d33206 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalAttemptExecution.java @@ -26,24 +26,16 @@ package io.airbyte.workers.temporal; import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.functional.CheckedConsumer; +import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.functional.CheckedSupplier; -import io.airbyte.commons.io.IOs; import io.airbyte.config.EnvConfigs; import io.airbyte.scheduler.models.JobRunConfig; -import io.airbyte.workers.Worker; -import io.airbyte.workers.WorkerException; +import io.airbyte.workers.WorkerConstants; import io.airbyte.workers.WorkerUtils; -import io.temporal.activity.Activity; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Duration; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; -import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,118 +44,47 @@ import org.slf4j.LoggerFactory; * outputs are passed to the selected worker. It also makes sures that the outputs of the worker are * persisted to the db. */ -public class TemporalAttemptExecution implements Supplier { +public class TemporalAttemptExecution implements CheckedSupplier { private static final Logger LOGGER = LoggerFactory.getLogger(TemporalAttemptExecution.class); - private static final Duration HEARTBEAT_INTERVAL = Duration.ofSeconds(10); - - public static String WORKFLOW_ID_FILENAME = "WORKFLOW_ID"; private final Path jobRoot; - private final CheckedSupplier, Exception> workerSupplier; - private final Supplier inputSupplier; + private final CheckedFunction execution; private final String jobId; private final BiConsumer mdcSetter; private final CheckedConsumer jobRootDirCreator; - private final CancellationHandler cancellationHandler; - private final Supplier workflowIdProvider; + + @VisibleForTesting + TemporalAttemptExecution(Path workspaceRoot, JobRunConfig jobRunConfig, CheckedFunction execution) { + this(workspaceRoot, jobRunConfig, execution, WorkerUtils::setJobMdc, Files::createDirectories); + } public TemporalAttemptExecution(Path workspaceRoot, JobRunConfig jobRunConfig, - CheckedSupplier, Exception> workerSupplier, - Supplier inputSupplier, - CancellationHandler cancellationHandler) { - this(workspaceRoot, jobRunConfig, workerSupplier, inputSupplier, WorkerUtils::setJobMdc, Files::createDirectories, cancellationHandler, - () -> Activity.getExecutionContext().getInfo().getWorkflowId()); - } - - @VisibleForTesting - TemporalAttemptExecution(Path workspaceRoot, - JobRunConfig jobRunConfig, - CheckedSupplier, Exception> workerSupplier, - Supplier inputSupplier, - BiConsumer mdcSetter, - CheckedConsumer jobRootDirCreator, - CancellationHandler cancellationHandler, - Supplier workflowIdProvider) { + CheckedFunction execution, + BiConsumer mdcSetter, + CheckedConsumer jobRootDirCreator) { this.jobRoot = WorkerUtils.getJobRoot(workspaceRoot, jobRunConfig.getJobId(), jobRunConfig.getAttemptId()); - this.workerSupplier = workerSupplier; - this.inputSupplier = inputSupplier; + this.execution = execution; this.jobId = jobRunConfig.getJobId(); this.mdcSetter = mdcSetter; this.jobRootDirCreator = jobRootDirCreator; - this.cancellationHandler = cancellationHandler; - this.workflowIdProvider = workflowIdProvider; } @Override - public OUTPUT get() { + public T get() throws TemporalJobException { try { mdcSetter.accept(jobRoot, jobId); LOGGER.info("Executing worker wrapper. Airbyte version: {}", new EnvConfigs().getAirbyteVersionOrWarning()); jobRootDirCreator.accept(jobRoot); - final String workflowId = workflowIdProvider.get(); - final Path workflowIdFile = jobRoot.getParent().resolve(WORKFLOW_ID_FILENAME); - IOs.writeFile(workflowIdFile, workflowId); - - final Worker worker = workerSupplier.get(); - final CompletableFuture outputFuture = new CompletableFuture<>(); - final Thread workerThread = getWorkerThread(worker, outputFuture); - final ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); - final Runnable cancellationChecker = getCancellationChecker(worker, workerThread, outputFuture); - - workerThread.start(); - scheduledExecutor.scheduleAtFixedRate(cancellationChecker, 0, HEARTBEAT_INTERVAL.toSeconds(), TimeUnit.SECONDS); - - try { - // block and wait for the output - return outputFuture.get(); - } finally { - LOGGER.info("Stopping cancellation check scheduling..."); - scheduledExecutor.shutdown(); - } + return execution.apply(jobRoot); + } catch (TemporalJobException e) { + throw e; } catch (Exception e) { - throw Activity.wrap(e); + throw new TemporalJobException(jobRoot.resolve(WorkerConstants.LOG_FILENAME), e); } } - private Thread getWorkerThread(Worker worker, CompletableFuture outputFuture) { - return new Thread(() -> { - mdcSetter.accept(jobRoot, jobId); - - try { - final OUTPUT output = worker.run(inputSupplier.get(), jobRoot); - outputFuture.complete(output); - } catch (Throwable e) { - LOGGER.info("Completing future exceptionally...", e); - outputFuture.completeExceptionally(e); - } - }); - } - - private Runnable getCancellationChecker(Worker worker, Thread workerThread, CompletableFuture outputFuture) { - return () -> { - try { - mdcSetter.accept(jobRoot, jobId); - - final Runnable onCancellationCallback = () -> { - LOGGER.info("Running sync worker cancellation..."); - worker.cancel(); - - LOGGER.info("Interrupting worker thread..."); - workerThread.interrupt(); - - LOGGER.info("Cancelling completable future..."); - outputFuture.cancel(false); - }; - - cancellationHandler.checkAndHandleCancellation(onCancellationCallback); - } catch (WorkerException e) { - LOGGER.error("Cancellation checker exception", e); - } - }; - } - } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalClient.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalClient.java index 00b937f5699..2caa89232bb 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalClient.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalClient.java @@ -24,7 +24,6 @@ package io.airbyte.workers.temporal; -import com.google.common.annotations.VisibleForTesting; import io.airbyte.config.JobCheckConnectionConfig; import io.airbyte.config.JobDiscoverCatalogConfig; import io.airbyte.config.JobGetSpecConfig; @@ -38,41 +37,33 @@ import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.scheduler.models.IntegrationLauncherConfig; import io.airbyte.scheduler.models.JobRunConfig; -import io.airbyte.workers.WorkerUtils; import io.temporal.client.WorkflowClient; -import java.nio.file.Path; import java.util.UUID; -import java.util.function.Supplier; public class TemporalClient { - private final Path workspaceRoot; private final WorkflowClient client; - public static TemporalClient production(Path workspaceRoot) { - return new TemporalClient(TemporalUtils.TEMPORAL_CLIENT, workspaceRoot); + public static TemporalClient production() { + return new TemporalClient(TemporalUtils.TEMPORAL_CLIENT); } - // todo (cgardens) - there are two sources of truth on workspace root. we need to get this down to - // one. either temporal decides and can report it or it is injected into temporal runs. - public TemporalClient(WorkflowClient client, Path workspaceRoot) { + public TemporalClient(WorkflowClient client) { this.client = client; - this.workspaceRoot = workspaceRoot; } - public TemporalResponse submitGetSpec(UUID jobId, int attempt, JobGetSpecConfig config) { + public ConnectorSpecification submitGetSpec(UUID jobId, int attempt, JobGetSpecConfig config) throws TemporalJobException { final JobRunConfig jobRunConfig = TemporalUtils.createJobRunConfig(jobId, attempt); final IntegrationLauncherConfig launcherConfig = new IntegrationLauncherConfig() .withJobId(jobId.toString()) .withAttemptId((long) attempt) .withDockerImage(config.getDockerImage()); - return execute(jobRunConfig, - () -> getWorkflowStub(SpecWorkflow.class, TemporalJobType.GET_SPEC).run(jobRunConfig, launcherConfig)); + return getWorkflowStub(SpecWorkflow.class, TemporalJobType.GET_SPEC).run(jobRunConfig, launcherConfig); } - public TemporalResponse submitCheckConnection(UUID jobId, int attempt, JobCheckConnectionConfig config) { + public StandardCheckConnectionOutput submitCheckConnection(UUID jobId, int attempt, JobCheckConnectionConfig config) throws TemporalJobException { final JobRunConfig jobRunConfig = TemporalUtils.createJobRunConfig(jobId, attempt); final IntegrationLauncherConfig launcherConfig = new IntegrationLauncherConfig() .withJobId(jobId.toString()) @@ -80,11 +71,10 @@ public class TemporalClient { .withDockerImage(config.getDockerImage()); final StandardCheckConnectionInput input = new StandardCheckConnectionInput().withConnectionConfiguration(config.getConnectionConfiguration()); - return execute(jobRunConfig, - () -> getWorkflowStub(CheckConnectionWorkflow.class, TemporalJobType.CHECK_CONNECTION).run(jobRunConfig, launcherConfig, input)); + return getWorkflowStub(CheckConnectionWorkflow.class, TemporalJobType.CHECK_CONNECTION).run(jobRunConfig, launcherConfig, input); } - public TemporalResponse submitDiscoverSchema(UUID jobId, int attempt, JobDiscoverCatalogConfig config) { + public AirbyteCatalog submitDiscoverSchema(UUID jobId, int attempt, JobDiscoverCatalogConfig config) throws TemporalJobException { final JobRunConfig jobRunConfig = TemporalUtils.createJobRunConfig(jobId, attempt); final IntegrationLauncherConfig launcherConfig = new IntegrationLauncherConfig() .withJobId(jobId.toString()) @@ -92,11 +82,10 @@ public class TemporalClient { .withDockerImage(config.getDockerImage()); final StandardDiscoverCatalogInput input = new StandardDiscoverCatalogInput().withConnectionConfiguration(config.getConnectionConfiguration()); - return execute(jobRunConfig, - () -> getWorkflowStub(DiscoverCatalogWorkflow.class, TemporalJobType.DISCOVER_SCHEMA).run(jobRunConfig, launcherConfig, input)); + return getWorkflowStub(DiscoverCatalogWorkflow.class, TemporalJobType.DISCOVER_SCHEMA).run(jobRunConfig, launcherConfig, input); } - public TemporalResponse submitSync(long jobId, int attempt, JobSyncConfig config) { + public StandardSyncOutput submitSync(long jobId, int attempt, JobSyncConfig config) throws TemporalJobException { final JobRunConfig jobRunConfig = TemporalUtils.createJobRunConfig(jobId, attempt); final IntegrationLauncherConfig sourceLauncherConfig = new IntegrationLauncherConfig() @@ -116,34 +105,15 @@ public class TemporalClient { .withCatalog(config.getConfiguredAirbyteCatalog()) .withState(config.getState()); - return execute(jobRunConfig, - () -> getWorkflowStub(SyncWorkflow.class, TemporalJobType.SYNC).run( - jobRunConfig, - sourceLauncherConfig, - destinationLauncherConfig, - input)); + return getWorkflowStub(SyncWorkflow.class, TemporalJobType.SYNC).run( + jobRunConfig, + sourceLauncherConfig, + destinationLauncherConfig, + input); } private T getWorkflowStub(Class workflowClass, TemporalJobType jobType) { return client.newWorkflowStub(workflowClass, TemporalUtils.getWorkflowOptions(jobType)); } - @VisibleForTesting - TemporalResponse execute(JobRunConfig jobRunConfig, Supplier executor) { - final Path jobRoot = WorkerUtils.getJobRoot(workspaceRoot, jobRunConfig); - final Path logPath = WorkerUtils.getLogPath(jobRoot); - - T operationOutput = null; - RuntimeException exception = null; - - try { - operationOutput = executor.get(); - } catch (RuntimeException e) { - exception = e; - } - - final JobMetadata metadata = new JobMetadata(exception == null, logPath); - return new TemporalResponse<>(operationOutput, metadata); - } - } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/JobMetadata.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalJobException.java similarity index 62% rename from airbyte-workers/src/main/java/io/airbyte/workers/temporal/JobMetadata.java rename to airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalJobException.java index 278c1234f8a..bd1679e1397 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/JobMetadata.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalJobException.java @@ -25,49 +25,27 @@ package io.airbyte.workers.temporal; import java.nio.file.Path; -import java.util.Objects; -public class JobMetadata { +public class TemporalJobException extends Exception { - private final boolean succeeded; private final Path logPath; - public JobMetadata(final boolean succeeded, final Path logPath) { - this.succeeded = succeeded; + public TemporalJobException(Path logPath) { this.logPath = logPath; } - public boolean isSucceeded() { - return succeeded; + public TemporalJobException(Path logPath, Throwable cause) { + super(cause); + this.logPath = logPath; + } + + public TemporalJobException(Path logPath, String message, Throwable cause) { + super(message, cause); + this.logPath = logPath; } public Path getLogPath() { return logPath; } - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final JobMetadata that = (JobMetadata) o; - return succeeded == that.succeeded && Objects.equals(logPath, that.logPath); - } - - @Override - public int hashCode() { - return Objects.hash(succeeded, logPath); - } - - @Override - public String toString() { - return "JobMetadata{" + - "succeeded=" + succeeded + - ", logPath=" + logPath + - '}'; - } - } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalResponse.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalResponse.java deleted file mode 100644 index efb6b94869f..00000000000 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalResponse.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.workers.temporal; - -import java.util.Objects; -import java.util.Optional; - -public class TemporalResponse { - - private final T output; - private final JobMetadata metadata; - - public static TemporalResponse error(JobMetadata metadata) { - return new TemporalResponse<>(null, metadata); - } - - public static TemporalResponse success(T output, JobMetadata metadata) { - return new TemporalResponse<>(output, metadata); - } - - public TemporalResponse(final T output, final JobMetadata metadata) { - this.output = output; - this.metadata = metadata; - } - - public boolean isSuccess() { - return metadata.isSucceeded(); - } - - /** - * Returns the output of the Temporal job. - * - * @return The output of the Temporal job. Empty if no output or if the job failed. - */ - public Optional getOutput() { - return Optional.ofNullable(output); - } - - public JobMetadata getMetadata() { - return metadata; - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final TemporalResponse that = (TemporalResponse) o; - return Objects.equals(output, that.output) && Objects.equals(metadata, that.metadata); - } - - @Override - public int hashCode() { - return Objects.hash(output, metadata); - } - - @Override - public String toString() { - return "TemporalResponse{" + - "output=" + output + - ", metadata=" + metadata + - '}'; - } - -} diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java index 769d266d2a2..dcb14292e20 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java @@ -46,8 +46,6 @@ public class TemporalUtils { public static final RetryOptions NO_RETRY = RetryOptions.newBuilder().setMaximumAttempts(1).build(); - public static final String DEFAULT_NAMESPACE = "default"; - @FunctionalInterface public interface TemporalJobCreator { diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalAttemptExecutionTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalAttemptExecutionTest.java index 5e10bbc299e..77c1f288775 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalAttemptExecutionTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalAttemptExecutionTest.java @@ -25,25 +25,22 @@ package io.airbyte.workers.temporal; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import io.airbyte.commons.functional.CheckedConsumer; -import io.airbyte.commons.functional.CheckedSupplier; +import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.scheduler.models.JobRunConfig; -import io.airbyte.workers.Worker; -import io.temporal.internal.common.CheckedExceptionWrapper; +import io.airbyte.workers.WorkerConstants; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.function.BiConsumer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.stubbing.Answer; class TemporalAttemptExecutionTest { @@ -52,63 +49,79 @@ class TemporalAttemptExecutionTest { private static final JobRunConfig JOB_RUN_CONFIG = new JobRunConfig().withJobId(JOB_ID).withAttemptId((long) ATTEMPT_ID); private Path jobRoot; + private TemporalJobException expectedException; - private CheckedSupplier, Exception> execution; + private CheckedFunction execution; private BiConsumer mdcSetter; private CheckedConsumer jobRootDirCreator; - private TemporalAttemptExecution attemptExecution; + private TemporalAttemptExecution attemptExecution; @SuppressWarnings("unchecked") @BeforeEach void setup() throws IOException { final Path workspaceRoot = Files.createTempDirectory(Path.of("/tmp"), "temporal_attempt_execution_test"); - jobRoot = workspaceRoot.resolve(JOB_ID).resolve(String.valueOf(ATTEMPT_ID)); jobRoot = workspaceRoot.resolve(String.valueOf(JOB_ID)).resolve(String.valueOf(ATTEMPT_ID)); + expectedException = new TemporalJobException(jobRoot.resolve(WorkerConstants.LOG_FILENAME)); - execution = mock(CheckedSupplier.class); + execution = mock(CheckedFunction.class); mdcSetter = mock(BiConsumer.class); - jobRootDirCreator = Files::createDirectories; + jobRootDirCreator = mock(CheckedConsumer.class); - attemptExecution = new TemporalAttemptExecution<>(workspaceRoot, JOB_RUN_CONFIG, execution, () -> "", mdcSetter, jobRootDirCreator, - mock(CancellationHandler.class), () -> "workflow_id"); + attemptExecution = new TemporalAttemptExecution<>(workspaceRoot, JOB_RUN_CONFIG, execution, mdcSetter, jobRootDirCreator); } @Test - void testSuccessfulSupplierRun() throws Exception { + void testGet() throws Exception { final String expected = "louis XVI"; - Worker worker = mock(Worker.class); - when(worker.run(any(), any())).thenReturn(expected); - - when(execution.get()).thenAnswer((Answer>) invocation -> worker); + when(execution.apply(jobRoot)).thenReturn(expected); final String actual = attemptExecution.get(); assertEquals(expected, actual); - - verify(execution).get(); - verify(mdcSetter, atLeast(2)).accept(jobRoot, JOB_ID); + verify(execution).apply(jobRoot); + verify(mdcSetter).accept(jobRoot, JOB_ID); + verify(jobRootDirCreator).accept(jobRoot); } @Test void testThrowsCheckedException() throws Exception { - when(execution.get()).thenThrow(new IOException()); + when(execution.apply(jobRoot)).thenThrow(new IOException()); - final CheckedExceptionWrapper actualException = assertThrows(CheckedExceptionWrapper.class, () -> attemptExecution.get()); - assertEquals(IOException.class, CheckedExceptionWrapper.unwrap(actualException).getClass()); + final TemporalJobException actualException = assertThrows(TemporalJobException.class, () -> attemptExecution.get()); + assertEquals(expectedException.getLogPath(), actualException.getLogPath()); + assertEquals(IOException.class, actualException.getCause().getClass()); - verify(execution).get(); + verify(execution).apply(jobRoot); verify(mdcSetter).accept(jobRoot, JOB_ID); + verify(jobRootDirCreator).accept(jobRoot); } @Test void testThrowsUnCheckedException() throws Exception { - when(execution.get()).thenThrow(new IllegalArgumentException()); + when(execution.apply(jobRoot)).thenThrow(new IllegalArgumentException()); - assertThrows(IllegalArgumentException.class, () -> attemptExecution.get()); + final TemporalJobException actualException = assertThrows(TemporalJobException.class, () -> attemptExecution.get()); + assertEquals(expectedException.getLogPath(), actualException.getLogPath()); + assertEquals(IllegalArgumentException.class, actualException.getCause().getClass()); - verify(execution).get(); + verify(execution).apply(jobRoot); verify(mdcSetter).accept(jobRoot, JOB_ID); + verify(jobRootDirCreator).accept(jobRoot); + } + + @Test + void testThrowsTemporalJobExceptionException() throws Exception { + final Path otherFilePath = jobRoot.resolve("other file path"); + when(execution.apply(jobRoot)).thenThrow(new TemporalJobException(otherFilePath)); + + final TemporalJobException actualException = assertThrows(TemporalJobException.class, () -> attemptExecution.get()); + assertEquals(otherFilePath, actualException.getLogPath()); + assertNull(actualException.getCause()); + + verify(execution).apply(jobRoot); + verify(mdcSetter).accept(jobRoot, JOB_ID); + verify(jobRootDirCreator).accept(jobRoot); } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java index 52ea0defed9..2716f89374a 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java @@ -24,10 +24,6 @@ package io.airbyte.workers.temporal; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -43,16 +39,9 @@ import io.airbyte.config.StandardSyncInput; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.scheduler.models.IntegrationLauncherConfig; import io.airbyte.scheduler.models.JobRunConfig; -import io.airbyte.workers.WorkerConstants; import io.temporal.client.WorkflowClient; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.UUID; -import java.util.function.Supplier; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; class TemporalClientTest { @@ -76,126 +65,82 @@ class TemporalClientTest { private WorkflowClient workflowClient; private TemporalClient temporalClient; - private Path logPath; @BeforeEach - void setup() throws IOException { - final Path workspaceRoot = Files.createTempDirectory(Path.of("/tmp"), "temporal_client_test"); - logPath = workspaceRoot.resolve(String.valueOf(JOB_ID)).resolve(String.valueOf(ATTEMPT_ID)).resolve(WorkerConstants.LOG_FILENAME); + void setup() { workflowClient = mock(WorkflowClient.class); - temporalClient = new TemporalClient(workflowClient, workspaceRoot); + temporalClient = new TemporalClient(workflowClient); } - @Nested - @DisplayName("Test execute method.") - class ExecuteJob { - - @SuppressWarnings("unchecked") - @Test - void testExecute() { - final Supplier supplier = mock(Supplier.class); - when(supplier.get()).thenReturn("hello"); - - final TemporalResponse response = temporalClient.execute(JOB_RUN_CONFIG, supplier); - - assertNotNull(response); - assertTrue(response.getOutput().isPresent()); - assertEquals("hello", response.getOutput().get()); - assertTrue(response.getMetadata().isSucceeded()); - assertEquals(logPath, response.getMetadata().getLogPath()); - } - - @SuppressWarnings("unchecked") - @Test - void testExecuteWithException() { - final Supplier supplier = mock(Supplier.class); - when(supplier.get()).thenThrow(IllegalStateException.class); - - final TemporalResponse response = temporalClient.execute(JOB_RUN_CONFIG, supplier); - - assertNotNull(response); - assertFalse(response.getOutput().isPresent()); - assertFalse(response.getMetadata().isSucceeded()); - assertEquals(logPath, response.getMetadata().getLogPath()); - } + @Test + void testSubmitGetSpec() throws TemporalJobException { + final SpecWorkflow specWorkflow = mock(SpecWorkflow.class); + when(workflowClient.newWorkflowStub(SpecWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.GET_SPEC))).thenReturn(specWorkflow); + final JobGetSpecConfig getSpecConfig = new JobGetSpecConfig().withDockerImage(IMAGE_NAME1); + temporalClient.submitGetSpec(JOB_UUID, ATTEMPT_ID, getSpecConfig); + specWorkflow.run(JOB_RUN_CONFIG, UUID_LAUNCHER_CONFIG); + verify(workflowClient).newWorkflowStub(SpecWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.GET_SPEC)); } - @Nested - @DisplayName("Test job creation for each configuration type.") - class TestJobSubmission { + @Test + void testSubmitCheckConnection() throws TemporalJobException { + final CheckConnectionWorkflow checkConnectionWorkflow = mock(CheckConnectionWorkflow.class); + when(workflowClient.newWorkflowStub(CheckConnectionWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.CHECK_CONNECTION))) + .thenReturn(checkConnectionWorkflow); + final JobCheckConnectionConfig checkConnectionConfig = new JobCheckConnectionConfig() + .withDockerImage(IMAGE_NAME1) + .withConnectionConfiguration(Jsons.emptyObject()); + final StandardCheckConnectionInput input = new StandardCheckConnectionInput() + .withConnectionConfiguration(checkConnectionConfig.getConnectionConfiguration()); - @Test - void testSubmitGetSpec() { - final SpecWorkflow specWorkflow = mock(SpecWorkflow.class); - when(workflowClient.newWorkflowStub(SpecWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.GET_SPEC))).thenReturn(specWorkflow); - final JobGetSpecConfig getSpecConfig = new JobGetSpecConfig().withDockerImage(IMAGE_NAME1); + temporalClient.submitCheckConnection(JOB_UUID, ATTEMPT_ID, checkConnectionConfig); + checkConnectionWorkflow.run(JOB_RUN_CONFIG, UUID_LAUNCHER_CONFIG, input); + verify(workflowClient).newWorkflowStub(CheckConnectionWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.CHECK_CONNECTION)); + } - temporalClient.submitGetSpec(JOB_UUID, ATTEMPT_ID, getSpecConfig); - specWorkflow.run(JOB_RUN_CONFIG, UUID_LAUNCHER_CONFIG); - verify(workflowClient).newWorkflowStub(SpecWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.GET_SPEC)); - } + @Test + void testSubmitDiscoverSchema() throws TemporalJobException { + final DiscoverCatalogWorkflow discoverCatalogWorkflow = mock(DiscoverCatalogWorkflow.class); + when(workflowClient.newWorkflowStub(DiscoverCatalogWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.DISCOVER_SCHEMA))) + .thenReturn(discoverCatalogWorkflow); + final JobDiscoverCatalogConfig checkConnectionConfig = new JobDiscoverCatalogConfig() + .withDockerImage(IMAGE_NAME1) + .withConnectionConfiguration(Jsons.emptyObject()); + final StandardDiscoverCatalogInput input = new StandardDiscoverCatalogInput() + .withConnectionConfiguration(checkConnectionConfig.getConnectionConfiguration()); - @Test - void testSubmitCheckConnection() { - final CheckConnectionWorkflow checkConnectionWorkflow = mock(CheckConnectionWorkflow.class); - when(workflowClient.newWorkflowStub(CheckConnectionWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.CHECK_CONNECTION))) - .thenReturn(checkConnectionWorkflow); - final JobCheckConnectionConfig checkConnectionConfig = new JobCheckConnectionConfig() - .withDockerImage(IMAGE_NAME1) - .withConnectionConfiguration(Jsons.emptyObject()); - final StandardCheckConnectionInput input = new StandardCheckConnectionInput() - .withConnectionConfiguration(checkConnectionConfig.getConnectionConfiguration()); + temporalClient.submitDiscoverSchema(JOB_UUID, ATTEMPT_ID, checkConnectionConfig); + discoverCatalogWorkflow.run(JOB_RUN_CONFIG, UUID_LAUNCHER_CONFIG, input); + verify(workflowClient).newWorkflowStub(DiscoverCatalogWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.DISCOVER_SCHEMA)); + } - temporalClient.submitCheckConnection(JOB_UUID, ATTEMPT_ID, checkConnectionConfig); - checkConnectionWorkflow.run(JOB_RUN_CONFIG, UUID_LAUNCHER_CONFIG, input); - verify(workflowClient).newWorkflowStub(CheckConnectionWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.CHECK_CONNECTION)); - } + @Test + void testSubmitSync() throws TemporalJobException { + final SyncWorkflow discoverCatalogWorkflow = mock(SyncWorkflow.class); + when(workflowClient.newWorkflowStub(SyncWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.SYNC))) + .thenReturn(discoverCatalogWorkflow); + final JobSyncConfig syncConfig = new JobSyncConfig() + .withSourceDockerImage(IMAGE_NAME1) + .withSourceDockerImage(IMAGE_NAME2) + .withSourceConfiguration(Jsons.emptyObject()) + .withDestinationConfiguration(Jsons.emptyObject()) + .withConfiguredAirbyteCatalog(new ConfiguredAirbyteCatalog()); + final StandardSyncInput input = new StandardSyncInput() + .withPrefix(syncConfig.getPrefix()) + .withSourceConfiguration(syncConfig.getSourceConfiguration()) + .withDestinationConfiguration(syncConfig.getDestinationConfiguration()) + .withCatalog(syncConfig.getConfiguredAirbyteCatalog()) + .withState(syncConfig.getState()); - @Test - void testSubmitDiscoverSchema() { - final DiscoverCatalogWorkflow discoverCatalogWorkflow = mock(DiscoverCatalogWorkflow.class); - when(workflowClient.newWorkflowStub(DiscoverCatalogWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.DISCOVER_SCHEMA))) - .thenReturn(discoverCatalogWorkflow); - final JobDiscoverCatalogConfig checkConnectionConfig = new JobDiscoverCatalogConfig() - .withDockerImage(IMAGE_NAME1) - .withConnectionConfiguration(Jsons.emptyObject()); - final StandardDiscoverCatalogInput input = new StandardDiscoverCatalogInput() - .withConnectionConfiguration(checkConnectionConfig.getConnectionConfiguration()); - - temporalClient.submitDiscoverSchema(JOB_UUID, ATTEMPT_ID, checkConnectionConfig); - discoverCatalogWorkflow.run(JOB_RUN_CONFIG, UUID_LAUNCHER_CONFIG, input); - verify(workflowClient).newWorkflowStub(DiscoverCatalogWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.DISCOVER_SCHEMA)); - } - - @Test - void testSubmitSync() { - final SyncWorkflow discoverCatalogWorkflow = mock(SyncWorkflow.class); - when(workflowClient.newWorkflowStub(SyncWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.SYNC))) - .thenReturn(discoverCatalogWorkflow); - final JobSyncConfig syncConfig = new JobSyncConfig() - .withSourceDockerImage(IMAGE_NAME1) - .withSourceDockerImage(IMAGE_NAME2) - .withSourceConfiguration(Jsons.emptyObject()) - .withDestinationConfiguration(Jsons.emptyObject()) - .withConfiguredAirbyteCatalog(new ConfiguredAirbyteCatalog()); - final StandardSyncInput input = new StandardSyncInput() - .withPrefix(syncConfig.getPrefix()) - .withSourceConfiguration(syncConfig.getSourceConfiguration()) - .withDestinationConfiguration(syncConfig.getDestinationConfiguration()) - .withCatalog(syncConfig.getConfiguredAirbyteCatalog()) - .withState(syncConfig.getState()); - - final IntegrationLauncherConfig destinationLauncherConfig = new IntegrationLauncherConfig() - .withJobId(String.valueOf(JOB_ID)) - .withAttemptId((long) ATTEMPT_ID) - .withDockerImage(IMAGE_NAME2); - - temporalClient.submitSync(JOB_ID, ATTEMPT_ID, syncConfig); - discoverCatalogWorkflow.run(JOB_RUN_CONFIG, LAUNCHER_CONFIG, destinationLauncherConfig, input); - verify(workflowClient).newWorkflowStub(SyncWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.SYNC)); - } + final IntegrationLauncherConfig destinationLauncherConfig = new IntegrationLauncherConfig() + .withJobId(String.valueOf(JOB_ID)) + .withAttemptId((long) ATTEMPT_ID) + .withDockerImage(IMAGE_NAME2); + temporalClient.submitSync(JOB_ID, ATTEMPT_ID, syncConfig); + discoverCatalogWorkflow.run(JOB_RUN_CONFIG, LAUNCHER_CONFIG, destinationLauncherConfig, input); + verify(workflowClient).newWorkflowStub(SyncWorkflow.class, TemporalUtils.getWorkflowOptions(TemporalJobType.SYNC)); } } diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 6c0d42bb63d..e7d602d62ea 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -29,7 +29,6 @@ * [Files](integrations/sources/file.md) * [Freshdesk](integrations/sources/freshdesk.md) * [GitHub](integrations/sources/github.md) - * [GitLab](integrations/sources/gitlab.md) * [Google Adwords](integrations/sources/google-adwords.md) * [Google Analytics](integrations/sources/googleanalytics.md) * [Google Directory](integrations/sources/google-directory.md) @@ -92,7 +91,7 @@ * [Developing Connectors](contributing-to-airbyte/building-new-connector/README.md) * [Best Practices](contributing-to-airbyte/building-new-connector/best-practices.md) * [Java Connectors](contributing-to-airbyte/building-new-connector/java-connectors.md) - * [Monorepo Python Development](contributing-to-airbyte/building-new-connector/monorepo-python-development.md) + * [Python Connectors](contributing-to-airbyte/building-new-connector/python-connectors.md) * [Testing Connectors](contributing-to-airbyte/building-new-connector/testing-connectors.md) * [Standard Source Test Suite](contributing-to-airbyte/building-new-connector/standard-source-tests.md) * [Code Style](contributing-to-airbyte/code-style.md) @@ -118,7 +117,7 @@ * [Strategy](company-handbook/strategy.md) * [Business Model](company-handbook/business-model.md) * [Career & Open Positions](career-and-open-positions/README.md) - * [Founding Developer Advocate](career-and-open-positions/founding-developer-advocate.md) + * [Founding Developer Advocate](career-and-open-positions/founding-developer-evangelist.md) * [Senior Software Engineer](career-and-open-positions/senior-software-engineer.md) * [Operations Manager](career-and-open-positions/operations-manager.md) * [License](license.md) diff --git a/docs/api/generated-api-html/index.html b/docs/api/generated-api-html/index.html index 2de149f9eb3..92e90b14d2f 100644 --- a/docs/api/generated-api-html/index.html +++ b/docs/api/generated-api-html/index.html @@ -261,7 +261,6 @@ font-style: italic;

Jobs

@@ -1903,94 +1902,6 @@ font-style: italic;

Jobs

-
-
- Up - 
post /v1/jobs/cancel
-
Cancels a job (cancelJob)
-
- - -

Consumes

- This API call consumes the following media types via the Content-Type request header: -
    -
  • application/json
    • -
- -

Request body

-
-
JobIdRequestBody JobIdRequestBody (required)
- -
Body Parameter —
- -
- - - - -

Return type

-
- JobInfoRead - -
- - - -

Example data

-
Content-Type: application/json
- 
{
-  "job" : {
-    "createdAt" : 6,
-    "configId" : "configId",
-    "id" : 0,
-    "updatedAt" : 1
-  },
-  "attempts" : [ {
-    "attempt" : {
-      "createdAt" : 5,
-      "bytesSynced" : 9,
-      "endedAt" : 7,
-      "id" : 5,
-      "recordsSynced" : 3,
-      "updatedAt" : 2
-    },
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    }
-  }, {
-    "attempt" : {
-      "createdAt" : 5,
-      "bytesSynced" : 9,
-      "endedAt" : 7,
-      "id" : 5,
-      "recordsSynced" : 3,
-      "updatedAt" : 2
-    },
-    "logs" : {
-      "logLines" : [ "logLines", "logLines" ]
-    }
-  } ]
-}
- -

Produces

- This API call produces the following media types according to the Accept request header; - the media type will be conveyed by the Content-Type response header. -
    -
  • application/json
    • -
- -

Responses

-

200

- Successful operation - JobInfoRead -

404

- Job not found - -

422

- Invalid Input - -
-
Up diff --git a/docs/architecture/basic-normalization.md b/docs/architecture/basic-normalization.md index e53652de905..739be45bb1e 100644 --- a/docs/architecture/basic-normalization.md +++ b/docs/architecture/basic-normalization.md @@ -89,7 +89,7 @@ CREATE TABLE "cars" ( "_airbyte_normalized_at" TIMESTAMP_WITH_TIMEZONE, "make" VARCHAR, - "model" VARCHAR + "model" VARCHAR, ); CREATE TABLE "limited_editions" ( @@ -122,7 +122,7 @@ CREATE TABLE "cars" ( "_airbyte_normalized_at" TIMESTAMP_WITH_TIMEZONE, "make" VARCHAR, - "model" VARCHAR + "model" VARCHAR, ); CREATE TABLE "limited_editions" ( @@ -156,7 +156,7 @@ CREATE TABLE "cars" ( "_airbyte_normalized_at" TIMESTAMP_WITH_TIMEZONE, "make" VARCHAR, - "model" VARCHAR + "model" VARCHAR, ); CREATE TABLE "powertrain_specs" ( @@ -170,86 +170,3 @@ CREATE TABLE "powertrain_specs" ( ); ``` -### Naming Collisions for un-nested objects - -When extracting nested objects or arrays, the Basic Normalization process needs to figure out new names for the expanded tables. - -For example, if we had a `cars` table with a nested column `cars` containing an object whose schema is identical to the parent table. - -```javascript -{ - "make": "alfa romeo", - "model": "4C coupe", - "cars": [ - { "make": "audi", "model": "A7" }, - { "make" : "lotus" , "model": "elise" } - { "make" : "chevrolet" , "model": "mustang" } - ] -} -``` - -The expanded table would have a conflict in terms of naming since both are named `cars`. -To avoid name collisions and ensure a more consistent naming scheme, Basic Normalization chooses the expanded name as follows: -- `cars` for the original parent table -- `cars_da3_cars` for the expanded nested columns following this naming scheme in 3 parts: `__` - -1. Json path: The entire json path string with '_' characters used as delimiters to reach the table that contains the nested column name. -2. Hash: Hash of the entire json path to reach the nested column reduced to 3 characters. This is to make sure we have a unique name (in case part of the name gets truncated, see below) -3. Nested column name: name of the column being expanded into its own table. - -By following this strategy, nested columns should "never" collide with other table names. -If it does, an exception will probably be thrown either by the normalization process or by DBT that runs afterward. - -```sql -CREATE TABLE "cars" ( - "_airbyte_cars_hashid" VARCHAR, - "_airbyte_emitted_at" TIMESTAMP_WITH_TIMEZONE, - "_airbyte_normalized_at" TIMESTAMP_WITH_TIMEZONE, - - "make" VARCHAR, - "model" VARCHAR -); - -CREATE TABLE "cars_da3_cars" ( - "_airbyte_cars_hashid" VARCHAR, - "_airbyte_cars_foreign_hashid" VARCHAR, - "_airbyte_emitted_at" TIMESTAMP_WITH_TIMEZONE, - "_airbyte_normalized_at" TIMESTAMP_WITH_TIMEZONE, - - "make" VARCHAR, - "model" VARCHAR -); -``` - -### Naming limitations & truncation - -Note that different destinations have various naming limitations, most commonly on how long names can be. -For instance, the Postgres documentation states: - -> The system uses no more than NAMEDATALEN-1 bytes of an identifier; -> longer names can be written in commands, but they will be truncated. -> By default, NAMEDATALEN is 64 so the maximum identifier length is 63 bytes - -Most modern data warehouses have name lengths limits on the longer side, so this should not affect us that often. -Basic Normalization will fallback to the following rules: - -1. No Truncate if under destination's character limits - -However, in the rare cases where these limits are reached: - -2. Truncate only the `Json path` to fit into destination's character limits -3. Truncate the `Json path` to at least the 10 first characters, then truncate the nested column name starting in the middle to preserve prefix/suffix substrings intact (whenever a truncate in the middle is made, two '__' characters are also inserted to denote where it happened) to fit into destination's character limits - -As an example from the hubspot source, we could have the following tables with nested columns: - -| Description | Example 1 | Example 2 | -| :--- | :--- | :--- | -| Original Stream Name | companies | deals | -| Json path to the nested column | `companies/property_engagements_last_meeting_booked_campaign` | `deals/properties/engagements_last_meeting_booked_medium` | -| Final table name of expanded nested column on BigQuery | companies\_2e8\_property\_engagements\_last\_meeting\_booked\_campaign | deals\_properties\_6e6\_engagements\_last\_meeting\_booked\_medium | -| Final table name of expanded nested column on Postgres | companies\_2e8\_property\_engag\_\_oked\_campaign | deals\_prop\_6e6\_engagements\_l\_\_booked\_medium | - -Note that all the choices made by Normalization as described in this documentation page in terms of naming could be overriden by your own custom choices. -To do so, you can follow the following tutorial -- to build a [custom SQL view](../tutorials/connecting-el-with-t-using-sql.md) with your own naming conventions -- to export, edit and run [custom DBT normalization](../tutorials/connecting-el-with-t-using-dbt.md) yourself diff --git a/docs/career-and-open-positions/founding-developer-advocate.md b/docs/career-and-open-positions/founding-developer-evangelist.md similarity index 100% rename from docs/career-and-open-positions/founding-developer-advocate.md rename to docs/career-and-open-positions/founding-developer-evangelist.md diff --git a/docs/contributing-to-airbyte/building-new-connector/README.md b/docs/contributing-to-airbyte/building-new-connector/README.md index d3fcb9b294b..f1535927ce6 100644 --- a/docs/contributing-to-airbyte/building-new-connector/README.md +++ b/docs/contributing-to-airbyte/building-new-connector/README.md @@ -18,7 +18,7 @@ To add a new connector you need to: 1. Implement & Package your connector in an Airbyte Protocol compliant Docker image 2. Add integration tests for your connector. At a minimum, all connectors must pass [Airbyte's standard test suite](testing-connectors.md), but you can also add your own tests. -3. Document how to build & test your connector +3. Add the appropriate Gradle tasks to build the image within the Airbyte monorepo and CI Each requirement has a subsection below. @@ -48,22 +48,15 @@ If you are developing a Python/Singer connector, you may find the [building a Py At a minimum, your connector must implement the standard tests described in [Testing Connectors](testing-connectors.md) -### 3. Document building & testing your connector -To merge your connector, Airbyte needs to know how to build & test it. If you're writing in Python or Java, skip this section -- it is provided -automatically. +### 3. Integrating with Gradle -If you're writing in another language, please document the commands needed to: -1. Build your connector docker image (usually this is just `docker build .` but let us know if there are necessary flags, gotchas, etc..) -2. Run any unit or integration tests _in a Docker image_. +Generated templates provide the following Gradle tasks: -Your integration and unit tests must be runnable entirely within a Docker image. This is important to guarantee consistent build environments. - -When you submit a PR to Airbyte with your connector, the reviewer will use the commands you provide to integrate your connector into Airbyte's build -system as follows: 1. `:airbyte-integrations:connectors:source-:build` should run unit tests and build the integration's Docker image 2. `:airbyte-integrations:connectors:source-:integrationTest` should run integration tests including Airbyte's Standard test suite. ### Best practices + Make sure to review the [Best Practices for Connector Development](best-practices.md) guide. Following best practices is **not** a requirement for merging your contribution to Airbyte, but it certainly doesn't hurt ;\) ## Updating a connector diff --git a/docs/contributing-to-airbyte/building-new-connector/monorepo-python-development.md b/docs/contributing-to-airbyte/building-new-connector/python-connectors.md similarity index 91% rename from docs/contributing-to-airbyte/building-new-connector/monorepo-python-development.md rename to docs/contributing-to-airbyte/building-new-connector/python-connectors.md index 89bba34738a..95a500634f6 100644 --- a/docs/contributing-to-airbyte/building-new-connector/monorepo-python-development.md +++ b/docs/contributing-to-airbyte/building-new-connector/python-connectors.md @@ -1,7 +1,8 @@ -# Monorepo Python Development +# Python Connectors -This guide contains instructions on how to setup Python with Gradle within the Airbyte Monorepo. If you are a contributor working on one or two connectors, -this page is most likely not relevant to you. Instead, you should use your standard Python development flow. +Although we support any language to build connectors, many of them are developed in `python`. + +In order to provide the best developer experience, here are some instructions on how to configure your python project. ## Python Connector Development diff --git a/docs/contributing-to-airbyte/building-new-connector/testing-connectors.md b/docs/contributing-to-airbyte/building-new-connector/testing-connectors.md index e0932dd52f4..74aa04e0360 100644 --- a/docs/contributing-to-airbyte/building-new-connector/testing-connectors.md +++ b/docs/contributing-to-airbyte/building-new-connector/testing-connectors.md @@ -70,9 +70,8 @@ Here are some example commands: 1. `/test connector=all` - Runs integration tests for all connectors in a single GitHub workflow. Some of our integration tests interact with rate-limited resources, so please use this judiciously. 2. `/test connector=source-sendgrid` - Runs integration tests for a single connector on the latest PR commit. -3. `/test connector=connectors/source-sendgrid` - Runs integration tests for a single connector on the latest PR commit. -4. `/test connector=source-sendgrid ref=master` - Runs integration tests for a single connector on a different branch. -5. `/test connector=source-sendgrid ref=d5c53102` - Runs integration tests for a single connector on a specific commit. +3. `/test connector=source-sendgrid ref=master` - Runs integration tests for a single connector on a different branch. +4. `/test connector=source-sendgrid ref=d5c53102` - Runs integration tests for a single connector on a specific commit. A command dispatcher GitHub workflow will launch on comment submission. This dispatcher will add an :eyes: reaction to the comment when it starts processing. If there is an error dispatching your request, an error will be appended to your comment. If it launches the test run successfully, a :rocket: reaction will appear on your comment. @@ -80,21 +79,7 @@ Once the integration test workflow launches, it will append a link to the workfl Integration tests can also be manually requested by clicking "[Run workflow](https://github.com/airbytehq/airbyte/actions?query=workflow%3Aintegration-test)" and specifying the connector and GitHub ref. -### 3. Requesting GitHub PR publishing Docker Images - -In order for users to reference the new versions of a connector, it needs to be published and available in the [dockerhub](https://hub.docker.com/r/airbyte/source-sendgrid/tags?page=1&ordering=last_updated) with the latest tag updated. - -As seen previously, GitHub workflow can be triggered by comment submission. -Publishing docker images to the dockerhub repository can also be submitted likewise: - -Note that integration tests can be triggered with a slightly different syntax for arguments. -This second set is required to distinguish between `connectors` and `bases` folders. -Thus, it is also easier to switch between the `/test` and `/publish` commands: - -- `/test connector=connectors/source-sendgrid` - Runs integration tests for a single connector on the latest PR commit. -- `/publish connector=connectors/source-sendgrid` - Publish the docker image if it doesn't exist for a single connector on the latest PR commit. - -### 4. Automatically Run From `master` +### 3. Automatically Run From `master` Commits to `master` attempt to launch integration tests. Two workflows launch for each commit: one is a launcher for integration tests, the other is the core build \(the same as the default for PR and branch builds\). diff --git a/docs/integrations/connector-health.md b/docs/integrations/connector-health.md index 7adb305293b..ad202d98c6e 100644 --- a/docs/integrations/connector-health.md +++ b/docs/integrations/connector-health.md @@ -24,16 +24,13 @@ Clicking on a status will take you to a summary of past tests that link to the G | Facebook Marketing | [![source-facebook-marketing](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-facebook-marketing%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-facebook-marketing) | ✅ | | | Files | [![source-file](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-file%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-file) | ✅ | | | Freshdesk | [![source-freshdesk](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-freshdesk%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-freshdesk) | | | -| GitHub | [![source-github-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-github-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-github-singer) | ✅ | | -| GitLab | [![source-gitlab-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-gitlab-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-gitlab-singer) | ✅ | | -| Google Adwords | [![source-google-adwords-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-google-adwords-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-google-adwords-singer) | ✅ | **March 22nd, 2021:** Airbyte's API token used for testing is getting throttled. We currently have an application with Google to increase our rate limit. | -| Google Analytics | [![source-googleanalytics-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-googleanalytics-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-googleanalytics-singer) | ✅ | | +| GitHub | [![source-github-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-github-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-github-singer) | | | +| Google Adwords | [![source-google-adwords-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-google-adwords-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-google-adwords-singer) | ✅ | **February 2nd, 2021:** Credentials used in CI were out of date. | +| Google Analytics | [![source-googleanalytics-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-googleanalytics-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-googleanalytics-singer) | | **January 25, 2021:** Data in the Google Analytics test account changed mid-test, causing the expected and actual results to diverge and the test to fail. | | Google Sheets | [![source-google-sheets](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-google-sheets%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-google-sheets) | ✅ | | -| Google Directory API | [![source-google-directory](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-google-directory%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-google-directory) | ✅ | | | Greenhouse | [![source-greenhouse](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-greenhouse%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-greenhouse) | | | | HTTP Request | [![source-http-request](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-http-request%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-http-request) | | | -| Hubspot | [![source-hubspot-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-hubspot%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-hubspot) | ✅ | | -| Instagram | [![source-instagram](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-instagram%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-instagram) | ✅ | | +| Hubspot | [![source-hubspot-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-hubspot-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-hubspot-singer) | ✅ | | | Intercom | [![source-intercom-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-intercom-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-intercom-singer) | | | | Jira | [![source-jira](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-jira%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-jira) | | | | Looker | [![source-looker](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-looker%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-looker) | | \*\*\*\* | @@ -42,7 +39,6 @@ Clicking on a status will take you to a summary of past tests that link to the G | Microsoft SQL Server \(MSSQL\) | [![source-mssql](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-mssql%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-mssql) | ⏱Certification in progress | | | Microsoft Teams | [![source-microsoft-teams](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-microsoft-teams%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-microsoft-teams) | | | | Mixpanel | [![source-mixpanel-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-mixpanel-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-mixpanel-singer) | | | -| Mongo DB | [![source-mongodb](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-mongodb%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-mongodb) | | | | MySQL | [![source-mysql](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-mysql%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-mysql) | ⏱Certification in progress | | | Plaid | [![source-plaid](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-plaid%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-plaid) | | | | Postgres | [![source-postgres](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-postgres%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-postgres) | ⏱Certification in progress | | @@ -53,8 +49,7 @@ Clicking on a status will take you to a summary of past tests that link to the G | Shopify | [![source-shopify-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-shopify-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-shopify-singer) | ✅ | | | Slack | [![source-slack-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-slack-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-slack-singer) | | **January 25, 2021:** The underlying Slack instance \(Airbyte's public slack\) receives enough traffic that data almost always changes mid-test, causing a discrepancy in expected and actual results. The fix will be to pull data between two specific dates in the past to avoid data changing in real time. | | Stripe | [![source-stripe-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-stripe-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-stripe-singer) | ✅ | | -| Twilio | [![source-twilio-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-twilio-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-tempo) | | | -| Tempo | [![source-tempo](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-tempo%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-tempo) | | | +| Twilio | [![source-twilio-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-twilio-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-twilio-singer) | | | | Zendesk Support | [![source-zendesk-support-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-zendesk-support-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-zendesk-support-singer) | | \*\*\*\* | | Zoom | [![source-zoom-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-zoom-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-zoom-singer) | | | diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md deleted file mode 100644 index 60738f72e21..00000000000 --- a/docs/integrations/sources/gitlab.md +++ /dev/null @@ -1,56 +0,0 @@ -# Gitlab - -## Overview - -The Gitlab source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. - -This Gitlab source wraps the [Singer Gitlab Tap](https://gitlab.com/meltano/tap-gitlab). - -### Output schema - -This connector outputs the following streams: - -* [Branches](https://docs.gitlab.com/ee/api/branches.html) -* [Commits](https://docs.gitlab.com/ee/api/commits.html) (Incremental) -* [Issues](https://docs.gitlab.com/ee/api/issues.html) (Incremental) -* [Pipelines](https://docs.gitlab.com/ee/api/pipelines.html) (Incremental) -* [Jobs](https://docs.gitlab.com/ee/api/jobs.html) -* [Projects](https://docs.gitlab.com/ee/api/projects.html) (Incremental) -* [Project Milestones](https://docs.gitlab.com/ee/api/milestones.html) -* [Project Merge Requests](https://docs.gitlab.com/ee/api/merge_requests.html) (Incremental) -* [Users](https://docs.gitlab.com/ee/api/users.html) -* [Groups](https://docs.gitlab.com/ee/api/groups.html) -* [Group Milestones](https://docs.gitlab.com/ee/api/group_milestones.html) -* [Group and Project members](https://docs.gitlab.com/ee/api/members.html) -* [Tags](https://docs.gitlab.com/ee/api/tags.html) -* [Releases](https://docs.gitlab.com/ee/api/releases/index.html) -* [Group Labels](https://docs.gitlab.com/ee/api/group_labels.html) -* [Project Labels](https://docs.gitlab.com/ee/api/labels.html) -* [Epics](https://docs.gitlab.com/ee/api/epics.html) (Incremental, only available for GitLab Ultimate and GitLab.com Gold accounts) -* [Epic Issues](https://docs.gitlab.com/ee/api/epic_issues.html) (only available for GitLab Ultimate and GitLab.com Gold accounts) - -### Features - -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Partially (not all streams) | -| SSL connection | Yes | - -### Performance considerations - -Gitlab has the [rate limits](https://docs.gitlab.com/ee/user/gitlab_com/index.html#gitlabcom-specific-rate-limits), but the Gitlab connector should not run into Gitlab API limitations under normal usage. -Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. - -## Getting started - -### Requirements - -* Gitlab Account -* Gitlab Personal Access Token wih the necessary permissions \(described below\) - -### Setup guide - -Log into Gitlab and then generate a [personal access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html). - -Your token should have the `read_api` scope, that Grants read access to the API, including all groups and projects, the container registry, and the package registry. diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index c2c2ec3e46a..baaf1b3da9b 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -2,27 +2,11 @@ ## Overview -The Hubspot connector can be used to sync your Hubspot data. It supports full refresh sync for all streams and incremental sync for Email Events and Subscription Changes streams. +The Hubspot connector can be used to sync your Hubspot data. It supports full refresh and incremental sync. It is based on the [Singer Hubspot Tap](https://github.com/singer-io/tap-hubspot). ### Output schema -Several output streams are available from this source: -* [Campaigns](https://developers.hubspot.com/docs/methods/email/get_campaign_data) -* [Companies](https://developers.hubspot.com/docs/api/crm/companies) -* [Contact Lists](http://developers.hubspot.com/docs/methods/lists/get_lists) -* [Contacts](https://developers.hubspot.com/docs/methods/contacts/get_contacts) -* [Deal Pipelines](https://developers.hubspot.com/docs/methods/pipelines/get_pipelines_for_object_type) -* [Deals](https://developers.hubspot.com/docs/api/crm/deals) -* [Email Events](https://developers.hubspot.com/docs/methods/email/get_events) (Incremental) -* [Engagements](https://legacydocs.hubspot.com/docs/methods/engagements/get-all-engagements) -* [Forms](https://developers.hubspot.com/docs/api/marketing/forms) -* [Line Items](https://developers.hubspot.com/docs/api/crm/line-items) -* [Owners](https://developers.hubspot.com/docs/methods/owners/get_owners) -* [Products](https://developers.hubspot.com/docs/api/crm/products) -* [Quotes](https://developers.hubspot.com/docs/api/crm/quotes) -* [Subscription Changes](https://developers.hubspot.com/docs/methods/email/get_subscriptions_timeline) (Incremental) -* [Tickets](https://developers.hubspot.com/docs/api/crm/tickets) -* [Workflows](https://legacydocs.hubspot.com/docs/methods/workflows/v3/get_workflows) +Several output streams are available from this source \(campaigns, contacts, deals, etc.\) For a comprehensive output schema [look at the Singer tap schema files](https://github.com/singer-io/tap-hubspot/tree/master/tap_hubspot/schemas). ### Features @@ -30,7 +14,7 @@ Several output streams are available from this source: | :--- | :--- | | Full Refresh Sync | Yes | | Incremental Sync | Yes | -| Replicate Incremental Deletes | No | +| Replicate Incremental Deletes | Coming soon | | SSL connection | Yes | ### Performance considerations @@ -46,7 +30,9 @@ The connector is restricted by normal Hubspot [rate limitations](https://legacyd ### Setup guide -This connector supports only authentication with API Key. -To obtain API key for the account go to settings -> integrations \(under the account banner\) -> api key. -If you already have an api key you can use that. Otherwise generated a new one. -See [docs](https://knowledge.hubspot.com/integrations/how-do-i-get-my-hubspot-api-key) for more details. +\*There are two ways of performing auth with hubspot \(api key and oauth\): + +* For api key auth, in Hubspot, for the account to go settings -> integrations \(under the account banner\) -> api key. If you already have an api key you can use that. Otherwise generated a new one. + * Note: The Hubspot [docs](https://legacydocs.hubspot.com/docs/methods/auth/oauth-overview) recommends that api key auth is only used for testing purposes. +* For oauth follow the [oauth instruction](https://developers.hubspot.com/docs/api/oauth-quickstart-guide) in Hubspot to get client\_id, client\_secret, redirect\_uri, and refresh\_token. + diff --git a/docs/tutorials/building-a-python-source.md b/docs/tutorials/building-a-python-source.md index b939e327909..2d7a8f11fcd 100644 --- a/docs/tutorials/building-a-python-source.md +++ b/docs/tutorials/building-a-python-source.md @@ -17,7 +17,7 @@ All the commands below assume that `python` points to a version of python 3. On ### Creating a Source * Step 1: Create the source using template -* Step 2: Build the newly generated source +* Step 2: Build the newly generated source `./gradlew :airbyte-integrations:connectors:source-:build` * Step 3: Set up your Airbyte development environment * Step 4: Implement `spec` \(and define the specification for the source `airbyte-integrations/connectors/source-/spec.json`\) * Step 5: Implement `check` @@ -42,8 +42,10 @@ All `./gradlew` commands must be run from the root of the airbyte project. * If you need help with any step of the process, feel free to submit a PR with your progress and any questions you have. * Submit a PR. * To run integration tests, Airbyte needs access to a test account/environment. Coordinate with an Airbyte engineer \(via the PR\) to add test credentials so that we can run tests for the integration in the CI. \(We will create our own test account once you let us know what source we need to create it for.\) -* Once the config is stored in Github Secrets, edit `.github/workflows/test-command.yml` and `.github/workflows/publish-command.yml` to inject the config into the build environment. +* Once the config is stored in Github Secrets, edit `.github/workflows/test-command.yml` to inject the config into the build environment. * Edit the `airbyte/tools/bin/ci_credentials.sh` script to pull the script from the build environment and write it to `secrets/config.json` during the build. +* From the `airbyte` project root, run `./gradlew :airbyte-integrations:connectors:source-:build` to make sure your module builds. +* Apply Airbyte auto formatting `./gradlew format` and commit any changes. {% hint style="info" %} If you have a question about a step the Submitting a Source to Airbyte checklist include it in your PR or ask it on [slack](https://slack.airbyte.io). @@ -66,16 +68,9 @@ Select the `python` template and then input the name of your connector. For this ### Step 2: Build the newly generated source -Build the source by running: +Build the source using: `./gradlew :airbyte-integrations:connectors:source-:build` -``` -cd airbyte-integrations/connectors/source- -python -m venv .venv # Create a virtual environment in the .venv directory -source .venv/bin/activate # enable the venv -pip install -r requirements.txt -``` - -This step sets up the initial python environment. **All** subsequent `python` or `pip` commands assume you have activated your virtual environment. +This step sets up the initial python environment. By sanity checking that the source builds at the beginning we have a good starting place for developing our source. ### Step 3: Set up your Airbyte development environment @@ -90,12 +85,11 @@ The generator creates a file `source_/source.py`. This will be wher Python dependencies for your source should be declared in `airbyte-integrations/connectors/source-/setup.py` in the `install_requires` field. You will notice that a couple of Airbyte dependencies are already declared there. Do not remove these; they give your source access to the helper interface that is provided by the generator. -You may notice that there is a `requirements.txt` in your source's directory as well. Do not touch this. It is autogenerated and used to provide Airbyte - dependencies. All your dependencies should be declared in `setup.py`. +You may notice that there is a `requirements.txt` in your source's directory as well. Do not touch this. It helps IDEs pull in local Airbyte dependencies to help with code completion. It is _not_ used outside of the development environment. All dependencies should be declared in `setup.py`. #### Development Environment -The commands we ran above created a virtual environment for your source. If you want your IDE to auto complete and resolve dependencies properly, point it at the virtual env `airbyte-integrations/connectors/source-/.venv`. Also anytime you change the dependencies in the `setup.py` make sure to re-run the build command. The build system will handle installing all dependencies in the `setup.py` into the virtual environment. +Running `./gradlew :airbyte-integrations:connectors:source-:build` creates a virtual environment for your source. If you want your IDE to auto complete and resolve dependencies properly, point it at the virtual env `airbyte-integrations/connectors/source-/.venv`. Also anytime you change the dependencies in the `setup.py` make sure to re-run the build command. The build system will handle installing all dependencies in the `setup.py` into the virtual environment. Pretty much all it takes to create a source is to implement the `Source` interface. The template fills in a lot of information for you and has extensive docstrings describing what you need to do to implement each method. The next 4 steps are just implementing that interface. @@ -123,21 +117,18 @@ The nice thing about this approach is that you can iterate completely within in **Run the source using docker** -If you want to run your source exactly as it will be run by Airbyte \(i.e. within a docker container\), you can use the following commands from the -connector module directory (`airbyte-integrations/connectors/source-example-python`): +If you want to run your source exactly as it will be run by Airbyte \(i.e. within a docker container\), you can use the following commands: ```text -# First build the container -docker build . -t airbyte/source-example-python:dev - -# Then use the following commands to run it +# in airbyte root directory +./gradlew :airbyte-integrations:connectors:source-example-python:airbyteDocker docker run --rm airbyte/source-example-python:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-example-python:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-example-python:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-example-python:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-example-python/secrets:/secrets airbyte/source-example-python:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-example-python/secrets:/secrets airbyte/source-example-python:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/airbyte-integrations/connectors/source-example-python/secrets:/secrets -v $(pwd)/airbyte-integrations/connectors/source-example-python/sample_files:/sample_files airbyte/source-example-python:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json ``` -Note: Each time you make a change to your implementation you need to re-build the connector image. `docker build . -t airbyte/source-example-python:dev`. This ensures the new python code is added into the docker container. +Note: Each time you make a change to your implementation you need to re-build the connector. `./gradlew :airbyte-integrations:connectors:source-:build`. This makes sure that the new python code is added into the docker container. The nice thing about this approach is that you are running your source exactly as it will be run by Airbyte. The tradeoff is that iteration is slightly slower, because you need to re-build the connector between each change. @@ -161,7 +152,7 @@ The generated code that Airbyte provides, handles implementing the `spec` method As described in the template code, this method takes in a json object called config that has the values described in the `spec.json` filled in. In other words if the `spec.json` said that the source requires a `username` and `password` the config object might be `{ "username": "airbyte", "password": "password123" }`. It returns a json object that reports, given the credentials in the config, whether we were able to connect to the source. For example, with the given credentials could the source connect to the database server. -While developing, we recommend storing this object in `secrets/config.json`. The `secrets` directory is gitignored by default. +While developing, we recommend storing this object in `secrets/config.json`. All tests assume that is where credentials will be stored. ### Step 6: Implement `discover` @@ -171,14 +162,13 @@ For a brief overview on the catalog check out [Beginner's Guide to the Airbyte C ### Step 7: Implement `read` -As described in the template code, this method takes in the same config object as the previous methods. It also takes in a "configured catalog". This object wraps the catalog emitted by the `discover` step and includes configuration on how the data should be replicated. For a brief overview on the configured catalog check out [Beginner's Guide to the Airbyte Catalog](beginners-guide-to-catalog.md). It then returns a generator which returns each record in the stream. +As described in the template code, this method takes in the same config object as the previous methods. It also takes in a "configured catalog". This object wraps the catalog emitted by the `discover` step and includes configuration on how the data should be replicated. For a brief overview on the configured catalog check out [Beginner's Guide to the Airbyte Catalog](beginners-guide-to-catalog.md). It then returns each record that it fetches from the source as a stream \(or generator\). ### Step 8: Set up Standard Tests -The Standard Tests are a set of tests that run against all sources. These tests are run in the Airbyte CI to prevent regressions. They also can help you sanity check that your source works as expected. The following [article](../contributing-to-airbyte/building-new-connector/testing-connectors.md) explains Standard Tests and how to run them. +The Standard Tests are a set of tests that run against all sources. These tests are run in the Airbyte CI to prevent regressions. They also can help you sanity check that your source works as expected. The following [article](../contributing-to-airbyte/building-new-connector/testing-connectors.md) gives a brief overview of the Standard Tests and explains what you need to do to set up these tests. -You can run the tests using `./gradlew :airbyte-integrations:connectors:source-:integrationTest`. Make sure to run this command from -the Airbyte repository root. +You can run the tests using `./gradlew :airbyte-integrations:connectors:source-:integrationTest` {% hint style="info" %} In some rare cases we make exceptions and allow a source to not need to pass all the standard tests. If for some reason you think your source cannot reasonably pass one of the tests cases, reach out to us on github or slack, and we can determine whether there's a change we can make so that the test will pass or if we should skip that test for your source. @@ -186,19 +176,17 @@ In some rare cases we make exceptions and allow a source to not need to pass all ### Step 9: Write unit tests and/or integration tests -The Standard Tests are meant to cover the basic functionality of a source. Think of it as the bare minimum required for us to add a source to Airbyte. In case you need to test additional functionality of your source, write unit or integration tests. +The Standard Tests are meant to cover the basic functionality of a source. Think of it as the bare minimum required for us to add a source to Airbyte. #### Unit Tests Add any relevant unit tests to the `unit_tests` directory. Unit tests should _not_ depend on any secrets. -You can run the tests using `python -m pytest -s unit_tests` +You can run the tests using `./gradlew :airbyte-integrations:connectors:source-:test` #### Integration Tests -Place any integration tests in the `integration_tests` directory such that they can be [discovered by pytest](https://docs.pytest.org/en/reorganize-docs/new-docs/user/naming_conventions.html). - -Run integration tests using `python -m pytest -s integration_tests`. +_coming soon_ #### Step 10: Update the `README.md` @@ -206,7 +194,7 @@ The template fills in most of the information for the readme for you. Unless the #### Step 11: Add the connector to the API/UI -Open the following file: `airbyte-config/init/src/main/resources/seed/source_definitions.yaml`. You'll find a list of all the connectors that Airbyte displays in the UI. Pattern match to add your own connector. Make sure to generate a new _unique_ UUIDv4 for the `sourceDefinitionId` field. You can get one [here](https://www.uuidgenerator.net/). +Open the following file: `airbyte-config/init/src/main/resources/seed/source_definitions.yaml`. You'll find a list of all the connectors that Airbyte displays in the UI. Pattern match to add your own connector. Make sure to generate a new _unique_ UUIDv4 for the `sourceDefinitionId` field. You can get one [here](https://www.uuidgenerator.net/). After you do, run `./gradlew :airbyte-config:init:build` \(this command generates some necessary configuration files\). Note that for simple and quick testing use cases, you can also do this step [using the UI](../integrations/custom-connectors.md#adding-your-connectors-in-the-ui). diff --git a/docs/tutorials/upgrading-airbyte.md b/docs/tutorials/upgrading-airbyte.md index bf3d91906c4..4722ef93ee5 100644 --- a/docs/tutorials/upgrading-airbyte.md +++ b/docs/tutorials/upgrading-airbyte.md @@ -34,7 +34,7 @@ On the other hand, if you don't mind losing your current Airbyte configuration o docker run --rm -v :/config airbyte/migration: --\ --input /config/airbyte_archive.tar.gz\ --output \ - --target-version + --target-version ``` Here's an example of what might look like with the values filled in. It assumes that the downloaded `airbyte_archive.tar.gz` is in `/tmp`. @@ -42,7 +42,8 @@ Here's an example of what might look like with the values filled in. It assumes ```bash docker run --rm -v /tmp:/config airbyte/migration:0.17.2-alpha --\ --input /config/airbyte_archive.tar.gz\ - --output /config/airbyte_archive_migrated.tar.gz + --output /config/airbyte_archive_migrated.tar.gz\ + --target-version 0.17.2-alpha ``` {% hint style="info" %} diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index c9f9d9b3134..9c007ec3bd8 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -27,7 +27,6 @@ write_standard_creds source-drift "$DRIFT_INTEGRATION_TEST_CREDS" write_standard_creds source-file "$AWS_S3_INTEGRATION_TEST_CREDS" "aws.json" write_standard_creds source-freshdesk "$FRESHDESK_TEST_CREDS" write_standard_creds source-facebook-marketing "$FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS" -write_standard_creds source-gitlab-singer "$GITLAB_INTEGRATION_TEST_CREDS" write_standard_creds source-github-singer "$GH_INTEGRATION_TEST_CREDS" write_standard_creds source-google-adwords-singer "$ADWORDS_INTEGRATION_TEST_CREDS" write_standard_creds source-googleanalytics-singer "$GOOGLE_ANALYTICS_TEST_CREDS" @@ -35,7 +34,7 @@ write_standard_creds source-googleanalytics-singer "$GOOGLE_ANALYTICS_TEST_TRACK write_standard_creds source-google-directory "$GOOGLE_DIRECTORY_TEST_CREDS" write_standard_creds source-google-sheets "$GSHEETS_INTEGRATION_TESTS_CREDS" "creds.json" write_standard_creds source-greenhouse "$GREENHOUSE_TEST_CREDS" -write_standard_creds source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS" +write_standard_creds source-hubspot-singer "$HUBSPOT_INTEGRATION_TESTS_CREDS" write_standard_creds source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS" write_standard_creds source-intercom-singer "$INTERCOM_INTEGRATION_TEST_CREDS" write_standard_creds source-jira "$JIRA_INTEGRATION_TEST_CREDS" diff --git a/tools/bin/ci_integration_test.sh b/tools/bin/ci_integration_test.sh index 41c9f8978d1..642db0f0c35 100755 --- a/tools/bin/ci_integration_test.sh +++ b/tools/bin/ci_integration_test.sh @@ -2,8 +2,6 @@ set -e -. tools/lib/lib.sh - # runs integration tests for an integration name connector="$1" @@ -13,14 +11,8 @@ if [[ "$connector" == "all" ]] ; then echo "Running: ./gradlew --no-daemon --scan integrationTest" ./gradlew --no-daemon --scan integrationTest else - if [[ "$connector" == *"connectors"* ]] || [[ "$connector" == *"bases"* ]]; then - connector_name=$(echo $connector | cut -d / -f 2) - selected_integration_test=$(echo "$all_integration_tests" | grep "^$connector_name$" || echo "") - integrationTestCommand="$(_to_gradle_path "airbyte-integrations/$connector" integrationTest)" - else - selected_integration_test=$(echo "$all_integration_tests" | grep "^$connector$" || echo "") - integrationTestCommand=":airbyte-integrations:connectors:$connector:integrationTest" - fi + selected_integration_test=$(echo "$all_integration_tests" | grep "^$connector$" || echo "") + integrationTestCommand=":airbyte-integrations:connectors:$connector:integrationTest" if [ -n "$selected_integration_test" ] ; then echo "Running: ./gradlew --no-daemon --scan $integrationTestCommand" ./gradlew --no-daemon --scan "$integrationTestCommand"