jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-23 21:03:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

Source Hubspot: enable SAT high strictness (#21775)

Browse Source 
* Source Hubspot: enable SAT high strictness

* Ignore stream companies for basic read because one of the fields are constantly changing

* Ignore streams that change often

* Remove contacts stream from empty_streams

* Updated expected records

* add not empty streams to expected records

* Updated expacted records

* Added allowedHosts

* Skiped property_history stream for basic read

* Resolve packages dependecy conflict

---------

Co-authored-by: Serhii Lazebnyi <serhii.lazebnyi@globallogic.com>
Co-authored-by: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com>
This commit is contained in:
Arsen Losenko
2023-02-07 00:11:30 +02:00
committed by GitHub
parent 369765f30c
commit dfdec6668d
6 changed files with 105 additions and 1649 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
airbyte-config/init/src/main/resources/seed/source_definitions.yaml
View File

@@ -801,6 +801,9 @@
icon: hubspot.svg
sourceType: api
releaseStage: generally_available
allowedHosts:
hosts:
- api.hubapi.com
- name: IP2Whois
sourceDefinitionId: f23b7b7c-d705-49a3-9042-09add3b104a5
dockerRepository: airbyte/source-ip2whois

2
airbyte-integrations/connectors/source-hubspot/README.md
View File

@@ -91,7 +91,7 @@ and place them into `secrets/config.json`.
python main.py spec
python main.py check --config secrets/config.json
python main.py discover --config secrets/config.json
python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
python main.py read --config secrets/config.json --catalog sample_files/basic_read_catalog.json
```
## Testing

132
airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml
View File

@@ -1,91 +1,69 @@
connector_image: airbyte/source-hubspot:dev
test_strictness_level: high
acceptance_tests:
spec:
tests:
- spec_path: "source_hubspot/spec.yaml"
backward_compatibility_tests_config:
# API Key authentication is deleted from authorization choices so it is not backward compatible with previous spec
disable_for_version: "0.2.3"
- spec_path: source_hubspot/spec.yaml
connection:
tests:
- config_path: "secrets/config.json"
status: "succeed"
- config_path: "integration_tests/invalid_config.json"
status: "failed"
- config_path: "integration_tests/invalid_config_oauth.json"
status: "failed"
- config_path: "integration_tests/invalid_config_wrong_title.json"
status: "failed"
- config_path: secrets/config_oauth.json
status: succeed
- config_path: integration_tests/invalid_config.json
status: failed
- config_path: integration_tests/invalid_config_oauth.json
status: failed
- config_path: integration_tests/invalid_config_wrong_title.json
status: failed
discovery:
tests:
- config_path: "secrets/config.json"
backward_compatibility_tests_config:
# Quote stream is deleted so it is not backward compatible with previous discovery
disable_for_version: "0.2.3"
- config_path: secrets/config_oauth.json
basic_read:
tests:
- config_path: "secrets/config.json"
timeout_seconds: 600
configured_catalog_path: "sample_files/basic_read_oauth_catalog.json"
- config_path: secrets/config_oauth.json
empty_streams:
- name: "form_submissions"
bypass_reason: "unable to populate"
- name: "form_submissions"
bypass_reason: "unable to populate"
- name: "ticket_pipelines"
bypass_reason: "unable to populate"
- name: "engagements_meetings"
bypass_reason: "unable to populate"
- name: "engagements_emails"
bypass_reason: "unable to populate"
- name: "engagements"
bypass_reason: "unable to populate"
- name: "engagements_calls"
bypass_reason: "unable to populate"
- name: "quotes"
bypass_reason: "unable to populate"
# This test commented out, since it produces errors during active testing
# expect_records:
# path: "integration_tests/expected_records.jsonl"
incremental:
tests:
- config_path: "secrets/config.json"
configured_catalog_path: "sample_files/incremental_catalog.json"
future_state:
future_state_path: "integration_tests/abnormal_state.json"
- name: property_history
bypass_reason: has randomly calculated records
- name: form_submissions
bypass_reason: unable to populate
- name: ticket_pipelines
bypass_reason: unable to populate
- name: engagements_meetings
bypass_reason: unable to populate
- name: engagements_emails
bypass_reason: unable to populate
- name: engagements_calls
bypass_reason: unable to populate
- name: quotes
bypass_reason: unable to populate
timeout_seconds: 3600
expect_records:
path: integration_tests/expected_records.jsonl
exact_order: yes
extra_fields: yes
extra_records: no
full_refresh:
tests:
- config_path: "secrets/config.json"
configured_catalog_path: "sample_files/full_refresh_oauth_catalog.json"
- config_path: secrets/config_oauth.json
configured_catalog_path: sample_files/full_refresh_oauth_catalog.json
ignored_fields:
"contact_lists": [ "properties", "ilsFilterBranch" ]
"companies": [ "properties", "hs_time_in_customer" ]
"companies": [ "properties", "hs_time_in_evangelist" ]
"companies": [ "properties", "hs_time_in_lead" ]
"companies": [ "properties", "hs_time_in_marketingqualifiedlead" ]
"companies": [ "properties", "hs_time_in_opportunity" ]
"companies": [ "properties", "hs_time_in_other" ]
"companies": [ "properties", "hs_time_in_salesqualifiedlead" ]
"companies": [ "properties", "hs_time_in_subscriber" ]
"contacts": [ "properties", "hs_time_in_customer" ]
"contacts": [ "properties", "hs_time_in_evangelist" ]
"contacts": [ "properties", "hs_time_in_lead" ]
"contacts": [ "properties", "hs_time_in_marketingqualifiedlead" ]
"contacts": [ "properties", "hs_time_in_opportunity" ]
"contacts": [ "properties", "hs_time_in_other" ]
"contacts": [ "properties", "hs_time_in_salesqualifiedlead" ]
"contacts": [ "properties", "hs_time_in_subscriber" ]
"deals": [ "properties", "hs_time_in_9567448" ]
"deals": [ "properties", "hs_time_in_9567449" ]
"deals": [ "properties", "hs_time_in_appointmentscheduled" ]
"deals": [ "properties", "hs_time_in_closedlost" ]
"deals": [ "properties", "hs_time_in_closedwon" ]
"deals": [ "properties", "hs_time_in_contractsent" ]
"deals": [ "properties", "hs_time_in_customclosedwonstage" ]
"deals": [ "properties", "hs_time_in_decisionmakerboughtin" ]
"deals": [ "properties", "hs_time_in_presentationscheduled" ]
"deals": [ "properties", "hs_time_in_qualifiedtobuy" ]
"tickets": [ "properties", "hs_time_in_1" ]
"tickets": [ "properties", "hs_time_in_2" ]
"tickets": [ "properties", "hs_time_in_3" ]
"tickets": [ "properties", "hs_time_in_4" ]
companies:
- properties
- hs_time_in_subscriber
contact_lists:
- properties
- ilsFilterBranch
contacts:
- properties
- hs_time_in_subscriber
deals:
- properties
- hs_time_in_qualifiedtobuy
tickets:
- properties
- hs_time_in_4
incremental:
tests:
- config_path: secrets/config_oauth.json
configured_catalog_path: sample_files/incremental_catalog.json
future_state:
future_state_path: integration_tests/abnormal_state.json

2
airbyte-integrations/connectors/source-hubspot/integration_tests/conftest.py
View File

@@ -9,7 +9,7 @@ import pytest
@pytest.fixture(scope="session", name="config")
def config_fixture():
with open("secrets/config.json", "r") as config_file:
with open("secrets/config_oauth.json", "r") as config_file:
return json.load(config_file)

1611
airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl
View File

File diff suppressed because one or more lines are too long

4
airbyte-integrations/connectors/source-hubspot/setup.py
View File

@@ -6,14 +6,14 @@
from setuptools import find_packages, setup
MAIN_REQUIREMENTS = [
"airbyte-cdk~=0.2",
"airbyte-cdk",
"backoff==1.11.1",
"pendulum==2.1.2",
"requests==2.26.0",
]
TEST_REQUIREMENTS = [
"pytest==6.1.2",
"pytest~=6.2",
"pytest-mock~=3.6",
"requests-mock~=1.9.3",
"connector-acceptance-test",