From 0e09ac3032771d1c8b00a4425bb72bfb1c79c72a Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Wed, 3 Nov 2021 01:06:38 -0300 Subject: [PATCH] Publish PR 6963: new source freshsales (#7585) * initial commit * finish implementing full_refresh * add other tables * cleaning up * add docs and use requests_native_auth * fix function return different number of values * change package author * fix schema * fix schema bug * linting * fix unit test * clean up * add null for schemas * remove fc_widget_collaboration col * change accpt test * add creds * run format * add config files * run format Co-authored-by: Tuan Nguyen --- .github/workflows/publish-command.yml | 1 + .github/workflows/test-command.yml | 1 + .../eca08d79-7b92-4065-b7f3-79c14836ebe7.json | 7 +++++ .../resources/seed/source_definitions.yaml | 6 +++++ ...ryptedOracleDestinationAcceptanceTest.java | 2 +- .../acceptance-test-config.yml | 2 +- .../integration_tests/invalid_config.json | 2 +- .../integration_tests/sample_config.json | 2 +- .../source_freshsales/schemas/accounts.json | 20 +++++++------- .../source_freshsales/schemas/contacts.json | 26 +++++++++---------- .../source-mixpanel/source_mixpanel/source.py | 1 - tools/bin/ci_credentials.sh | 1 + 12 files changed, 43 insertions(+), 28 deletions(-) create mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eca08d79-7b92-4065-b7f3-79c14836ebe7.json diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 40a166ff644..56ade97c3d2 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -199,6 +199,7 @@ jobs: SOURCE_DELIGHTED_TEST_CREDS: ${{ secrets.SOURCE_DELIGHTED_TEST_CREDS }} SOURCE_RETENTLY_TEST_CREDS: ${{ secrets.SOURCE_RETENTLY_TEST_CREDS }} SOURCE_SENTRY_TEST_CREDS: ${{ secrets.SOURCE_SENTRY_TEST_CREDS }} + SOURCE_FRESHSALES_TEST_CREDS: ${{ secrets.SOURCE_FRESHSALES_TEST_CREDS }} - run: | echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD} ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 674d8657388..94e50a3f3ae 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -194,6 +194,7 @@ jobs: SOURCE_DELIGHTED_TEST_CREDS: ${{ secrets.SOURCE_DELIGHTED_TEST_CREDS }} SOURCE_RETENTLY_TEST_CREDS: ${{ secrets.SOURCE_RETENTLY_TEST_CREDS }} SOURCE_SENTRY_TEST_CREDS: ${{ secrets.SOURCE_SENTRY_TEST_CREDS }} + SOURCE_FRESHSALES_TEST_CREDS: ${{ secrets.SOURCE_FRESHSALES_TEST_CREDS }} - run: | ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }} name: test ${{ github.event.inputs.connector }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eca08d79-7b92-4065-b7f3-79c14836ebe7.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eca08d79-7b92-4065-b7f3-79c14836ebe7.json new file mode 100644 index 00000000000..d74256e9d28 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eca08d79-7b92-4065-b7f3-79c14836ebe7.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "eca08d79-7b92-4065-b7f3-79c14836ebe7", + "name": "Freshsales", + "dockerRepository": "airbyte/source-freshsales", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/freshsales" +} diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 17d6acc05d1..324edc44f9b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -156,6 +156,12 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/freshdesk icon: freshdesk.svg sourceType: api +- name: Freshsales + sourceDefinitionId: eca08d79-7b92-4065-b7f3-79c14836ebe7 + dockerRepository: airbyte/source-freshsales + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/freshsales + sourceType: api - name: Freshservice sourceDefinitionId: 9bb85338-ea95-4c93-b267-6be89125b267 dockerRepository: airbyte/source-freshservice diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java index 963e1277a2e..8e57e31ef7f 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java @@ -13,8 +13,8 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.db.Database; import io.airbyte.db.Databases; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.destination.ExtendedNameTransformer; import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/source-freshsales/acceptance-test-config.yml b/airbyte-integrations/connectors/source-freshsales/acceptance-test-config.yml index dfd392cb6e6..d8678f3f244 100644 --- a/airbyte-integrations/connectors/source-freshsales/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-freshsales/acceptance-test-config.yml @@ -14,7 +14,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + empty_streams: ["lost_deals", "won_deals"] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-freshsales/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-freshsales/integration_tests/invalid_config.json index 8b7f2e2e3e0..31299549a84 100644 --- a/airbyte-integrations/connectors/source-freshsales/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-freshsales/integration_tests/invalid_config.json @@ -1 +1 @@ -{"domain_name": "", "api_key": "ghiklmn"} \ No newline at end of file +{ "domain_name": "", "api_key": "ghiklmn" } diff --git a/airbyte-integrations/connectors/source-freshsales/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-freshsales/integration_tests/sample_config.json index 1e106e9658f..690c9240911 100644 --- a/airbyte-integrations/connectors/source-freshsales/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-freshsales/integration_tests/sample_config.json @@ -1 +1 @@ -{"domain_name": "testabc.myfreshworks.com", "api_key": "ghiklmn"} \ No newline at end of file +{ "domain_name": "testabc.myfreshworks.com", "api_key": "ghiklmn" } diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/accounts.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/accounts.json index ec1193d483c..d43ae83240c 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/accounts.json @@ -24,21 +24,21 @@ "created_at": { "type": ["null", "string"] }, "updated_at": { "type": ["null", "string"] }, "parent_sales_account_id": { "type": ["null", "integer"] }, - "first_name": { "type": ["null", "string"] }, - "last_name": { "type": ["null", "string"] }, + "first_name": { "type": ["null", "string"] }, + "last_name": { "type": ["null", "string"] }, "display_name": { "type": ["null", "string"] }, - "avatar": { "type": ["null", "string"] }, - "job_title": { "type": ["null", "string"] }, - "email": { "type": ["null", "string"] }, - "emails": { "type": ["null", "array"] }, - "time_zone": { "type": ["null", "string"] }, - "work_number": { "type": ["null", "string"] }, + "avatar": { "type": ["null", "string"] }, + "job_title": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] }, + "emails": { "type": ["null", "array"] }, + "time_zone": { "type": ["null", "string"] }, + "work_number": { "type": ["null", "string"] }, "mobile_number": { "type": ["null", "string"] }, - "last_seen": { "type": ["null", "string"] }, + "last_seen": { "type": ["null", "string"] }, "lead_score": { "type": ["null", "integer"] }, "last_contacted": { "type": ["null", "string"] }, "open_deals_amount": { "type": ["null", "number"] }, - "won_deals_amount": { "type": ["null", "number"] }, + "won_deals_amount": { "type": ["null", "number"] }, "links": { "type": ["null", "object"] }, "last_contacted_sales_activity_mode": { "type": ["null", "string"] }, "custom_field": { "type": ["null", "object"] }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/contacts.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/contacts.json index cf32ac6abe1..e7cc9ffc162 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/contacts.json @@ -36,28 +36,28 @@ "sales_account_id": { "type": ["null", "integer"] }, "lifecycle_stage_id": { "type": ["null", "integer"] }, "display_name": { "type": ["null", "string"] }, - "avatar": { "type": ["null", "string"] }, - "last_seen": { "type": ["null", "string"] }, + "avatar": { "type": ["null", "string"] }, + "last_seen": { "type": ["null", "string"] }, "lead_score": { "type": ["null", "integer"] }, - "last_contacted": { "type": ["null", "string"] }, - "open_deals_amount": { "type": ["null", "number"] }, - "won_deals_amount": { "type": ["null", "number"] }, + "last_contacted": { "type": ["null", "string"] }, + "open_deals_amount": { "type": ["null", "number"] }, + "won_deals_amount": { "type": ["null", "number"] }, "links": { "type": ["null", "object"] }, "last_contacted_sales_activity_mode": { "type": ["null", "string"] }, - "custom_field": { "type": ["null", "object"] }, - "last_contacted_mode": { "type": ["null", "string"] }, - "recent_note": { "type": ["null", "string"] }, + "custom_field": { "type": ["null", "object"] }, + "last_contacted_mode": { "type": ["null", "string"] }, + "recent_note": { "type": ["null", "string"] }, "won_deals_count": { "type": ["null", "integer"] }, "last_contacted_via_sales_activity": { "type": ["null", "string"] }, - "completed_sales_sequences": { "type": ["null", "string"] }, - "active_sales_sequences": { "type": ["null", "string"] }, - "web_form_ids": { "type": ["null", "string"] }, + "completed_sales_sequences": { "type": ["null", "string"] }, + "active_sales_sequences": { "type": ["null", "string"] }, + "web_form_ids": { "type": ["null", "string"] }, "open_deals_count": { "type": ["null", "integer"] }, "last_assigned_at": { "type": ["null", "string"] }, "tags": { "type": ["null", "array"] }, - "is_deleted": { "type": ["null", "boolean"] }, + "is_deleted": { "type": ["null", "boolean"] }, "team_user_ids": { "type": ["null", "string"] }, - "work_email": { "type": ["null", "string"] }, + "work_email": { "type": ["null", "string"] }, "customer_fit": { "type": ["null", "integer"] }, "whatsapp_subscription_status": { "type": ["null", "integer"] }, "phone_numbers": { "type": ["null", "array"] } diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py index 894301292ad..7c3916d342c 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py @@ -729,7 +729,6 @@ class Export(DateSlicesMixin, IncrementalMixpanelStream): yield item - def get_json_schema(self) -> Mapping[str, Any]: """ :return: A dict of the JSON schema representing this stream. diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 9b1dd92577a..03e35719baa 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -74,6 +74,7 @@ write_standard_creds source-file "$AZURE_STORAGE_INTEGRATION_TEST_CREDS" "azblob write_standard_creds source-file "$FILE_SECURE_HTTPS_TEST_CREDS" write_standard_creds source-file-secure "$FILE_SECURE_HTTPS_TEST_CREDS" write_standard_creds source-freshdesk "$FRESHDESK_TEST_CREDS" +write_standard_creds source-freshsales "$SOURCE_FRESHSALES_TEST_CREDS" write_standard_creds source-freshservice "$SOURCE_FRESHSERVICE_TEST_CREDS" write_standard_creds source-facebook-marketing "$FACEBOOK_MARKETING_TEST_INTEGRATION_CREDS" write_standard_creds source-facebook-pages "$FACEBOOK_PAGES_INTEGRATION_TEST_CREDS"