diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index a5e9b8e4e6c..931b45c102a 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -147,6 +147,7 @@ jobs: SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }} SENDGRID_INTEGRATION_TEST_CREDS: ${{ secrets.SENDGRID_INTEGRATION_TEST_CREDS }} SHOPIFY_INTEGRATION_TEST_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_CREDS }} + SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS }} SOURCE_ASANA_TEST_CREDS: ${{ secrets.SOURCE_ASANA_TEST_CREDS }} SOURCE_OKTA_TEST_CREDS: ${{ secrets.SOURCE_OKTA_TEST_CREDS }} SOURCE_SLACK_TEST_CREDS: ${{ secrets.SOURCE_SLACK_TEST_CREDS }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 31c5d3cba9f..657eccd72ef 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -143,6 +143,7 @@ jobs: SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }} SENDGRID_INTEGRATION_TEST_CREDS: ${{ secrets.SENDGRID_INTEGRATION_TEST_CREDS }} SHOPIFY_INTEGRATION_TEST_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_CREDS }} + SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS: ${{ secrets.SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS }} SOURCE_OKTA_TEST_CREDS: ${{ secrets.SOURCE_OKTA_TEST_CREDS }} SOURCE_SLACK_TEST_CREDS: ${{ secrets.SOURCE_SLACK_TEST_CREDS }} SOURCE_SLACK_OAUTH_TEST_CREDS: ${{ secrets.SOURCE_SLACK_OAUTH_TEST_CREDS }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json index 3d84caee2ca..71bd12225a4 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9", "name": "Shopify", "dockerRepository": "airbyte/source-shopify", - "dockerImageTag": "0.1.18", + "dockerImageTag": "0.1.19", "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 1f4ad60cc0d..6b1dbace180 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -174,7 +174,7 @@ - sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9 name: Shopify dockerRepository: airbyte/source-shopify - dockerImageTag: 0.1.18 + dockerImageTag: 0.1.19 documentationUrl: https://docs.airbyte.io/integrations/sources/shopify sourceType: api - sourceDefinitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6 diff --git a/airbyte-integrations/connectors/source-shopify/Dockerfile b/airbyte-integrations/connectors/source-shopify/Dockerfile index 5631218d7bd..beb50a9f899 100644 --- a/airbyte-integrations/connectors/source-shopify/Dockerfile +++ b/airbyte-integrations/connectors/source-shopify/Dockerfile @@ -28,5 +28,5 @@ COPY source_shopify ./source_shopify ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.18 +LABEL io.airbyte.version=0.1.19 LABEL io.airbyte.name=airbyte/source-shopify diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml index a8b2ef8d61a..045f6bc1c10 100644 --- a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml @@ -7,8 +7,13 @@ tests: status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "integration_tests/invalid_oauth_config.json" + status: "failed" discovery: - config_path: "secrets/config.json" + - config_path: "secrets/config_oauth.json" basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_config.json index b4baf9ece36..baa4462bd98 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_config.json @@ -1,5 +1,8 @@ { "shop": "SHOP_NAME", "start_date": "2020-11-01", - "api_password": "API_PASSWORD" -} + "auth_method": { + "auth_method": "api_password", + "api_password": "SOME_API_PASSWORD" + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_oauth_config.json b/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_oauth_config.json new file mode 100644 index 00000000000..baee6c78d4e --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_oauth_config.json @@ -0,0 +1,10 @@ +{ + "shop": "SHOP_NAME", + "start_date": "2020-11-01", + "auth_method": { + "auth_method": "access_token", + "client_id": "SOME_CLIENT_ID", + "client_secret": "SOME_CLIENT_SECRET", + "access_token": "SOME_ACCESS_TOKEN" + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py new file mode 100644 index 00000000000..47f0ff8e5c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Dict, Mapping + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + + +class NotImplementedAuth(Exception): + """ Not implemented Auth option error""" + + logger = AirbyteLogger() + + def __init__(self, auth_method: str = None): + self.message = f"Not implemented Auth method = {auth_method}" + super().__init__(self.logger.error(self.message)) + + +class ShopifyAuthenticator(TokenAuthenticator): + + """ + Making Authenticator to be able to accept Header-Based authentication. + """ + + def __init__(self, config: Mapping[str, Any]): + self.config = config + + def get_auth_header(self) -> Mapping[str, Any]: + + auth_header: str = "X-Shopify-Access-Token" + auth_method: Dict = self.config["auth_method"] + auth_option: str = auth_method.get("auth_method") + + if auth_option == "access_token": + return {auth_header: auth_method.get("access_token")} + elif auth_option == "api_password": + return {auth_header: auth_method.get("api_password")} + else: + raise NotImplementedAuth(auth_option) diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py index 1df2c7990a6..e7a4944b367 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -12,8 +12,8 @@ from airbyte_cdk import AirbyteLogger from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from .auth import ShopifyAuthenticator from .transform import DataTypeEnforcer from .utils import EagerlyCachedStreamState as stream_state_cache from .utils import ShopifyRateLimiter as limiter @@ -325,25 +325,14 @@ class DiscountCodes(ChildSubstream): return f"price_rules/{price_rule_id}/{self.data_field}.json" -class ShopifyAuthenticator(TokenAuthenticator): - - """ - Making Authenticator to be able to accept Header-Based authentication. - """ - - def get_auth_header(self) -> Mapping[str, Any]: - return {"X-Shopify-Access-Token": f"{self._token}"} - - class SourceShopify(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: """ Testing connection availability for the connector. """ - auth = ShopifyAuthenticator(token=config["api_password"]).get_auth_header() + auth = ShopifyAuthenticator(config).get_auth_header() api_version = "2021-07" # Latest Stable Release - url = f"https://{config['shop']}.myshopify.com/admin/api/{api_version}/shop.json" try: @@ -359,8 +348,7 @@ class SourceShopify(AbstractSource): Mapping a input config of the user input configuration as defined in the connector spec. Defining streams to run. """ - # config["ex_state"] = super().exposed_state - config["authenticator"] = ShopifyAuthenticator(token=config["api_password"]) + config["authenticator"] = ShopifyAuthenticator(config) return [ Customers(config), diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json b/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json index 3af46170f8d..1e04937dabb 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Shopify Source CDK Specifications", "type": "object", - "required": ["shop", "start_date", "api_password"], + "required": ["shop", "start_date", "auth_method"], "additionalProperties": false, "properties": { "shop": { @@ -17,11 +17,68 @@ "examples": ["2021-01-01"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" }, - "api_password": { - "type": "string", - "description": "The API PASSWORD for a private application in Shopify shop.", - "airbyte_secret": true + "auth_method": { + "title": "Shopify Authorization Method", + "type": "object", + "oneOf": [ + { + "type": "object", + "title": "OAuth2.0", + "required": ["client_id","client_secret","access_token"], + "properties": { + "auth_method": { + "type": "string", + "const": "access_token", + "enum": ["access_token"], + "default": "access_token", + "order": 0 + }, + "client_id": { + "type": "string", + "description": "The API Key of the Shopify developer application.", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "description": "The API Secret the Shopify developer application.", + "airbyte_secret": true + }, + "access_token": { + "type": "string", + "description": "Access Token for making authenticated requests.", + "airbyte_secret": true + } + } + }, + { + "title": "API Password", + "type": "object", + "required": ["api_password"], + "properties": { + "auth_method": { + "type": "string", + "const": "api_password", + "enum": ["api_password"], + "default": "api_password", + "order": 0 + }, + "api_password": { + "type": "string", + "description": "The API PASSWORD for your private application in `Shopify` shop.", + "airbyte_secret": true + } + } + } + ] } + } + }, + "authSpecification": { + "auth_type": "oauth2.0", + "oauth2Specification": { + "rootObject": ["auth_method", 0], + "oauthFlowInitParameters": [["client_id"], ["client_secret"]], + "oauthFlowOutputParameters": [["access_token"]] } } } diff --git a/docs/integrations/sources/shopify.md b/docs/integrations/sources/shopify.md index a8d4e5aa7c1..f9ee3e0d973 100644 --- a/docs/integrations/sources/shopify.md +++ b/docs/integrations/sources/shopify.md @@ -74,6 +74,9 @@ This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Erro ## Getting started +This connector support both: `OAuth 2.0` and `API PASSWORD` (for private applications) athentication methods. + +### Connect using `API PASSWORD` option: 1. Go to `https://YOURSTORE.myshopify.com/admin/apps/private` 2. Enable private development if it isn't enabled. 3. Create a private application. @@ -82,10 +85,17 @@ This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Erro 5. The password under the `Admin API` section is what you'll use as the `api_password` for the integration. 6. You're ready to set up Shopify in Airbyte! +### Connect using `OAuth 2.0` option: +1. Select `OAuth 2.0` in `Shopify Authorization Method` +2. Click on `authenticate` +2. Proceed the authentication using your credentials for your Shopify account. + + ## Changelog | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.19 | 2021-10-11 | [6951](https://github.com/airbytehq/airbyte/pull/6951) | Added support of `OAuth 2.0` authorisation option | | 0.1.18 | 2021-09-21 | [6056](https://github.com/airbytehq/airbyte/pull/6056) | Added `pre_tax_price` to the `orders/line_items` schema | | 0.1.17 | 2021-09-17 | [5244](https://github.com/airbytehq/airbyte/pull/5244) | Created data type enforcer for converting prices into numbers | | 0.1.16 | 2021-09-09 | [5965](https://github.com/airbytehq/airbyte/pull/5945) | Fixed the connector's performance for `Incremental refresh` | diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 86498ee35a1..1f35898bfcf 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -123,6 +123,7 @@ write_standard_creds source-salesforce "$SALESFORCE_BULK_INTEGRATION_TESTS_CREDS write_standard_creds source-salesforce "$SALESFORCE_INTEGRATION_TESTS_CREDS" write_standard_creds source-sendgrid "$SENDGRID_INTEGRATION_TEST_CREDS" write_standard_creds source-shopify "$SHOPIFY_INTEGRATION_TEST_CREDS" +write_standard_creds source-shopify "$SHOPIFY_INTEGRATION_TEST_OAUTH_CREDS" "config_oauth.json" write_standard_creds source-shortio "$SOURCE_SHORTIO_TEST_CREDS" write_standard_creds source-slack "$SOURCE_SLACK_TEST_CREDS" write_standard_creds source-slack "$SOURCE_SLACK_OAUTH_TEST_CREDS" "config_oauth.json"