jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

Source Amazon Seller Partner: OAuth support (#13119)

Browse Source 
* Update docs

* add oauth

* add spec.py

* test

* fix

* fix

* add lwa_app_id

* test

* test

* fix

* add beta version

* fix

* test

* test

* test

* merge

* fix

* fix

* fix

* test

* test

* test

* test

* sort AWSRegion

* up

* add import

* up

* up

* up

* fix

* fix tests

* bump version

* upd

* auto-bump connector version

* format

Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
This commit is contained in:
Anna Lvova
2022-05-30 16:24:10 +03:00
committed by GitHub
parent d6d52c5d99
commit 1dbd7ab99c
13 changed files with 584 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
airbyte-config/init/src/main/resources/seed/source_definitions.yaml
View File

@@ -25,7 +25,7 @@
- name: Amazon Seller Partner
sourceDefinitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460
dockerRepository: airbyte/source-amazon-seller-partner
dockerImageTag: 0.2.18
dockerImageTag: 0.2.19
sourceType: api
documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-seller-partner
icon: amazonsellerpartner.svg

165
airbyte-config/init/src/main/resources/seed/source_specs.yaml
View File

@@ -213,7 +213,7 @@
type: "string"
path_in_connector_config:
- "client_secret"
- dockerImage: "airbyte/source-amazon-seller-partner:0.2.18"
- dockerImage: "airbyte/source-amazon-seller-partner:0.2.19"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/amazon-seller-partner"
changelogUrl: "https://docs.airbyte.io/integrations/sources/amazon-seller-partner"
@@ -221,6 +221,56 @@
title: "Amazon Seller Partner Spec"
type: "object"
properties:
app_id:
title: "App Id *"
description: "Your Amazon App ID"
airbyte_secret: true
order: 0
type: "string"
auth_type:
title: "Auth Type"
const: "oauth2.0"
order: 1
type: "string"
lwa_app_id:
title: "LWA Client Id"
description: "Your Login with Amazon Client ID."
order: 2
type: "string"
lwa_client_secret:
title: "LWA Client Secret"
description: "Your Login with Amazon Client Secret."
airbyte_secret: true
order: 3
type: "string"
refresh_token:
title: "Refresh Token"
description: "The Refresh Token obtained via OAuth flow authorization."
airbyte_secret: true
order: 4
type: "string"
aws_access_key:
title: "AWS Access Key"
description: "Specifies the AWS access key used as part of the credentials\
\ to authenticate the user."
airbyte_secret: true
order: 5
type: "string"
aws_secret_key:
title: "AWS Secret Access Key"
description: "Specifies the AWS secret key used as part of the credentials\
\ to authenticate the user."
airbyte_secret: true
order: 6
type: "string"
role_arn:
title: "Role ARN"
description: "Specifies the Amazon Resource Name (ARN) of an IAM role that\
\ you want to use to perform operations requested using this profile.\
\ (Needs permission to 'Assume Role' STS)."
airbyte_secret: true
order: 7
type: "string"
replication_start_date:
title: "Start Date"
description: "UTC date and time in the format 2017-01-25T00:00:00Z. Any\
@@ -257,40 +307,6 @@
- "500"
- "1980"
type: "integer"
refresh_token:
title: "Refresh Token"
description: "The Refresh Token obtained via OAuth flow authorization."
airbyte_secret: true
type: "string"
lwa_app_id:
title: "LwA App Id"
description: "Your Login with Amazon App ID"
airbyte_secret: true
type: "string"
lwa_client_secret:
title: "LwA Client Secret"
description: "Your Login with Amazon Client Secret."
airbyte_secret: true
type: "string"
aws_access_key:
title: "AWS Access Key"
description: "Specifies the AWS access key used as part of the credentials\
\ to authenticate the user."
airbyte_secret: true
type: "string"
aws_secret_key:
title: "AWS Secret Access Key"
description: "Specifies the AWS secret key used as part of the credentials\
\ to authenticate the user."
airbyte_secret: true
type: "string"
role_arn:
title: "Role ARN"
description: "Specifies the Amazon Resource Name (ARN) of an IAM role that\
\ you want to use to perform operations requested using this profile.\
\ (Needs permission to 'Assume Role' STS)."
airbyte_secret: true
type: "string"
aws_environment:
title: "AWSEnvironment"
description: "An enumeration."
@@ -303,37 +319,38 @@
description: "An enumeration."
enum:
- "AE"
- "AU"
- "BR"
- "CA"
- "DE"
- "PL"
- "EG"
- "ES"
- "FR"
- "GB"
- "IN"
- "IT"
- "JP"
- "MX"
- "NL"
- "PL"
- "SA"
- "SE"
- "SG"
- "TR"
- "UK"
- "AU"
- "JP"
- "SG"
- "US"
- "BR"
- "CA"
- "MX"
- "GB"
type: "string"
required:
- "replication_start_date"
- "refresh_token"
- "lwa_app_id"
- "lwa_client_secret"
- "refresh_token"
- "aws_access_key"
- "aws_secret_key"
- "role_arn"
- "replication_start_date"
- "aws_environment"
- "region"
additionalProperties: true
definitions:
AWSEnvironment:
title: "AWSEnvironment"
@@ -347,30 +364,72 @@
description: "An enumeration."
enum:
- "AE"
- "AU"
- "BR"
- "CA"
- "DE"
- "PL"
- "EG"
- "ES"
- "FR"
- "GB"
- "IN"
- "IT"
- "JP"
- "MX"
- "NL"
- "PL"
- "SA"
- "SE"
- "SG"
- "TR"
- "UK"
- "AU"
- "JP"
- "SG"
- "US"
- "BR"
- "CA"
- "MX"
- "GB"
type: "string"
supportsNormalization: false
supportsDBT: false
supported_destination_sync_modes: []
advanced_auth:
auth_flow_type: "oauth2.0"
predicate_key:
- "auth_type"
predicate_value: "oauth2.0"
oauth_config_specification:
oauth_user_input_from_connector_config_specification:
type: "object"
additionalProperties: false
properties:
app_id:
type: "string"
path_in_connector_config:
- "app_id"
complete_oauth_output_specification:
type: "object"
additionalProperties: false
properties:
refresh_token:
type: "string"
path_in_connector_config:
- "refresh_token"
complete_oauth_server_input_specification:
type: "object"
additionalProperties: false
properties:
lwa_app_id:
type: "string"
lwa_client_secret:
type: "string"
complete_oauth_server_output_specification:
type: "object"
additionalProperties: false
properties:
lwa_app_id:
type: "string"
path_in_connector_config:
- "lwa_app_id"
lwa_client_secret:
type: "string"
path_in_connector_config:
- "lwa_client_secret"
- dockerImage: "airbyte/source-amazon-sqs:0.1.0"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/amazon-sqs"

2
airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile
View File

@@ -12,5 +12,5 @@ RUN pip install .
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.2.18
LABEL io.airbyte.version=0.2.19
LABEL io.airbyte.name=airbyte/source-amazon-seller-partner

2
airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml
View File

@@ -1,7 +1,7 @@
connector_image: airbyte/source-amazon-seller-partner:dev
tests:
spec:
- spec_path: "source_amazon_seller_partner/spec.json"
- spec_path: "integration_tests/spec.json"
connection:
# - config_path: "secrets/config.json"
# status: "succeed"

184
airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json → airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json
View File

@@ -5,6 +5,60 @@
"title": "Amazon Seller Partner Spec",
"type": "object",
"properties": {
"app_id": {
"title": "App Id *",
"description": "Your Amazon App ID",
"airbyte_secret": true,
"order": 0,
"type": "string"
},
"auth_type": {
"title": "Auth Type",
"const": "oauth2.0",
"order": 1,
"type": "string"
},
"lwa_app_id": {
"title": "LWA Client Id",
"description": "Your Login with Amazon Client ID.",
"order": 2,
"type": "string"
},
"lwa_client_secret": {
"title": "LWA Client Secret",
"description": "Your Login with Amazon Client Secret.",
"airbyte_secret": true,
"order": 3,
"type": "string"
},
"refresh_token": {
"title": "Refresh Token",
"description": "The Refresh Token obtained via OAuth flow authorization.",
"airbyte_secret": true,
"order": 4,
"type": "string"
},
"aws_access_key": {
"title": "AWS Access Key",
"description": "Specifies the AWS access key used as part of the credentials to authenticate the user.",
"airbyte_secret": true,
"order": 5,
"type": "string"
},
"aws_secret_key": {
"title": "AWS Secret Access Key",
"description": "Specifies the AWS secret key used as part of the credentials to authenticate the user.",
"airbyte_secret": true,
"order": 6,
"type": "string"
},
"role_arn": {
"title": "Role ARN",
"description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. (Needs permission to 'Assume Role' STS).",
"airbyte_secret": true,
"order": 7,
"type": "string"
},
"replication_start_date": {
"title": "Start Date",
"description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.",
@@ -35,42 +89,6 @@
"examples": ["500", "1980"],
"type": "integer"
},
"refresh_token": {
"title": "Refresh Token",
"description": "The Refresh Token obtained via OAuth flow authorization.",
"airbyte_secret": true,
"type": "string"
},
"lwa_app_id": {
"title": "LwA App Id",
"description": "Your Login with Amazon App ID",
"airbyte_secret": true,
"type": "string"
},
"lwa_client_secret": {
"title": "LwA Client Secret",
"description": "Your Login with Amazon Client Secret.",
"airbyte_secret": true,
"type": "string"
},
"aws_access_key": {
"title": "AWS Access Key",
"description": "Specifies the AWS access key used as part of the credentials to authenticate the user.",
"airbyte_secret": true,
"type": "string"
},
"aws_secret_key": {
"title": "AWS Secret Access Key",
"description": "Specifies the AWS secret key used as part of the credentials to authenticate the user.",
"airbyte_secret": true,
"type": "string"
},
"role_arn": {
"title": "Role ARN",
"description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. (Needs permission to 'Assume Role' STS).",
"airbyte_secret": true,
"type": "string"
},
"aws_environment": {
"title": "AWSEnvironment",
"description": "An enumeration.",
@@ -82,41 +100,42 @@
"description": "An enumeration.",
"enum": [
"AE",
"AU",
"BR",
"CA",
"DE",
"PL",
"EG",
"ES",
"FR",
"GB",
"IN",
"IT",
"JP",
"MX",
"NL",
"PL",
"SA",
"SE",
"SG",
"TR",
"UK",
"AU",
"JP",
"SG",
"US",
"BR",
"CA",
"MX",
"GB"
"US"
],
"type": "string"
}
},
"required": [
"replication_start_date",
"refresh_token",
"lwa_app_id",
"lwa_client_secret",
"refresh_token",
"aws_access_key",
"aws_secret_key",
"role_arn",
"replication_start_date",
"aws_environment",
"region"
],
"additionalProperties": true,
"definitions": {
"AWSEnvironment": {
"title": "AWSEnvironment",
@@ -129,29 +148,82 @@
"description": "An enumeration.",
"enum": [
"AE",
"AU",
"BR",
"CA",
"DE",
"PL",
"EG",
"ES",
"FR",
"GB",
"IN",
"IT",
"JP",
"MX",
"NL",
"PL",
"SA",
"SE",
"SG",
"TR",
"UK",
"AU",
"JP",
"SG",
"US",
"BR",
"CA",
"MX",
"GB"
"US"
],
"type": "string"
}
}
},
"advanced_auth": {
"auth_flow_type": "oauth2.0",
"predicate_key": ["auth_type"],
"predicate_value": "oauth2.0",
"oauth_config_specification": {
"oauth_user_input_from_connector_config_specification": {
"type": "object",
"additionalProperties": false,
"properties": {
"app_id": {
"type": "string",
"path_in_connector_config": ["app_id"]
}
}
},
"complete_oauth_output_specification": {
"type": "object",
"additionalProperties": false,
"properties": {
"refresh_token": {
"type": "string",
"path_in_connector_config": ["refresh_token"]
}
}
},
"complete_oauth_server_input_specification": {
"type": "object",
"additionalProperties": false,
"properties": {
"lwa_app_id": {
"type": "string"
},
"lwa_client_secret": {
"type": "string"
}
}
},
"complete_oauth_server_output_specification": {
"type": "object",
"additionalProperties": false,
"properties": {
"lwa_app_id": {
"type": "string",
"path_in_connector_config": ["lwa_app_id"]
},
"lwa_client_secret": {
"type": "string",
"path_in_connector_config": ["lwa_client_secret"]
}
}
}
}
}
}

4
airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/__init__.py
View File

@@ -21,6 +21,6 @@
# SOFTWARE.
from .source import ConnectorConfig, SourceAmazonSellerPartner
from .source import SourceAmazonSellerPartner
__all__ = ["SourceAmazonSellerPartner", "ConnectorConfig"]
__all__ = ["SourceAmazonSellerPartner"]

16
airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py
View File

@@ -40,26 +40,26 @@ class AWSEnvironment(str, Enum):
class AWSRegion(str, Enum):
AE = "AE"
AU = "AU"
BR = "BR"
CA = "CA"
DE = "DE"
PL = "PL"
EG = "EG"
ES = "ES"
FR = "FR"
GB = "GB"
IN = "IN"
IT = "IT"
JP = "JP"
MX = "MX"
NL = "NL"
PL = "PL"
SA = "SA"
SE = "SE"
SG = "SG"
TR = "TR"
UK = "UK"
AU = "AU"
JP = "JP"
SG = "SG"
US = "US"
BR = "BR"
CA = "CA"
MX = "MX"
GB = "GB"
def get_aws_base_url(aws_env: AWSEnvironment) -> str:

80
airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py
View File

@@ -5,15 +5,13 @@
from typing import Any, List, Mapping, Tuple
import boto3
import requests
from airbyte_cdk.logger import AirbyteLogger
from airbyte_cdk.models import ConnectorSpecification, SyncMode
from airbyte_cdk.sources import AbstractSource
from airbyte_cdk.sources.streams import Stream
from pydantic import Field
from pydantic.main import BaseModel
from source_amazon_seller_partner.auth import AWSAuthenticator, AWSSignature
from source_amazon_seller_partner.constants import AWSEnvironment, AWSRegion, get_marketplaces
from source_amazon_seller_partner.constants import get_marketplaces
from source_amazon_seller_partner.spec import AmazonSellerPartnerConfig, advanced_auth
from source_amazon_seller_partner.streams import (
BrandAnalyticsAlternatePurchaseReports,
BrandAnalyticsItemComparisonReports,
@@ -37,60 +35,8 @@ from source_amazon_seller_partner.streams import (
)
class ConnectorConfig(BaseModel):
class Config:
title = "Amazon Seller Partner Spec"
replication_start_date: str = Field(
description="UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.",
title="Start Date",
pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
examples=["2017-01-25T00:00:00Z"],
)
period_in_days: int = Field(
30,
description="Will be used for stream slicing for initial full_refresh sync when no updated state is present for reports that support sliced incremental sync.",
examples=["30", "365"],
)
report_options: str = Field(
None,
description="Additional information passed to reports. This varies by report type. Must be a valid json string.",
examples=['{"GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT": {"reportPeriod": "WEEK"}}', '{"GET_SOME_REPORT": {"custom": "true"}}'],
)
max_wait_seconds: int = Field(
500,
title="Max wait time for reports (in seconds)",
description="Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.",
examples=["500", "1980"],
)
refresh_token: str = Field(
description="The Refresh Token obtained via OAuth flow authorization.",
title="Refresh Token",
airbyte_secret=True,
)
lwa_app_id: str = Field(description="Your Login with Amazon App ID", title="LwA App Id", airbyte_secret=True)
lwa_client_secret: str = Field(description="Your Login with Amazon Client Secret.", title="LwA Client Secret", airbyte_secret=True)
aws_access_key: str = Field(
description="Specifies the AWS access key used as part of the credentials to authenticate the user.",
title="AWS Access Key",
airbyte_secret=True,
)
aws_secret_key: str = Field(
description="Specifies the AWS secret key used as part of the credentials to authenticate the user.",
title="AWS Secret Access Key",
airbyte_secret=True,
)
role_arn: str = Field(
description="Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. (Needs permission to 'Assume Role' STS).",
title="Role ARN",
airbyte_secret=True,
)
aws_environment: AWSEnvironment = Field(description="Select the AWS Environment.", title="AWS Environment")
region: AWSRegion = Field(description="Select the AWS Region.", title="AWS Region")
class SourceAmazonSellerPartner(AbstractSource):
def _get_stream_kwargs(self, config: ConnectorConfig) -> Mapping[str, Any]:
def _get_stream_kwargs(self, config: AmazonSellerPartnerConfig) -> Mapping[str, Any]:
endpoint, marketplace_id, region = get_marketplaces(config.aws_environment)[config.region]
sts_credentials = self.get_sts_credentials(config)
@@ -104,10 +50,11 @@ class SourceAmazonSellerPartner(AbstractSource):
)
auth = AWSAuthenticator(
token_refresh_endpoint="https://api.amazon.com/auth/o2/token",
client_secret=config.lwa_client_secret,
client_id=config.lwa_app_id,
client_secret=config.lwa_client_secret,
refresh_token=config.refresh_token,
host=endpoint.replace("https://", ""),
refresh_access_token_headers={"Content-Type": "application/x-www-form-urlencoded"},
)
stream_kwargs = {
"url_base": endpoint,
@@ -121,7 +68,8 @@ class SourceAmazonSellerPartner(AbstractSource):
}
return stream_kwargs
def get_sts_credentials(self, config: ConnectorConfig) -> dict:
@staticmethod
def get_sts_credentials(config: AmazonSellerPartnerConfig) -> dict:
"""
We can only use a IAM User arn entity or a IAM Role entity.
If we use an IAM user arn entity in the connector configuration we need to get the credentials directly from the boto3 sts client
@@ -146,20 +94,23 @@ class SourceAmazonSellerPartner(AbstractSource):
Show error message in case of request exception or unexpected response.
"""
try:
config = ConnectorConfig.parse_obj(config) # FIXME: this will be not need after we fix CDK
config = AmazonSellerPartnerConfig.parse_obj(config) # FIXME: this will be not need after we fix CDK
stream_kwargs = self._get_stream_kwargs(config)
orders_stream = Orders(**stream_kwargs)
next(orders_stream.read_records(sync_mode=SyncMode.full_refresh))
return True, None
except StopIteration or requests.exceptions.RequestException as e:
except Exception as e:
if isinstance(e, StopIteration):
e = "Could not check connection without data for Orders stream. " "Please change value for replication start date field."
logger.error(
"Could not check connection without data for Orders stream. Please change value for replication start date field."
)
return False, e
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
"""
:param config: A Mapping of the user input configuration as defined in the connector spec.
"""
config = ConnectorConfig.parse_obj(config) # FIXME: this will be not need after we fix CDK
config = AmazonSellerPartnerConfig.parse_obj(config) # FIXME: this will be not need after we fix CDK
stream_kwargs = self._get_stream_kwargs(config)
return [
@@ -190,7 +141,7 @@ class SourceAmazonSellerPartner(AbstractSource):
configurations (e.g: username and password) required to run this integration.
"""
# FIXME: airbyte-cdk does not parse pydantic $ref correctly. This override won't be needed after the fix
schema = ConnectorConfig.schema()
schema = AmazonSellerPartnerConfig.schema()
schema["properties"]["aws_environment"] = schema["definitions"]["AWSEnvironment"]
schema["properties"]["region"] = schema["definitions"]["AWSRegion"]
@@ -198,4 +149,5 @@ class SourceAmazonSellerPartner(AbstractSource):
documentationUrl="https://docs.airbyte.io/integrations/sources/amazon-seller-partner",
changelogUrl="https://docs.airbyte.io/integrations/sources/amazon-seller-partner",
connectionSpecification=schema,
advanced_auth=advanced_auth,
)

106
airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.py Normal file
View File

@@ -0,0 +1,106 @@
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
from airbyte_cdk.models import AdvancedAuth, AuthFlowType, OAuthConfigSpecification
from pydantic import BaseModel, Field
from source_amazon_seller_partner.constants import AWSEnvironment, AWSRegion
class AmazonSellerPartnerConfig(BaseModel):
class Config:
title = "Amazon Seller Partner Spec"
schema_extra = {"additionalProperties": True}
app_id: str = Field(None, description="Your Amazon App ID", title="App Id *", airbyte_secret=True, order=0)
auth_type: str = Field(default="oauth2.0", const=True, order=1)
lwa_app_id: str = Field(description="Your Login with Amazon Client ID.", title="LWA Client Id", order=2)
lwa_client_secret: str = Field(
description="Your Login with Amazon Client Secret.", title="LWA Client Secret", airbyte_secret=True, order=3
)
refresh_token: str = Field(
description="The Refresh Token obtained via OAuth flow authorization.", title="Refresh Token", airbyte_secret=True, order=4
)
aws_access_key: str = Field(
description="Specifies the AWS access key used as part of the credentials to authenticate the user.",
title="AWS Access Key",
airbyte_secret=True,
order=5,
)
aws_secret_key: str = Field(
description="Specifies the AWS secret key used as part of the credentials to authenticate the user.",
title="AWS Secret Access Key",
airbyte_secret=True,
order=6,
)
role_arn: str = Field(
description="Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. (Needs permission to 'Assume Role' STS).",
title="Role ARN",
airbyte_secret=True,
order=7,
)
replication_start_date: str = Field(
description="UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.",
title="Start Date",
pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
examples=["2017-01-25T00:00:00Z"],
)
period_in_days: int = Field(
30,
description="Will be used for stream slicing for initial full_refresh sync when no updated state is present for reports that support sliced incremental sync.",
examples=["30", "365"],
)
report_options: str = Field(
None,
description="Additional information passed to reports. This varies by report type. Must be a valid json string.",
examples=['{"GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT": {"reportPeriod": "WEEK"}}', '{"GET_SOME_REPORT": {"custom": "true"}}'],
)
max_wait_seconds: int = Field(
500,
title="Max wait time for reports (in seconds)",
description="Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.",
examples=["500", "1980"],
)
aws_environment: AWSEnvironment = Field(description="Select the AWS Environment.", title="AWS Environment")
region: AWSRegion = Field(description="Select the AWS Region.", title="AWS Region")
advanced_auth = AdvancedAuth(
auth_flow_type=AuthFlowType.oauth2_0,
predicate_key=["auth_type"],
predicate_value="oauth2.0",
oauth_config_specification=OAuthConfigSpecification(
complete_oauth_output_specification={
"type": "object",
"additionalProperties": False,
"properties": {"refresh_token": {"type": "string", "path_in_connector_config": ["refresh_token"]}},
},
complete_oauth_server_input_specification={
"type": "object",
"additionalProperties": False,
"properties": {"lwa_app_id": {"type": "string"}, "lwa_client_secret": {"type": "string"}},
},
complete_oauth_server_output_specification={
"type": "object",
"additionalProperties": False,
"properties": {
"lwa_app_id": {"type": "string", "path_in_connector_config": ["lwa_app_id"]},
"lwa_client_secret": {"type": "string", "path_in_connector_config": ["lwa_client_secret"]},
},
},
oauth_user_input_from_connector_config_specification={
"type": "object",
"additionalProperties": False,
"properties": {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}},
},
),
)

5
airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py
View File

@@ -7,8 +7,9 @@ from unittest.mock import MagicMock
import pytest
from airbyte_cdk.models import ConnectorSpecification
from airbyte_cdk.sources.streams import Stream
from source_amazon_seller_partner import ConnectorConfig, SourceAmazonSellerPartner
from source_amazon_seller_partner import SourceAmazonSellerPartner
from source_amazon_seller_partner.source import boto3
from source_amazon_seller_partner.spec import AmazonSellerPartnerConfig
@pytest.fixture
@@ -18,7 +19,7 @@ def connector_source():
@pytest.fixture
def connector_config():
return ConnectorConfig(
return AmazonSellerPartnerConfig(
replication_start_date="2017-01-25T00:00:00Z",
refresh_token="Atzr|IwEBIP-abc123",
lwa_app_id="amzn1.application-oa2-client.abc123",

1
airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java
View File

@@ -21,6 +21,7 @@ public class OAuthImplementationFactory {
public OAuthImplementationFactory(final ConfigRepository configRepository, final HttpClient httpClient) {
OAUTH_FLOW_MAPPING = ImmutableMap.<String, OAuthFlowImplementation>builder()
.put("airbyte/source-amazon-ads", new AmazonAdsOAuthFlow(configRepository, httpClient))
.put("airbyte/source-amazon-seller-partner", new AmazonSellerPartnerOAuthFlow(configRepository, httpClient))
.put("airbyte/source-asana", new AsanaOAuthFlow(configRepository, httpClient))
.put("airbyte/source-facebook-marketing", new FacebookMarketingOAuthFlow(configRepository, httpClient))
.put("airbyte/source-facebook-pages", new FacebookPagesOAuthFlow(configRepository, httpClient))

110
airbyte-oauth/src/main/java/io/airbyte/oauth/flows/AmazonSellerPartnerOAuthFlow.java Normal file
View File

@@ -0,0 +1,110 @@
/*
* Copyright (c) 2022 Airbyte, Inc., all rights reserved.
*/
package io.airbyte.oauth.flows;
import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.collect.ImmutableMap;
import io.airbyte.config.persistence.ConfigRepository;
import io.airbyte.oauth.BaseOAuth2Flow;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.http.HttpClient;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.function.Supplier;
import org.apache.http.client.utils.URIBuilder;
public class AmazonSellerPartnerOAuthFlow extends BaseOAuth2Flow {
private static final String AUTHORIZE_URL = "https://sellercentral.amazon.com/apps/authorize/consent";
private static final String ACCESS_TOKEN_URL = "https://api.amazon.com/auth/o2/token";
protected String getClientIdUnsafe(final JsonNode oauthConfig) {
return getConfigValueUnsafe(oauthConfig, "lwa_app_id");
}
protected String getClientSecretUnsafe(final JsonNode oauthConfig) {
return getConfigValueUnsafe(oauthConfig, "lwa_client_secret");
}
public AmazonSellerPartnerOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) {
super(configRepository, httpClient);
}
public AmazonSellerPartnerOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier<String> stateSupplier) {
super(configRepository, httpClient, stateSupplier);
}
/**
* Depending on the OAuth flow implementation, the URL to grant user's consent may differ,
* especially in the query parameters to be provided. This function should generate such consent URL
* accordingly.
*
* @param definitionId The configured definition ID of this client
* @param clientId The configured client ID
* @param redirectUrl the redirect URL
*/
@Override
protected String formatConsentUrl(final UUID definitionId,
final String clientId,
final String redirectUrl,
final JsonNode inputOAuthConfiguration)
throws IOException {
// getting application_id value from user's config
final String application_id = getConfigValueUnsafe(inputOAuthConfiguration, "app_id");
try {
return new URIBuilder(AUTHORIZE_URL)
.addParameter("application_id", application_id)
.addParameter("redirect_uri", redirectUrl)
.addParameter("state", getState())
.addParameter("version", "beta")
.build().toString();
} catch (final URISyntaxException e) {
throw new IOException("Failed to format Consent URL for OAuth flow", e);
}
}
@Override
protected String extractCodeParameter(Map<String, Object> queryParams) throws IOException {
if (queryParams.containsKey("spapi_oauth_code")) {
return (String) queryParams.get("spapi_oauth_code");
} else {
throw new IOException("Undefined 'spapi_oauth_code' from consent redirected url.");
}
}
@Override
protected Map<String, String> getAccessTokenQueryParameters(final String clientId,
final String clientSecret,
final String authCode,
final String redirectUrl) {
return ImmutableMap.<String, String>builder()
// required
.put("client_id", clientId)
.put("redirect_uri", redirectUrl)
.put("client_secret", clientSecret)
.put("code", authCode)
.put("grant_type", "authorization_code")
.build();
}
/**
* Returns the URL where to retrieve the access token from.
*
*/
@Override
protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) {
return ACCESS_TOKEN_URL;
}
@Override
public List<String> getDefaultOAuthOutputPath() {
return List.of();
}
}

168
docs/integrations/sources/amazon-seller-partner.md
View File

@@ -1,92 +1,114 @@
# Amazon Seller Partner
## Features
This page guides you through the process of setting up the Amazon Seller Partner source connector.
| Feature | Supported?\(Yes/No\) | Notes |
| :---------------- | :------------------- | :---- |
| Full Refresh Sync | Yes | |
| Incremental Sync | Yes | |
## Prerequisites
This source syncs data from the [Amazon Seller Partner API](https://github.com/amzn/selling-partner-api-docs/blob/main/guides/en-US/developer-guide/SellingPartnerApiDeveloperGuide.md).
## Supported Tables
This source is capable of syncing the following streams:
- [Order Report (by order date and by last update)](https://sellercentral.amazon.com/gp/help/help.html?itemID=201648780)
- [All Listings](https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reporttype-values.md#inventory-reports)
- [FBA Inventory Reports](https://sellercentral.amazon.com/gp/help/200740930)
- [FBA Replacements Report](https://sellercentral.amazon.com/help/hub/reference/200453300)
- [Amazon-Fulfilled Shipments Report](https://sellercentral.amazon.com/gp/help/help.html?itemID=200453120)
- [Open Listings Report](https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reporttype-values.md#inventory-reports)
- [Removal Order Detail Report (overview)](https://sellercentral.amazon.com/gp/help/help.html?itemID=200989110)
- [Removal Shipment Detail Report](https://sellercentral.amazon.com/gp/help/help.html?itemID=200989100)
- [Inventory Health & Planning Report](https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reporttype-values.md#vendor-retail-analytics-reports)
- [Orders](https://github.com/amzn/selling-partner-api-docs/blob/main/references/orders-api/ordersV0.md) \(incremental\)
- [VendorDirectFulfillmentShipping](https://github.com/amzn/selling-partner-api-docs/blob/main/references/vendor-direct-fulfillment-shipping-api/vendorDirectFulfillmentShippingV1.md)
- [Seller Feedback Report](https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reporttype-values.md#performance-reports)
- [Brand Analytics Alternate Purchase Report](https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reporttype-values.md#brand-analytics-reports)
- [Brand Analytics Item Comparison Report](https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reporttype-values.md#brand-analytics-reports)
- [Brand Analytics Market Basket Report](https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reporttype-values.md#brand-analytics-reports)
- [Brand Analytics Repeat Purchase Report](https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reporttype-values.md#brand-analytics-reports)
- [Brand Analytics Search Terms Report](https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reporttype-values.md#brand-analytics-reports)
- [Browse tree report](https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reporttype-values.md#browse-tree-report)
## Getting started
**Requirements**
- replication_start_date
- refresh_token
- app_id
- lwa_app_id
- lwa_client_secret
- refresh_token
- aws_access_key
- aws_secret_key
- role_arn
- aws_environment
- region
- replication_start_date
**Setup guide**
## Step 1: Set up Amazon Seller Partner
Information about how to get credentials you may find [here](https://github.com/amzn/selling-partner-api-docs/blob/main/guides/en-US/developer-guide/SellingPartnerApiDeveloperGuide.md).
[Register](https://developer-docs.amazon.com/sp-api/docs/registering-your-application) Amazon Seller Partner application.
[Create](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html) IAM user.
## Step 2: Set up the source connector in Airbyte
**For Airbyte Cloud:**
1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account.
2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**.
3. On the source setup page, select **Amazon Seller Partner** from the Source type dropdown and enter a name for this connector.
4. Click `Authenticate your account`.
5. Log in and Authorize to the Amazon Seller Partner account.
6. Paste all other data to required fields using your IAM user.
7. Click `Set up source`.
**For Airbyte OSS:**
1. Using developer application from Step 1, [generate](https://developer-docs.amazon.com/sp-api/docs/self-authorization) refresh token.
2. Go to local Airbyte page.
3. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**.
4. On the Set up the source page, enter the name for the Amazon Seller Partner connector and select **Amazon Seller Partner** from the Source type dropdown.
5. Paste all data to required fields using your IAM user and developer account.
6. Click `Set up source`.
## Supported sync modes
The Amazon Seller Partner source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-mode):
- Full Refresh
- Incremental
## Performance considerations
Information about rate limits you may find [here](https://developer-docs.amazon.com/sp-api/docs/usage-plans-and-rate-limits-in-the-sp-api).
## Supported streams
This source is capable of syncing the following tables and their data:
- [FBA Inventory Reports](https://sellercentral.amazon.com/gp/help/200740930)
- [FBA Orders Reports](https://sellercentral.amazon.com/gp/help/help.html?itemID=200989110)
- [FBA Shipments Reports](https://sellercentral.amazon.com/gp/help/help.html?itemID=200989100)
- [FBA Replacements Reports](https://sellercentral.amazon.com/help/hub/reference/200453300)
- [Flat File Open Listings Reports](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference)
- [Flat File Orders Reports](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference)
- [Flat File Orders Reports By Last Update](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) \(incremental\)
- [Amazon-Fulfilled Shipments Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference)
- [Merchant Listings Reports](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference)
- [Vendor Direct Fulfillment Shipping](https://developer-docs.amazon.com/sp-api/docs/vendor-direct-fulfillment-shipping-api-v1-reference)
- [Vendor Inventory Health Reports](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference)
- [Orders](https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference) \(incremental\)
- [Seller Feedback Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) \(incremental\)
- [Brand Analytics Alternate Purchase Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports)
- [Brand Analytics Item Comparison Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports)
- [Brand Analytics Market Basket Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports)
- [Brand Analytics Repeat Purchase Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports)
- [Brand Analytics Search Terms Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports)
- [Browse tree report](https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reporttype-values.md#browse-tree-report)
## Data type mapping
| Integration Type | Airbyte Type | Notes |
| :----------------------- | :----------- | :---- |
| `string` | `string` | |
| `int`, `float`, `number` | `number` | |
| `date` | `date` | |
| `datetime` | `datetime` | |
| `array` | `array` | |
| `object` | `object` | |
| Integration Type | Airbyte Type |
| :----------------------- | :----------- |
| `string` | `string` |
| `int`, `float`, `number` | `number` |
| `date` | `date` |
| `datetime` | `datetime` |
| `array` | `array` |
| `object` | `object` |
### Performance Considerations (Airbyte Open-Source)
Information about rate limits you may find [here](https://github.com/amzn/selling-partner-api-docs/blob/main/guides/en-US/usage-plans-rate-limits/Usage-Plans-and-Rate-Limits.md).
## Changelog
## CHANGELOG
| Version | Date | Pull Request | Subject |
|:---------|:-----------|:---------------------------------------------------------| :--------------------------------------------------------------------- |
| `0.2.18` | 2022-05-06 | [\#12663](https://github.com/airbytehq/airbyte/pull/12663)| Add GET_XML_BROWSE_TREE_DATA report |
| `0.2.17` | 2022-05-19 | [\#12946](https://github.com/airbytehq/airbyte/pull/12946)| Add throttling exception managing in Orders streams |
| `0.2.16` | 2022-05-04 | [\#12523](https://github.com/airbytehq/airbyte/pull/12523)| allow to use IAM user arn or IAM role arn |
| `0.2.15` | 2022-01-25 | [\#9789](https://github.com/airbytehq/airbyte/pull/9789) | Add stream FbaReplacementsReports |
| `0.2.14` | 2022-01-19 | [\#9621](https://github.com/airbytehq/airbyte/pull/9621) | Add GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL report |
| `0.2.13` | 2022-01-18 | [\#9581](https://github.com/airbytehq/airbyte/pull/9581) | Change createdSince parameter to dataStartTime |
| `0.2.12` | 2022-01-05 | [\#9312](https://github.com/airbytehq/airbyte/pull/9312) | Add all remaining brand analytics report streams |
| `0.2.11` | 2022-01-05 | [\#9115](https://github.com/airbytehq/airbyte/pull/9115) | Fix reading only 100 orders |
| `0.2.10` | 2021-12-31 | [\#9236](https://github.com/airbytehq/airbyte/pull/9236) | Fix NoAuth deprecation warning |
| `0.2.9` | 2021-12-30 | [\#9212](https://github.com/airbytehq/airbyte/pull/9212) | Normalize GET_SELLER_FEEDBACK_DATA header field names |
| `0.2.8` | 2021-12-22 | [\#8810](https://github.com/airbytehq/airbyte/pull/8810) | Fix GET_SELLER_FEEDBACK_DATA Date cursor field format |
| `0.2.7` | 2021-12-21 | [\#9002](https://github.com/airbytehq/airbyte/pull/9002) | Extract REPORTS_MAX_WAIT_SECONDS to configurable parameter |
| `0.2.6` | 2021-12-10 | [\#8179](https://github.com/airbytehq/airbyte/pull/8179) | Add GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT report |
| `0.2.5` | 2021-12-06 | [\#8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec |
| `0.2.4` | 2021-11-08 | [\#8021](https://github.com/airbytehq/airbyte/pull/8021) | Added GET_SELLER_FEEDBACK_DATA report with incremental sync capability |
| `0.2.3` | 2021-11-08 | [\#7828](https://github.com/airbytehq/airbyte/pull/7828) | Remove datetime format from all streams |
| `0.2.2` | 2021-11-08 | [\#7752](https://github.com/airbytehq/airbyte/pull/7752) | Change `check_connection` function to use stream Orders |
| `0.2.1` | 2021-09-17 | [\#5248](https://github.com/airbytehq/airbyte/pull/5248) | `Added extra stream support. Updated reports streams logics` |
| `0.2.0` | 2021-08-06 | [\#4863](https://github.com/airbytehq/airbyte/pull/4863) | `Rebuild source with airbyte-cdk` |
| `0.1.3` | 2021-06-23 | [\#4288](https://github.com/airbytehq/airbyte/pull/4288) | `Bugfix failing connection check` |
| `0.1.2` | 2021-06-15 | [\#4108](https://github.com/airbytehq/airbyte/pull/4108) | `Fixed: Sync fails with timeout when create report is CANCELLED` |
| Version | Date | Pull Request | Subject |
|:---------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------|
| `0.2.19` | 2022-05-24 | [\#13119](https://github.com/airbytehq/airbyte/pull/13119) | Add OAuth2.0 support |
| `0.2.18` | 2022-05-06 | [\#12663](https://github.com/airbytehq/airbyte/pull/12663) | Add GET_XML_BROWSE_TREE_DATA report |
| `0.2.17` | 2022-05-19 | [\#12946](https://github.com/airbytehq/airbyte/pull/12946) | Add throttling exception managing in Orders streams |
| `0.2.16` | 2022-05-04 | [\#12523](https://github.com/airbytehq/airbyte/pull/12523) | allow to use IAM user arn or IAM role arn |
| `0.2.15` | 2022-01-25 | [\#9789](https://github.com/airbytehq/airbyte/pull/9789) | Add stream FbaReplacementsReports |
| `0.2.14` | 2022-01-19 | [\#9621](https://github.com/airbytehq/airbyte/pull/9621) | Add GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL report |
| `0.2.13` | 2022-01-18 | [\#9581](https://github.com/airbytehq/airbyte/pull/9581) | Change createdSince parameter to dataStartTime |
| `0.2.12` | 2022-01-05 | [\#9312](https://github.com/airbytehq/airbyte/pull/9312) | Add all remaining brand analytics report streams |
| `0.2.11` | 2022-01-05 | [\#9115](https://github.com/airbytehq/airbyte/pull/9115) | Fix reading only 100 orders |
| `0.2.10` | 2021-12-31 | [\#9236](https://github.com/airbytehq/airbyte/pull/9236) | Fix NoAuth deprecation warning |
| `0.2.9` | 2021-12-30 | [\#9212](https://github.com/airbytehq/airbyte/pull/9212) | Normalize GET_SELLER_FEEDBACK_DATA header field names |
| `0.2.8` | 2021-12-22 | [\#8810](https://github.com/airbytehq/airbyte/pull/8810) | Fix GET_SELLER_FEEDBACK_DATA Date cursor field format |
| `0.2.7` | 2021-12-21 | [\#9002](https://github.com/airbytehq/airbyte/pull/9002) | Extract REPORTS_MAX_WAIT_SECONDS to configurable parameter |
| `0.2.6` | 2021-12-10 | [\#8179](https://github.com/airbytehq/airbyte/pull/8179) | Add GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT report |
| `0.2.5` | 2021-12-06 | [\#8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec |
| `0.2.4` | 2021-11-08 | [\#8021](https://github.com/airbytehq/airbyte/pull/8021) | Added GET_SELLER_FEEDBACK_DATA report with incremental sync capability |
| `0.2.3` | 2021-11-08 | [\#7828](https://github.com/airbytehq/airbyte/pull/7828) | Remove datetime format from all streams |
| `0.2.2` | 2021-11-08 | [\#7752](https://github.com/airbytehq/airbyte/pull/7752) | Change `check_connection` function to use stream Orders |
| `0.2.1` | 2021-09-17 | [\#5248](https://github.com/airbytehq/airbyte/pull/5248) | `Added extra stream support. Updated reports streams logics` |
| `0.2.0` | 2021-08-06 | [\#4863](https://github.com/airbytehq/airbyte/pull/4863) | `Rebuild source with airbyte-cdk` |
| `0.1.3` | 2021-06-23 | [\#4288](https://github.com/airbytehq/airbyte/pull/4288) | `Bugfix failing connection check` |
| `0.1.2` | 2021-06-15 | [\#4108](https://github.com/airbytehq/airbyte/pull/4108) | `Fixed: Sync fails with timeout when create report is CANCELLED` |