jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-25 02:09:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

🎉 Source Recharge: increase unit_test cov, fix schemas (#14902)

Browse Source
This commit is contained in:
Baz
2022-07-22 22:58:25 +03:00
committed by GitHub
parent e838cd6822
commit 67d1a138f7
18 changed files with 565 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
airbyte-integrations/connectors/source-recharge/Dockerfile
View File

@@ -12,5 +12,5 @@ RUN pip install .
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
LABEL io.airbyte.version=0.1.5
LABEL io.airbyte.version=0.1.6
LABEL io.airbyte.name=airbyte/source-recharge

6
airbyte-integrations/connectors/source-recharge/README.md
View File

@@ -26,8 +26,7 @@ If you are in an IDE, follow your IDE's instructions to activate the virtualenv.
Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is
used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`.
If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything
should work as you expect.
If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect.
#### Building via Gradle
You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow.
@@ -101,7 +100,8 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Source Acce
If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
To run your integration tests with acceptance tests, from the connector root, run
```
python -m pytest integration_tests -p integration_tests.acceptance
docker build . --no-cache -t airbyte/source-recharge:dev \
&& python -m pytest -p integration_tests.acceptance
```
To run your integration tests with docker

1
airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml
View File

@@ -13,6 +13,7 @@ tests:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/streams_with_output_records_catalog.json"
timeout_seconds: 1200
empty_streams: ["collections", "discounts"]
incremental:
- config_path: "secrets/config.json"
configured_catalog_path: "integration_tests/streams_with_output_records_catalog.json"

2
airbyte-integrations/connectors/source-recharge/integration_tests/acceptance.py
View File

@@ -11,6 +11,4 @@ pytest_plugins = ("source_acceptance_test.plugin",)
@pytest.fixture(scope="session", autouse=True)
def connector_setup():
"""This fixture is a placeholder for external resources that acceptance test might require."""
# TODO: setup test dependencies if needed. otherwise remove the TODO comments
yield
# TODO: clean up test dependencies

1
airbyte-integrations/connectors/source-recharge/setup.py
View File

@@ -11,6 +11,7 @@ MAIN_REQUIREMENTS = [
TEST_REQUIREMENTS = [
"pytest~=6.1",
"requests-mock",
]
setup(

4
airbyte-integrations/connectors/source-recharge/source_recharge/api.py
View File

@@ -74,6 +74,10 @@ class IncrementalRechargeStream(RechargeStream, ABC):
super().__init__(**kwargs)
self._start_date = pendulum.parse(start_date)
@property
def state_checkpoint_interval(self):
return self.limit
def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]:
latest_benchmark = latest_record[self.cursor_field]
if current_stream_state.get(self.cursor_field):

40
airbyte-integrations/connectors/source-recharge/source_recharge/schemas/charges.json
View File

@@ -88,7 +88,7 @@
"discount_codes": {
"type": ["null", "array"],
"items": {
"type": "object"
"type": ["null", "object"]
}
},
"email": {
@@ -214,7 +214,7 @@
"type": ["null", "string"]
},
"shopify_variant_id_not_found": {
"type": ["null", "integer", "string"]
"type": ["null", "integer"]
},
"status": {
"type": ["null", "string"]
@@ -226,13 +226,34 @@
"type": ["null", "string"]
},
"tax_lines": {
"type": ["null", "string", "number"]
"oneOf": [
{
"type": ["null", "number"]
},
{
"type": ["null", "string"]
}
]
},
"total_discounts": {
"type": ["null", "string"]
"oneOf": [
{
"type": ["null", "number"]
},
{
"type": ["null", "string"]
}
]
},
"total_line_items_price": {
"type": ["null", "string"]
"oneOf": [
{
"type": ["null", "number"]
},
{
"type": ["null", "string"]
}
]
},
"total_price": {
"type": ["null", "string"]
@@ -241,7 +262,14 @@
"type": ["null", "string"]
},
"total_tax": {
"type": ["null", "string", "number"]
"oneOf": [
{
"type": ["null", "number"]
},
{
"type": ["null", "string"]
}
]
},
"total_weight": {
"type": ["null", "integer"]

18
airbyte-integrations/connectors/source-recharge/source_recharge/schemas/metafields.json
View File

@@ -19,7 +19,14 @@
"type": ["null", "string"]
},
"owner_id": {
"type": ["null", "integer", "string"]
"oneOf": [
{
"type": ["null", "number"]
},
{
"type": ["null", "string"]
}
]
},
"owner_resource": {
"type": ["null", "string"]
@@ -29,7 +36,14 @@
"format": "date-time"
},
"value": {
"type": ["null", "string", "number", "integer"]
"oneOf": [
{
"type": ["null", "number"]
},
{
"type": ["null", "string"]
}
]
},
"value_type": {
"type": ["null", "string"]

27
airbyte-integrations/connectors/source-recharge/source_recharge/schemas/onetimes.json
View File

@@ -6,21 +6,42 @@
"type": ["null", "integer"]
},
"address_id": {
"type": ["null", "integer", "string"]
"oneOf": [
{
"type": ["null", "number"]
},
{
"type": ["null", "string"]
}
]
},
"created_at": {
"type": ["null", "string"],
"format": "date-time"
},
"customer_id": {
"type": ["null", "integer", "string"]
"oneOf": [
{
"type": ["null", "number"]
},
{
"type": ["null", "string"]
}
]
},
"next_charge_scheduled_at": {
"type": ["null", "string"],
"format": "date-time"
},
"price": {
"type": ["null", "integer", "string", "number"]
"oneOf": [
{
"type": ["null", "number"]
},
{
"type": ["null", "string"]
}
]
},
"product_title": {
"type": ["null", "string"]

37
airbyte-integrations/connectors/source-recharge/source_recharge/schemas/orders.json
View File

@@ -210,7 +210,7 @@
"type": ["null", "object"]
},
"price": {
"type": ["null", "string"]
"type": ["null", "number"]
},
"properties": {
"type": ["null", "array"]
@@ -219,10 +219,24 @@
"type": ["null", "integer"]
},
"shopify_product_id": {
"type": ["null", "string", "integer"]
"oneOf": [
{
"type": ["null", "number"]
},
{
"type": ["null", "string"]
}
]
},
"shopify_variant_id": {
"type": ["null", "string", "integer"]
"oneOf": [
{
"type": ["null", "number"]
},
{
"type": ["null", "string"]
}
]
},
"sku": {
"type": ["null", "string"]
@@ -343,7 +357,7 @@
"type": ["null", "string"]
},
"subtotal_price": {
"type": ["null", "number", "string"]
"type": ["null", "number"]
},
"tags": {
"type": ["null", "string"]
@@ -355,19 +369,26 @@
"type": ["null", "string"]
},
"total_discounts": {
"type": ["null", "string"]
"type": ["null", "number"]
},
"total_line_items_price": {
"type": ["null", "string"]
"type": ["null", "number"]
},
"total_price": {
"type": ["null", "string"]
"type": ["null", "number"]
},
"total_refunds": {
"type": ["null", "string"]
},
"total_tax": {
"type": ["null", "number", "string"]
"oneOf": [
{
"type": ["null", "number"]
},
{
"type": ["null", "string"]
}
]
},
"total_weight": {
"type": ["null", "integer"]

9
airbyte-integrations/connectors/source-recharge/source_recharge/schemas/subscriptions.json
View File

@@ -85,7 +85,14 @@
"type": "string"
},
"value": {
"type": ["string", "integer"]
"oneOf": [
{
"type": ["null", "number"]
},
{
"type": ["null", "string"]
}
]
}
}
}

8
airbyte-integrations/connectors/source-recharge/source_recharge/source.py
View File

@@ -21,10 +21,12 @@ class RechargeTokenAuthenticator(TokenAuthenticator):
class SourceRecharge(AbstractSource):
def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]:
auth = RechargeTokenAuthenticator(token=config["access_token"])
stream = Shop(authenticator=auth)
try:
auth = RechargeTokenAuthenticator(token=config["access_token"])
list(Shop(authenticator=auth).read_records(SyncMode.full_refresh))
return True, None
result = list(stream.read_records(SyncMode.full_refresh))[0]
if stream.name in result.keys():
return True, None
except Exception as error:
return False, f"Unable to connect to Recharge API with the provided credentials - {repr(error)}"

339
airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py Normal file
View File

@@ -0,0 +1,339 @@
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
from http import HTTPStatus
from unittest.mock import MagicMock, patch
import pytest
import requests
from source_recharge.api import (
Addresses,
Charges,
Collections,
Customers,
Discounts,
Metafields,
Onetimes,
Orders,
Products,
RechargeStream,
Shop,
Subscriptions,
)
# config
@pytest.fixture(name="config")
def config():
return {
"authenticator": None,
"access_token": "access_token",
"start_date": "2021-08-15T00:00:00Z",
}
class TestCommon:
main = RechargeStream()
@pytest.mark.parametrize(
"stream_cls, expected",
[
(Addresses, "id"),
(Charges, "id"),
(Collections, "id"),
(Customers, "id"),
(Discounts, "id"),
(Metafields, "id"),
(Onetimes, "id"),
(Orders, "id"),
(Products, "id"),
(Shop, ["shop", "store"]),
(Subscriptions, "id"),
],
)
def test_primary_key(self, stream_cls, expected):
assert expected == stream_cls.primary_key
@pytest.mark.parametrize(
"stream_cls",
[
(Addresses),
(Charges),
(Collections),
(Customers),
(Discounts),
(Metafields),
(Onetimes),
(Orders),
(Products),
(Shop),
(Subscriptions),
],
)
def test_url_base(self, stream_cls):
expected = self.main.url_base
result = stream_cls.url_base
assert expected == result
@pytest.mark.parametrize(
"stream_cls",
[
(Addresses),
(Charges),
(Collections),
(Customers),
(Discounts),
(Metafields),
(Onetimes),
(Orders),
(Products),
(Shop),
(Subscriptions),
],
)
def test_limit(self, stream_cls):
expected = self.main.limit
result = stream_cls.limit
assert expected == result
@pytest.mark.parametrize(
"stream_cls",
[
(Addresses),
(Charges),
(Collections),
(Customers),
(Discounts),
(Metafields),
(Onetimes),
(Orders),
(Products),
(Shop),
(Subscriptions),
],
)
def test_page_num(self, stream_cls):
expected = self.main.page_num
result = stream_cls.page_num
assert expected == result
@pytest.mark.parametrize(
"stream_cls, stream_type, expected",
[
(Addresses, "incremental", "addresses"),
(Charges, "incremental", "charges"),
(Collections, "full-refresh", "collections"),
(Customers, "incremental", "customers"),
(Discounts, "incremental", "discounts"),
(Metafields, "full-refresh", "metafields"),
(Onetimes, "incremental", "onetimes"),
(Orders, "incremental", "orders"),
(Products, "full-refresh", "products"),
(Shop, "full-refresh", None),
(Subscriptions, "incremental", "subscriptions"),
],
)
def test_data_path(self, config, stream_cls, stream_type, expected):
if stream_type == "incremental":
result = stream_cls(start_date=config["start_date"]).data_path
else:
result = stream_cls().data_path
assert expected == result
@pytest.mark.parametrize(
"stream_cls, stream_type, expected",
[
(Addresses, "incremental", "addresses"),
(Charges, "incremental", "charges"),
(Collections, "full-refresh", "collections"),
(Customers, "incremental", "customers"),
(Discounts, "incremental", "discounts"),
(Metafields, "full-refresh", "metafields"),
(Onetimes, "incremental", "onetimes"),
(Orders, "incremental", "orders"),
(Products, "full-refresh", "products"),
(Shop, "full-refresh", "shop"),
(Subscriptions, "incremental", "subscriptions"),
],
)
def test_path(self, config, stream_cls, stream_type, expected):
if stream_type == "incremental":
result = stream_cls(start_date=config["start_date"]).path()
else:
result = stream_cls().path()
assert expected == result
@pytest.mark.parametrize(
("http_status", "should_retry"),
[
(HTTPStatus.OK, True),
(HTTPStatus.BAD_REQUEST, False),
(HTTPStatus.TOO_MANY_REQUESTS, True),
(HTTPStatus.INTERNAL_SERVER_ERROR, True),
],
)
def test_should_retry(patch_base_class, http_status, should_retry):
response_mock = MagicMock()
response_mock.status_code = http_status
stream = RechargeStream()
assert stream.should_retry(response_mock) == should_retry
class TestFullRefreshStreams:
def generate_records(self, stream_name, count):
result = []
for i in range(0, count):
result.append({f"record_{i}": f"test_{i}"})
return {stream_name: result}
@pytest.mark.parametrize(
"stream_cls, rec_limit, expected",
[
(Collections, 1, {"page": 2}),
(Metafields, 2, {"page": 2}),
(Products, 1, {"page": 2}),
(Shop, 1, {"page": 2}),
],
)
def test_next_page_token(self, stream_cls, rec_limit, requests_mock, expected):
stream = stream_cls()
stream.limit = rec_limit
url = f"{stream.url_base}{stream.path()}"
requests_mock.get(url, json=self.generate_records(stream.name, rec_limit))
response = requests.get(url)
assert stream.next_page_token(response) == expected
@pytest.mark.parametrize(
"stream_cls, next_page_token, stream_state, stream_slice, expected",
[
(Collections, None, {}, {}, {"limit": 250}),
(Metafields, {"page": 2}, {"updated_at": "2030-01-01"}, {}, {"limit": 250, "page": 2}),
(Products, None, {}, {}, {"limit": 250}),
(Shop, None, {}, {}, {"limit": 250}),
],
)
def test_request_params(self, stream_cls, next_page_token, stream_state, stream_slice, expected):
stream = stream_cls()
result = stream.request_params(stream_state, stream_slice, next_page_token)
assert result == expected
@pytest.mark.parametrize(
"stream_cls, data, expected",
[
(Collections, [{"test": 123}], [{"test": 123}]),
(Metafields, [{"test2": 234}], [{"test2": 234}]),
(Products, [{"test3": 345}], [{"test3": 345}]),
(Shop, {"test4": 456}, [{"test4": 456}]),
],
)
def test_parse_response(self, stream_cls, data, requests_mock, expected):
stream = stream_cls()
url = f"{stream.url_base}{stream.path()}"
data = {stream.data_path: data} if stream.data_path else data
requests_mock.get(url, json=data)
response = requests.get(url)
assert list(stream.parse_response(response)) == expected
@pytest.mark.parametrize(
"stream_cls, data, expected",
[
(Collections, [{"test": 123}], [{"test": 123}]),
(Metafields, [{"test2": 234}], [{"test2": 234}]),
(Products, [{"test3": 345}], [{"test3": 345}]),
(Shop, {"test4": 456}, [{"test4": 456}]),
],
)
def get_stream_data(self, stream_cls, data, requests_mock, expected):
stream = stream_cls()
url = f"{stream.url_base}{stream.path()}"
data = {stream.data_path: data} if stream.data_path else data
requests_mock.get(url, json=data)
response = requests.get(url)
assert list(stream.parse_response(response)) == expected
@pytest.mark.parametrize("owner_resource, expected", [({"customer": {"id": 123}}, {"customer": {"id": 123}})])
def test_metafields_read_records(self, owner_resource, expected):
with patch.object(Metafields, "read_records", return_value=owner_resource):
result = Metafields().read_records(stream_slice={"owner_resource": owner_resource})
assert result == expected
class TestIncrementalStreams:
def generate_records(self, stream_name, count):
result = []
for i in range(0, count):
result.append({f"record_{i}": f"test_{i}"})
return {stream_name: result}
@pytest.mark.parametrize(
"stream_cls, expected",
[
(Addresses, "updated_at"),
(Charges, "updated_at"),
(Customers, "updated_at"),
(Discounts, "updated_at"),
(Onetimes, "updated_at"),
(Orders, "updated_at"),
(Subscriptions, "updated_at"),
],
)
def test_cursor_field(self, config, stream_cls, expected):
stream = stream_cls(start_date=config["start_date"])
result = stream.cursor_field
assert result == expected
@pytest.mark.parametrize(
"stream_cls, rec_limit, expected",
[
(Addresses, 1, {"page": 2}),
(Charges, 2, {"page": 2}),
(Customers, 1, {"page": 2}),
(Discounts, 1, {"page": 2}),
(Onetimes, 1, {"page": 2}),
(Orders, 1, {"page": 2}),
(Subscriptions, 1, {"page": 2}),
],
)
def test_next_page_token(self, config, stream_cls, rec_limit, requests_mock, expected):
stream = stream_cls(start_date=config["start_date"])
stream.limit = rec_limit
url = f"{stream.url_base}{stream.path()}"
requests_mock.get(url, json=self.generate_records(stream.name, rec_limit))
response = requests.get(url)
assert stream.next_page_token(response) == expected
@pytest.mark.parametrize(
"stream_cls, next_page_token, stream_state, stream_slice, expected",
[
(Addresses, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15 00:00:00"}),
(Charges, {"page": 2}, {"updated_at": "2030-01-01"}, {}, {"limit": 250, "page": 2, "updated_at_min": "2030-01-01 00:00:00"}),
(Customers, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15 00:00:00"}),
(Discounts, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15 00:00:00"}),
(Onetimes, {"page": 2}, {"updated_at": "2030-01-01"}, {}, {"limit": 250, "page": 2, "updated_at_min": "2030-01-01 00:00:00"}),
(Orders, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15 00:00:00"}),
(Subscriptions, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15 00:00:00"}),
],
)
def test_request_params(self, config, stream_cls, next_page_token, stream_state, stream_slice, expected):
stream = stream_cls(start_date=config["start_date"])
result = stream.request_params(stream_state, stream_slice, next_page_token)
assert result == expected
@pytest.mark.parametrize(
"stream_cls, current_state, latest_record, expected",
[
(Addresses, {}, {"updated_at": 2}, {"updated_at": 2}),
(Charges, {"updated_at": 2}, {"updated_at": 3}, {"updated_at": 3}),
(Customers, {"updated_at": 3}, {"updated_at": 4}, {"updated_at": 4}),
(Discounts, {}, {"updated_at": 2}, {"updated_at": 2}),
(Onetimes, {}, {"updated_at": 2}, {"updated_at": 2}),
(Orders, {"updated_at": 5}, {"updated_at": 5}, {"updated_at": 5}),
(Subscriptions, {"updated_at": 6}, {"updated_at": 7}, {"updated_at": 7}),
],
)
def test_get_updated_state(self, config, stream_cls, current_state, latest_record, expected):
stream = stream_cls(start_date=config["start_date"])
result = stream.get_updated_state(current_state, latest_record)
assert result == expected

58
airbyte-integrations/connectors/source-recharge/unit_tests/test_source.py Normal file
View File

@@ -0,0 +1,58 @@
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
from unittest.mock import patch
import pytest
from requests.exceptions import HTTPError
from source_recharge.api import Shop
from source_recharge.source import RechargeTokenAuthenticator, SourceRecharge
# config
@pytest.fixture(name="config")
def config():
return {
"authenticator": None,
"access_token": "access_token",
"start_date": "2021-08-15T00:00:00Z",
}
# logger
@pytest.fixture(name="logger_mock")
def logger_mock_fixture():
return patch("source_recharge.source.AirbyteLogger")
def test_get_auth_header(config):
expected = {"X-Recharge-Access-Token": config.get("access_token")}
actual = RechargeTokenAuthenticator(token=config["access_token"]).get_auth_header()
assert actual == expected
@pytest.mark.parametrize(
"patch, expected",
[
(
patch.object(Shop, "read_records", return_value=[{"shop": {"id": 123}}]),
(True, None),
),
(
patch.object(Shop, "read_records", side_effect=HTTPError(403)),
(False, "Unable to connect to Recharge API with the provided credentials - HTTPError(403)"),
),
],
ids=["success", "fail"],
)
def test_check_connection(logger_mock, config, patch, expected):
with patch:
result = SourceRecharge().check_connection(logger_mock, config=config)
assert result == expected
def test_streams(config):
streams = SourceRecharge().streams(config)
assert len(streams) == 11

7
airbyte-integrations/connectors/source-recharge/unit_tests/unit_test.py
View File

@@ -1,7 +0,0 @@
#
# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
#
def test_example_method():
assert True