diff --git a/airbyte-integrations/connector-templates/generator/generate.sh b/airbyte-integrations/connector-templates/generator/generate.sh index 361bb747b3e..0f79661fcd3 100755 --- a/airbyte-integrations/connector-templates/generator/generate.sh +++ b/airbyte-integrations/connector-templates/generator/generate.sh @@ -1,5 +1,7 @@ #!/usr/bin/env sh +UID=$(id -u) +GID=$(id -g) # Remove container if already exist echo "Removing previous generator if it exists..." docker container rm -f airbyte-connector-bootstrap >/dev/null 2>&1 @@ -8,7 +10,7 @@ docker container rm -f airbyte-connector-bootstrap >/dev/null 2>&1 # Specify the host system user UID and GID to chown the generated files to host system user. # This is done because all generated files in container with mounted folders has root owner echo "Building generator docker image..." -docker build --build-arg UID="$(id -u)" --build-arg GID="$(id -g)" . -t airbyte/connector-bootstrap +docker build --build-arg UID="$UID" --build-arg GID="$GID" . -t airbyte/connector-bootstrap # Run the container and mount the airbyte folder if [ $# -eq 2 ]; then @@ -16,7 +18,7 @@ if [ $# -eq 2 ]; then docker run --name airbyte-connector-bootstrap -e package_desc="$1" -e package_name="$2" -v "$(pwd)/../../../.":/airbyte airbyte/connector-bootstrap else echo "Running generator..." - docker run --rm -it --name airbyte-connector-bootstrap -v "$(pwd)/../../../.":/airbyte airbyte/connector-bootstrap + docker run --rm -it --name airbyte-connector-bootstrap --user $UID:$GID -v "$(pwd)/../../../.":/airbyte airbyte/connector-bootstrap fi echo "Finished running generator" diff --git a/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs index 5482f6491ed..e971fac5483 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs @@ -31,6 +31,7 @@ MAIN_REQUIREMENTS = [ TEST_REQUIREMENTS = [ "pytest~=6.1", + "pytest-mock~=3.6.1", "source-acceptance-test", ] diff --git a/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/unit_test.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/__init__.py similarity index 96% rename from airbyte-integrations/connector-templates/source-python-http-api/unit_tests/unit_test.py.hbs rename to airbyte-integrations/connector-templates/source-python-http-api/unit_tests/__init__.py index b8a8150b507..9db886e0930 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/unit_test.py.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/__init__.py @@ -21,7 +21,3 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. # - - -def test_example_method(): - assert True diff --git a/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_incremental_streams.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_incremental_streams.py.hbs new file mode 100644 index 00000000000..81348eee504 --- /dev/null +++ b/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_incremental_streams.py.hbs @@ -0,0 +1,80 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from pytest import fixture +from unittest.mock import MagicMock + +from airbyte_cdk.models import SyncMode +from source_{{snakeCase name}}.source import Incremental{{properCase name}}Stream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(Incremental{{properCase name}}Stream, "path", "v0/example_endpoint") + mocker.patch.object(Incremental{{properCase name}}Stream, "primary_key", "test_primary_key") + mocker.patch.object(Incremental{{properCase name}}Stream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = Incremental{{properCase name}}Stream() + # TODO: replace this with your expected cursor field + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = Incremental{{properCase name}}Stream() + expected_cursor_field = [] + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = Incremental{{properCase name}}Stream() + expected_cursor_field = [] + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(Incremental{{properCase name}}Stream, "cursor_field", "dummy_field") + stream = Incremental{{properCase name}}Stream() + assert stream.supports_incremental + +def test_source_defined_cursor(patch_incremental_base_class): + stream = Incremental{{properCase name}}Stream() + assert stream.source_defined_cursor + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = Incremental{{properCase name}}Stream() + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_source.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_source.py.hbs new file mode 100644 index 00000000000..d8459dadc93 --- /dev/null +++ b/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_source.py.hbs @@ -0,0 +1,40 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from source_{{snakeCase name}}.source import Source{{properCase name}} +from unittest.mock import MagicMock + + +def test_check_connection(mocker): + source = Source{{properCase name}}() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + +def test_streams(mocker): + source = Source{{properCase name}}() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_streams.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_streams.py.hbs new file mode 100644 index 00000000000..2257856c63e --- /dev/null +++ b/airbyte-integrations/connector-templates/source-python-http-api/unit_tests/test_streams.py.hbs @@ -0,0 +1,103 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from http import HTTPStatus +from unittest.mock import MagicMock +import pytest + +from source_{{snakeCase name}}.source import {{properCase name}}Stream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object({{properCase name}}Stream, "path", "v0/example_endpoint") + mocker.patch.object({{properCase name}}Stream, "primary_key", "test_primary_key") + mocker.patch.object({{properCase name}}Stream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = {{properCase name}}Stream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = {{properCase name}}Stream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = {{properCase name}}Stream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = {{properCase name}}Stream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == {} + + +def test_http_method(patch_base_class): + stream = {{properCase name}}Stream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = {{properCase name}}Stream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = {{properCase name}}Stream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time