diff --git a/airbyte-integrations/bases/base-singer/build.gradle b/airbyte-integrations/bases/base-singer/build.gradle index d9314256612..70d9d88c2aa 100644 --- a/airbyte-integrations/bases/base-singer/build.gradle +++ b/airbyte-integrations/bases/base-singer/build.gradle @@ -7,6 +7,7 @@ airbytePython { moduleDirectory 'base_singer' } + dependencies { implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connector-templates/source-python/Dockerfile b/airbyte-integrations/connector-templates/source-python/Dockerfile index 3b065eb5913..92c3c29a814 100644 --- a/airbyte-integrations/connector-templates/source-python/Dockerfile +++ b/airbyte-integrations/connector-templates/source-python/Dockerfile @@ -10,7 +10,7 @@ ENV AIRBYTE_IMPL_PATH="Source{{properCase name}}" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.1.0 LABEL io.airbyte.name=airbyte/source-{{dashCase name}} diff --git a/airbyte-integrations/connector-templates/source-singer/Dockerfile b/airbyte-integrations/connector-templates/source-singer/Dockerfile index 84af3d4a126..3e114d1c5d2 100644 --- a/airbyte-integrations/connector-templates/source-singer/Dockerfile +++ b/airbyte-integrations/connector-templates/source-singer/Dockerfile @@ -10,7 +10,7 @@ ENV AIRBYTE_IMPL_PATH="Source{{properCase name}}Singer" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.1.0 LABEL io.airbyte.name=airbyte/source-{{dashCase name}}-singer diff --git a/airbyte-integrations/connectors/source-appstore-singer/Dockerfile b/airbyte-integrations/connectors/source-appstore-singer/Dockerfile index c8ba354d534..9065969cad0 100644 --- a/airbyte-integrations/connectors/source-appstore-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-appstore-singer/Dockerfile @@ -10,7 +10,7 @@ ENV AIRBYTE_IMPL_PATH="SourceAppstoreSinger" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-appstore-singer diff --git a/airbyte-integrations/connectors/source-appstore-singer/requirements.txt b/airbyte-integrations/connectors/source-appstore-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-appstore-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-appstore-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-appstore-singer/setup.py b/airbyte-integrations/connectors/source-appstore-singer/setup.py index 8ca28b30567..23b266502ef 100644 --- a/airbyte-integrations/connectors/source-appstore-singer/setup.py +++ b/airbyte-integrations/connectors/source-appstore-singer/setup.py @@ -36,12 +36,8 @@ setup( "base-singer", "base-python", "pyjwt==1.6.4", # required by appstore connect + "pytest==6.1.2", "tap-appstore @ https://github.com/airbytehq/tap-appstore/tarball/v0.2.1-airbyte", ], package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - "tests": ["airbyte-python-test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-braintree-singer/Dockerfile b/airbyte-integrations/connectors/source-braintree-singer/Dockerfile index 4aa3c5cac29..a2feb0281e8 100644 --- a/airbyte-integrations/connectors/source-braintree-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-braintree-singer/Dockerfile @@ -12,7 +12,7 @@ LABEL io.airbyte.name=airbyte/source-braintree-singer WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-braintree-singer/Dockerfile.test b/airbyte-integrations/connectors/source-braintree-singer/Dockerfile.test deleted file mode 100644 index c8f444fdf9f..00000000000 --- a/airbyte-integrations/connectors/source-braintree-singer/Dockerfile.test +++ /dev/null @@ -1,25 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y bash jq && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="standardtest" -ENV AIRBYTE_TEST_MODULE="standardtest" -ENV AIRBYTE_TEST_PATH="BraintreeStandardSourceTest" -ENV AIRBYTE_TEST_CASE=true - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-braintree-singer-standard-test - -WORKDIR /airbyte/integration_code -COPY $CODE_PATH ./$CODE_PATH - -# copy all resources to the code path. -COPY secrets ./$CODE_PATH -COPY source_braintree_singer/*.json ./$CODE_PATH -COPY sample_files/*.json ./$CODE_PATH/ - -COPY setup.py ./ -RUN pip install -v ".[integration_tests]" - -WORKDIR /airbyte - diff --git a/airbyte-integrations/connectors/source-braintree-singer/build.gradle b/airbyte-integrations/connectors/source-braintree-singer/build.gradle index 4b62e53be89..a35e0b713c2 100644 --- a/airbyte-integrations/connectors/source-braintree-singer/build.gradle +++ b/airbyte-integrations/connectors/source-braintree-singer/build.gradle @@ -1,14 +1,20 @@ plugins { id 'airbyte-docker' - id 'airbyte-integration-test-java' id 'airbyte-python' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_braintree_singer' } +airbyteStandardSourceTestFile { + specPath = "source_braintree_singer/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-braintree-singer/requirements.standard-test.txt b/airbyte-integrations/connectors/source-braintree-singer/requirements.standard-test.txt deleted file mode 100644 index bf756121ea4..00000000000 --- a/airbyte-integrations/connectors/source-braintree-singer/requirements.standard-test.txt +++ /dev/null @@ -1,2 +0,0 @@ --e ../../bases/base-python-test --e . diff --git a/airbyte-integrations/connectors/source-braintree-singer/requirements.txt b/airbyte-integrations/connectors/source-braintree-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-braintree-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-braintree-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-braintree-singer/setup.py b/airbyte-integrations/connectors/source-braintree-singer/setup.py index ed054e2179d..fe0b6e05c57 100644 --- a/airbyte-integrations/connectors/source-braintree-singer/setup.py +++ b/airbyte-integrations/connectors/source-braintree-singer/setup.py @@ -29,10 +29,7 @@ setup( description="Source implementation for Braintree.", author="Airbyte", author_email="contact@airbyte.io", + install_requires=["tap-braintree==0.9.1", "python-dateutil==2.8.1", "airbyte-protocol", "base-singer", "base-python", "pytest==6.1.2"], packages=find_packages(), package_data={"": ["*.json"]}, - extras_require={ - "main": ["tap-braintree==0.9.1", "python-dateutil==2.8.1", "airbyte-protocol", "base-singer", "base-python"], - "standardtest": ["airbyte_python_test"], - }, ) diff --git a/airbyte-integrations/connectors/source-braintree-singer/standardtest/__init__.py b/airbyte-integrations/connectors/source-braintree-singer/standardtest/__init__.py deleted file mode 100644 index 49e55766d88..00000000000 --- a/airbyte-integrations/connectors/source-braintree-singer/standardtest/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import BraintreeStandardSourceTest - -__all__ = ["BraintreeStandardSourceTest"] diff --git a/airbyte-integrations/connectors/source-braintree-singer/standardtest/standard_source_test.py b/airbyte-integrations/connectors/source-braintree-singer/standardtest/standard_source_test.py deleted file mode 100644 index c53866de774..00000000000 --- a/airbyte-integrations/connectors/source-braintree-singer/standardtest/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class BraintreeStandardSourceTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-drift/Dockerfile.test b/airbyte-integrations/connectors/source-drift/Dockerfile.test deleted file mode 100644 index a58310cfd1a..00000000000 --- a/airbyte-integrations/connectors/source-drift/Dockerfile.test +++ /dev/null @@ -1,22 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceDriftStandardTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-drift-standard-test - -WORKDIR /airbyte/integration_code -COPY source_drift source_drift -COPY $CODE_PATH $CODE_PATH -COPY sample_files/*.json $CODE_PATH/ -COPY secrets/* $CODE_PATH -COPY source_drift/*.json $CODE_PATH -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-drift/build.gradle b/airbyte-integrations/connectors/source-drift/build.gradle index ef5eb40d6d0..cbea73a13ed 100644 --- a/airbyte-integrations/connectors/source-drift/build.gradle +++ b/airbyte-integrations/connectors/source-drift/build.gradle @@ -1,14 +1,20 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' - id 'airbyte-integration-test-java' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_drift' } +airbyteStandardSourceTestFile { + specPath = "source_drift/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/__init__.py b/airbyte-integrations/connectors/source-drift/integration_tests/__init__.py deleted file mode 100644 index 6aac035c4f1..00000000000 --- a/airbyte-integrations/connectors/source-drift/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceDriftStandardTest - -__all__ = ["SourceDriftStandardTest"] diff --git a/airbyte-integrations/connectors/source-drift/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-drift/integration_tests/standard_source_test.py deleted file mode 100644 index f626b875fe6..00000000000 --- a/airbyte-integrations/connectors/source-drift/integration_tests/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class SourceDriftStandardTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-drift/requirements.txt b/airbyte-integrations/connectors/source-drift/requirements.txt index e0281021aed..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-drift/requirements.txt +++ b/airbyte-integrations/connectors/source-drift/requirements.txt @@ -1,5 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-drift/setup.py b/airbyte-integrations/connectors/source-drift/setup.py index 538be142dfc..460aec34b3b 100644 --- a/airbyte-integrations/connectors/source-drift/setup.py +++ b/airbyte-integrations/connectors/source-drift/setup.py @@ -30,15 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=[ - "airbyte-protocol", - "base-python", - "requests~=2.22", - ], + install_requires=["airbyte-protocol", "base-python", "requests~=2.22", "pytest==6.1.2"], package_data={"": ["*.json", "schemas/*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - "tests": ["airbyte_python_test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/Dockerfile.test b/airbyte-integrations/connectors/source-exchangeratesapi-singer/Dockerfile.test deleted file mode 100644 index 4a03d035cb4..00000000000 --- a/airbyte-integrations/connectors/source-exchangeratesapi-singer/Dockerfile.test +++ /dev/null @@ -1,22 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV MODULE_NAME="source_exchangeratesapi_singer" -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="ExchangeratesApiStandardSourceTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-exchangeratesapi-singer-standard-test - -WORKDIR /airbyte/integration_code -COPY $MODULE_NAME $MODULE_NAME -COPY $CODE_PATH $CODE_PATH -COPY $MODULE_NAME/*.json $CODE_PATH/ -COPY sample_files/*.json $CODE_PATH/ -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle b/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle index 1840cf0d8c5..277087dcc2c 100644 --- a/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle +++ b/airbyte-integrations/connectors/source-exchangeratesapi-singer/build.gradle @@ -1,13 +1,22 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_exchangeratesapi_singer' } +airbyteStandardSourceTestFile { + specPath = "source_exchangeratesapi_singer/spec.json" + configPath = "sample_files/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/integration_tests/__init__.py b/airbyte-integrations/connectors/source-exchangeratesapi-singer/integration_tests/__init__.py deleted file mode 100644 index bafbbd1843d..00000000000 --- a/airbyte-integrations/connectors/source-exchangeratesapi-singer/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import ExchangeratesApiStandardSourceTest - -__all__ = ["ExchangeratesApiStandardSourceTest"] diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-exchangeratesapi-singer/integration_tests/standard_source_test.py deleted file mode 100644 index 375a7c554b3..00000000000 --- a/airbyte-integrations/connectors/source-exchangeratesapi-singer/integration_tests/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class ExchangeratesApiStandardSourceTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-exchangeratesapi-singer/setup.py b/airbyte-integrations/connectors/source-exchangeratesapi-singer/setup.py index 22f2bcc968a..b495d7cec58 100644 --- a/airbyte-integrations/connectors/source-exchangeratesapi-singer/setup.py +++ b/airbyte-integrations/connectors/source-exchangeratesapi-singer/setup.py @@ -31,12 +31,5 @@ setup( author_email="contact@airbyte.io", packages=find_packages(), package_data={"": ["*.json"]}, - install_requires=[ - "tap-exchangeratesapi==0.1.1", - "base-python", - "airbyte-protocol", - ], - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={"main": ["base-singer"], "tests": ["airbyte_python_test", "pytest"]}, + install_requires=["tap-exchangeratesapi==0.1.1", "base-python", "airbyte-protocol", "pytest==6.1.2"], ) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/build.gradle b/airbyte-integrations/connectors/source-facebook-marketing/build.gradle index ec2d61d66f6..d3a09bc0ba8 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/build.gradle +++ b/airbyte-integrations/connectors/source-facebook-marketing/build.gradle @@ -22,5 +22,6 @@ task("pythonIntegrationTests", type: PythonTask, dependsOn: installTestReqs) { integrationTest.dependsOn("pythonIntegrationTests") dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt b/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt index e0281021aed..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt +++ b/airbyte-integrations/connectors/source-facebook-marketing/requirements.txt @@ -1,5 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-facebook-marketing/setup.py b/airbyte-integrations/connectors/source-facebook-marketing/setup.py index b0594631db0..b5dfd23eef0 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/setup.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/setup.py @@ -37,10 +37,7 @@ setup( "backoff==1.10.0", "pendulum==1.2.0", "cached_property==1.5.2", + "pytest==6.1.2", ], package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, - tests_require=["pytest==6.1.2"], - extras_require={ - "tests": ["airbyte_python_test==0.0.0", "pytest==6.1.2"], - }, ) diff --git a/airbyte-integrations/connectors/source-file/Dockerfile b/airbyte-integrations/connectors/source-file/Dockerfile index 74f88daed6d..db47c9d5d12 100644 --- a/airbyte-integrations/connectors/source-file/Dockerfile +++ b/airbyte-integrations/connectors/source-file/Dockerfile @@ -9,7 +9,7 @@ ENV AIRBYTE_IMPL_PATH="SourceFile" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-file diff --git a/airbyte-integrations/connectors/source-file/Dockerfile.test b/airbyte-integrations/connectors/source-file/Dockerfile.test deleted file mode 100644 index 4fc6b15e378..00000000000 --- a/airbyte-integrations/connectors/source-file/Dockerfile.test +++ /dev/null @@ -1,21 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceFileStandardTest" -ENV AIRBYTE_TEST_CASE=true - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-file-standard-test - -WORKDIR /airbyte/integration_code -COPY source_file source_file -COPY $CODE_PATH $CODE_PATH -COPY source_file/*.json $CODE_PATH -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-file/build.gradle b/airbyte-integrations/connectors/source-file/build.gradle index 8c3e54cf7b0..ddcf4ba0e9e 100644 --- a/airbyte-integrations/connectors/source-file/build.gradle +++ b/airbyte-integrations/connectors/source-file/build.gradle @@ -3,13 +3,20 @@ import ru.vyarus.gradle.plugin.python.task.PythonTask plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_file' } + +airbyteStandardSourceTestFile { + specPath = "source_file/spec.json" + configPath = "integration_tests/config.json" + configuredCatalogPath = "integration_tests/configured_catalog.json" +} + task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs){ module = "pytest" command = "-s integration_tests" @@ -17,6 +24,8 @@ task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs integrationTest.dependsOn("customIntegrationTestPython") + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-file/requirements.txt b/airbyte-integrations/connectors/source-file/requirements.txt index 8e9bd19acc1..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-file/requirements.txt +++ b/airbyte-integrations/connectors/source-file/requirements.txt @@ -1,7 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . -pytest -boto3 diff --git a/airbyte-integrations/connectors/source-file/setup.py b/airbyte-integrations/connectors/source-file/setup.py index 2b5f5be3d09..f5181fb3f0c 100644 --- a/airbyte-integrations/connectors/source-file/setup.py +++ b/airbyte-integrations/connectors/source-file/setup.py @@ -43,7 +43,6 @@ MAIN_REQUIREMENTS = [ ] TEST_REQUIREMENTS = [ - "airbyte-python-test", "boto3==1.16.57", "pytest==6.1.2", "pytest-docker==0.10.1", @@ -57,13 +56,7 @@ setup( packages=find_packages(), install_requires=MAIN_REQUIREMENTS, package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], extras_require={ - # Dependencies required by the main package but not integration tests should go in main. Deps required by - # integration tests but not the main package go in integration_tests. Deps required by both should go in - # install_requires. - "main": [], "tests": TEST_REQUIREMENTS, }, ) diff --git a/airbyte-integrations/connectors/source-freshdesk/build.gradle b/airbyte-integrations/connectors/source-freshdesk/build.gradle index b0186677096..07e64afab65 100644 --- a/airbyte-integrations/connectors/source-freshdesk/build.gradle +++ b/airbyte-integrations/connectors/source-freshdesk/build.gradle @@ -16,7 +16,9 @@ airbyteStandardSourceTestFile { configuredCatalogPath = "sample_files/configured_catalog.json" } + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-freshdesk/requirements.txt b/airbyte-integrations/connectors/source-freshdesk/requirements.txt index e0281021aed..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-freshdesk/requirements.txt +++ b/airbyte-integrations/connectors/source-freshdesk/requirements.txt @@ -1,5 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-freshdesk/setup.py b/airbyte-integrations/connectors/source-freshdesk/setup.py index fddc60ab246..8645e6c1922 100644 --- a/airbyte-integrations/connectors/source-freshdesk/setup.py +++ b/airbyte-integrations/connectors/source-freshdesk/setup.py @@ -33,7 +33,6 @@ MAIN_REQUIREMENTS = [ ] TEST_REQUIREMENTS = [ - "airbyte_python_test", "pytest==6.1.2", "requests_mock==1.8.0", ] diff --git a/airbyte-integrations/connectors/source-github-singer/Dockerfile b/airbyte-integrations/connectors/source-github-singer/Dockerfile index fb61387af68..2ca3beda922 100644 --- a/airbyte-integrations/connectors/source-github-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-github-singer/Dockerfile @@ -14,7 +14,7 @@ LABEL io.airbyte.name=airbyte/source-github-singer WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-github-singer/build.gradle b/airbyte-integrations/connectors/source-github-singer/build.gradle index 45332e487ff..f209452686f 100644 --- a/airbyte-integrations/connectors/source-github-singer/build.gradle +++ b/airbyte-integrations/connectors/source-github-singer/build.gradle @@ -26,6 +26,8 @@ airbyteStandardSourceTestFile { // command = "-s integration_tests" //} + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-github-singer/requirements.txt b/airbyte-integrations/connectors/source-github-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-github-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-github-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-github-singer/setup.py b/airbyte-integrations/connectors/source-github-singer/setup.py index 9e328dc1fd6..77c832b43f5 100644 --- a/airbyte-integrations/connectors/source-github-singer/setup.py +++ b/airbyte-integrations/connectors/source-github-singer/setup.py @@ -38,8 +38,4 @@ setup( "base-python", ], package_data={"": ["*.json"]}, - # two sets of dependencies: 1) for main 2) for standard test deps. 2 does not have all of the dependencies of 1, which is we cannot use install_requires. - extras_require={ - "standardtest": ["airbyte_python_test"], - }, ) diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile b/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile index fbe9e4902b6..c9126e1a004 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile @@ -10,7 +10,7 @@ WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ RUN pip install tap-adwords==1.12.0 -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-google-adwords-singer diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle b/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle index c1d4f02ddf0..92b5ef9aeff 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle +++ b/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle @@ -9,7 +9,9 @@ airbytePython { moduleDirectory 'source_google_adwords_singer' } + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/requirements.txt b/airbyte-integrations/connectors/source-google-adwords-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-google-adwords-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/setup.py b/airbyte-integrations/connectors/source-google-adwords-singer/setup.py index 5eecc2bd799..8e4a40ebbe0 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/setup.py +++ b/airbyte-integrations/connectors/source-google-adwords-singer/setup.py @@ -30,15 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol"], + install_requires=["airbyte-protocol", "base-singer", "base-python", "pytest==6.1.2"], package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - # Dependencies required by the main package but not integration tests should go in main. Deps required by - # integration tests but not the main package go in integration_tests. Deps required by both should go in - # install_requires. - "main": ["base-singer", "base-python"], - "tests": ["airbyte-python-test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-google-sheets/build.gradle b/airbyte-integrations/connectors/source-google-sheets/build.gradle index 264b9aaafa3..998b905999d 100644 --- a/airbyte-integrations/connectors/source-google-sheets/build.gradle +++ b/airbyte-integrations/connectors/source-google-sheets/build.gradle @@ -8,6 +8,8 @@ airbytePython { moduleDirectory 'google_sheets_source' } + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-google-sheets/setup.py b/airbyte-integrations/connectors/source-google-sheets/setup.py index e260cd19bc6..9ead61f0d39 100644 --- a/airbyte-integrations/connectors/source-google-sheets/setup.py +++ b/airbyte-integrations/connectors/source-google-sheets/setup.py @@ -39,10 +39,9 @@ setup( "google-api-python-client", "PyYAML==5.3.1", "pydantic==1.6.1", + "pytest==6.1.2", ], package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], extras_require={ "integration_tests": ["airbyte_python_test"], }, diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile b/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile index 3f8e8f08aea..c73e9b3b5e8 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile @@ -14,6 +14,6 @@ LABEL io.airbyte.name=airbyte/source-googleanalytics-singer WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle b/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle index 58d618e39b2..446aef2c59d 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle @@ -8,7 +8,9 @@ airbytePython { moduleDirectory 'source_googleanalytics_signger' } + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/requirements.standard-test.txt b/airbyte-integrations/connectors/source-googleanalytics-singer/requirements.standard-test.txt deleted file mode 100644 index bf756121ea4..00000000000 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/requirements.standard-test.txt +++ /dev/null @@ -1,2 +0,0 @@ --e ../../bases/base-python-test --e . diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/requirements.txt b/airbyte-integrations/connectors/source-googleanalytics-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/setup.py b/airbyte-integrations/connectors/source-googleanalytics-singer/setup.py index 3a0ae59d46b..bae49f57a57 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/setup.py +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/setup.py @@ -29,17 +29,13 @@ setup( description="Airbyte Source for Google Analytics (singer-based)", author="Airbyte", author_email="contact@airbyte.io", + install_requires=[ + "pipelinewise-tap-google-analytics @ https://github.com/airbytehq/pipelinewise-tap-google-analytics/tarball/master#egg=pipelinewise-tap-google-analytics-1.3.0-airbyte", + "pydantic==1.6.1", + "base-singer", + "base-python", + "airbyte-protocol", + ], packages=find_packages(), package_data={"": ["*.json", "*.txt"]}, - # two sets of dependencies: 1) for main 2) for standard test deps. 2 does not have all of the dependencies of 1, which is we cannot use install_requires. - extras_require={ - "main": [ - "pipelinewise-tap-google-analytics @ https://github.com/airbytehq/pipelinewise-tap-google-analytics/tarball/master#egg=pipelinewise-tap-google-analytics-1.3.0-airbyte", - "pydantic==1.6.1", - "base-singer", - "base-python", - "airbyte-protocol", - ], - "standardtest": ["airbyte-python-test", "requests"], - }, ) diff --git a/airbyte-integrations/connectors/source-greenhouse/Dockerfile.test b/airbyte-integrations/connectors/source-greenhouse/Dockerfile.test deleted file mode 100644 index d52d2b3f155..00000000000 --- a/airbyte-integrations/connectors/source-greenhouse/Dockerfile.test +++ /dev/null @@ -1,22 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceGreenhouseStandardTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-greenhouse-standard-test - -WORKDIR /airbyte/integration_code -COPY source_greenhouse source_greenhouse -COPY $CODE_PATH $CODE_PATH -COPY sample_files/*.json $CODE_PATH/ -COPY secrets/* $CODE_PATH -COPY source_greenhouse/*.json $CODE_PATH -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-greenhouse/build.gradle b/airbyte-integrations/connectors/source-greenhouse/build.gradle index 14a9858490f..44c9744c2d8 100644 --- a/airbyte-integrations/connectors/source-greenhouse/build.gradle +++ b/airbyte-integrations/connectors/source-greenhouse/build.gradle @@ -1,14 +1,21 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' - id 'airbyte-integration-test-java' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_greenhouse' } +airbyteStandardSourceTestFile { + specPath = "source_greenhouse/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/__init__.py b/airbyte-integrations/connectors/source-greenhouse/integration_tests/__init__.py deleted file mode 100644 index 8b684162fa9..00000000000 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceGreenhouseStandardTest - -__all__ = ["SourceGreenhouseStandardTest"] diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-greenhouse/integration_tests/standard_source_test.py deleted file mode 100644 index 6a5196f9215..00000000000 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/standard_source_test.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import json -import pkgutil - -from airbyte_protocol import ConfiguredAirbyteCatalog, ConnectorSpecification -from base_python_test import StandardSourceTestIface - - -class SourceGreenhouseStandardTest(StandardSourceTestIface): - def get_spec(self) -> ConnectorSpecification: - raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "spec.json") - return ConnectorSpecification.parse_obj(json.loads(raw_spec)) - - def get_config(self) -> object: - return json.loads(pkgutil.get_data(self.__class__.__module__.split(".")[0], "config.json")) - - def get_catalog(self) -> ConfiguredAirbyteCatalog: - raw_catalog = pkgutil.get_data(self.__class__.__module__.split(".")[0], "configured_catalog.json") - return ConfiguredAirbyteCatalog.parse_obj(json.loads(raw_catalog)) - - def setup(self) -> None: - pass - - def teardown(self) -> None: - pass diff --git a/airbyte-integrations/connectors/source-greenhouse/requirements.txt b/airbyte-integrations/connectors/source-greenhouse/requirements.txt index e0281021aed..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-greenhouse/requirements.txt +++ b/airbyte-integrations/connectors/source-greenhouse/requirements.txt @@ -1,5 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-greenhouse/setup.py b/airbyte-integrations/connectors/source-greenhouse/setup.py index 38733b2888d..25631f12e99 100644 --- a/airbyte-integrations/connectors/source-greenhouse/setup.py +++ b/airbyte-integrations/connectors/source-greenhouse/setup.py @@ -30,16 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=[ - "airbyte-protocol", - "base-python", - "six==1.15.0", - "grnhse-api==0.1.1", - ], + install_requires=["airbyte-protocol", "base-python", "six==1.15.0", "grnhse-api==0.1.1", "pytest==6.1.2"], package_data={"": ["*.json", "schemas/*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - "tests": ["airbyte_python_test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-http-request/Dockerfile b/airbyte-integrations/connectors/source-http-request/Dockerfile index 0884039f7ed..b1bf9c36d3f 100644 --- a/airbyte-integrations/connectors/source-http-request/Dockerfile +++ b/airbyte-integrations/connectors/source-http-request/Dockerfile @@ -9,7 +9,7 @@ ENV AIRBYTE_IMPL_PATH="SourceHttpRequest" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-http-request diff --git a/airbyte-integrations/connectors/source-http-request/Dockerfile.test b/airbyte-integrations/connectors/source-http-request/Dockerfile.test deleted file mode 100644 index 5e5b4e3f663..00000000000 --- a/airbyte-integrations/connectors/source-http-request/Dockerfile.test +++ /dev/null @@ -1,21 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceHttpRequestStandardTest" -ENV AIRBYTE_TEST_CASE=true - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-http-request-standard-test - -WORKDIR /airbyte/integration_code -COPY source_http_request source_http_request -COPY $CODE_PATH $CODE_PATH -COPY source_http_request/*.json $CODE_PATH -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-http-request/build.gradle b/airbyte-integrations/connectors/source-http-request/build.gradle index 30efbc4c8ae..27654aa3baf 100644 --- a/airbyte-integrations/connectors/source-http-request/build.gradle +++ b/airbyte-integrations/connectors/source-http-request/build.gradle @@ -2,13 +2,21 @@ plugins { id 'java' id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_http_request' } +airbyteStandardSourceTestFile { + specPath = "source_http_request/spec.json" + configPath = "sample_files/config.json" + configuredCatalogPath = "integration_tests/configured_catalog.json" +} + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-http-request/integration_tests/__init__.py b/airbyte-integrations/connectors/source-http-request/integration_tests/__init__.py deleted file mode 100644 index 260939f2b29..00000000000 --- a/airbyte-integrations/connectors/source-http-request/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceHttpRequestStandardTest - -__all__ = ["SourceHttpRequestStandardTest"] diff --git a/airbyte-integrations/connectors/source-http-request/integration_tests/config.json b/airbyte-integrations/connectors/source-http-request/integration_tests/config.json new file mode 100644 index 00000000000..af5c0301295 --- /dev/null +++ b/airbyte-integrations/connectors/source-http-request/integration_tests/config.json @@ -0,0 +1,5 @@ +{ + "http_method": "get", + "url": "http://api.bart.gov/api/route.aspx?cmd=routes&key=MW9S-E7SL-26DU-VV8V&json=y", + "headers": "{\"Content-Type\": \"application/json\"}" +} diff --git a/airbyte-integrations/connectors/source-http-request/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-http-request/integration_tests/standard_source_test.py deleted file mode 100644 index 77adb16b5d3..00000000000 --- a/airbyte-integrations/connectors/source-http-request/integration_tests/standard_source_test.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import json -import pkgutil - -from airbyte_protocol import ConfiguredAirbyteCatalog, ConnectorSpecification -from base_python_test import StandardSourceTestIface - - -class SourceHttpRequestStandardTest(StandardSourceTestIface): - def __init__(self): - pass - - def get_spec(self) -> ConnectorSpecification: - raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "spec.json") - return ConnectorSpecification.parse_obj(json.loads(raw_spec)) - - def get_config(self) -> object: - return { - "http_method": "get", - "url": "http://api.bart.gov/api/route.aspx?cmd=routes&key=MW9S-E7SL-26DU-VV8V&json=y", - "headers": '{"Content-Type": "application/json"}', - } - - def get_catalog(self) -> ConfiguredAirbyteCatalog: - raw_catalog = pkgutil.get_data(self.__class__.__module__.split(".")[0], "configured_catalog.json") - return ConfiguredAirbyteCatalog.parse_obj(json.loads(raw_catalog)) - - def setup(self) -> None: - pass - - def teardown(self) -> None: - pass diff --git a/airbyte-integrations/connectors/source-http-request/requirements.txt b/airbyte-integrations/connectors/source-http-request/requirements.txt index e0281021aed..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-http-request/requirements.txt +++ b/airbyte-integrations/connectors/source-http-request/requirements.txt @@ -1,5 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-http-request/setup.py b/airbyte-integrations/connectors/source-http-request/setup.py index e79c6799ea8..7c3a0819601 100644 --- a/airbyte-integrations/connectors/source-http-request/setup.py +++ b/airbyte-integrations/connectors/source-http-request/setup.py @@ -30,12 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "requests"], + install_requires=["airbyte-protocol", "base-python", "requests", "pytest==6.1.2"], package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - "main": ["base-python"], - "tests": ["airbyte-python-test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile b/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile index d30e5740be3..7820f0a218a 100644 --- a/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile @@ -14,7 +14,7 @@ LABEL io.airbyte.name=airbyte/source-hubspot-singer WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile.test b/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile.test deleted file mode 100644 index 66dd0f31d4b..00000000000 --- a/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile.test +++ /dev/null @@ -1,24 +0,0 @@ - -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y \ - bash \ - && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="standardtest" -ENV AIRBYTE_TEST_MODULE="standardtest" -ENV AIRBYTE_TEST_PATH="HubspotStandardSourceTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-hubspot-singer-standard-test - -WORKDIR /airbyte/integration_code -COPY secrets ./$CODE_PATH -COPY resourcesstandardtest ./$CODE_PATH -COPY $CODE_PATH ./$CODE_PATH -COPY source_hubspot_singer/*.json $CODE_PATH/ -COPY setup.py ./ -RUN pip install ".[standardtest]" - -WORKDIR /airbyte - diff --git a/airbyte-integrations/connectors/source-hubspot-singer/build.gradle b/airbyte-integrations/connectors/source-hubspot-singer/build.gradle index 4867fb90e76..ae533e155bc 100644 --- a/airbyte-integrations/connectors/source-hubspot-singer/build.gradle +++ b/airbyte-integrations/connectors/source-hubspot-singer/build.gradle @@ -1,13 +1,22 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_hubspot_singer' } +airbyteStandardSourceTestFile { + specPath = "source_hubspot_singer/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-hubspot-singer/requirements.standard-test.txt b/airbyte-integrations/connectors/source-hubspot-singer/requirements.standard-test.txt deleted file mode 100644 index bf756121ea4..00000000000 --- a/airbyte-integrations/connectors/source-hubspot-singer/requirements.standard-test.txt +++ /dev/null @@ -1,2 +0,0 @@ --e ../../bases/base-python-test --e . diff --git a/airbyte-integrations/connectors/source-hubspot-singer/requirements.txt b/airbyte-integrations/connectors/source-hubspot-singer/requirements.txt index 837ada1dcee..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-hubspot-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-hubspot-singer/requirements.txt @@ -1,6 +1,5 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-singer --e ../../bases/base-python-test -e ../../bases/base-python -e . diff --git a/airbyte-integrations/connectors/source-hubspot-singer/resourcesstandardtest/configured_catalog.json b/airbyte-integrations/connectors/source-hubspot-singer/sample_files/configured_catalog.json similarity index 100% rename from airbyte-integrations/connectors/source-hubspot-singer/resourcesstandardtest/configured_catalog.json rename to airbyte-integrations/connectors/source-hubspot-singer/sample_files/configured_catalog.json diff --git a/airbyte-integrations/connectors/source-hubspot-singer/setup.py b/airbyte-integrations/connectors/source-hubspot-singer/setup.py index b0d9afcc3a9..f46a326087c 100644 --- a/airbyte-integrations/connectors/source-hubspot-singer/setup.py +++ b/airbyte-integrations/connectors/source-hubspot-singer/setup.py @@ -29,10 +29,7 @@ setup( description="Source implementation for Github.", author="Airbyte", author_email="contact@airbyte.io", + install_requires=["tap-hubspot==2.8.1", "requests", "airbyte-protocol", "base-singer"], packages=find_packages(), package_data={"": ["*.json"]}, - extras_require={ - "main": ["tap-hubspot==2.8.1", "requests", "airbyte-protocol", "base-singer"], - "standardtest": ["airbyte_python_test"], - }, ) diff --git a/airbyte-integrations/connectors/source-hubspot-singer/standardtest/__init__.py b/airbyte-integrations/connectors/source-hubspot-singer/standardtest/__init__.py deleted file mode 100644 index 758a9ddc400..00000000000 --- a/airbyte-integrations/connectors/source-hubspot-singer/standardtest/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import HubspotStandardSourceTest - -__all__ = ["HubspotStandardSourceTest"] diff --git a/airbyte-integrations/connectors/source-hubspot-singer/standardtest/standard_source_test.py b/airbyte-integrations/connectors/source-hubspot-singer/standardtest/standard_source_test.py deleted file mode 100644 index c4e540e1625..00000000000 --- a/airbyte-integrations/connectors/source-hubspot-singer/standardtest/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class HubspotStandardSourceTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-intercom-singer/Dockerfile.test b/airbyte-integrations/connectors/source-intercom-singer/Dockerfile.test deleted file mode 100644 index a697a0aabe2..00000000000 --- a/airbyte-integrations/connectors/source-intercom-singer/Dockerfile.test +++ /dev/null @@ -1,28 +0,0 @@ -FROM airbyte/base-python-test:dev - -# Bash is installed for convenient debugging. GCC is required by tap-intercom to compile ciso8601. -ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update \ - && apt-get install -y bash \ - && apt-get install -y gcc \ - && rm -rf /var/lib/apt/lists/* - -ENV MODULE_NAME="source_intercom_singer" -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceIntercomSingerStandardTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-intercom-singer-standard-test - -WORKDIR /airbyte/integration_code -COPY $MODULE_NAME $MODULE_NAME -COPY $CODE_PATH $CODE_PATH -COPY secrets/config.json $CODE_PATH/config.json -COPY $MODULE_NAME/*.json $CODE_PATH/ -COPY sample_files/*.json $CODE_PATH/ -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-intercom-singer/build.gradle b/airbyte-integrations/connectors/source-intercom-singer/build.gradle index a32cb3875f4..756a22177ae 100644 --- a/airbyte-integrations/connectors/source-intercom-singer/build.gradle +++ b/airbyte-integrations/connectors/source-intercom-singer/build.gradle @@ -1,13 +1,23 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_intercom_singer' } + +airbyteStandardSourceTestFile { + specPath = "source_intercom_singer/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-intercom-singer/integration_tests/__init__.py b/airbyte-integrations/connectors/source-intercom-singer/integration_tests/__init__.py deleted file mode 100644 index 076044665cb..00000000000 --- a/airbyte-integrations/connectors/source-intercom-singer/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceIntercomSingerStandardTest - -__all__ = ["SourceIntercomSingerStandardTest"] diff --git a/airbyte-integrations/connectors/source-intercom-singer/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-intercom-singer/integration_tests/standard_source_test.py deleted file mode 100644 index 7d40e7743fe..00000000000 --- a/airbyte-integrations/connectors/source-intercom-singer/integration_tests/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class SourceIntercomSingerStandardTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-intercom-singer/requirements.txt b/airbyte-integrations/connectors/source-intercom-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-intercom-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-intercom-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-intercom-singer/setup.py b/airbyte-integrations/connectors/source-intercom-singer/setup.py index bb20d600703..ad8d0b9e6f5 100644 --- a/airbyte-integrations/connectors/source-intercom-singer/setup.py +++ b/airbyte-integrations/connectors/source-intercom-singer/setup.py @@ -30,12 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-python", "tap-intercom==1.1.1"], + install_requires=["airbyte-protocol", "base-python", "base-singer", "tap-intercom==1.1.1", "pytest==6.1.2"], package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - "main": ["base-singer"], - "tests": ["airbyte_python_test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-jira/Dockerfile.test b/airbyte-integrations/connectors/source-jira/Dockerfile.test deleted file mode 100644 index 9b967166d8f..00000000000 --- a/airbyte-integrations/connectors/source-jira/Dockerfile.test +++ /dev/null @@ -1,22 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceJiraStandardTest" - -LABEL io.airbyte.version=0.1.1 -LABEL io.airbyte.name=airbyte/source-jira-standard-test - -WORKDIR /airbyte/integration_code -COPY source_jira source_jira -COPY $CODE_PATH $CODE_PATH -COPY sample_files/*.json $CODE_PATH/ -COPY secrets/* $CODE_PATH -COPY source_jira/*.json $CODE_PATH -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-jira/build.gradle b/airbyte-integrations/connectors/source-jira/build.gradle index 4ab6aad783a..7db4ece5321 100644 --- a/airbyte-integrations/connectors/source-jira/build.gradle +++ b/airbyte-integrations/connectors/source-jira/build.gradle @@ -1,14 +1,21 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' - id 'airbyte-integration-test-java' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_jira' } +airbyteStandardSourceTestFile { + specPath = "source_jira/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/__init__.py b/airbyte-integrations/connectors/source-jira/integration_tests/__init__.py deleted file mode 100644 index cc8cf1f763a..00000000000 --- a/airbyte-integrations/connectors/source-jira/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceJiraStandardTest - -__all__ = ["SourceJiraStandardTest"] diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-jira/integration_tests/standard_source_test.py deleted file mode 100644 index 4c4ac033b5f..00000000000 --- a/airbyte-integrations/connectors/source-jira/integration_tests/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class SourceJiraStandardTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-jira/requirements.txt b/airbyte-integrations/connectors/source-jira/requirements.txt index e0281021aed..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-jira/requirements.txt +++ b/airbyte-integrations/connectors/source-jira/requirements.txt @@ -1,5 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-jira/setup.py b/airbyte-integrations/connectors/source-jira/setup.py index b154c3c08f1..916bcfef2da 100644 --- a/airbyte-integrations/connectors/source-jira/setup.py +++ b/airbyte-integrations/connectors/source-jira/setup.py @@ -30,15 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=[ - "airbyte-protocol", - "base-python", - "requests", - ], + install_requires=["airbyte-protocol", "base-python", "requests", "pytest==6.1.2"], package_data={"": ["*.json", "schemas/*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - "tests": ["airbyte_python_test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-looker/Dockerfile b/airbyte-integrations/connectors/source-looker/Dockerfile index 82f371213ef..f4e14381446 100644 --- a/airbyte-integrations/connectors/source-looker/Dockerfile +++ b/airbyte-integrations/connectors/source-looker/Dockerfile @@ -10,7 +10,7 @@ ENV AIRBYTE_IMPL_PATH="SourceLooker" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-looker diff --git a/airbyte-integrations/connectors/source-looker/Dockerfile.test b/airbyte-integrations/connectors/source-looker/Dockerfile.test deleted file mode 100644 index daf5775ab46..00000000000 --- a/airbyte-integrations/connectors/source-looker/Dockerfile.test +++ /dev/null @@ -1,21 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceLookerStandardTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-looker-standard-test - -WORKDIR /airbyte/integration_code -COPY source_looker source_looker -COPY $CODE_PATH $CODE_PATH -COPY secrets/* $CODE_PATH/ -COPY source_looker/*.json $CODE_PATH/ -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-looker/build.gradle b/airbyte-integrations/connectors/source-looker/build.gradle index 26627d7cd7a..69e55f85900 100644 --- a/airbyte-integrations/connectors/source-looker/build.gradle +++ b/airbyte-integrations/connectors/source-looker/build.gradle @@ -1,13 +1,21 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_looker' } +airbyteStandardSourceTestFile { + specPath = "source_looker/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "integration_tests/configured_catalog.json" +} + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-looker/integration_tests/__init__.py b/airbyte-integrations/connectors/source-looker/integration_tests/__init__.py deleted file mode 100644 index c2957fca25b..00000000000 --- a/airbyte-integrations/connectors/source-looker/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceLookerStandardTest - -__all__ = ["SourceLookerStandardTest"] diff --git a/airbyte-integrations/connectors/source-looker/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-looker/integration_tests/standard_source_test.py deleted file mode 100644 index e6b263038e7..00000000000 --- a/airbyte-integrations/connectors/source-looker/integration_tests/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class SourceLookerStandardTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-looker/requirements.txt b/airbyte-integrations/connectors/source-looker/requirements.txt index e0281021aed..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-looker/requirements.txt +++ b/airbyte-integrations/connectors/source-looker/requirements.txt @@ -1,5 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-looker/setup.py b/airbyte-integrations/connectors/source-looker/setup.py index 07993d2510c..1ddf7b030c0 100644 --- a/airbyte-integrations/connectors/source-looker/setup.py +++ b/airbyte-integrations/connectors/source-looker/setup.py @@ -30,15 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-python", "requests", "backoff"], + install_requires=["airbyte-protocol", "base-python", "requests", "backoff", "pytest==6.1.2"], package_data={"": ["*.json", "schemas/*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - # Dependencies required by the main package but not integration tests should go in main. Deps required by - # integration tests but not the main package go in integration_tests. Deps required by both should go in - # install_requires. - "main": [], - "tests": ["airbyte-python-test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-mailchimp/Dockerfile b/airbyte-integrations/connectors/source-mailchimp/Dockerfile index dd7ef757b9a..4d74c9ec5e3 100644 --- a/airbyte-integrations/connectors/source-mailchimp/Dockerfile +++ b/airbyte-integrations/connectors/source-mailchimp/Dockerfile @@ -10,7 +10,7 @@ ENV AIRBYTE_IMPL_PATH="SourceMailchimp" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-mailchimp diff --git a/airbyte-integrations/connectors/source-mailchimp/Dockerfile.test b/airbyte-integrations/connectors/source-mailchimp/Dockerfile.test deleted file mode 100644 index d1aea70297f..00000000000 --- a/airbyte-integrations/connectors/source-mailchimp/Dockerfile.test +++ /dev/null @@ -1,22 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceMailchimpStandardTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-mailchimp-standard-test - -WORKDIR /airbyte/integration_code -COPY source_mailchimp source_mailchimp -COPY $CODE_PATH $CODE_PATH -COPY secrets/* $CODE_PATH -COPY sample_files/*.json $CODE_PATH/ -COPY source_mailchimp/*.json $CODE_PATH -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-mailchimp/build.gradle b/airbyte-integrations/connectors/source-mailchimp/build.gradle index 9d9752c40be..46edd086a2b 100644 --- a/airbyte-integrations/connectors/source-mailchimp/build.gradle +++ b/airbyte-integrations/connectors/source-mailchimp/build.gradle @@ -1,13 +1,23 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_mailchimp' } + +airbyteStandardSourceTestFile { + specPath = "source_mailchimp/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/__init__.py b/airbyte-integrations/connectors/source-mailchimp/integration_tests/__init__.py deleted file mode 100644 index 89d6d2a2d31..00000000000 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceMailchimpStandardTest - -__all__ = ["SourceMailchimpStandardTest"] diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-mailchimp/integration_tests/standard_source_test.py deleted file mode 100644 index 2b063ede667..00000000000 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/standard_source_test.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import json -import pkgutil - -from airbyte_protocol import ConfiguredAirbyteCatalog, ConnectorSpecification -from base_python_test import StandardSourceTestIface - - -class SourceMailchimpStandardTest(StandardSourceTestIface): - def __init__(self): - pass - - def get_spec(self) -> ConnectorSpecification: - raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "spec.json") - return ConnectorSpecification.parse_obj(json.loads(raw_spec)) - - def get_config(self) -> object: - return json.loads(pkgutil.get_data(self.__class__.__module__.split(".")[0], "config.json")) - - def get_catalog(self) -> ConfiguredAirbyteCatalog: - raw_catalog = pkgutil.get_data(self.__class__.__module__.split(".")[0], "configured_catalog.json") - return ConfiguredAirbyteCatalog.parse_obj(json.loads(raw_catalog)) - - def setup(self) -> None: - pass - - def teardown(self) -> None: - pass diff --git a/airbyte-integrations/connectors/source-mailchimp/requirements.txt b/airbyte-integrations/connectors/source-mailchimp/requirements.txt index e0281021aed..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-mailchimp/requirements.txt +++ b/airbyte-integrations/connectors/source-mailchimp/requirements.txt @@ -1,5 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-mailchimp/setup.py b/airbyte-integrations/connectors/source-mailchimp/setup.py index b733d34bf1b..28c5605dffc 100644 --- a/airbyte-integrations/connectors/source-mailchimp/setup.py +++ b/airbyte-integrations/connectors/source-mailchimp/setup.py @@ -30,15 +30,13 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-python", "pydantic==1.6.1", "mailchimp3==3.0.14", "python-dateutil==2.8.1"], + install_requires=[ + "airbyte-protocol", + "base-python", + "pydantic==1.6.1", + "mailchimp3==3.0.14", + "python-dateutil==2.8.1", + "pytest==6.1.2", + ], package_data={"": ["*.json", "schemas/*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - # Dependencies required by the main package but not integration tests should go in main. Deps required by - # integration tests but not the main package go in integration_tests. Deps required by both should go in - # install_requires. - "main": [], - "tests": ["airbyte_python_test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-marketo-singer/Dockerfile b/airbyte-integrations/connectors/source-marketo-singer/Dockerfile index b25e749d08e..0d4237ba3e1 100644 --- a/airbyte-integrations/connectors/source-marketo-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-marketo-singer/Dockerfile @@ -12,7 +12,7 @@ ENV AIRBYTE_IMPL_PATH="SourceMarketoSinger" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-marketo-singer diff --git a/airbyte-integrations/connectors/source-marketo-singer/Dockerfile.test b/airbyte-integrations/connectors/source-marketo-singer/Dockerfile.test deleted file mode 100644 index 2b84eb90f99..00000000000 --- a/airbyte-integrations/connectors/source-marketo-singer/Dockerfile.test +++ /dev/null @@ -1,23 +0,0 @@ -FROM airbyte/base-python-test:dev - -# Bash is installed for convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV MODULE_NAME="source_marketo_singer" -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceMarketoSingerStandardTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-marketo-singer-standard-test - -WORKDIR /airbyte/integration_code -COPY $MODULE_NAME $MODULE_NAME -COPY $CODE_PATH $CODE_PATH -COPY secrets/* $CODE_PATH/ -COPY $MODULE_NAME/*.json $CODE_PATH -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-marketo-singer/build.gradle b/airbyte-integrations/connectors/source-marketo-singer/build.gradle index 03f195e8d50..c107518dc0e 100644 --- a/airbyte-integrations/connectors/source-marketo-singer/build.gradle +++ b/airbyte-integrations/connectors/source-marketo-singer/build.gradle @@ -1,13 +1,23 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_marketo_singer' } + +airbyteStandardSourceTestFile { + specPath = "source_marketo_singer/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "integration_tests/configured_catalog.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-marketo-singer/integration_tests/__init__.py b/airbyte-integrations/connectors/source-marketo-singer/integration_tests/__init__.py deleted file mode 100644 index 7a6b6ad4b2d..00000000000 --- a/airbyte-integrations/connectors/source-marketo-singer/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceMarketoSingerStandardTest - -__all__ = ["SourceMarketoSingerStandardTest"] diff --git a/airbyte-integrations/connectors/source-marketo-singer/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-marketo-singer/integration_tests/standard_source_test.py deleted file mode 100644 index 0dc9cf2e8f2..00000000000 --- a/airbyte-integrations/connectors/source-marketo-singer/integration_tests/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class SourceMarketoSingerStandardTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-marketo-singer/requirements.txt b/airbyte-integrations/connectors/source-marketo-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-marketo-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-marketo-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-marketo-singer/setup.py b/airbyte-integrations/connectors/source-marketo-singer/setup.py index 237b076e4ea..3379529dd24 100644 --- a/airbyte-integrations/connectors/source-marketo-singer/setup.py +++ b/airbyte-integrations/connectors/source-marketo-singer/setup.py @@ -30,15 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol"], + install_requires=["airbyte-protocol", "base-singer", "base-python", "tap-marketo==2.4.1", "pytest==6.1.2"], package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - # Dependencies required by the main package but not integration tests should go in main. Deps required by - # integration tests but not the main package go in integration_tests. Deps required by both should go in - # install_requires. - "main": ["base-singer", "base-python", "tap-marketo==2.4.1"], - "tests": ["airbyte_python_test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-marketo-singer/singer_catalog.json b/airbyte-integrations/connectors/source-marketo-singer/singer_catalog.json deleted file mode 100644 index 5cb82befe71..00000000000 --- a/airbyte-integrations/connectors/source-marketo-singer/singer_catalog.json +++ /dev/null @@ -1,6401 +0,0 @@ -{ - "streams": [ - { - "tap_stream_id": "leads", - "stream": "leads", - "key_properties": ["id"], - "metadata": [ - { - "breadcrumb": ["properties", "company"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "site"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "billingStreet"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "billingCity"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "billingState"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "billingCountry"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "billingPostalCode"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "website"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "mainPhone"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "annualRevenue"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "numberOfEmployees"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "industry"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "sicCode"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "mktoCompanyNotes"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "externalCompanyId"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "mktoName"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "personType"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "mktoIsPartner"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "isLead"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "mktoIsCustomer"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "isAnonymous"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "salutation"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "firstName"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "middleName"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "lastName"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "email"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "phone"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "mobilePhone"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "fax"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "title"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "contactCompany"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "dateOfBirth"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "address"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "city"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "state"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "country"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "postalCode"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "personTimeZone"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "originalSourceType"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "originalSourceInfo"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "registrationSourceType"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "registrationSourceInfo"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "originalSearchEngine"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "originalSearchPhrase"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "originalReferrer"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "emailInvalid"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "emailInvalidCause"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "unsubscribed"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "unsubscribedReason"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "doNotCall"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "mktoDoNotCallCause"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "doNotCallReason"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "marketingSuspended"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "marketingSuspendedCause"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "blackListed"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "blackListedCause"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "mktoPersonNotes"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "anonymousIP"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "inferredCompany"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "inferredCountry"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "inferredCity"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "inferredStateRegion"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "inferredPostalCode"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "inferredMetropolitanArea"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "inferredPhoneAreaCode"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "emailSuspended"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "emailSuspendedCause"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "emailSuspendedAt"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "department"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "createdAt"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "updatedAt"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "cookies"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "externalSalesPersonId"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "leadPerson"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "leadRole"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "leadSource"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "leadStatus"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "leadScore"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "urgency"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "priority"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "relativeScore"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "relativeUrgency"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "rating"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "personPrimaryLeadInterest"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "leadPartitionId"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "leadRevenueCycleModelId"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "leadRevenueStageId"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "acquisitionProgramId"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "mktoAcquisitionDate"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [], - "metadata": { - "table-key-properties": ["id"] - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "company": { - "type": ["string", "null"] - }, - "site": { - "type": ["string", "null"] - }, - "billingStreet": { - "type": ["string", "null"] - }, - "billingCity": { - "type": ["string", "null"] - }, - "billingState": { - "type": ["string", "null"] - }, - "billingCountry": { - "type": ["string", "null"] - }, - "billingPostalCode": { - "type": ["string", "null"] - }, - "website": { - "type": ["string", "null"] - }, - "mainPhone": { - "type": ["string", "null"] - }, - "annualRevenue": { - "type": ["number", "null"] - }, - "numberOfEmployees": { - "type": ["integer", "null"] - }, - "industry": { - "type": ["string", "null"] - }, - "sicCode": { - "type": ["string", "null"] - }, - "mktoCompanyNotes": { - "type": ["string", "null"] - }, - "externalCompanyId": { - "type": ["string", "null"] - }, - "id": { - "type": "integer" - }, - "mktoName": { - "type": ["string", "null"] - }, - "personType": { - "type": ["string", "null"] - }, - "mktoIsPartner": { - "type": ["boolean", "null"] - }, - "isLead": { - "type": ["boolean", "null"] - }, - "mktoIsCustomer": { - "type": ["boolean", "null"] - }, - "isAnonymous": { - "type": ["boolean", "null"] - }, - "salutation": { - "type": ["string", "null"] - }, - "firstName": { - "type": ["string", "null"] - }, - "middleName": { - "type": ["string", "null"] - }, - "lastName": { - "type": ["string", "null"] - }, - "email": { - "type": ["string", "null"] - }, - "phone": { - "type": ["string", "null"] - }, - "mobilePhone": { - "type": ["string", "null"] - }, - "fax": { - "type": ["string", "null"] - }, - "title": { - "type": ["string", "null"] - }, - "contactCompany": { - "type": ["string", "null"] - }, - "dateOfBirth": { - "type": ["string", "null"], - "format": "date-time" - }, - "address": { - "type": ["string", "null"] - }, - "city": { - "type": ["string", "null"] - }, - "state": { - "type": ["string", "null"] - }, - "country": { - "type": ["string", "null"] - }, - "postalCode": { - "type": ["string", "null"] - }, - "personTimeZone": { - "type": ["string", "null"] - }, - "originalSourceType": { - "type": ["string", "null"] - }, - "originalSourceInfo": { - "type": ["string", "null"] - }, - "registrationSourceType": { - "type": ["string", "null"] - }, - "registrationSourceInfo": { - "type": ["string", "null"] - }, - "originalSearchEngine": { - "type": ["string", "null"] - }, - "originalSearchPhrase": { - "type": ["string", "null"] - }, - "originalReferrer": { - "type": ["string", "null"] - }, - "emailInvalid": { - "type": ["boolean", "null"] - }, - "emailInvalidCause": { - "type": ["string", "null"] - }, - "unsubscribed": { - "type": ["boolean", "null"] - }, - "unsubscribedReason": { - "type": ["string", "null"] - }, - "doNotCall": { - "type": ["boolean", "null"] - }, - "mktoDoNotCallCause": { - "type": ["string", "null"] - }, - "doNotCallReason": { - "type": ["string", "null"] - }, - "marketingSuspended": { - "type": ["boolean", "null"] - }, - "marketingSuspendedCause": { - "type": ["string", "null"] - }, - "blackListed": { - "type": ["boolean", "null"] - }, - "blackListedCause": { - "type": ["string", "null"] - }, - "mktoPersonNotes": { - "type": ["string", "null"] - }, - "anonymousIP": { - "type": ["string", "null"] - }, - "inferredCompany": { - "type": ["string", "null"] - }, - "inferredCountry": { - "type": ["string", "null"] - }, - "inferredCity": { - "type": ["string", "null"] - }, - "inferredStateRegion": { - "type": ["string", "null"] - }, - "inferredPostalCode": { - "type": ["string", "null"] - }, - "inferredMetropolitanArea": { - "type": ["string", "null"] - }, - "inferredPhoneAreaCode": { - "type": ["string", "null"] - }, - "emailSuspended": { - "type": ["boolean", "null"] - }, - "emailSuspendedCause": { - "type": ["string", "null"] - }, - "emailSuspendedAt": { - "type": ["string", "null"], - "format": "date-time" - }, - "department": { - "type": ["string", "null"] - }, - "createdAt": { - "type": "string", - "format": "date-time" - }, - "updatedAt": { - "type": "string", - "format": "date-time" - }, - "cookies": { - "type": ["string", "null"] - }, - "externalSalesPersonId": { - "type": ["string", "null"] - }, - "leadPerson": { - "type": ["string", "null"] - }, - "leadRole": { - "type": ["string", "null"] - }, - "leadSource": { - "type": ["string", "null"] - }, - "leadStatus": { - "type": ["string", "null"] - }, - "leadScore": { - "type": ["integer", "null"] - }, - "urgency": { - "type": ["number", "null"] - }, - "priority": { - "type": ["integer", "null"] - }, - "relativeScore": { - "type": ["integer", "null"] - }, - "relativeUrgency": { - "type": ["integer", "null"] - }, - "rating": { - "type": ["string", "null"] - }, - "personPrimaryLeadInterest": { - "type": ["string", "null"] - }, - "leadPartitionId": { - "type": ["string", "null"] - }, - "leadRevenueCycleModelId": { - "type": ["string", "null"] - }, - "leadRevenueStageId": { - "type": ["string", "null"] - }, - "acquisitionProgramId": { - "type": ["string", "null"] - }, - "mktoAcquisitionDate": { - "type": ["string", "null"], - "format": "date-time" - } - } - } - }, - { - "tap_stream_id": "activity_types", - "stream": "activity_types", - "key_properties": ["id"], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "description": { - "type": ["string", "null"] - }, - "primaryAttribute": { - "type": ["object", "null"], - "properties": { - "name": { - "type": "string" - }, - "dataType": { - "type": "string" - } - } - }, - "attributes": { - "type": ["array", "null"], - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "dataType": { - "type": "string" - } - } - } - } - } - }, - "metadata": [ - { - "breadcrumb": ["properties", "id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "description"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "primaryAttribute"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "attributes"], - "metadata": { - "inclusion": "unsupported" - } - }, - { - "breadcrumb": [], - "metadata": { - "inclusion": "automatic", - "table-key-properties": ["id"] - } - } - ] - }, - { - "tap_stream_id": "activities_visit_webpage", - "stream": "activities_visit_webpage", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "webpage_id", - "marketo.activity-id": 1, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "client_ip_address"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "query_parameters"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "referrer_url"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "search_engine"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "search_query"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "user_agent"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "webpage_url"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "client_ip_address": { - "type": ["string", "null"] - }, - "query_parameters": { - "type": ["string", "null"] - }, - "referrer_url": { - "type": ["string", "null"] - }, - "search_engine": { - "type": ["string", "null"] - }, - "search_query": { - "type": ["string", "null"] - }, - "user_agent": { - "type": ["string", "null"] - }, - "webpage_url": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_fill_out_form", - "stream": "activities_fill_out_form", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "webform_id", - "marketo.activity-id": 2, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "client_ip_address"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "form_fields"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "query_parameters"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "referrer_url"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "user_agent"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "webpage_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "client_ip_address": { - "type": ["string", "null"] - }, - "form_fields": { - "type": ["string", "null"] - }, - "query_parameters": { - "type": ["string", "null"] - }, - "referrer_url": { - "type": ["string", "null"] - }, - "user_agent": { - "type": ["string", "null"] - }, - "webpage_id": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_click_link", - "stream": "activities_click_link", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "link_id", - "marketo.activity-id": 3, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "client_ip_address"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "query_parameters"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "referrer_url"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "user_agent"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "webpage_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "client_ip_address": { - "type": ["string", "null"] - }, - "query_parameters": { - "type": ["string", "null"] - }, - "referrer_url": { - "type": ["string", "null"] - }, - "user_agent": { - "type": ["string", "null"] - }, - "webpage_id": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_send_email", - "stream": "activities_send_email", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "mailing_id", - "marketo.activity-id": 6, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "campaign_run_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "choice_number"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "has_predictive"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "step_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "test_variant"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "campaign_run_id": { - "type": ["integer", "null"] - }, - "choice_number": { - "type": ["integer", "null"] - }, - "has_predictive": { - "type": ["boolean", "null"] - }, - "step_id": { - "type": ["integer", "null"] - }, - "test_variant": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_email_delivered", - "stream": "activities_email_delivered", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "mailing_id", - "marketo.activity-id": 7, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "campaign_run_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "choice_number"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "has_predictive"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "step_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "test_variant"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "campaign_run_id": { - "type": ["integer", "null"] - }, - "choice_number": { - "type": ["integer", "null"] - }, - "has_predictive": { - "type": ["boolean", "null"] - }, - "step_id": { - "type": ["integer", "null"] - }, - "test_variant": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_email_bounced", - "stream": "activities_email_bounced", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "mailing_id", - "marketo.activity-id": 8, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "campaign_run_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "category"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "choice_number"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "details"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "email"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "has_predictive"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "step_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "subcategory"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "test_variant"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "campaign_run_id": { - "type": ["integer", "null"] - }, - "category": { - "type": ["string", "null"] - }, - "choice_number": { - "type": ["integer", "null"] - }, - "details": { - "type": ["string", "null"] - }, - "email": { - "type": ["string", "null"] - }, - "has_predictive": { - "type": ["boolean", "null"] - }, - "step_id": { - "type": ["integer", "null"] - }, - "subcategory": { - "type": ["string", "null"] - }, - "test_variant": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_unsubscribe_email", - "stream": "activities_unsubscribe_email", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "mailing_id", - "marketo.activity-id": 9, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "campaign_run_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "client_ip_address"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "form_fields"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "has_predictive"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "query_parameters"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "referrer_url"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "test_variant"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "user_agent"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "webform_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "webpage_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "campaign_run_id": { - "type": ["string", "null"] - }, - "client_ip_address": { - "type": ["string", "null"] - }, - "form_fields": { - "type": ["string", "null"] - }, - "has_predictive": { - "type": ["boolean", "null"] - }, - "query_parameters": { - "type": ["string", "null"] - }, - "referrer_url": { - "type": ["string", "null"] - }, - "test_variant": { - "type": ["integer", "null"] - }, - "user_agent": { - "type": ["string", "null"] - }, - "webform_id": { - "type": ["integer", "null"] - }, - "webpage_id": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_open_email", - "stream": "activities_open_email", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "mailing_id", - "marketo.activity-id": 10, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "campaign_run_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "choice_number"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "device"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "has_predictive"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "is_mobile_device"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "platform"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "step_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "test_variant"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "user_agent"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "campaign_run_id": { - "type": ["integer", "null"] - }, - "choice_number": { - "type": ["integer", "null"] - }, - "device": { - "type": ["string", "null"] - }, - "has_predictive": { - "type": ["boolean", "null"] - }, - "is_mobile_device": { - "type": ["boolean", "null"] - }, - "platform": { - "type": ["string", "null"] - }, - "step_id": { - "type": ["integer", "null"] - }, - "test_variant": { - "type": ["integer", "null"] - }, - "user_agent": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_click_email", - "stream": "activities_click_email", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "mailing_id", - "marketo.activity-id": 11, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "campaign_run_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "choice_number"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "device"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "is_mobile_device"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "is_predictive"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "link"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "link_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "platform"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "step_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "test_variant"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "user_agent"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "campaign_run_id": { - "type": ["integer", "null"] - }, - "choice_number": { - "type": ["integer", "null"] - }, - "device": { - "type": ["string", "null"] - }, - "is_mobile_device": { - "type": ["boolean", "null"] - }, - "is_predictive": { - "type": ["boolean", "null"] - }, - "link": { - "type": ["string", "null"] - }, - "link_id": { - "type": ["string", "null"] - }, - "platform": { - "type": ["string", "null"] - }, - "step_id": { - "type": ["integer", "null"] - }, - "test_variant": { - "type": ["integer", "null"] - }, - "user_agent": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_new_lead", - "stream": "activities_new_lead", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "created_date"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "form_name"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "lead_source"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "list_name"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "sfdc_type"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "source_type"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "api_method_name"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "modifying_user"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "request_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.activity-id": 12, - "table-key-properties": ["marketoGUID"] - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "created_date": { - "type": ["string", "null"], - "format": "date-time" - }, - "form_name": { - "type": ["string", "null"] - }, - "lead_source": { - "type": ["string", "null"] - }, - "list_name": { - "type": ["string", "null"] - }, - "sfdc_type": { - "type": ["string", "null"] - }, - "source_type": { - "type": ["string", "null"] - }, - "api_method_name": { - "type": ["string", "null"] - }, - "modifying_user": { - "type": ["string", "null"] - }, - "request_id": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_change_data_value", - "stream": "activities_change_data_value", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "attribute_name", - "marketo.activity-id": 13, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "new_value"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "old_value"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "reason"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "source"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "api_method_name"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "modifying_user"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "request_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "new_value": { - "type": ["string", "null"] - }, - "old_value": { - "type": ["string", "null"] - }, - "reason": { - "type": ["string", "null"] - }, - "source": { - "type": ["string", "null"] - }, - "api_method_name": { - "type": ["string", "null"] - }, - "modifying_user": { - "type": ["string", "null"] - }, - "request_id": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_change_score", - "stream": "activities_change_score", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "score_name", - "marketo.activity-id": 22, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "change_value"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "new_value"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "old_value"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "reason"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "change_value": { - "type": ["string", "null"] - }, - "new_value": { - "type": ["integer", "null"] - }, - "old_value": { - "type": ["integer", "null"] - }, - "reason": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_add_to_list", - "stream": "activities_add_to_list", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "list_id", - "marketo.activity-id": 24, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "source"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "source": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_remove_from_list", - "stream": "activities_remove_from_list", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "list_id", - "marketo.activity-id": 25, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "source"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "source": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_email_bounced_soft", - "stream": "activities_email_bounced_soft", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "mailing_id", - "marketo.activity-id": 27, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "campaign_run_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "category"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "choice_number"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "details"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "email"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "has_predictive"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "step_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "subcategory"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "test_variant"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "campaign_run_id": { - "type": ["integer", "null"] - }, - "category": { - "type": ["string", "null"] - }, - "choice_number": { - "type": ["integer", "null"] - }, - "details": { - "type": ["string", "null"] - }, - "email": { - "type": ["string", "null"] - }, - "has_predictive": { - "type": ["boolean", "null"] - }, - "step_id": { - "type": ["integer", "null"] - }, - "subcategory": { - "type": ["string", "null"] - }, - "test_variant": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_merge_leads", - "stream": "activities_merge_leads", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "lead_id", - "marketo.activity-id": 32, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "merge_ids"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "master_updated"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "merged_in_sales"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "merge_source"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "api_method_name"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "modifying_user"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "request_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "merge_ids": { - "type": ["array", "null"], - "items": { - "type": ["integer", "number", "string", "null"] - } - }, - "master_updated": { - "type": ["boolean", "null"] - }, - "merged_in_sales": { - "type": ["boolean", "null"] - }, - "merge_source": { - "type": ["string", "null"] - }, - "api_method_name": { - "type": ["string", "null"] - }, - "modifying_user": { - "type": ["string", "null"] - }, - "request_id": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_add_to_opportunity", - "stream": "activities_add_to_opportunity", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "oppty_id", - "marketo.activity-id": 34, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "is_primary"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "role"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "is_primary": { - "type": ["boolean", "null"] - }, - "role": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_remove_from_opportunity", - "stream": "activities_remove_from_opportunity", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "oppty_id", - "marketo.activity-id": 35, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "is_primary"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "role"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "is_primary": { - "type": ["boolean", "null"] - }, - "role": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_update_opportunity", - "stream": "activities_update_opportunity", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "oppty_id", - "marketo.activity-id": 36, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "attribute_name"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "data_value_changes"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "new_value"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "old_value"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "attribute_name": { - "type": ["string", "null"] - }, - "data_value_changes": { - "type": ["string", "null"] - }, - "new_value": { - "type": ["string", "null"] - }, - "old_value": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_delete_lead", - "stream": "activities_delete_lead", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "lead_id", - "marketo.activity-id": 37, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "remove_from_crm"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "api_method_name"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "modifying_user"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "request_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "remove_from_crm": { - "type": ["boolean", "null"] - }, - "api_method_name": { - "type": ["string", "null"] - }, - "modifying_user": { - "type": ["string", "null"] - }, - "request_id": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_send_alert", - "stream": "activities_send_alert", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "mailing_id", - "marketo.activity-id": 38, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "send_to_list"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "send_to_owner"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "send_to_smart_list"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "send_to_list": { - "type": ["string", "null"] - }, - "send_to_owner": { - "type": ["string", "null"] - }, - "send_to_smart_list": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_send_sales_email", - "stream": "activities_send_sales_email", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "artifact_id", - "marketo.activity-id": 39, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "campaign_run_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "has_predictive"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "sent_by"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "source"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "template_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "campaign_run_id": { - "type": ["integer", "null"] - }, - "has_predictive": { - "type": ["boolean", "null"] - }, - "sent_by": { - "type": ["string", "null"] - }, - "source": { - "type": ["string", "null"] - }, - "template_id": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_open_sales_email", - "stream": "activities_open_sales_email", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "artifact_id", - "marketo.activity-id": 40, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "campaign_run_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "has_predictive"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "sent_by"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "source"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "template_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "campaign_run_id": { - "type": ["integer", "null"] - }, - "has_predictive": { - "type": ["boolean", "null"] - }, - "sent_by": { - "type": ["string", "null"] - }, - "source": { - "type": ["string", "null"] - }, - "template_id": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_click_sales_email", - "stream": "activities_click_sales_email", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "artifact_id", - "marketo.activity-id": 41, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "campaign_run_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "is_predictive"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "link"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "sent_by"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "source"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "template_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "campaign_run_id": { - "type": ["integer", "null"] - }, - "is_predictive": { - "type": ["boolean", "null"] - }, - "link": { - "type": ["string", "null"] - }, - "sent_by": { - "type": ["string", "null"] - }, - "source": { - "type": ["string", "null"] - }, - "template_id": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_receive_sales_email", - "stream": "activities_receive_sales_email", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "artifact_id", - "marketo.activity-id": 45, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "received_by"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "source"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "received_by": { - "type": ["string", "null"] - }, - "source": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_request_campaign", - "stream": "activities_request_campaign", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "campaign_id", - "marketo.activity-id": 47, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "source"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "source": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_sales_email_bounced", - "stream": "activities_sales_email_bounced", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "artifact_id", - "marketo.activity-id": 48, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "campaign_run_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "category"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "details"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "email"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "has_predictive"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "sent_by"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "subcategory"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "campaign_run_id": { - "type": ["integer", "null"] - }, - "category": { - "type": ["string", "null"] - }, - "details": { - "type": ["string", "null"] - }, - "email": { - "type": ["string", "null"] - }, - "has_predictive": { - "type": ["boolean", "null"] - }, - "sent_by": { - "type": ["string", "null"] - }, - "subcategory": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_change_lead_partition", - "stream": "activities_change_lead_partition", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "new_partition_id", - "marketo.activity-id": 100, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "old_partition_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "reason"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "old_partition_id": { - "type": ["integer", "null"] - }, - "reason": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_change_revenue_stage", - "stream": "activities_change_revenue_stage", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "model_id", - "marketo.activity-id": 101, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "new_stage_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "old_stage_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "reason"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "sla_expiration"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "new_stage"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "old_stage"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "new_stage_id": { - "type": ["integer", "null"] - }, - "old_stage_id": { - "type": ["integer", "null"] - }, - "reason": { - "type": ["string", "null"] - }, - "sla_expiration": { - "type": ["string", "null"], - "format": "date-time" - }, - "new_stage": { - "type": ["string", "null"] - }, - "old_stage": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_change_revenue_stage_manually", - "stream": "activities_change_revenue_stage_manually", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.activity-id": 102, - "table-key-properties": ["marketoGUID"] - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - } - } - } - }, - { - "tap_stream_id": "activities_change_status_in_progression", - "stream": "activities_change_status_in_progression", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "program_id", - "marketo.activity-id": 104, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "acquired_by"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "new_status_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "old_status_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "program_member_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "reason"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "success"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "new_status"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "old_status"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "acquired_by": { - "type": ["boolean", "null"] - }, - "new_status_id": { - "type": ["integer", "null"] - }, - "old_status_id": { - "type": ["integer", "null"] - }, - "program_member_id": { - "type": ["integer", "null"] - }, - "reason": { - "type": ["string", "null"] - }, - "success": { - "type": ["boolean", "null"] - }, - "new_status": { - "type": ["string", "null"] - }, - "old_status": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_change_segment", - "stream": "activities_change_segment", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "segmentation_id", - "marketo.activity-id": 108, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "new_segment_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "new_segment_id": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_call_webhook", - "stream": "activities_call_webhook", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "webhook", - "marketo.activity-id": 110, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "error_type"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "response"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "error_type": { - "type": ["integer", "null"] - }, - "response": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_sent_forward_to_friend_email", - "stream": "activities_sent_forward_to_friend_email", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "mailing_id", - "marketo.activity-id": 111, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "choice_number"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "lead_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "step_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "choice_number": { - "type": ["integer", "null"] - }, - "lead_id": { - "type": ["string", "null"] - }, - "step_id": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_received_forward_to_friend_email", - "stream": "activities_received_forward_to_friend_email", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "mailing_id", - "marketo.activity-id": 112, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "choice_number"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "lead_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "step_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "choice_number": { - "type": ["integer", "null"] - }, - "lead_id": { - "type": ["string", "null"] - }, - "step_id": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_add_to_nurture", - "stream": "activities_add_to_nurture", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "program_id", - "marketo.activity-id": 113, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "track_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "track_id": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_change_nurture_track", - "stream": "activities_change_nurture_track", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "program_id", - "marketo.activity-id": 114, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "new_track_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "previous_track_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "new_track_id": { - "type": ["integer", "null"] - }, - "previous_track_id": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_change_nurture_cadence", - "stream": "activities_change_nurture_cadence", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "program_id", - "marketo.activity-id": 115, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "new_nurture_cadence"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "previous_nurture_cadence"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "new_nurture_cadence": { - "type": ["string", "null"] - }, - "previous_nurture_cadence": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_change_program_member_data", - "stream": "activities_change_program_member_data", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "program_id", - "marketo.activity-id": 123, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "attribute_display_name"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "attribute_name"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "new_value"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "old_value"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "reason"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "source"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "attribute_display_name": { - "type": ["string", "null"] - }, - "attribute_name": { - "type": ["integer", "null"] - }, - "new_value": { - "type": ["string", "null"] - }, - "old_value": { - "type": ["string", "null"] - }, - "reason": { - "type": ["string", "null"] - }, - "source": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_push_lead_to_marketo", - "stream": "activities_push_lead_to_marketo", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "program_id", - "marketo.activity-id": 145, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "reason"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "source"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "api_method_name"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "modifying_user"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "request_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "reason": { - "type": ["string", "null"] - }, - "source": { - "type": ["string", "null"] - }, - "api_method_name": { - "type": ["string", "null"] - }, - "modifying_user": { - "type": ["string", "null"] - }, - "request_id": { - "type": ["string", "null"] - } - } - } - }, - { - "tap_stream_id": "activities_share_content", - "stream": "activities_share_content", - "key_properties": ["marketoGUID"], - "metadata": [ - { - "breadcrumb": ["properties", "marketoGUID"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "leadId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityDate"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "activityTypeId"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "primary_attribute_value_id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": [], - "metadata": { - "marketo.primary-attribute-name": "social_app_id", - "marketo.activity-id": 400, - "table-key-properties": ["marketoGUID"] - } - }, - { - "breadcrumb": ["properties", "share_message"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "social_app_type_id"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "social_network"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "webpage_id"], - "metadata": { - "inclusion": "available" - } - } - ], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "marketoGUID": { - "type": ["null", "string"] - }, - "leadId": { - "type": ["null", "integer"] - }, - "activityDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "activityTypeId": { - "type": ["null", "integer"] - }, - "primary_attribute_value": { - "type": ["null", "string"] - }, - "primary_attribute_name": { - "type": ["null", "string"] - }, - "primary_attribute_value_id": { - "type": ["null", "string"] - }, - "share_message": { - "type": ["string", "null"] - }, - "social_app_type_id": { - "type": ["integer", "null"] - }, - "social_network": { - "type": ["string", "null"] - }, - "webpage_id": { - "type": ["integer", "null"] - } - } - } - }, - { - "tap_stream_id": "campaigns", - "stream": "campaigns", - "key_properties": ["id"], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "id": { - "type": "integer" - }, - "createdAt": { - "type": "string", - "format": "date-time" - }, - "updatedAt": { - "type": "string", - "format": "date-time" - }, - "active": { - "type": ["boolean", "null"] - }, - "description": { - "type": ["string", "null"] - }, - "name": { - "type": "string" - }, - "programId": { - "type": ["integer", "null"] - }, - "programName": { - "type": ["string", "null"] - }, - "type": { - "type": "string" - }, - "workspaceName": { - "type": ["string", "null"] - } - } - }, - "metadata": [ - { - "breadcrumb": ["properties", "id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "createdAt"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "updatedAt"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "active"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "description"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "name"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "programId"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "programName"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "type"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "workspaceName"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [], - "metadata": { - "table-key-properties": ["id"] - } - } - ] - }, - { - "tap_stream_id": "lists", - "stream": "lists", - "key_properties": ["id"], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": "string" - }, - "createdAt": { - "type": "string", - "format": "date-time" - }, - "updatedAt": { - "type": "string", - "format": "date-time" - }, - "description": { - "type": ["string", "null"] - }, - "programName": { - "type": ["string", "null"] - }, - "workspaceName": { - "type": ["string", "null"] - } - } - }, - "metadata": [ - { - "breadcrumb": ["properties", "id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "name"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "createdAt"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "updatedAt"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "description"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "programName"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "workspaceName"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [], - "metadata": { - "table-key-properties": ["id"] - } - } - ] - }, - { - "tap_stream_id": "programs", - "stream": "programs", - "key_properties": ["id"], - "schema": { - "type": "object", - "additionalProperties": false, - "properties": { - "id": { - "type": "integer" - }, - "createdAt": { - "type": "string", - "format": "date-time" - }, - "updatedAt": { - "type": "string", - "format": "date-time" - }, - "name": { - "type": "string" - }, - "description": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "channel": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "workspace": { - "type": ["null", "string"] - }, - "folder": { - "type": "object", - "properties": { - "type": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "integer"] - }, - "folderName": { - "type": ["null", "string"] - } - } - } - } - }, - "metadata": [ - { - "breadcrumb": ["properties", "id"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "createdAt"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "updatedAt"], - "metadata": { - "inclusion": "automatic" - } - }, - { - "breadcrumb": ["properties", "name"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "description"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "url"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "type"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "channel"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "status"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "workspace"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": ["properties", "folder"], - "metadata": { - "inclusion": "available" - } - }, - { - "breadcrumb": [], - "metadata": { - "table-key-properties": ["id"] - } - } - ] - } - ] -} diff --git a/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile b/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile index e661f6176cc..8096060e41a 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile +++ b/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile @@ -10,7 +10,7 @@ ENV AIRBYTE_IMPL_PATH="SourceMicrosoftTeams" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-microsoft-teams diff --git a/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile.test b/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile.test deleted file mode 100644 index 398bf265f20..00000000000 --- a/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile.test +++ /dev/null @@ -1,22 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceMicrosoftTeamsStandardTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-microsoft-teams-standard-test - -WORKDIR /airbyte/integration_code -COPY source_microsoft_teams source_microsoft_teams -COPY $CODE_PATH $CODE_PATH -COPY sample_files/configured_catalog.json $CODE_PATH/ -COPY secrets/* $CODE_PATH/ -COPY source_microsoft_teams/*.json $CODE_PATH/ -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-microsoft-teams/build.gradle b/airbyte-integrations/connectors/source-microsoft-teams/build.gradle index 362034d6076..3f0ebb1c6f8 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/build.gradle +++ b/airbyte-integrations/connectors/source-microsoft-teams/build.gradle @@ -1,13 +1,23 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_microsoft_teams' } + +airbyteStandardSourceTestFile { + specPath = "source_microsoft_teams/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/__init__.py b/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/__init__.py deleted file mode 100644 index 77b96d6aa77..00000000000 --- a/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceMicrosoftTeamsStandardTest - -__all__ = ["SourceMicrosoftTeamsStandardTest"] diff --git a/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/standard_source_test.py deleted file mode 100644 index 85347245a35..00000000000 --- a/airbyte-integrations/connectors/source-microsoft-teams/integration_tests/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class SourceMicrosoftTeamsStandardTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-microsoft-teams/requirements.txt b/airbyte-integrations/connectors/source-microsoft-teams/requirements.txt index e0281021aed..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/requirements.txt +++ b/airbyte-integrations/connectors/source-microsoft-teams/requirements.txt @@ -1,5 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-microsoft-teams/setup.py b/airbyte-integrations/connectors/source-microsoft-teams/setup.py index da6d04affd1..09f9ad37d44 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/setup.py +++ b/airbyte-integrations/connectors/source-microsoft-teams/setup.py @@ -30,15 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-python", "requests", "msal==1.7.0", "backoff"], + install_requires=["airbyte-protocol", "base-python", "requests", "msal==1.7.0", "backoff", "pytest==6.1.2"], package_data={"": ["*.json", "schemas/*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - # Dependencies required by the main package but not integration tests should go in main. Deps required by - # integration tests but not the main package go in integration_tests. Deps required by both should go in - # install_requires. - "main": [], - "tests": ["airbyte-python-test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-mixpanel-singer/Dockerfile.test b/airbyte-integrations/connectors/source-mixpanel-singer/Dockerfile.test deleted file mode 100644 index e868a1179dd..00000000000 --- a/airbyte-integrations/connectors/source-mixpanel-singer/Dockerfile.test +++ /dev/null @@ -1,28 +0,0 @@ -FROM airbyte/base-python-test:dev - -# Bash is installed for convenient debugging. -ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update \ - && apt-get install -y bash \ - && apt-get install -y gcc \ - && rm -rf /var/lib/apt/lists/* - -ENV MODULE_NAME="source_mixpanel_singer" -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceMixpanelSingerStandardTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-mixpanel-singer-standard-test - -WORKDIR /airbyte/integration_code -COPY $MODULE_NAME $MODULE_NAME -COPY $CODE_PATH $CODE_PATH -COPY secrets/* $CODE_PATH/ -COPY $MODULE_NAME/*.json $CODE_PATH/ -COPY sample_files/*.json $CODE_PATH/ -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-mixpanel-singer/build.gradle b/airbyte-integrations/connectors/source-mixpanel-singer/build.gradle index 2540c271065..4fd521b08bb 100644 --- a/airbyte-integrations/connectors/source-mixpanel-singer/build.gradle +++ b/airbyte-integrations/connectors/source-mixpanel-singer/build.gradle @@ -1,13 +1,23 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_mixpanel_singer' } + +airbyteStandardSourceTestFile { + specPath = "source_mixpanel_singer/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-mixpanel-singer/integration_tests/__init__.py b/airbyte-integrations/connectors/source-mixpanel-singer/integration_tests/__init__.py deleted file mode 100644 index 27ee1c39498..00000000000 --- a/airbyte-integrations/connectors/source-mixpanel-singer/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceMixpanelSingerStandardTest - -__all__ = ["SourceMixpanelSingerStandardTest"] diff --git a/airbyte-integrations/connectors/source-mixpanel-singer/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-mixpanel-singer/integration_tests/standard_source_test.py deleted file mode 100644 index 696009e46ae..00000000000 --- a/airbyte-integrations/connectors/source-mixpanel-singer/integration_tests/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class SourceMixpanelSingerStandardTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-mixpanel-singer/requirements.txt b/airbyte-integrations/connectors/source-mixpanel-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-mixpanel-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-mixpanel-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-mixpanel-singer/setup.py b/airbyte-integrations/connectors/source-mixpanel-singer/setup.py index 40285c337ed..ca96b5d9909 100644 --- a/airbyte-integrations/connectors/source-mixpanel-singer/setup.py +++ b/airbyte-integrations/connectors/source-mixpanel-singer/setup.py @@ -30,12 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-python", "tap-mixpanel==1.2.0"], + install_requires=["airbyte-protocol", "base-python", "base-singer", "tap-mixpanel==1.2.0", "pytest==6.1.2"], package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - "main": ["base-singer"], - "tests": ["airbyte-python-test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-recurly/Dockerfile b/airbyte-integrations/connectors/source-recurly/Dockerfile index 048f73af904..81b26a9b4f8 100644 --- a/airbyte-integrations/connectors/source-recurly/Dockerfile +++ b/airbyte-integrations/connectors/source-recurly/Dockerfile @@ -10,7 +10,7 @@ ENV AIRBYTE_IMPL_PATH="SourceRecurly" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-recurly diff --git a/airbyte-integrations/connectors/source-recurly/Dockerfile.test b/airbyte-integrations/connectors/source-recurly/Dockerfile.test deleted file mode 100644 index dda1bdf0ac2..00000000000 --- a/airbyte-integrations/connectors/source-recurly/Dockerfile.test +++ /dev/null @@ -1,22 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="RecurlyStandardSourceTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-recurly-standard-test - - -WORKDIR /airbyte/integration_code -COPY source_recurly source_recurly -COPY $CODE_PATH $CODE_PATH -COPY secrets/* $CODE_PATH -COPY source_recurly/*.json $CODE_PATH -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-recurly/build.gradle b/airbyte-integrations/connectors/source-recurly/build.gradle index d0c1e5126a9..e1db5870ce9 100644 --- a/airbyte-integrations/connectors/source-recurly/build.gradle +++ b/airbyte-integrations/connectors/source-recurly/build.gradle @@ -1,14 +1,23 @@ plugins { id 'airbyte-docker' - id 'airbyte-integration-test-java' id 'airbyte-python' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_recurly' } + +airbyteStandardSourceTestFile { + specPath = "source_recurly/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "integration_tests/configured_catalog.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-recurly/integration_tests/__init__.py b/airbyte-integrations/connectors/source-recurly/integration_tests/__init__.py deleted file mode 100644 index 391e72e7b68..00000000000 --- a/airbyte-integrations/connectors/source-recurly/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import RecurlyStandardSourceTest - -__all__ = ["RecurlyStandardSourceTest"] diff --git a/airbyte-integrations/connectors/source-recurly/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-recurly/integration_tests/standard_source_test.py deleted file mode 100644 index ff3deb92056..00000000000 --- a/airbyte-integrations/connectors/source-recurly/integration_tests/standard_source_test.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import json -import pkgutil - -from airbyte_protocol import ConfiguredAirbyteCatalog, ConnectorSpecification -from base_python_test import StandardSourceTestIface - - -class RecurlyStandardSourceTest(StandardSourceTestIface): - def get_spec(self) -> ConnectorSpecification: - raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "spec.json") - return ConnectorSpecification.parse_obj(json.loads(raw_spec)) - - def get_config(self) -> object: - return json.loads(pkgutil.get_data(self.__class__.__module__.split(".")[0], "config.json")) - - def get_catalog(self) -> ConfiguredAirbyteCatalog: - raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "configured_catalog.json") - return ConfiguredAirbyteCatalog.parse_obj(json.loads(raw_spec)) - - def setup(self) -> None: - pass - - def teardown(self) -> None: - pass diff --git a/airbyte-integrations/connectors/source-recurly/requirements.standard-test.txt b/airbyte-integrations/connectors/source-recurly/requirements.standard-test.txt deleted file mode 100644 index bf756121ea4..00000000000 --- a/airbyte-integrations/connectors/source-recurly/requirements.standard-test.txt +++ /dev/null @@ -1,2 +0,0 @@ --e ../../bases/base-python-test --e . diff --git a/airbyte-integrations/connectors/source-recurly/requirements.txt b/airbyte-integrations/connectors/source-recurly/requirements.txt index e0281021aed..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-recurly/requirements.txt +++ b/airbyte-integrations/connectors/source-recurly/requirements.txt @@ -1,5 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-recurly/setup.py b/airbyte-integrations/connectors/source-recurly/setup.py index 44a94b30ee4..4580ddbf56f 100644 --- a/airbyte-integrations/connectors/source-recurly/setup.py +++ b/airbyte-integrations/connectors/source-recurly/setup.py @@ -30,15 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-python", "recurly==3.14.0", "requests"], + install_requires=["airbyte-protocol", "base-python", "recurly==3.14.0", "requests", "pytest==6.1.2"], package_data={"": ["*.json", "schemas/*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - # Dependencies required by the main package but not integration tests should go in main. Deps required by - # integration tests but not the main package go in integration_tests. Deps required by both should go in - # install_requires. - "main": [], - "tests": ["airbyte-python-test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-salesforce-singer/build.gradle b/airbyte-integrations/connectors/source-salesforce-singer/build.gradle index 5930cb754cf..44e3bbd44b0 100644 --- a/airbyte-integrations/connectors/source-salesforce-singer/build.gradle +++ b/airbyte-integrations/connectors/source-salesforce-singer/build.gradle @@ -22,7 +22,9 @@ airbytePython { // configuredCatalogPath = "sample_files/configured_catalog.json" //} + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-salesforce-singer/integration_tests/__init__.py b/airbyte-integrations/connectors/source-salesforce-singer/integration_tests/__init__.py deleted file mode 100644 index f74beac9e2d..00000000000 --- a/airbyte-integrations/connectors/source-salesforce-singer/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SalesforceStandardSourceTest - -__all__ = ["SalesforceStandardSourceTest"] diff --git a/airbyte-integrations/connectors/source-salesforce-singer/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-salesforce-singer/integration_tests/standard_source_test.py deleted file mode 100644 index 2d758f4ce54..00000000000 --- a/airbyte-integrations/connectors/source-salesforce-singer/integration_tests/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class SalesforceStandardSourceTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-salesforce-singer/requirements.txt b/airbyte-integrations/connectors/source-salesforce-singer/requirements.txt index 275d422e3c8..f72756bb215 100644 --- a/airbyte-integrations/connectors/source-salesforce-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-salesforce-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-python -e ../../bases/base-singer --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-salesforce-singer/setup.py b/airbyte-integrations/connectors/source-salesforce-singer/setup.py index 3c03281a281..8fd46b09df4 100644 --- a/airbyte-integrations/connectors/source-salesforce-singer/setup.py +++ b/airbyte-integrations/connectors/source-salesforce-singer/setup.py @@ -32,13 +32,10 @@ setup( packages=find_packages(), package_data={"": ["*.json"]}, install_requires=[ - "tap-salesforce==1.4.34", - "requests", "airbyte-protocol", "base-python", "base-singer", + "pytest==6.1.2" "requests", + "tap-salesforce==1.4.34", ], - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={"main": ["base-singer"], "tests": ["airbyte_python_test", "pytest"]}, ) diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/Dockerfile b/airbyte-integrations/connectors/source-scaffold-source-python/Dockerfile index 453b36d1eee..16aba9ed5f0 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/Dockerfile +++ b/airbyte-integrations/connectors/source-scaffold-source-python/Dockerfile @@ -10,7 +10,7 @@ ENV AIRBYTE_IMPL_PATH="SourceScaffoldSourcePython" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.1.0 LABEL io.airbyte.name=airbyte/source-scaffold-source-python diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/requirements.txt b/airbyte-integrations/connectors/source-scaffold-source-python/requirements.txt index e0281021aed..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/requirements.txt +++ b/airbyte-integrations/connectors/source-scaffold-source-python/requirements.txt @@ -1,5 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py index 8546531edee..bae3705e361 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/setup.py +++ b/airbyte-integrations/connectors/source-scaffold-source-python/setup.py @@ -30,15 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-python"], + install_requires=["airbyte-protocol", "base-python", "pytest==6.1.2"], package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - # Dependencies required by the main package but not integration tests should go in main. Deps required by - # integration tests but not the main package go in tests. Deps required by both should go in - # install_requires. - "main": [], - "tests": ["airbyte-python-test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-sendgrid/Dockerfile b/airbyte-integrations/connectors/source-sendgrid/Dockerfile index 606dad82bb4..38d0a1cfad3 100644 --- a/airbyte-integrations/connectors/source-sendgrid/Dockerfile +++ b/airbyte-integrations/connectors/source-sendgrid/Dockerfile @@ -10,7 +10,7 @@ ENV AIRBYTE_IMPL_PATH="SourceSendgrid" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-sendgrid diff --git a/airbyte-integrations/connectors/source-sendgrid/Dockerfile.test b/airbyte-integrations/connectors/source-sendgrid/Dockerfile.test deleted file mode 100644 index 2b015218586..00000000000 --- a/airbyte-integrations/connectors/source-sendgrid/Dockerfile.test +++ /dev/null @@ -1,21 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceSendgridStandardTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-sendgrid-standard-test - -WORKDIR /airbyte/integration_code -COPY source_sendgrid source_sendgrid -COPY $CODE_PATH $CODE_PATH -COPY secrets/* $CODE_PATH -COPY source_sendgrid/*.json $CODE_PATH -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-sendgrid/build.gradle b/airbyte-integrations/connectors/source-sendgrid/build.gradle index d2a31934961..2313bb5979d 100644 --- a/airbyte-integrations/connectors/source-sendgrid/build.gradle +++ b/airbyte-integrations/connectors/source-sendgrid/build.gradle @@ -1,14 +1,22 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' - id 'airbyte-integration-test-java' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_sendgrid' } +airbyteStandardSourceTestFile { + specPath = "source_sendgrid/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "integration_tests/configured_catalog.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-sendgrid/integration_tests/__init__.py b/airbyte-integrations/connectors/source-sendgrid/integration_tests/__init__.py deleted file mode 100644 index dce26eb17a5..00000000000 --- a/airbyte-integrations/connectors/source-sendgrid/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceSendgridStandardTest - -__all__ = ["SourceSendgridStandardTest"] diff --git a/airbyte-integrations/connectors/source-sendgrid/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-sendgrid/integration_tests/standard_source_test.py deleted file mode 100644 index 677d8250683..00000000000 --- a/airbyte-integrations/connectors/source-sendgrid/integration_tests/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class SourceSendgridStandardTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-sendgrid/requirements.txt b/airbyte-integrations/connectors/source-sendgrid/requirements.txt index e0281021aed..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-sendgrid/requirements.txt +++ b/airbyte-integrations/connectors/source-sendgrid/requirements.txt @@ -1,5 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-sendgrid/setup.py b/airbyte-integrations/connectors/source-sendgrid/setup.py index 1b3655f3716..78c0b0bd45e 100644 --- a/airbyte-integrations/connectors/source-sendgrid/setup.py +++ b/airbyte-integrations/connectors/source-sendgrid/setup.py @@ -30,15 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-python", "backoff", "requests"], + install_requires=["airbyte-protocol", "base-python", "backoff", "requests", "pytest==6.1.2"], package_data={"": ["*.json", "schemas/*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - # Dependencies required by the main package but not integration tests should go in main. Deps required by - # integration tests but not the main package go in integration_tests. Deps required by both should go in - # install_requires. - "main": [], - "tests": ["airbyte-python-test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile index 1bedbec71af..4f9eeb1458f 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile @@ -11,7 +11,7 @@ WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ RUN pip install https://github.com/airbytehq/tap-shopify/tarball/master -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-shopify-singer diff --git a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile.test b/airbyte-integrations/connectors/source-shopify-singer/Dockerfile.test deleted file mode 100644 index 7900e75dc5b..00000000000 --- a/airbyte-integrations/connectors/source-shopify-singer/Dockerfile.test +++ /dev/null @@ -1,24 +0,0 @@ -FROM airbyte/base-python-test:dev - -# Bash is installed for convenient debugging. -RUN apt-get update && apt-get install -y gcc bash && rm -rf /var/lib/apt/lists/* - -ENV MODULE_NAME="source_shopify_singer" -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceShopifySingerStandardTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-shopify-singer-standard-test - -WORKDIR /airbyte/integration_code -COPY $MODULE_NAME $MODULE_NAME -COPY $CODE_PATH $CODE_PATH -COPY secrets/config.json $CODE_PATH/config.json -COPY $MODULE_NAME/*.json $CODE_PATH/ -COPY setup.py ./ - -RUN pip install https://github.com/airbytehq/tap-shopify/tarball/master -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-shopify-singer/build.gradle b/airbyte-integrations/connectors/source-shopify-singer/build.gradle index c3a0578575f..c8e73881728 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/build.gradle +++ b/airbyte-integrations/connectors/source-shopify-singer/build.gradle @@ -1,14 +1,23 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_shopify_singer' } + +airbyteStandardSourceTestFile { + specPath = "source_shopify_singer/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "integration_tests/configured_catalog.json" +} + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-shopify-singer/integration_tests/__init__.py b/airbyte-integrations/connectors/source-shopify-singer/integration_tests/__init__.py deleted file mode 100644 index d72318cc989..00000000000 --- a/airbyte-integrations/connectors/source-shopify-singer/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceShopifySingerStandardTest - -__all__ = ["SourceShopifySingerStandardTest"] diff --git a/airbyte-integrations/connectors/source-shopify-singer/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-shopify-singer/integration_tests/standard_source_test.py deleted file mode 100644 index 4763ef41006..00000000000 --- a/airbyte-integrations/connectors/source-shopify-singer/integration_tests/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class SourceShopifySingerStandardTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-shopify-singer/requirements.txt b/airbyte-integrations/connectors/source-shopify-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-shopify-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-shopify-singer/setup.py b/airbyte-integrations/connectors/source-shopify-singer/setup.py index 6341945eb25..aa4447ab370 100644 --- a/airbyte-integrations/connectors/source-shopify-singer/setup.py +++ b/airbyte-integrations/connectors/source-shopify-singer/setup.py @@ -30,15 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "ShopifyAPI==8.2.0"], + install_requires=["airbyte-protocol", "base-singer", "base-python", "ShopifyAPI==8.2.0", "pytest==6.1.2"], package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - # Dependencies required by the main package but not integration tests should go in main. Deps required by - # integration tests but not the main package go in integration_tests. Deps required by both should go in - # install_requires. - "main": ["base-singer", "base-python"], - "tests": ["airbyte-python-test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-slack-singer/Dockerfile b/airbyte-integrations/connectors/source-slack-singer/Dockerfile index a0d4a7416c6..f4d0a92ee96 100644 --- a/airbyte-integrations/connectors/source-slack-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-slack-singer/Dockerfile @@ -12,7 +12,7 @@ LABEL io.airbyte.name=airbyte/source-slack-singer WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-slack-singer/Dockerfile.test b/airbyte-integrations/connectors/source-slack-singer/Dockerfile.test deleted file mode 100644 index dff9b1e628d..00000000000 --- a/airbyte-integrations/connectors/source-slack-singer/Dockerfile.test +++ /dev/null @@ -1,25 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y gcc bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="standardtest" -ENV AIRBYTE_TEST_MODULE="standardtest" -ENV AIRBYTE_TEST_PATH="SlackStandardSourceTest" -ENV AIRBYTE_TEST_CASE=true - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-slack-singer-standard-test - -WORKDIR /airbyte/integration_code -COPY $CODE_PATH ./$CODE_PATH - -# copy all resources to the code path. -COPY secrets ./$CODE_PATH -COPY source_slack_singer/*.json ./$CODE_PATH -COPY sample_files/*.json ./$CODE_PATH/ - -COPY setup.py ./ -RUN pip install -v ".[integration_tests]" - -WORKDIR /airbyte - diff --git a/airbyte-integrations/connectors/source-slack-singer/build.gradle b/airbyte-integrations/connectors/source-slack-singer/build.gradle index 68732399487..597f224879f 100644 --- a/airbyte-integrations/connectors/source-slack-singer/build.gradle +++ b/airbyte-integrations/connectors/source-slack-singer/build.gradle @@ -1,14 +1,22 @@ plugins { id 'airbyte-docker' - id 'airbyte-integration-test-java' id 'airbyte-python' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_slack_singer' } +airbyteStandardSourceTestFile { + specPath = "source_slack_singer/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-slack-singer/requirements.standard-test.txt b/airbyte-integrations/connectors/source-slack-singer/requirements.standard-test.txt deleted file mode 100644 index bf756121ea4..00000000000 --- a/airbyte-integrations/connectors/source-slack-singer/requirements.standard-test.txt +++ /dev/null @@ -1,2 +0,0 @@ --e ../../bases/base-python-test --e . diff --git a/airbyte-integrations/connectors/source-slack-singer/requirements.txt b/airbyte-integrations/connectors/source-slack-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-slack-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-slack-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-slack-singer/setup.py b/airbyte-integrations/connectors/source-slack-singer/setup.py index d652374f3e8..09cf7264981 100644 --- a/airbyte-integrations/connectors/source-slack-singer/setup.py +++ b/airbyte-integrations/connectors/source-slack-singer/setup.py @@ -29,12 +29,7 @@ setup( description="Source implementation for Slack API, built on the Singer tap implementation", author="Airbyte", author_email="contact@airbyte.io", + install_requires=["tap-slack==1.0.0", "slack-sdk==3.0.0", "airbyte-protocol", "base-singer", "base-python"], packages=find_packages(), package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - "main": ["tap-slack==1.0.0", "slack-sdk==3.0.0", "airbyte-protocol", "base-singer", "base-python"], - "standardtest": ["airbyte_python_test"], - }, ) diff --git a/airbyte-integrations/connectors/source-slack-singer/standardtest/__init__.py b/airbyte-integrations/connectors/source-slack-singer/standardtest/__init__.py deleted file mode 100644 index 1705ab1b16c..00000000000 --- a/airbyte-integrations/connectors/source-slack-singer/standardtest/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SlackStandardSourceTest - -__all__ = ["SlackStandardSourceTest"] diff --git a/airbyte-integrations/connectors/source-slack-singer/standardtest/standard_source_test.py b/airbyte-integrations/connectors/source-slack-singer/standardtest/standard_source_test.py deleted file mode 100644 index 8f8693a6b13..00000000000 --- a/airbyte-integrations/connectors/source-slack-singer/standardtest/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class SlackStandardSourceTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-stripe-singer/setup.py b/airbyte-integrations/connectors/source-stripe-singer/setup.py index defe606695f..2bf599f802a 100644 --- a/airbyte-integrations/connectors/source-stripe-singer/setup.py +++ b/airbyte-integrations/connectors/source-stripe-singer/setup.py @@ -32,8 +32,8 @@ setup( packages=find_packages(), package_data={"": ["*.json"]}, install_requires=[ - "tap-stripe @ https://github.com/airbytehq/tap-stripe/tarball/master#egg=tap-stripe-1.4.5", "airbyte-protocol", "base-singer", + "tap-stripe @ https://github.com/airbytehq/tap-stripe/tarball/master#egg=tap-stripe-1.4.5", ], ) diff --git a/airbyte-integrations/connectors/source-tempo/Dockerfile.test b/airbyte-integrations/connectors/source-tempo/Dockerfile.test deleted file mode 100644 index ec88ad23219..00000000000 --- a/airbyte-integrations/connectors/source-tempo/Dockerfile.test +++ /dev/null @@ -1,22 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceTempoStandardTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-tempo-standard-test - -WORKDIR /airbyte/integration_code -COPY source_tempo source_tempo -COPY $CODE_PATH $CODE_PATH -COPY sample_files/*.json $CODE_PATH/ -COPY secrets/* $CODE_PATH/ -COPY source_tempo/*.json $CODE_PATH/ -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-tempo/build.gradle b/airbyte-integrations/connectors/source-tempo/build.gradle index af7d80e9679..521c8459c91 100644 --- a/airbyte-integrations/connectors/source-tempo/build.gradle +++ b/airbyte-integrations/connectors/source-tempo/build.gradle @@ -1,14 +1,22 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' - id 'airbyte-integration-test-java' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_tempo' } +airbyteStandardSourceTestFile { + specPath = "source_tempo/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-tempo/integration_tests/__init__.py b/airbyte-integrations/connectors/source-tempo/integration_tests/__init__.py deleted file mode 100644 index d49ad903c19..00000000000 --- a/airbyte-integrations/connectors/source-tempo/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceTempoStandardTest - -__all__ = ["SourceTempoStandardTest"] diff --git a/airbyte-integrations/connectors/source-tempo/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-tempo/integration_tests/standard_source_test.py deleted file mode 100644 index 7ed8eeda33d..00000000000 --- a/airbyte-integrations/connectors/source-tempo/integration_tests/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class SourceTempoStandardTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-tempo/requirements.txt b/airbyte-integrations/connectors/source-tempo/requirements.txt index e0281021aed..dd447512e62 100644 --- a/airbyte-integrations/connectors/source-tempo/requirements.txt +++ b/airbyte-integrations/connectors/source-tempo/requirements.txt @@ -1,5 +1,4 @@ # This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. -e ../../bases/airbyte-protocol -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-tempo/setup.py b/airbyte-integrations/connectors/source-tempo/setup.py index b0902b34920..f103fece393 100644 --- a/airbyte-integrations/connectors/source-tempo/setup.py +++ b/airbyte-integrations/connectors/source-tempo/setup.py @@ -30,15 +30,6 @@ setup( author="Thomas van Latum", author_email="thomas@gcompany.nl", packages=find_packages(), - install_requires=[ - "airbyte-protocol", - "base-python", - "requests", - ], + install_requires=["airbyte-protocol", "base-python", "requests", "pytest==6.1.2"], package_data={"": ["*.json", "schemas/*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - "tests": ["airbyte_python_test", "pytest"], - }, ) diff --git a/airbyte-integrations/connectors/source-twilio-singer/Dockerfile b/airbyte-integrations/connectors/source-twilio-singer/Dockerfile index af7d5cd2ccf..ac695d5ac93 100644 --- a/airbyte-integrations/connectors/source-twilio-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-twilio-singer/Dockerfile @@ -12,7 +12,7 @@ LABEL io.airbyte.name=airbyte/source-twilio-singer WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-twilio-singer/Dockerfile.test b/airbyte-integrations/connectors/source-twilio-singer/Dockerfile.test deleted file mode 100644 index c8bfdff6b4e..00000000000 --- a/airbyte-integrations/connectors/source-twilio-singer/Dockerfile.test +++ /dev/null @@ -1,25 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && apt-get install -y gcc bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="standardtest" -ENV AIRBYTE_TEST_MODULE="standardtest" -ENV AIRBYTE_TEST_PATH="TwilioStandardSourceTest" -ENV AIRBYTE_TEST_CASE=true - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-twilio-singer-standard-test - -WORKDIR /airbyte/integration_code -COPY $CODE_PATH ./$CODE_PATH - -# copy all resources to the code path. -COPY secrets ./$CODE_PATH -COPY source_twilio_singer/*.json ./$CODE_PATH/ -COPY sample_files/*.json ./$CODE_PATH/ - -COPY setup.py ./ -RUN pip install -v ".[integration_tests]" - -WORKDIR /airbyte - diff --git a/airbyte-integrations/connectors/source-twilio-singer/build.gradle b/airbyte-integrations/connectors/source-twilio-singer/build.gradle index 5ec392b4b41..f8241826798 100644 --- a/airbyte-integrations/connectors/source-twilio-singer/build.gradle +++ b/airbyte-integrations/connectors/source-twilio-singer/build.gradle @@ -1,14 +1,22 @@ plugins { id 'airbyte-docker' - id 'airbyte-integration-test-java' id 'airbyte-python' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_twilio_singer' } +airbyteStandardSourceTestFile { + specPath = "source_twilio_singer/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-twilio-singer/requirements.standard-test.txt b/airbyte-integrations/connectors/source-twilio-singer/requirements.standard-test.txt deleted file mode 100644 index bf756121ea4..00000000000 --- a/airbyte-integrations/connectors/source-twilio-singer/requirements.standard-test.txt +++ /dev/null @@ -1,2 +0,0 @@ --e ../../bases/base-python-test --e . diff --git a/airbyte-integrations/connectors/source-twilio-singer/requirements.txt b/airbyte-integrations/connectors/source-twilio-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-twilio-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-twilio-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-twilio-singer/setup.py b/airbyte-integrations/connectors/source-twilio-singer/setup.py index 4e4aa9da407..f05939355a3 100644 --- a/airbyte-integrations/connectors/source-twilio-singer/setup.py +++ b/airbyte-integrations/connectors/source-twilio-singer/setup.py @@ -29,12 +29,7 @@ setup( description="Source implementation for Twilio API, built on the Singer tap implementation", author="Airbyte", author_email="contact@airbyte.io", + install_requires=["tap-twilio==0.0.1", "twilio==6.48.0", "airbyte-protocol", "base-singer", "base-python"], packages=find_packages(), package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - "main": ["tap-twilio==0.0.1", "twilio==6.48.0", "airbyte-protocol", "base-singer", "base-python"], - "standardtest": ["airbyte_python_test"], - }, ) diff --git a/airbyte-integrations/connectors/source-zendesk-support-singer/Dockerfile b/airbyte-integrations/connectors/source-zendesk-support-singer/Dockerfile index 8bda37aeceb..4015750f192 100644 --- a/airbyte-integrations/connectors/source-zendesk-support-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-support-singer/Dockerfile @@ -9,7 +9,7 @@ ENV AIRBYTE_IMPL_PATH="SourceZendeskSupportSinger" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-zendesk-support-singer diff --git a/airbyte-integrations/connectors/source-zendesk-support-singer/Dockerfile.test b/airbyte-integrations/connectors/source-zendesk-support-singer/Dockerfile.test deleted file mode 100644 index e9061e77570..00000000000 --- a/airbyte-integrations/connectors/source-zendesk-support-singer/Dockerfile.test +++ /dev/null @@ -1,22 +0,0 @@ -FROM airbyte/base-python-test:dev - -RUN apt-get update && rm -rf /var/lib/apt/lists/* - -ENV MODULE_NAME="source_zendesk_support_singer" -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceZendeskSupportSingerStandardTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-zendesk-support-singer-support-standard-test - -WORKDIR /airbyte/integration_code -COPY $MODULE_NAME $MODULE_NAME -COPY $CODE_PATH $CODE_PATH -COPY secrets/* $CODE_PATH -COPY $MODULE_NAME/*.json $CODE_PATH -COPY setup.py ./ - -RUN pip install ".[integration_tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-zendesk-support-singer/build.gradle b/airbyte-integrations/connectors/source-zendesk-support-singer/build.gradle index 3514e8ab0fd..86229f7b0ff 100644 --- a/airbyte-integrations/connectors/source-zendesk-support-singer/build.gradle +++ b/airbyte-integrations/connectors/source-zendesk-support-singer/build.gradle @@ -1,13 +1,22 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_zendesk_support_singer' } +airbyteStandardSourceTestFile { + specPath = "source_zendesk_support_singer/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "integration_tests/catalog_subset.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-zendesk-support-singer/integration_tests/__init__.py b/airbyte-integrations/connectors/source-zendesk-support-singer/integration_tests/__init__.py deleted file mode 100644 index 7d031772d15..00000000000 --- a/airbyte-integrations/connectors/source-zendesk-support-singer/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceZendeskSupportSingerStandardTest - -__all__ = ["SourceZendeskSupportSingerStandardTest"] diff --git a/airbyte-integrations/connectors/source-zendesk-support-singer/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-zendesk-support-singer/integration_tests/standard_source_test.py deleted file mode 100644 index 3a3be5f6131..00000000000 --- a/airbyte-integrations/connectors/source-zendesk-support-singer/integration_tests/standard_source_test.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -import json -import pkgutil - -from airbyte_protocol import ConfiguredAirbyteCatalog, ConnectorSpecification -from base_python_test import StandardSourceTestIface - - -class SourceZendeskSupportSingerStandardTest(StandardSourceTestIface): - def get_spec(self) -> ConnectorSpecification: - raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "spec.json") - return ConnectorSpecification.parse_obj(json.loads(raw_spec)) - - def get_config(self) -> object: - return json.loads(pkgutil.get_data(self.__class__.__module__.split(".")[0], "config.json")) - - def get_catalog(self) -> ConfiguredAirbyteCatalog: - raw_catalog = pkgutil.get_data(self.__class__.__module__.split(".")[0], "catalog_subset.json") - return ConfiguredAirbyteCatalog.parse_obj(json.loads(raw_catalog)) - - def setup(self) -> None: - pass - - def teardown(self) -> None: - pass diff --git a/airbyte-integrations/connectors/source-zendesk-support-singer/requirements.txt b/airbyte-integrations/connectors/source-zendesk-support-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-zendesk-support-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-zendesk-support-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-zendesk-support-singer/setup.py b/airbyte-integrations/connectors/source-zendesk-support-singer/setup.py index 1321f10e50b..8febf2b3ebd 100644 --- a/airbyte-integrations/connectors/source-zendesk-support-singer/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-support-singer/setup.py @@ -30,9 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol"], + install_requires=["airbyte-protocol", "base-singer", "base-python", "tap-zendesk==1.5.3"], package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={"main": ["base-singer", "base-python", "tap-zendesk==1.5.3"], "standardtest": ["airbyte_python_test"]}, ) diff --git a/airbyte-integrations/connectors/source-zoom-singer/Dockerfile.test b/airbyte-integrations/connectors/source-zoom-singer/Dockerfile.test deleted file mode 100644 index 2c7189c7fd0..00000000000 --- a/airbyte-integrations/connectors/source-zoom-singer/Dockerfile.test +++ /dev/null @@ -1,28 +0,0 @@ -FROM airbyte/base-python-test:dev - -# Bash is installed for convenient debugging. GCC is required by tap-zoom to compile ciso8601. -ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update \ - && apt-get install -y bash \ - && apt-get install -y gcc \ - && rm -rf /var/lib/apt/lists/* - -ENV MODULE_NAME="source_zoom_singer" -ENV CODE_PATH="integration_tests" -ENV AIRBYTE_TEST_MODULE="integration_tests" -ENV AIRBYTE_TEST_PATH="SourceZoomSingerStandardTest" - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-zoom-singer-standard-test - -WORKDIR /airbyte/integration_code -COPY $MODULE_NAME $MODULE_NAME -COPY $CODE_PATH $CODE_PATH -COPY secrets/* $CODE_PATH -COPY $MODULE_NAME/*.json $CODE_PATH/ -COPY sample_files/*.json $CODE_PATH/ -COPY setup.py ./ - -RUN pip install ".[tests]" - -WORKDIR /airbyte diff --git a/airbyte-integrations/connectors/source-zoom-singer/build.gradle b/airbyte-integrations/connectors/source-zoom-singer/build.gradle index 118336d0ec7..553e55b7840 100644 --- a/airbyte-integrations/connectors/source-zoom-singer/build.gradle +++ b/airbyte-integrations/connectors/source-zoom-singer/build.gradle @@ -1,13 +1,22 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-source-test' + id 'airbyte-standard-source-test-file' } airbytePython { moduleDirectory 'source_zoom_singer' } +airbyteStandardSourceTestFile { + specPath = "source_zoom_singer/spec.json" + configPath = "secrets/config.json" + configuredCatalogPath = "sample_files/configured_catalog.json" +} + + + dependencies { + implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-zoom-singer/integration_tests/__init__.py b/airbyte-integrations/connectors/source-zoom-singer/integration_tests/__init__.py deleted file mode 100644 index f19063fdeb8..00000000000 --- a/airbyte-integrations/connectors/source-zoom-singer/integration_tests/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .standard_source_test import SourceZoomSingerStandardTest - -__all__ = ["SourceZoomSingerStandardTest"] diff --git a/airbyte-integrations/connectors/source-zoom-singer/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-zoom-singer/integration_tests/standard_source_test.py deleted file mode 100644 index d745375f419..00000000000 --- a/airbyte-integrations/connectors/source-zoom-singer/integration_tests/standard_source_test.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from base_python_test import DefaultStandardSourceTest - - -class SourceZoomSingerStandardTest(DefaultStandardSourceTest): - pass diff --git a/airbyte-integrations/connectors/source-zoom-singer/requirements.txt b/airbyte-integrations/connectors/source-zoom-singer/requirements.txt index 484253df1df..a0f3f089952 100644 --- a/airbyte-integrations/connectors/source-zoom-singer/requirements.txt +++ b/airbyte-integrations/connectors/source-zoom-singer/requirements.txt @@ -2,5 +2,4 @@ -e ../../bases/airbyte-protocol -e ../../bases/base-singer -e ../../bases/base-python --e ../../bases/base-python-test -e . diff --git a/airbyte-integrations/connectors/source-zoom-singer/setup.py b/airbyte-integrations/connectors/source-zoom-singer/setup.py index 7f388476898..90be690ab55 100644 --- a/airbyte-integrations/connectors/source-zoom-singer/setup.py +++ b/airbyte-integrations/connectors/source-zoom-singer/setup.py @@ -30,12 +30,6 @@ setup( author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-python", "tap-zoom==0.0.3"], + install_requires=["airbyte-protocol", "base-python", "base-singer", "tap-zoom==0.0.3", "pytest==6.1.2"], package_data={"": ["*.json"]}, - setup_requires=["pytest-runner"], - tests_require=["pytest"], - extras_require={ - "main": ["base-singer"], - "tests": ["airbyte_python_test", "pytest"], - }, )