Python sources refactoring (#592)

.gitignore

@@ -4,3 +4,7 @@ build
 !tools/build
 .DS_Store
 data
+
+# Python
+*.egg-info
+__pycache__

.python-version

@@ -0,0 +1 @@
+3.7.9

airbyte-integrations/base-python/.dockerignore

@@ -1,4 +1 @@
-*
+build
-!Dockerfile
-!base.py
-!airbyte_protocol

airbyte-integrations/base-python/.gitignore

@@ -0,0 +1 @@
+airbyte_protocol/models/yaml

airbyte-integrations/base-python/Dockerfile

@@ -1,22 +1,15 @@
 FROM python:3.7-slim
 COPY --from=airbyte/integration-base:dev /airbyte /airbyte
 
-WORKDIR /airbyte
+WORKDIR /airbyte/base_python_code
-ENV VIRTUAL_ENV=/airbyte/env
+COPY airbyte_protocol ./airbyte_protocol
-RUN python -m venv $VIRTUAL_ENV
+COPY setup.py ./
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-
-WORKDIR /airbyte/airbyte_protocol
-COPY airbyte_protocol .
 RUN pip install .
 
-WORKDIR /airbyte/base-python
+ENV AIRBYTE_SPEC_CMD "base-python spec"
-COPY base.py .
+ENV AIRBYTE_CHECK_CMD "base-python check"
-
+ENV AIRBYTE_DISCOVER_CMD "base-python discover"
-ENV AIRBYTE_SPEC_CMD "python3 /airbyte/base-python/base.py spec"
+ENV AIRBYTE_READ_CMD "base-python read"
-ENV AIRBYTE_CHECK_CMD "python3 /airbyte/base-python/base.py check"
-ENV AIRBYTE_DISCOVER_CMD "python3 /airbyte/base-python/base.py discover"
-ENV AIRBYTE_READ_CMD "python3 /airbyte/base-python/base.py read"
 
 ENTRYPOINT ["/airbyte/base.sh"]
 

airbyte-integrations/base-python/airbyte_protocol/__init__.py

@@ -0,0 +1,12 @@
+from .integration import *
+from .logger import AirbyteLogger
+from .models import AirbyteCatalog
+from .models import AirbyteLogMessage
+from .models import AirbyteMessage
+from .models import AirbyteRecordMessage
+from .models import AirbyteStateMessage
+from .models import AirbyteStream
+
+# Must be the last one because the way we load the connector module creates a circular
+# dependency and models might not have been loaded yet
+from .entrypoint import AirbyteEntrypoint

airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/.gitignore

@@ -1 +0,0 @@
-types

airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py

@@ -1,125 +0,0 @@
-from typing import Generator
-import yaml
-import json
-import pkgutil
-import warnings
-import python_jsonschema_objects as pjs
-from dataclasses import dataclass
-
-
-def _load_classes(yaml_path: str):
-    data = yaml.load(pkgutil.get_data(__name__, yaml_path), Loader=yaml.FullLoader)
-    builder = pjs.ObjectBuilder(data)
-    return builder.build_classes(standardize_names=False)
-
-
-# hide json schema version warnings
-with warnings.catch_warnings():
-    warnings.filterwarnings("ignore", category=UserWarning)
-    message_classes = _load_classes("types/airbyte_message.yaml")
-    AirbyteMessage = message_classes.AirbyteMessage
-    AirbyteLogMessage = message_classes.AirbyteLogMessage
-    AirbyteRecordMessage = message_classes.AirbyteRecordMessage
-    AirbyteStateMessage = message_classes.AirbyteStateMessage
-
-    catalog_classes = _load_classes("types/airbyte_catalog.yaml")
-    AirbyteCatalog = catalog_classes.AirbyteCatalog
-    AirbyteStream = catalog_classes.AirbyteStream
-
-
-class AirbyteSpec(object):
-    def __init__(self, spec_string):
-        self.spec_string = spec_string
-
-
-class AirbyteCheckResponse(object):
-    def __init__(self, successful, field_to_error):
-        self.successful = successful
-        self.field_to_error = field_to_error
-
-
-class Integration(object):
-    def __init__(self):
-        pass
-
-    def spec(self) -> AirbyteSpec:
-        raise Exception("Not Implemented")
-
-    def read_config(self, config_path):
-        with open(config_path, 'r') as file:
-            contents = file.read()
-        return json.loads(contents)
-
-    # can be overridden to change an input file config
-    def transform_config(self, raw_config):
-        return raw_config
-
-    def write_config(self, config_object, path):
-        with open(path, 'w') as fh:
-            fh.write(json.dumps(config_object))
-
-    def check(self, logger, config_container) -> AirbyteCheckResponse:
-        raise Exception("Not Implemented")
-
-    def discover(self, logger, config_container) -> AirbyteCatalog:
-        raise Exception("Not Implemented")
-
-
-class Source(Integration):
-    def __init__(self):
-        pass
-
-    # Iterator<AirbyteMessage>
-    def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]:
-        raise Exception("Not Implemented")
-
-
-class Destination(Integration):
-    def __init__(self):
-        pass
-
-
-class AirbyteLogger:
-    def __init__(self):
-        self.valid_log_types = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"]
-
-    def log_by_prefix(self, message, default_level):
-        split_line = message.split()
-        first_word = next(iter(split_line), None)
-        if first_word in self.valid_log_types:
-            log_level = first_word
-            rendered_message = " ".join(split_line[1:])
-        else:
-            log_level = default_level
-            rendered_message = message
-        self.log(log_level, rendered_message)
-
-    def log(self, level, message):
-        log_record = AirbyteLogMessage(level=level, message=message)
-        log_message = AirbyteMessage(type="LOG", log=log_record)
-        print(log_message.serialize())
-
-    def fatal(self, message):
-        self.log("FATAL", message)
-
-    def error(self, message):
-        self.log("ERROR", message)
-
-    def warn(self, message):
-        self.log("WARN", message)
-
-    def info(self, message):
-        self.log("INFO", message)
-
-    def debug(self, message):
-        self.log("DEBUG", message)
-
-    def trace(self, message):
-        self.log("TRACE", message)
-
-@dataclass
-class ConfigContainer:
-    raw_config: object
-    rendered_config: object
-    raw_config_path: str
-    rendered_config_path: str

airbyte-integrations/base-python/airbyte_protocol/entrypoint.py

@@ -1,28 +1,25 @@
 import argparse
+import importlib
+import os.path
 import sys
 import tempfile
-import os.path
-import importlib
 
-from airbyte_protocol import ConfigContainer
+from .integration import ConfigContainer, Source
-from airbyte_protocol import Source
+from .logger import AirbyteLogger
-from airbyte_protocol import AirbyteLogger
-from airbyte_protocol import AirbyteLogMessage
-from airbyte_protocol import AirbyteMessage
-
-impl_module = os.environ['AIRBYTE_IMPL_MODULE']
-impl_class = os.environ['AIRBYTE_IMPL_PATH']
 
+impl_module = os.environ.get('AIRBYTE_IMPL_MODULE', Source.__module__)
+impl_class = os.environ.get('AIRBYTE_IMPL_PATH', Source.__name__)
 module = importlib.import_module(impl_module)
 impl = getattr(module, impl_class)
 
 logger = AirbyteLogger()
 
+
 class AirbyteEntrypoint(object):
     def __init__(self, source):
         self.source = source
 
-    def start(self):
+    def start(self, args):
         # set up parent parsers
         parent_parser = argparse.ArgumentParser(add_help=False)
         main_parser = argparse.ArgumentParser()
@@ -57,23 +54,25 @@ class AirbyteEntrypoint(object):
                                           help='path to the catalog used to determine which data to read')
 
         # parse the args
-        parsed_args = main_parser.parse_args()
+        parsed_args = main_parser.parse_args(args)
 
         # execute
         cmd = parsed_args.command
+        if not cmd:
+            raise Exception("No command passed")
 
         # todo: add try catch for exceptions with different exit codes
 
         with tempfile.TemporaryDirectory() as temp_dir:
             if cmd == "spec":
                 # todo: output this as a JSON formatted message
-                print(source.spec().spec_string)
+                print(self.source.spec(logger).spec_string)
                 sys.exit(0)
 
             rendered_config_path = os.path.join(temp_dir, 'config.json')
-            raw_config = source.read_config(parsed_args.config)
+            raw_config = self.source.read_config(parsed_args.config)
-            rendered_config = source.transform_config(raw_config)
+            rendered_config = self.source.transform_config(raw_config)
-            source.write_config(rendered_config, rendered_config_path)
+            self.source.write_config(rendered_config, rendered_config_path)
 
             config_container = ConfigContainer(
                 raw_config=raw_config,
@@ -82,7 +81,7 @@ class AirbyteEntrypoint(object):
                 rendered_config_path=rendered_config_path)
 
             if cmd == "check":
-                check_result = source.check(logger, config_container)
+                check_result = self.source.check(logger, config_container)
                 if check_result.successful:
                     logger.info("Check succeeded")
                     sys.exit(0)
@@ -90,11 +89,11 @@ class AirbyteEntrypoint(object):
                     logger.error("Check failed")
                     sys.exit(1)
             elif cmd == "discover":
-                catalog = source.discover(logger, config_container)
+                catalog = self.source.discover(logger, config_container)
                 print(catalog.serialize())
                 sys.exit(0)
             elif cmd == "read":
-                generator = source.read(logger, config_container, parsed_args.catalog, parsed_args.state)
+                generator = self.source.read(logger, config_container, parsed_args.catalog, parsed_args.state)
                 for message in generator:
                     print(message.serialize())
                 sys.exit(0)
@@ -102,10 +101,15 @@ class AirbyteEntrypoint(object):
                 raise Exception("Unexpected command " + cmd)
 
 
-# set up and run entrypoint
+def launch(source, args):
-source = impl()
+    AirbyteEntrypoint(source).start(args)
 
-if not isinstance(source, Source):
-    raise Exception("Source implementation provided does not implement Source class!")
 
-AirbyteEntrypoint(source).start()
+def main():
+    # set up and run entrypoint
+    source = impl()
+
+    if not isinstance(source, Source):
+        raise Exception("Source implementation provided does not implement Source class!")
+
+    launch(source, sys.argv[1:])

airbyte-integrations/base-python/airbyte_protocol/integration.py

@@ -0,0 +1,75 @@
+import json
+import pkgutil
+from dataclasses import dataclass
+from typing import Generator
+
+from .models import AirbyteCatalog, AirbyteMessage
+
+
+class AirbyteSpec(object):
+    @staticmethod
+    def from_file(file):
+        with open(file) as file:
+            spec_text = file.read()
+        return AirbyteSpec(spec_text)
+
+    def __init__(self, spec_string):
+        self.spec_string = spec_string
+
+
+class AirbyteCheckResponse(object):
+    def __init__(self, successful, field_to_error):
+        self.successful = successful
+        self.field_to_error = field_to_error
+
+
+@dataclass
+class ConfigContainer:
+    raw_config: object
+    rendered_config: object
+    raw_config_path: str
+    rendered_config_path: str
+
+
+class Integration(object):
+    def __init__(self):
+        pass
+
+    def spec(self, logger) -> AirbyteSpec:
+        raw_spec = pkgutil.get_data(self.__class__.__module__.split('.')[0], 'spec.json')
+        # we need to output a spec on a single line
+        flattened_json = json.dumps(json.loads(raw_spec))
+        return AirbyteSpec(flattened_json)
+
+    def read_config(self, config_path):
+        with open(config_path, 'r') as file:
+            contents = file.read()
+        return json.loads(contents)
+
+    # can be overridden to change an input file config
+    def transform_config(self, raw_config):
+        return raw_config
+
+    def write_config(self, config_object, path):
+        with open(path, 'w') as fh:
+            fh.write(json.dumps(config_object))
+
+    def check(self, logger, config_container) -> AirbyteCheckResponse:
+        raise Exception("Not Implemented")
+
+    def discover(self, logger, config_container) -> AirbyteCatalog:
+        raise Exception("Not Implemented")
+
+
+class Source(Integration):
+    def __init__(self):
+        super().__init__()
+
+    # Iterator<AirbyteMessage>
+    def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]:
+        raise Exception("Not Implemented")
+
+
+class Destination(Integration):
+    def __init__(self):
+        super().__init__()

airbyte-integrations/base-python/airbyte_protocol/logger.py

@@ -0,0 +1,40 @@
+from .models import AirbyteLogMessage, AirbyteMessage
+
+
+class AirbyteLogger:
+    def __init__(self):
+        self.valid_log_types = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"]
+
+    def log_by_prefix(self, message, default_level):
+        split_line = message.split()
+        first_word = next(iter(split_line), None)
+        if first_word in self.valid_log_types:
+            log_level = first_word
+            rendered_message = " ".join(split_line[1:])
+        else:
+            log_level = default_level
+            rendered_message = message
+        self.log(log_level, rendered_message)
+
+    def log(self, level, message):
+        log_record = AirbyteLogMessage(level=level, message=message)
+        log_message = AirbyteMessage(type="LOG", log=log_record)
+        print(log_message.serialize())
+
+    def fatal(self, message):
+        self.log("FATAL", message)
+
+    def error(self, message):
+        self.log("ERROR", message)
+
+    def warn(self, message):
+        self.log("WARN", message)
+
+    def info(self, message):
+        self.log("INFO", message)
+
+    def debug(self, message):
+        self.log("DEBUG", message)
+
+    def trace(self, message):
+        self.log("TRACE", message)

airbyte-integrations/base-python/airbyte_protocol/models/__init__.py

@@ -0,0 +1,25 @@
+import pkgutil
+import warnings
+
+import python_jsonschema_objects as pjs
+import yaml
+
+
+def _load_classes(yaml_path: str):
+    data = yaml.load(pkgutil.get_data(__name__, yaml_path), Loader=yaml.FullLoader)
+    builder = pjs.ObjectBuilder(data)
+    return builder.build_classes(standardize_names=False)
+
+
+# hide json schema version warnings
+with warnings.catch_warnings():
+    warnings.filterwarnings("ignore", category=UserWarning)
+    message_classes = _load_classes("yaml/airbyte_message.yaml")
+    AirbyteMessage = message_classes.AirbyteMessage
+    AirbyteLogMessage = message_classes.AirbyteLogMessage
+    AirbyteRecordMessage = message_classes.AirbyteRecordMessage
+    AirbyteStateMessage = message_classes.AirbyteStateMessage
+
+    catalog_classes = _load_classes("yaml/airbyte_catalog.yaml")
+    AirbyteCatalog = catalog_classes.AirbyteCatalog
+    AirbyteStream = catalog_classes.AirbyteStream

airbyte-integrations/base-python/airbyte_protocol/setup.py

@@ -1,12 +0,0 @@
-from setuptools import setup
-
-setup(
-    name='airbyte_protocol',
-    description='Contains classes representing the schema of the Airbyte protocol.',
-    author='Airbyte',
-    author_email='contact@airbyte.io',
-    packages=['airbyte_protocol'],
-    install_requires=['PyYAML==5.3.1', 'python-jsonschema-objects==0.3.13'],
-    package_data={'': ['types/*.yaml']},
-    include_package_data=True
-)

airbyte-integrations/base-python/build.gradle

@@ -1,15 +1,16 @@
 apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
 
+def typesDir = 'airbyte_protocol/models/yaml'
 task deleteProtocolDefinitions(type: Delete) {
-    delete 'airbyte_protocol/airbyte_protocol/types'
+    delete typesDir
 }
 
 task copyProtocolDefinitions(type: Copy) {
     from file("$rootDir/airbyte-protocol/models/src/main/resources/airbyte_protocol").absolutePath
-    into "airbyte_protocol/airbyte_protocol/types"
+    into typesDir
     dependsOn deleteProtocolDefinitions
 }
 
+assemble.dependsOn copyProtocolDefinitions
 clean.dependsOn deleteProtocolDefinitions
-buildImage.dependsOn copyProtocolDefinitions
 buildImage.dependsOn ':airbyte-integrations:base:buildImage'

airbyte-integrations/base-python/main_dev.py

@@ -0,0 +1,4 @@
+from airbyte_protocol.entrypoint import main
+
+if __name__ == "__main__":
+    main()

airbyte-integrations/base-python/requirements.txt

@@ -0,0 +1 @@
+-e .

airbyte-integrations/base-python/setup.py

@@ -0,0 +1,24 @@
+import setuptools
+
+setuptools.setup(
+    name='airbyte-protocol',
+    description='Contains classes representing the schema of the Airbyte protocol.',
+    author='Airbyte',
+    author_email='contact@airbyte.io',
+    url='https://github.com/airbytehq/airbyte',
+
+    packages=setuptools.find_packages(),
+    package_data={
+        '': ['models/yaml/*.yaml']
+    },
+
+    install_requires=[
+        'PyYAML==5.3.1',
+        'python-jsonschema-objects==0.3.13'
+    ],
+    entry_points={
+        'console_scripts': [
+            'base-python=airbyte_protocol.entrypoint:main'
+        ],
+    }
+)

airbyte-integrations/singer/base-singer/.dockerignore

@@ -1,5 +1 @@
-*
+build
-!Dockerfile
-!base_singer/__init__.py
-!base_singer/singer_helpers.py
-!setup.py

airbyte-integrations/singer/base-singer/Dockerfile

@@ -1,9 +1,8 @@
 FROM airbyte/integration-base-python:dev
 
-WORKDIR /airbyte/base_singer
+WORKDIR /airbyte/base_singer_code
-COPY base_singer/__init__.py base_singer/__init__.py
+COPY base_singer ./base_singer
-COPY base_singer/singer_helpers.py base_singer/singer_helpers.py
+COPY setup.py ./
-COPY setup.py .
 RUN pip install .
 
 LABEL io.airbyte.version=0.1.0

airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py

@@ -3,16 +3,16 @@ import os
 import selectors
 import subprocess
 import tempfile
-from airbyte_protocol import AirbyteSpec
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Generator
+
 from airbyte_protocol import AirbyteCatalog
 from airbyte_protocol import AirbyteMessage
-from airbyte_protocol import AirbyteLogMessage
 from airbyte_protocol import AirbyteRecordMessage
+from airbyte_protocol import AirbyteSpec
 from airbyte_protocol import AirbyteStateMessage
 from airbyte_protocol import AirbyteStream
-from typing import Generator
-from datetime import datetime
-from dataclasses import dataclass
 
 
 def to_json(string):
@@ -36,14 +36,6 @@ class Catalogs:
 
 
 class SingerHelper:
-    @staticmethod
-    def spec_from_file(spec_path) -> AirbyteSpec:
-        with open(spec_path) as file:
-            spec_text = file.read()
-        # we need to output a spec on a single line
-        flattened_json = json.dumps(json.loads(spec_text))
-        return AirbyteSpec(flattened_json)
-
     @staticmethod
     def get_catalogs(logger, shell_command, singer_transform=(lambda catalog: catalog), airbyte_transform=(lambda catalog: catalog)) -> Catalogs:
         completed_process = subprocess.run(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,

airbyte-integrations/singer/base-singer/build.gradle

@@ -1,3 +1,4 @@
 apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
+
 buildImage.dependsOn ':airbyte-integrations:base-python:buildImage'
 

airbyte-integrations/singer/base-singer/requirements.txt

@@ -0,0 +1,2 @@
+-e ../../base-python
+-e .

airbyte-integrations/singer/base-singer/setup.py

@@ -1,10 +1,12 @@
-from setuptools import setup
+from setuptools import setup, find_packages
 
 setup(
-    name='base_singer',
+    name='base-singer',
     description='Contains helpers for handling Singer sources and destinations.',
     author='Airbyte',
     author_email='contact@airbyte.io',
-    packages=['base_singer'],
+
-    install_requires=['airbyte_protocol']
+    packages=find_packages(),
+
+    install_requires=['airbyte-protocol']
 )

airbyte-integrations/singer/exchangeratesapi/source/.dockerignore

@@ -1,7 +1 @@
-*
+build
-!Dockerfile
-!entrypoint.sh
-!source_exchangeratesapi_singer/source_exchangeratesapi_singer.py
-!source_exchangeratesapi_singer/__init__.py
-!spec.json
-!setup.py

airbyte-integrations/singer/exchangeratesapi/source/Dockerfile

@@ -1,21 +1,19 @@
 FROM airbyte/integration-base-singer:dev
 
-RUN apt-get update && apt-get install -y jq
+RUN apt-get update && apt-get install -y \
-
+    jq \
-COPY spec.json /airbyte/exchangeratesapi-files/spec.json
+    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /airbyte/source_exchangeratesapi_singer
-
-COPY source_exchangeratesapi_singer/__init__.py ./source_exchangeratesapi_singer/__init__.py
-COPY source_exchangeratesapi_singer/source_exchangeratesapi_singer.py ./source_exchangeratesapi_singer/source_exchangeratesapi_singer.py
-
-COPY setup.py .
-RUN pip install .
-
-WORKDIR /airbyte
 
+ENV CODE_PATH="source_exchangeratesapi_singer"
 ENV AIRBYTE_IMPL_MODULE="source_exchangeratesapi_singer"
 ENV AIRBYTE_IMPL_PATH="SourceExchangeRatesApiSinger"
 
 LABEL io.airbyte.version=0.1.2
 LABEL io.airbyte.name=airbyte/source-exchangeratesapi-singer
+
+WORKDIR /airbyte/integration_code
+COPY $CODE_PATH ./$CODE_PATH
+COPY setup.py ./
+RUN pip install .
+
+WORKDIR /airbyte

airbyte-integrations/singer/exchangeratesapi/source/build.gradle

@@ -3,7 +3,6 @@ plugins {
 }
 
 apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
-buildImage.dependsOn ":airbyte-integrations:singer:base-singer:buildImage"
 apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle')
 
 dependencies {
@@ -13,3 +12,4 @@ dependencies {
 }
 
 integrationTest.dependsOn(buildImage)
+buildImage.dependsOn ":airbyte-integrations:singer:base-singer:buildImage"

airbyte-integrations/singer/exchangeratesapi/source/main_dev.py

@@ -0,0 +1,8 @@
+import sys
+from airbyte_protocol.entrypoint import launch
+
+from source_exchangeratesapi_singer import SourceExchangeRatesApiSinger
+
+if __name__ == "__main__":
+    source = SourceExchangeRatesApiSinger()
+    launch(source, sys.argv[1:])

airbyte-integrations/singer/exchangeratesapi/source/requirements.txt

@@ -0,0 +1,3 @@
+-e ../../../base-python
+-e ../../base-singer
+-e .

airbyte-integrations/singer/exchangeratesapi/source/setup.py

@@ -1,10 +1,19 @@
-from setuptools import setup
+from setuptools import setup, find_packages
 
 setup(
-    name='source_exchangeratesapi_singer',
+    name='source-exchangeratesapi-singer',
     description='Source implementation for the exchange rates API.',
     author='Airbyte',
     author_email='contact@airbyte.io',
-    packages=['source_exchangeratesapi_singer'],
+
-    install_requires=['tap-exchangeratesapi==0.1.1', 'base_singer', 'airbyte_protocol']
+    packages=find_packages(),
+    package_data={
+        '': ['*.json']
+    },
+
+    install_requires=[
+        'tap-exchangeratesapi==0.1.1',
+        'base_singer',
+        'airbyte_protocol'
+    ]
 )

airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/__init__.py

@@ -1 +1 @@
-from .source_exchangeratesapi_singer import *
+from .source import *

airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source.py

@@ -1,27 +1,25 @@
-from airbyte_protocol import Source
-from airbyte_protocol import AirbyteSpec
-from airbyte_protocol import AirbyteCheckResponse
-from airbyte_protocol import AirbyteCatalog
-from airbyte_protocol import AirbyteMessage
 import urllib.request
 from typing import Generator
+
+from airbyte_protocol import AirbyteCatalog
+from airbyte_protocol import AirbyteCheckResponse
+from airbyte_protocol import AirbyteMessage
+from airbyte_protocol import Source
 from base_singer import SingerHelper
-from base_singer import Catalogs
+
 
 class SourceExchangeRatesApiSinger(Source):
     def __init__(self):
         pass
 
-    def spec(self) -> AirbyteSpec:
-        return SingerHelper.spec_from_file("/airbyte/exchangeratesapi-files/spec.json")
-
     def check(self, logger, config_container) -> AirbyteCheckResponse:
         code = urllib.request.urlopen("https://api.exchangeratesapi.io/").getcode()
         logger.info(f"Ping response code: {code}")
         return AirbyteCheckResponse(code == 200, {})
 
     def discover(self, logger, config_container) -> AirbyteCatalog:
-        catalogs = SingerHelper.get_catalogs(logger, "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'")
+        cmd = "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'"
+        catalogs = SingerHelper.get_catalogs(logger, cmd)
         return catalogs.airbyte_catalog
 
     def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]:

airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceDataModelTest.java

@@ -70,7 +70,7 @@ public class SingerExchangeRatesApiSourceDataModelTest {
   }
 
   @Test
-  void stripeSchemaMessageIsValid() throws IOException {
+  void schemaMessageIsValid() throws IOException {
     final String input = MoreResources.readResource("schema_message.json");
     assertTrue(new SingerProtocolPredicate().test(Jsons.deserialize(input)));
   }

airbyte-integrations/singer/stripe_abprotocol/source/.dockerignore

@@ -1,7 +1 @@
-*
+build
-!Dockerfile
-!entrypoint.sh
-!source_stripe_singer/*.py
-!spec.json
-!setup.py
-

airbyte-integrations/singer/stripe_abprotocol/source/Dockerfile

@@ -1,20 +1,22 @@
 FROM airbyte/integration-base-singer:dev
 
-RUN apt-get update && apt-get install -y jq curl bash
+RUN apt-get update && apt-get install -y \
-
+    jq \
-COPY spec.json /airbyte/stripe-files/spec.json
+    curl \
-
+    bash \
-WORKDIR /airbyte/source_stripe_singer
+    && rm -rf /var/lib/apt/lists/*
-
-COPY source_stripe_singer/*.py ./source_stripe_singer/
-
-COPY setup.py .
-RUN pip install .
-
-WORKDIR /airbyte
 
+ENV CODE_PATH="source_stripe_singer"
 ENV AIRBYTE_IMPL_MODULE="source_stripe_singer"
 ENV AIRBYTE_IMPL_PATH="SourceStripeSinger"
 
 LABEL io.airbyte.version=0.1.3
 LABEL io.airbyte.name=airbyte/source-stripe-abprotocol-singer
+
+WORKDIR /airbyte/integration_code
+COPY $CODE_PATH ./$CODE_PATH
+COPY setup.py ./
+RUN pip install .
+
+WORKDIR /airbyte
+

airbyte-integrations/singer/stripe_abprotocol/source/main_dev.py

@@ -0,0 +1,8 @@
+import sys
+from airbyte_protocol.entrypoint import launch
+
+from source_stripe_singer import SourceStripeSinger
+
+if __name__ == "__main__":
+    source = SourceStripeSinger()
+    launch(source, sys.argv[1:])

airbyte-integrations/singer/stripe_abprotocol/source/requirements.txt

@@ -0,0 +1,3 @@
+-e ../../../base-python
+-e ../../base-singer
+-e .

airbyte-integrations/singer/stripe_abprotocol/source/setup.py

@@ -1,15 +1,20 @@
-from setuptools import setup
+from setuptools import setup, find_packages
 
 setup(
     name='source_stripe_singer',
     description='Source implementation for Stripe.',
     author='Airbyte',
     author_email='contact@airbyte.io',
-    packages=['source_stripe_singer'],
+
+    packages=find_packages(),
+    package_data={
+        '': ['*.json']
+    },
+
     install_requires=[
         'tap-stripe==1.4.4',
         'requests',
         'base_singer',
-        'airbyte_protocol']
+        'airbyte_protocol'
+    ]
 )
-

airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/__init__.py

@@ -1,2 +1,2 @@
-from .source_stripe_singer import *
+from .source import *
 

airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source.py

@@ -1,20 +1,16 @@
-from airbyte_protocol import Source
-from airbyte_protocol import AirbyteSpec
-from airbyte_protocol import AirbyteCheckResponse
-from airbyte_protocol import AirbyteCatalog
-from airbyte_protocol import AirbyteMessage
 import requests
-from typing import Generator
+from airbyte_protocol import AirbyteCatalog
+from airbyte_protocol import AirbyteCheckResponse
+from airbyte_protocol import AirbyteMessage
+from airbyte_protocol import Source
 from base_singer import SingerHelper
+from typing import Generator
 
 
 class SourceStripeSinger(Source):
     def __init__(self):
         pass
 
-    def spec(self) -> AirbyteSpec:
-        return SingerHelper.spec_from_file('/airbyte/stripe-files/spec.json')
-
     def check(self, logger, config_container) -> AirbyteCheckResponse:
         json_config = config_container.rendered_config
         r = requests.get('https://api.stripe.com/v1/customers', auth=(json_config['client_secret'], ''))
@@ -26,8 +22,10 @@ class SourceStripeSinger(Source):
         return catalogs.airbyte_catalog
 
     def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]:
+        discover_cmd = f"tap-stripe --config {config_container.rendered_config_path} --discover"
+        discovered_singer_catalog = SingerHelper.get_catalogs(logger, discover_cmd).singer_catalog
+
         masked_airbyte_catalog = self.read_config(catalog_path)
-        discovered_singer_catalog = SingerHelper.get_catalogs(logger, f"tap-stripe --config {config_container.rendered_config_path} --discover").singer_catalog
         selected_singer_catalog = SingerHelper.create_singer_catalog_with_selection(masked_airbyte_catalog, discovered_singer_catalog)
 
         config_option = f"--config {config_container.rendered_config_path}"

airbyte-integrations/template/python-source/.dockerignore

@@ -0,0 +1 @@
+build

airbyte-integrations/template/python-source/Dockerfile

@@ -0,0 +1,15 @@
+FROM airbyte/integration-base-python:dev
+
+ENV CODE_PATH="template_python_source"
+ENV AIRBYTE_IMPL_MODULE="template_python_source"
+ENV AIRBYTE_IMPL_PATH="TemplatePythonSource"
+
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/source-template-python
+
+WORKDIR /airbyte/integration_code
+COPY $CODE_PATH ./$CODE_PATH
+COPY setup.py ./
+RUN pip install .
+
+WORKDIR /airbyte

airbyte-integrations/template/python-source/README.md

@@ -0,0 +1,31 @@
+# Python Airbyte Source Development
+
+Prepare development environment:
+```
+cd airbyte-integrations/template/python-source
+
+# create & activate virtualenv
+virtualenv build/venv
+source build/venv/bin/activate
+
+# install necessary dependencies
+pip install -r requirements.txt
+```
+
+Test locally:
+```
+python main_dev.py spec
+python main_dev.py check --config sample_files/test_config.json
+python main_dev.py discover --config sample_files/test_config.json
+python main_dev.py read --config sample_files/test_config.json --catalog sample_files/test_catalog.json
+```
+
+Test image:
+```
+# in airbyte root directory
+./gradlew :airbyte-integrations:template:python-source:buildImage
+docker run --rm -v $(pwd)/airbyte-integrations/template/python-source/sample_files:/sample_files airbyte/source-template-python:dev spec
+docker run --rm -v $(pwd)/airbyte-integrations/template/python-source/sample_files:/sample_files airbyte/source-template-python:dev check --config /sample_files/test_config.json
+docker run --rm -v $(pwd)/airbyte-integrations/template/python-source/sample_files:/sample_files airbyte/source-template-python:dev discover --config /sample_files/test_config.json
+docker run --rm -v $(pwd)/airbyte-integrations/template/python-source/sample_files:/sample_files airbyte/source-template-python:dev read --config /sample_files/test_config.json --catalog /sample_files/test_catalog.json
+```

airbyte-integrations/template/python-source/build.gradle

@@ -0,0 +1,3 @@
+apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
+
+buildImage.dependsOn ":airbyte-integrations:base-python:buildImage"

airbyte-integrations/template/python-source/main_dev.py

@@ -0,0 +1,9 @@
+import sys
+
+from airbyte_protocol.entrypoint import launch
+
+from template_python_source import TemplatePythonSource
+
+if __name__ == "__main__":
+    source = TemplatePythonSource()
+    launch(source, sys.argv[1:])

airbyte-integrations/template/python-source/requirements.txt

@@ -0,0 +1,2 @@
+-e ../../base-python
+-e .

airbyte-integrations/template/python-source/sample_files/test_catalog.json

@@ -0,0 +1,16 @@
+{
+  "streams": [
+    {
+      "name": "love_airbyte",
+      "schema": {
+        "type": "object",
+        "required": ["love"],
+        "properties": {
+          "love": {
+            "type": "boolean"
+          }
+        }
+      }
+    }
+  ]
+}

airbyte-integrations/template/python-source/sample_files/test_config.json

@@ -0,0 +1,3 @@
+{
+  "love_airbyte": true
+}

airbyte-integrations/template/python-source/setup.py

@@ -0,0 +1,15 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='template-python-source',
+    description='Source implementation template',
+    author='Airbyte',
+    author_email='contact@airbyte.io',
+
+    packages=find_packages(),
+    package_data={
+        '': ['*.json']
+    },
+
+    install_requires=['airbyte-protocol']
+)

airbyte-integrations/template/python-source/template_python_source/__init__.py

@@ -0,0 +1 @@
+from .source import *

airbyte-integrations/template/python-source/template_python_source/catalog.json

@@ -0,0 +1,16 @@
+{
+  "streams": [
+    {
+      "name": "love_airbyte",
+      "schema": {
+        "type": "object",
+        "required": ["love"],
+        "properties": {
+          "love": {
+            "type": "boolean"
+          }
+        }
+      }
+    }
+  ]
+}

airbyte-integrations/template/python-source/template_python_source/source.py

@@ -0,0 +1,36 @@
+import pkgutil
+import time
+from typing import Generator
+
+from airbyte_protocol import AirbyteCatalog
+from airbyte_protocol import AirbyteCheckResponse
+from airbyte_protocol import AirbyteMessage
+from airbyte_protocol import AirbyteRecordMessage
+from airbyte_protocol import AirbyteSpec
+from airbyte_protocol import AirbyteStateMessage
+from airbyte_protocol import Source
+
+
+class TemplatePythonSource(Source):
+    def __init__(self):
+        pass
+
+    def check(self, logger, config_container) -> AirbyteCheckResponse:
+        logger.info(f'Checking configuration ({config_container.rendered_config_path})...')
+        return AirbyteCheckResponse(True, {})
+
+    def discover(self, logger, config_container) -> AirbyteCatalog:
+        logger.info(f'Discovering ({config_container.rendered_config_path})...')
+        return AirbyteCatalog.from_json(pkgutil.get_data(__name__, 'catalog.json'))
+
+    def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]:
+        logger.info(f'Reading ({config_container.rendered_config_path}, {catalog_path}, {state})...')
+
+        message = AirbyteRecordMessage(
+            stream='love_airbyte',
+            data={'love': True},
+            emitted_at=int(time.time() * 1000))
+        yield AirbyteMessage(type='RECORD', record=message)
+
+        state = AirbyteStateMessage(data={'love_cursor': 'next_version'})
+        yield AirbyteMessage(type='STATE', state=state)

airbyte-integrations/template/python-source/template_python_source/spec.json

@@ -0,0 +1,17 @@
+{
+  "documentationUrl": "https://docs.airbyte.io/integrations/sources/template-python-source",
+  "connectionSpecification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Template python source Spec",
+    "type": "object",
+    "required": ["love_airbyte"],
+    "additionalProperties": false,
+    "properties": {
+      "love_airbyte": {
+        "type": "boolean",
+        "description": "Do you love Airbyte",
+        "examples": ["true"]
+      }
+    }
+  }
+}

airbyte-integrations/template/singer-source/.dockerignore

@@ -0,0 +1 @@
+build

airbyte-integrations/template/singer-source/Dockerfile

@@ -0,0 +1,19 @@
+FROM airbyte/integration-base-singer:dev
+
+RUN apt-get update && apt-get install -y \
+    jq \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV CODE_PATH="template_singer_source"
+ENV AIRBYTE_IMPL_MODULE="template_singer_source"
+ENV AIRBYTE_IMPL_PATH="TemplateSingerSource"
+
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/source-template-singer
+
+WORKDIR /airbyte/integration_code
+COPY $CODE_PATH ./$CODE_PATH
+COPY setup.py ./
+RUN pip install .
+
+WORKDIR /airbyte

airbyte-integrations/template/singer-source/README.md

@@ -0,0 +1 @@
+# This is a demo source

airbyte-integrations/template/singer-source/build.gradle

@@ -0,0 +1,5 @@
+apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
+apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle')
+
+integrationTest.dependsOn(buildImage)
+buildImage.dependsOn ":airbyte-integrations:singer:base-singer:buildImage"

airbyte-integrations/template/singer-source/main_dev.py

@@ -0,0 +1,8 @@
+import sys
+from airbyte_protocol.entrypoint import launch
+
+from template_singer_source import TemplateSingerSource
+
+if __name__ == "__main__":
+    source = TemplateSingerSource()
+    launch(source, sys.argv[1:])

airbyte-integrations/template/singer-source/requirements.txt

@@ -0,0 +1,3 @@
+-e ../../base-python
+-e ../../singer/base-singer
+-e .

airbyte-integrations/template/singer-source/setup.py

@@ -0,0 +1,19 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='template-singer-source',
+    description='Singer source implementation template',
+    author='Airbyte',
+    author_email='contact@airbyte.io',
+
+    packages=find_packages(),
+    package_data={
+        '': ['*.json']
+    },
+
+    install_requires=[
+        'tap-exchangeratesapi==0.1.1',
+        'base_singer',
+        'airbyte_protocol'
+    ]
+)

airbyte-integrations/template/singer-source/template_singer_source/__init__.py

@@ -0,0 +1 @@
+from .source import *

airbyte-integrations/template/singer-source/template_singer_source/source.py

@@ -0,0 +1,28 @@
+import urllib.request
+from typing import Generator
+
+from airbyte_protocol import AirbyteCatalog
+from airbyte_protocol import AirbyteCheckResponse
+from airbyte_protocol import AirbyteMessage
+from airbyte_protocol import Source
+from base_singer import SingerHelper
+
+
+class TemplateSingerSource(Source):
+    def __init__(self):
+        pass
+
+    def check(self, logger, config_container) -> AirbyteCheckResponse:
+        code = urllib.request.urlopen("https://api.exchangeratesapi.io/").getcode()
+        logger.info(f"Ping response code: {code}")
+        return AirbyteCheckResponse(code == 200, {})
+
+    def discover(self, logger, config_container) -> AirbyteCatalog:
+        cmd = "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'"
+        catalogs = SingerHelper.get_catalogs(logger, cmd)
+        return catalogs.airbyte_catalog
+
+    def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]:
+        config_option = f"--config {config_container.rendered_config_path}"
+        state_option = f"--state {state}" if state else ""
+        return SingerHelper.read(logger, f"tap-exchangeratesapi {config_option} {state_option}")

airbyte-integrations/template/singer-source/template_singer_source/spec.json

@@ -0,0 +1,21 @@
+{
+  "documentationUrl": "https://docs.airbyte.io/integrations/sources/exchangeratesapi-io",
+  "connectionSpecification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Template Singer Source Spec",
+    "type": "object",
+    "required": ["start_date", "base"],
+    "additionalProperties": false,
+    "properties": {
+      "start_date": {
+        "type": "string",
+        "description": "Start getting data from that date.",
+        "examples": ["YYYY-MM-DD"]
+      },
+      "base": {
+        "type": "string",
+        "description": "ISO reference currency. See <a href=\"https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html\">here</a>."
+      }
+    }
+  }
+}

build.gradle

@@ -2,7 +2,7 @@ plugins {
     id 'base'
     id 'java'
     id 'pmd'
-    id 'com.diffplug.spotless' version '5.4.0'
+    id 'com.diffplug.spotless' version '5.6.1'
     // id 'de.aaschmid.cpd' version '3.1'
 }