jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-19 18:14:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

Python sources refactoring (#592)

Browse Source
This commit is contained in:
Michel Tricot
2020-10-16 10:56:08 -07:00
committed by GitHub
parent bd696014d5
commit bc56f02c41
65 changed files with 608 additions and 281 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@@ -4,3 +4,7 @@ build
!tools/build
.DS_Store
data
# Python
*.egg-info
__pycache__

1
.python-version Normal file
View File

@@ -0,0 +1 @@
3.7.9

5
airbyte-integrations/base-python/.dockerignore
View File

@@ -1,4 +1 @@
*
!Dockerfile
!base.py
!airbyte_protocol
build

1
airbyte-integrations/base-python/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
airbyte_protocol/models/yaml

21
airbyte-integrations/base-python/Dockerfile
View File

@@ -1,22 +1,15 @@
FROM python:3.7-slim
COPY --from=airbyte/integration-base:dev /airbyte /airbyte
WORKDIR /airbyte
ENV VIRTUAL_ENV=/airbyte/env
RUN python -m venv $VIRTUAL_ENV
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
WORKDIR /airbyte/airbyte_protocol
COPY airbyte_protocol .
WORKDIR /airbyte/base_python_code
COPY airbyte_protocol ./airbyte_protocol
COPY setup.py ./
RUN pip install .
WORKDIR /airbyte/base-python
COPY base.py .
ENV AIRBYTE_SPEC_CMD "python3 /airbyte/base-python/base.py spec"
ENV AIRBYTE_CHECK_CMD "python3 /airbyte/base-python/base.py check"
ENV AIRBYTE_DISCOVER_CMD "python3 /airbyte/base-python/base.py discover"
ENV AIRBYTE_READ_CMD "python3 /airbyte/base-python/base.py read"
ENV AIRBYTE_SPEC_CMD "base-python spec"
ENV AIRBYTE_CHECK_CMD "base-python check"
ENV AIRBYTE_DISCOVER_CMD "base-python discover"
ENV AIRBYTE_READ_CMD "base-python read"
ENTRYPOINT ["/airbyte/base.sh"]

12
airbyte-integrations/base-python/airbyte_protocol/__init__.py Normal file
View File

@@ -0,0 +1,12 @@
from .integration import *
from .logger import AirbyteLogger
from .models import AirbyteCatalog
from .models import AirbyteLogMessage
from .models import AirbyteMessage
from .models import AirbyteRecordMessage
from .models import AirbyteStateMessage
from .models import AirbyteStream
# Must be the last one because the way we load the connector module creates a circular
# dependency and models might not have been loaded yet
from .entrypoint import AirbyteEntrypoint

1
airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/.gitignore vendored
View File

@@ -1 +0,0 @@
types

125
airbyte-integrations/base-python/airbyte_protocol/airbyte_protocol/__init__.py
View File

@@ -1,125 +0,0 @@
from typing import Generator
import yaml
import json
import pkgutil
import warnings
import python_jsonschema_objects as pjs
from dataclasses import dataclass
def _load_classes(yaml_path: str):
data = yaml.load(pkgutil.get_data(__name__, yaml_path), Loader=yaml.FullLoader)
builder = pjs.ObjectBuilder(data)
return builder.build_classes(standardize_names=False)
# hide json schema version warnings
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=UserWarning)
message_classes = _load_classes("types/airbyte_message.yaml")
AirbyteMessage = message_classes.AirbyteMessage
AirbyteLogMessage = message_classes.AirbyteLogMessage
AirbyteRecordMessage = message_classes.AirbyteRecordMessage
AirbyteStateMessage = message_classes.AirbyteStateMessage
catalog_classes = _load_classes("types/airbyte_catalog.yaml")
AirbyteCatalog = catalog_classes.AirbyteCatalog
AirbyteStream = catalog_classes.AirbyteStream
class AirbyteSpec(object):
def __init__(self, spec_string):
self.spec_string = spec_string
class AirbyteCheckResponse(object):
def __init__(self, successful, field_to_error):
self.successful = successful
self.field_to_error = field_to_error
class Integration(object):
def __init__(self):
pass
def spec(self) -> AirbyteSpec:
raise Exception("Not Implemented")
def read_config(self, config_path):
with open(config_path, 'r') as file:
contents = file.read()
return json.loads(contents)
# can be overridden to change an input file config
def transform_config(self, raw_config):
return raw_config
def write_config(self, config_object, path):
with open(path, 'w') as fh:
fh.write(json.dumps(config_object))
def check(self, logger, config_container) -> AirbyteCheckResponse:
raise Exception("Not Implemented")
def discover(self, logger, config_container) -> AirbyteCatalog:
raise Exception("Not Implemented")
class Source(Integration):
def __init__(self):
pass
# Iterator<AirbyteMessage>
def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]:
raise Exception("Not Implemented")
class Destination(Integration):
def __init__(self):
pass
class AirbyteLogger:
def __init__(self):
self.valid_log_types = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"]
def log_by_prefix(self, message, default_level):
split_line = message.split()
first_word = next(iter(split_line), None)
if first_word in self.valid_log_types:
log_level = first_word
rendered_message = " ".join(split_line[1:])
else:
log_level = default_level
rendered_message = message
self.log(log_level, rendered_message)
def log(self, level, message):
log_record = AirbyteLogMessage(level=level, message=message)
log_message = AirbyteMessage(type="LOG", log=log_record)
print(log_message.serialize())
def fatal(self, message):
self.log("FATAL", message)
def error(self, message):
self.log("ERROR", message)
def warn(self, message):
self.log("WARN", message)
def info(self, message):
self.log("INFO", message)
def debug(self, message):
self.log("DEBUG", message)
def trace(self, message):
self.log("TRACE", message)
@dataclass
class ConfigContainer:
raw_config: object
rendered_config: object
raw_config_path: str
rendered_config_path: str

52
airbyte-integrations/base-python/base.py → airbyte-integrations/base-python/airbyte_protocol/entrypoint.py
View File

@@ -1,28 +1,25 @@
import argparse
import importlib
import os.path
import sys
import tempfile
import os.path
import importlib
from airbyte_protocol import ConfigContainer
from airbyte_protocol import Source
from airbyte_protocol import AirbyteLogger
from airbyte_protocol import AirbyteLogMessage
from airbyte_protocol import AirbyteMessage
impl_module = os.environ['AIRBYTE_IMPL_MODULE']
impl_class = os.environ['AIRBYTE_IMPL_PATH']
from .integration import ConfigContainer, Source
from .logger import AirbyteLogger
impl_module = os.environ.get('AIRBYTE_IMPL_MODULE', Source.__module__)
impl_class = os.environ.get('AIRBYTE_IMPL_PATH', Source.__name__)
module = importlib.import_module(impl_module)
impl = getattr(module, impl_class)
logger = AirbyteLogger()
class AirbyteEntrypoint(object):
def __init__(self, source):
self.source = source
def start(self):
def start(self, args):
# set up parent parsers
parent_parser = argparse.ArgumentParser(add_help=False)
main_parser = argparse.ArgumentParser()
@@ -57,23 +54,25 @@ class AirbyteEntrypoint(object):
help='path to the catalog used to determine which data to read')
# parse the args
parsed_args = main_parser.parse_args()
parsed_args = main_parser.parse_args(args)
# execute
cmd = parsed_args.command
if not cmd:
raise Exception("No command passed")
# todo: add try catch for exceptions with different exit codes
with tempfile.TemporaryDirectory() as temp_dir:
if cmd == "spec":
# todo: output this as a JSON formatted message
print(source.spec().spec_string)
print(self.source.spec(logger).spec_string)
sys.exit(0)
rendered_config_path = os.path.join(temp_dir, 'config.json')
raw_config = source.read_config(parsed_args.config)
rendered_config = source.transform_config(raw_config)
source.write_config(rendered_config, rendered_config_path)
raw_config = self.source.read_config(parsed_args.config)
rendered_config = self.source.transform_config(raw_config)
self.source.write_config(rendered_config, rendered_config_path)
config_container = ConfigContainer(
raw_config=raw_config,
@@ -82,7 +81,7 @@ class AirbyteEntrypoint(object):
rendered_config_path=rendered_config_path)
if cmd == "check":
check_result = source.check(logger, config_container)
check_result = self.source.check(logger, config_container)
if check_result.successful:
logger.info("Check succeeded")
sys.exit(0)
@@ -90,11 +89,11 @@ class AirbyteEntrypoint(object):
logger.error("Check failed")
sys.exit(1)
elif cmd == "discover":
catalog = source.discover(logger, config_container)
catalog = self.source.discover(logger, config_container)
print(catalog.serialize())
sys.exit(0)
elif cmd == "read":
generator = source.read(logger, config_container, parsed_args.catalog, parsed_args.state)
generator = self.source.read(logger, config_container, parsed_args.catalog, parsed_args.state)
for message in generator:
print(message.serialize())
sys.exit(0)
@@ -102,10 +101,15 @@ class AirbyteEntrypoint(object):
raise Exception("Unexpected command " + cmd)
# set up and run entrypoint
source = impl()
def launch(source, args):
AirbyteEntrypoint(source).start(args)
if not isinstance(source, Source):
raise Exception("Source implementation provided does not implement Source class!")
AirbyteEntrypoint(source).start()
def main():
# set up and run entrypoint
source = impl()
if not isinstance(source, Source):
raise Exception("Source implementation provided does not implement Source class!")
launch(source, sys.argv[1:])

75
airbyte-integrations/base-python/airbyte_protocol/integration.py Normal file
View File

@@ -0,0 +1,75 @@
import json
import pkgutil
from dataclasses import dataclass
from typing import Generator
from .models import AirbyteCatalog, AirbyteMessage
class AirbyteSpec(object):
@staticmethod
def from_file(file):
with open(file) as file:
spec_text = file.read()
return AirbyteSpec(spec_text)
def __init__(self, spec_string):
self.spec_string = spec_string
class AirbyteCheckResponse(object):
def __init__(self, successful, field_to_error):
self.successful = successful
self.field_to_error = field_to_error
@dataclass
class ConfigContainer:
raw_config: object
rendered_config: object
raw_config_path: str
rendered_config_path: str
class Integration(object):
def __init__(self):
pass
def spec(self, logger) -> AirbyteSpec:
raw_spec = pkgutil.get_data(self.__class__.__module__.split('.')[0], 'spec.json')
# we need to output a spec on a single line
flattened_json = json.dumps(json.loads(raw_spec))
return AirbyteSpec(flattened_json)
def read_config(self, config_path):
with open(config_path, 'r') as file:
contents = file.read()
return json.loads(contents)
# can be overridden to change an input file config
def transform_config(self, raw_config):
return raw_config
def write_config(self, config_object, path):
with open(path, 'w') as fh:
fh.write(json.dumps(config_object))
def check(self, logger, config_container) -> AirbyteCheckResponse:
raise Exception("Not Implemented")
def discover(self, logger, config_container) -> AirbyteCatalog:
raise Exception("Not Implemented")
class Source(Integration):
def __init__(self):
super().__init__()
# Iterator<AirbyteMessage>
def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]:
raise Exception("Not Implemented")
class Destination(Integration):
def __init__(self):
super().__init__()

40
airbyte-integrations/base-python/airbyte_protocol/logger.py Normal file
View File

@@ -0,0 +1,40 @@
from .models import AirbyteLogMessage, AirbyteMessage
class AirbyteLogger:
def __init__(self):
self.valid_log_types = ["FATAL", "ERROR", "WARN", "INFO", "DEBUG", "TRACE"]
def log_by_prefix(self, message, default_level):
split_line = message.split()
first_word = next(iter(split_line), None)
if first_word in self.valid_log_types:
log_level = first_word
rendered_message = " ".join(split_line[1:])
else:
log_level = default_level
rendered_message = message
self.log(log_level, rendered_message)
def log(self, level, message):
log_record = AirbyteLogMessage(level=level, message=message)
log_message = AirbyteMessage(type="LOG", log=log_record)
print(log_message.serialize())
def fatal(self, message):
self.log("FATAL", message)
def error(self, message):
self.log("ERROR", message)
def warn(self, message):
self.log("WARN", message)
def info(self, message):
self.log("INFO", message)
def debug(self, message):
self.log("DEBUG", message)
def trace(self, message):
self.log("TRACE", message)

25
airbyte-integrations/base-python/airbyte_protocol/models/__init__.py Normal file
View File

@@ -0,0 +1,25 @@
import pkgutil
import warnings
import python_jsonschema_objects as pjs
import yaml
def _load_classes(yaml_path: str):
data = yaml.load(pkgutil.get_data(__name__, yaml_path), Loader=yaml.FullLoader)
builder = pjs.ObjectBuilder(data)
return builder.build_classes(standardize_names=False)
# hide json schema version warnings
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=UserWarning)
message_classes = _load_classes("yaml/airbyte_message.yaml")
AirbyteMessage = message_classes.AirbyteMessage
AirbyteLogMessage = message_classes.AirbyteLogMessage
AirbyteRecordMessage = message_classes.AirbyteRecordMessage
AirbyteStateMessage = message_classes.AirbyteStateMessage
catalog_classes = _load_classes("yaml/airbyte_catalog.yaml")
AirbyteCatalog = catalog_classes.AirbyteCatalog
AirbyteStream = catalog_classes.AirbyteStream

12
airbyte-integrations/base-python/airbyte_protocol/setup.py
View File

@@ -1,12 +0,0 @@
from setuptools import setup
setup(
name='airbyte_protocol',
description='Contains classes representing the schema of the Airbyte protocol.',
author='Airbyte',
author_email='contact@airbyte.io',
packages=['airbyte_protocol'],
install_requires=['PyYAML==5.3.1', 'python-jsonschema-objects==0.3.13'],
package_data={'': ['types/*.yaml']},
include_package_data=True
)

7
airbyte-integrations/base-python/build.gradle
View File

@@ -1,15 +1,16 @@
apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
def typesDir = 'airbyte_protocol/models/yaml'
task deleteProtocolDefinitions(type: Delete) {
delete 'airbyte_protocol/airbyte_protocol/types'
delete typesDir
}
task copyProtocolDefinitions(type: Copy) {
from file("$rootDir/airbyte-protocol/models/src/main/resources/airbyte_protocol").absolutePath
into "airbyte_protocol/airbyte_protocol/types"
into typesDir
dependsOn deleteProtocolDefinitions
}
assemble.dependsOn copyProtocolDefinitions
clean.dependsOn deleteProtocolDefinitions
buildImage.dependsOn copyProtocolDefinitions
buildImage.dependsOn ':airbyte-integrations:base:buildImage'

4
airbyte-integrations/base-python/main_dev.py Normal file
View File

@@ -0,0 +1,4 @@
from airbyte_protocol.entrypoint import main
if __name__ == "__main__":
main()

1
airbyte-integrations/base-python/requirements.txt Normal file
View File

@@ -0,0 +1 @@
-e .

24
airbyte-integrations/base-python/setup.py Normal file
View File

@@ -0,0 +1,24 @@
import setuptools
setuptools.setup(
name='airbyte-protocol',
description='Contains classes representing the schema of the Airbyte protocol.',
author='Airbyte',
author_email='contact@airbyte.io',
url='https://github.com/airbytehq/airbyte',
packages=setuptools.find_packages(),
package_data={
'': ['models/yaml/*.yaml']
},
install_requires=[
'PyYAML==5.3.1',
'python-jsonschema-objects==0.3.13'
],
entry_points={
'console_scripts': [
'base-python=airbyte_protocol.entrypoint:main'
],
}
)

6
airbyte-integrations/singer/base-singer/.dockerignore
View File

@@ -1,5 +1 @@
*
!Dockerfile
!base_singer/__init__.py
!base_singer/singer_helpers.py
!setup.py
build

7
airbyte-integrations/singer/base-singer/Dockerfile
View File

@@ -1,9 +1,8 @@
FROM airbyte/integration-base-python:dev
WORKDIR /airbyte/base_singer
COPY base_singer/__init__.py base_singer/__init__.py
COPY base_singer/singer_helpers.py base_singer/singer_helpers.py
COPY setup.py .
WORKDIR /airbyte/base_singer_code
COPY base_singer ./base_singer
COPY setup.py ./
RUN pip install .
LABEL io.airbyte.version=0.1.0

18
airbyte-integrations/singer/base-singer/base_singer/singer_helpers.py
View File

@@ -3,16 +3,16 @@ import os
import selectors
import subprocess
import tempfile
from airbyte_protocol import AirbyteSpec
from dataclasses import dataclass
from datetime import datetime
from typing import Generator
from airbyte_protocol import AirbyteCatalog
from airbyte_protocol import AirbyteMessage
from airbyte_protocol import AirbyteLogMessage
from airbyte_protocol import AirbyteRecordMessage
from airbyte_protocol import AirbyteSpec
from airbyte_protocol import AirbyteStateMessage
from airbyte_protocol import AirbyteStream
from typing import Generator
from datetime import datetime
from dataclasses import dataclass
def to_json(string):
@@ -36,14 +36,6 @@ class Catalogs:
class SingerHelper:
@staticmethod
def spec_from_file(spec_path) -> AirbyteSpec:
with open(spec_path) as file:
spec_text = file.read()
# we need to output a spec on a single line
flattened_json = json.dumps(json.loads(spec_text))
return AirbyteSpec(flattened_json)
@staticmethod
def get_catalogs(logger, shell_command, singer_transform=(lambda catalog: catalog), airbyte_transform=(lambda catalog: catalog)) -> Catalogs:
completed_process = subprocess.run(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE,

1
airbyte-integrations/singer/base-singer/build.gradle
View File

@@ -1,3 +1,4 @@
apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
buildImage.dependsOn ':airbyte-integrations:base-python:buildImage'

2
airbyte-integrations/singer/base-singer/requirements.txt Normal file
View File

@@ -0,0 +1,2 @@
-e ../../base-python
-e .

10
airbyte-integrations/singer/base-singer/setup.py
View File

@@ -1,10 +1,12 @@
from setuptools import setup
from setuptools import setup, find_packages
setup(
name='base_singer',
name='base-singer',
description='Contains helpers for handling Singer sources and destinations.',
author='Airbyte',
author_email='contact@airbyte.io',
packages=['base_singer'],
install_requires=['airbyte_protocol']
packages=find_packages(),
install_requires=['airbyte-protocol']
)

8
airbyte-integrations/singer/exchangeratesapi/source/.dockerignore
View File

@@ -1,7 +1 @@
*
!Dockerfile
!entrypoint.sh
!source_exchangeratesapi_singer/source_exchangeratesapi_singer.py
!source_exchangeratesapi_singer/__init__.py
!spec.json
!setup.py
build

24
airbyte-integrations/singer/exchangeratesapi/source/Dockerfile
View File

@@ -1,21 +1,19 @@
FROM airbyte/integration-base-singer:dev
RUN apt-get update && apt-get install -y jq
COPY spec.json /airbyte/exchangeratesapi-files/spec.json
WORKDIR /airbyte/source_exchangeratesapi_singer
COPY source_exchangeratesapi_singer/__init__.py ./source_exchangeratesapi_singer/__init__.py
COPY source_exchangeratesapi_singer/source_exchangeratesapi_singer.py ./source_exchangeratesapi_singer/source_exchangeratesapi_singer.py
COPY setup.py .
RUN pip install .
WORKDIR /airbyte
RUN apt-get update && apt-get install -y \
jq \
&& rm -rf /var/lib/apt/lists/*
ENV CODE_PATH="source_exchangeratesapi_singer"
ENV AIRBYTE_IMPL_MODULE="source_exchangeratesapi_singer"
ENV AIRBYTE_IMPL_PATH="SourceExchangeRatesApiSinger"
LABEL io.airbyte.version=0.1.2
LABEL io.airbyte.name=airbyte/source-exchangeratesapi-singer
WORKDIR /airbyte/integration_code
COPY $CODE_PATH ./$CODE_PATH
COPY setup.py ./
RUN pip install .
WORKDIR /airbyte

2
airbyte-integrations/singer/exchangeratesapi/source/build.gradle
View File

@@ -3,7 +3,6 @@ plugins {
}
apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
buildImage.dependsOn ":airbyte-integrations:singer:base-singer:buildImage"
apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle')
dependencies {
@@ -13,3 +12,4 @@ dependencies {
}
integrationTest.dependsOn(buildImage)
buildImage.dependsOn ":airbyte-integrations:singer:base-singer:buildImage"

8
airbyte-integrations/singer/exchangeratesapi/source/main_dev.py Normal file
View File

@@ -0,0 +1,8 @@
import sys
from airbyte_protocol.entrypoint import launch
from source_exchangeratesapi_singer import SourceExchangeRatesApiSinger
if __name__ == "__main__":
source = SourceExchangeRatesApiSinger()
launch(source, sys.argv[1:])

3
airbyte-integrations/singer/exchangeratesapi/source/requirements.txt Normal file
View File

@@ -0,0 +1,3 @@
-e ../../../base-python
-e ../../base-singer
-e .

17
airbyte-integrations/singer/exchangeratesapi/source/setup.py
View File

@@ -1,10 +1,19 @@
from setuptools import setup
from setuptools import setup, find_packages
setup(
name='source_exchangeratesapi_singer',
name='source-exchangeratesapi-singer',
description='Source implementation for the exchange rates API.',
author='Airbyte',
author_email='contact@airbyte.io',
packages=['source_exchangeratesapi_singer'],
install_requires=['tap-exchangeratesapi==0.1.1', 'base_singer', 'airbyte_protocol']
packages=find_packages(),
package_data={
'': ['*.json']
},
install_requires=[
'tap-exchangeratesapi==0.1.1',
'base_singer',
'airbyte_protocol'
]
)

2
airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/__init__.py
View File

@@ -1 +1 @@
from .source_exchangeratesapi_singer import *
from .source import *

18
airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source_exchangeratesapi_singer.py → airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/source.py
View File

@@ -1,27 +1,25 @@
from airbyte_protocol import Source
from airbyte_protocol import AirbyteSpec
from airbyte_protocol import AirbyteCheckResponse
from airbyte_protocol import AirbyteCatalog
from airbyte_protocol import AirbyteMessage
import urllib.request
from typing import Generator
from airbyte_protocol import AirbyteCatalog
from airbyte_protocol import AirbyteCheckResponse
from airbyte_protocol import AirbyteMessage
from airbyte_protocol import Source
from base_singer import SingerHelper
from base_singer import Catalogs
class SourceExchangeRatesApiSinger(Source):
def __init__(self):
pass
def spec(self) -> AirbyteSpec:
return SingerHelper.spec_from_file("/airbyte/exchangeratesapi-files/spec.json")
def check(self, logger, config_container) -> AirbyteCheckResponse:
code = urllib.request.urlopen("https://api.exchangeratesapi.io/").getcode()
logger.info(f"Ping response code: {code}")
return AirbyteCheckResponse(code == 200, {})
def discover(self, logger, config_container) -> AirbyteCatalog:
catalogs = SingerHelper.get_catalogs(logger, "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'")
cmd = "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'"
catalogs = SingerHelper.get_catalogs(logger, cmd)
return catalogs.airbyte_catalog
def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]:

0
airbyte-integrations/singer/exchangeratesapi/source/spec.json → airbyte-integrations/singer/exchangeratesapi/source/source_exchangeratesapi_singer/spec.json
View File

2
airbyte-integrations/singer/exchangeratesapi/source/src/test-integration/java/io/airbyte/integration_tests/sources/SingerExchangeRatesApiSourceDataModelTest.java
View File

@@ -70,7 +70,7 @@ public class SingerExchangeRatesApiSourceDataModelTest {
}
@Test
void stripeSchemaMessageIsValid() throws IOException {
void schemaMessageIsValid() throws IOException {
final String input = MoreResources.readResource("schema_message.json");
assertTrue(new SingerProtocolPredicate().test(Jsons.deserialize(input)));
}

8
airbyte-integrations/singer/stripe_abprotocol/source/.dockerignore
View File

@@ -1,7 +1 @@
*
!Dockerfile
!entrypoint.sh
!source_stripe_singer/*.py
!spec.json
!setup.py
build

26
airbyte-integrations/singer/stripe_abprotocol/source/Dockerfile
View File

@@ -1,20 +1,22 @@
FROM airbyte/integration-base-singer:dev
RUN apt-get update && apt-get install -y jq curl bash
COPY spec.json /airbyte/stripe-files/spec.json
WORKDIR /airbyte/source_stripe_singer
COPY source_stripe_singer/*.py ./source_stripe_singer/
COPY setup.py .
RUN pip install .
WORKDIR /airbyte
RUN apt-get update && apt-get install -y \
jq \
curl \
bash \
&& rm -rf /var/lib/apt/lists/*
ENV CODE_PATH="source_stripe_singer"
ENV AIRBYTE_IMPL_MODULE="source_stripe_singer"
ENV AIRBYTE_IMPL_PATH="SourceStripeSinger"
LABEL io.airbyte.version=0.1.3
LABEL io.airbyte.name=airbyte/source-stripe-abprotocol-singer
WORKDIR /airbyte/integration_code
COPY $CODE_PATH ./$CODE_PATH
COPY setup.py ./
RUN pip install .
WORKDIR /airbyte

8
airbyte-integrations/singer/stripe_abprotocol/source/main_dev.py Normal file
View File

@@ -0,0 +1,8 @@
import sys
from airbyte_protocol.entrypoint import launch
from source_stripe_singer import SourceStripeSinger
if __name__ == "__main__":
source = SourceStripeSinger()
launch(source, sys.argv[1:])

3
airbyte-integrations/singer/stripe_abprotocol/source/requirements.txt Normal file
View File

@@ -0,0 +1,3 @@
-e ../../../base-python
-e ../../base-singer
-e .

13
airbyte-integrations/singer/stripe_abprotocol/source/setup.py
View File

@@ -1,15 +1,20 @@
from setuptools import setup
from setuptools import setup, find_packages
setup(
name='source_stripe_singer',
description='Source implementation for Stripe.',
author='Airbyte',
author_email='contact@airbyte.io',
packages=['source_stripe_singer'],
packages=find_packages(),
package_data={
'': ['*.json']
},
install_requires=[
'tap-stripe==1.4.4',
'requests',
'base_singer',
'airbyte_protocol']
'airbyte_protocol'
]
)

2
airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/__init__.py
View File

@@ -1,2 +1,2 @@
from .source_stripe_singer import *
from .source import *

18
airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source_stripe_singer.py → airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/source.py
View File

@@ -1,20 +1,16 @@
from airbyte_protocol import Source
from airbyte_protocol import AirbyteSpec
from airbyte_protocol import AirbyteCheckResponse
from airbyte_protocol import AirbyteCatalog
from airbyte_protocol import AirbyteMessage
import requests
from typing import Generator
from airbyte_protocol import AirbyteCatalog
from airbyte_protocol import AirbyteCheckResponse
from airbyte_protocol import AirbyteMessage
from airbyte_protocol import Source
from base_singer import SingerHelper
from typing import Generator
class SourceStripeSinger(Source):
def __init__(self):
pass
def spec(self) -> AirbyteSpec:
return SingerHelper.spec_from_file('/airbyte/stripe-files/spec.json')
def check(self, logger, config_container) -> AirbyteCheckResponse:
json_config = config_container.rendered_config
r = requests.get('https://api.stripe.com/v1/customers', auth=(json_config['client_secret'], ''))
@@ -26,8 +22,10 @@ class SourceStripeSinger(Source):
return catalogs.airbyte_catalog
def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]:
discover_cmd = f"tap-stripe --config {config_container.rendered_config_path} --discover"
discovered_singer_catalog = SingerHelper.get_catalogs(logger, discover_cmd).singer_catalog
masked_airbyte_catalog = self.read_config(catalog_path)
discovered_singer_catalog = SingerHelper.get_catalogs(logger, f"tap-stripe --config {config_container.rendered_config_path} --discover").singer_catalog
selected_singer_catalog = SingerHelper.create_singer_catalog_with_selection(masked_airbyte_catalog, discovered_singer_catalog)
config_option = f"--config {config_container.rendered_config_path}"

0
airbyte-integrations/singer/stripe_abprotocol/source/spec.json → airbyte-integrations/singer/stripe_abprotocol/source/source_stripe_singer/spec.json
View File

1
airbyte-integrations/template/python-source/.dockerignore Normal file
View File

@@ -0,0 +1 @@
build

15
airbyte-integrations/template/python-source/Dockerfile Normal file
View File

@@ -0,0 +1,15 @@
FROM airbyte/integration-base-python:dev
ENV CODE_PATH="template_python_source"
ENV AIRBYTE_IMPL_MODULE="template_python_source"
ENV AIRBYTE_IMPL_PATH="TemplatePythonSource"
LABEL io.airbyte.version=0.1.0
LABEL io.airbyte.name=airbyte/source-template-python
WORKDIR /airbyte/integration_code
COPY $CODE_PATH ./$CODE_PATH
COPY setup.py ./
RUN pip install .
WORKDIR /airbyte

31
airbyte-integrations/template/python-source/README.md Normal file
View File

@@ -0,0 +1,31 @@
# Python Airbyte Source Development
Prepare development environment:
```
cd airbyte-integrations/template/python-source
# create & activate virtualenv
virtualenv build/venv
source build/venv/bin/activate
# install necessary dependencies
pip install -r requirements.txt
```
Test locally:
```
python main_dev.py spec
python main_dev.py check --config sample_files/test_config.json
python main_dev.py discover --config sample_files/test_config.json
python main_dev.py read --config sample_files/test_config.json --catalog sample_files/test_catalog.json
```
Test image:
```
# in airbyte root directory
./gradlew :airbyte-integrations:template:python-source:buildImage
docker run --rm -v $(pwd)/airbyte-integrations/template/python-source/sample_files:/sample_files airbyte/source-template-python:dev spec
docker run --rm -v $(pwd)/airbyte-integrations/template/python-source/sample_files:/sample_files airbyte/source-template-python:dev check --config /sample_files/test_config.json
docker run --rm -v $(pwd)/airbyte-integrations/template/python-source/sample_files:/sample_files airbyte/source-template-python:dev discover --config /sample_files/test_config.json
docker run --rm -v $(pwd)/airbyte-integrations/template/python-source/sample_files:/sample_files airbyte/source-template-python:dev read --config /sample_files/test_config.json --catalog /sample_files/test_catalog.json
```

3
airbyte-integrations/template/python-source/build.gradle Normal file
View File

@@ -0,0 +1,3 @@
apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
buildImage.dependsOn ":airbyte-integrations:base-python:buildImage"

9
airbyte-integrations/template/python-source/main_dev.py Normal file
View File

@@ -0,0 +1,9 @@
import sys
from airbyte_protocol.entrypoint import launch
from template_python_source import TemplatePythonSource
if __name__ == "__main__":
source = TemplatePythonSource()
launch(source, sys.argv[1:])

2
airbyte-integrations/template/python-source/requirements.txt Normal file
View File

@@ -0,0 +1,2 @@
-e ../../base-python
-e .

16
airbyte-integrations/template/python-source/sample_files/test_catalog.json Normal file
View File

@@ -0,0 +1,16 @@
{
"streams": [
{
"name": "love_airbyte",
"schema": {
"type": "object",
"required": ["love"],
"properties": {
"love": {
"type": "boolean"
}
}
}
}
]
}

3
airbyte-integrations/template/python-source/sample_files/test_config.json Normal file
View File

@@ -0,0 +1,3 @@
{
"love_airbyte": true
}

15
airbyte-integrations/template/python-source/setup.py Normal file
View File

@@ -0,0 +1,15 @@
from setuptools import setup, find_packages
setup(
name='template-python-source',
description='Source implementation template',
author='Airbyte',
author_email='contact@airbyte.io',
packages=find_packages(),
package_data={
'': ['*.json']
},
install_requires=['airbyte-protocol']
)

1
airbyte-integrations/template/python-source/template_python_source/__init__.py Normal file
View File

@@ -0,0 +1 @@
from .source import *

16
airbyte-integrations/template/python-source/template_python_source/catalog.json Normal file
View File

@@ -0,0 +1,16 @@
{
"streams": [
{
"name": "love_airbyte",
"schema": {
"type": "object",
"required": ["love"],
"properties": {
"love": {
"type": "boolean"
}
}
}
}
]
}

36
airbyte-integrations/template/python-source/template_python_source/source.py Normal file
View File

@@ -0,0 +1,36 @@
import pkgutil
import time
from typing import Generator
from airbyte_protocol import AirbyteCatalog
from airbyte_protocol import AirbyteCheckResponse
from airbyte_protocol import AirbyteMessage
from airbyte_protocol import AirbyteRecordMessage
from airbyte_protocol import AirbyteSpec
from airbyte_protocol import AirbyteStateMessage
from airbyte_protocol import Source
class TemplatePythonSource(Source):
def __init__(self):
pass
def check(self, logger, config_container) -> AirbyteCheckResponse:
logger.info(f'Checking configuration ({config_container.rendered_config_path})...')
return AirbyteCheckResponse(True, {})
def discover(self, logger, config_container) -> AirbyteCatalog:
logger.info(f'Discovering ({config_container.rendered_config_path})...')
return AirbyteCatalog.from_json(pkgutil.get_data(__name__, 'catalog.json'))
def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]:
logger.info(f'Reading ({config_container.rendered_config_path}, {catalog_path}, {state})...')
message = AirbyteRecordMessage(
stream='love_airbyte',
data={'love': True},
emitted_at=int(time.time() * 1000))
yield AirbyteMessage(type='RECORD', record=message)
state = AirbyteStateMessage(data={'love_cursor': 'next_version'})
yield AirbyteMessage(type='STATE', state=state)

17
airbyte-integrations/template/python-source/template_python_source/spec.json Normal file
View File

@@ -0,0 +1,17 @@
{
"documentationUrl": "https://docs.airbyte.io/integrations/sources/template-python-source",
"connectionSpecification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Template python source Spec",
"type": "object",
"required": ["love_airbyte"],
"additionalProperties": false,
"properties": {
"love_airbyte": {
"type": "boolean",
"description": "Do you love Airbyte",
"examples": ["true"]
}
}
}
}

1
airbyte-integrations/template/singer-source/.dockerignore Normal file
View File

@@ -0,0 +1 @@
build

19
airbyte-integrations/template/singer-source/Dockerfile Normal file
View File

@@ -0,0 +1,19 @@
FROM airbyte/integration-base-singer:dev
RUN apt-get update && apt-get install -y \
jq \
&& rm -rf /var/lib/apt/lists/*
ENV CODE_PATH="template_singer_source"
ENV AIRBYTE_IMPL_MODULE="template_singer_source"
ENV AIRBYTE_IMPL_PATH="TemplateSingerSource"
LABEL io.airbyte.version=0.1.0
LABEL io.airbyte.name=airbyte/source-template-singer
WORKDIR /airbyte/integration_code
COPY $CODE_PATH ./$CODE_PATH
COPY setup.py ./
RUN pip install .
WORKDIR /airbyte

1
airbyte-integrations/template/singer-source/README.md Normal file
View File

@@ -0,0 +1 @@
# This is a demo source

5
airbyte-integrations/template/singer-source/build.gradle Normal file
View File

@@ -0,0 +1,5 @@
apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle')
integrationTest.dependsOn(buildImage)
buildImage.dependsOn ":airbyte-integrations:singer:base-singer:buildImage"

8
airbyte-integrations/template/singer-source/main_dev.py Normal file
View File

@@ -0,0 +1,8 @@
import sys
from airbyte_protocol.entrypoint import launch
from template_singer_source import TemplateSingerSource
if __name__ == "__main__":
source = TemplateSingerSource()
launch(source, sys.argv[1:])

3
airbyte-integrations/template/singer-source/requirements.txt Normal file
View File

@@ -0,0 +1,3 @@
-e ../../base-python
-e ../../singer/base-singer
-e .

19
airbyte-integrations/template/singer-source/setup.py Normal file
View File

@@ -0,0 +1,19 @@
from setuptools import setup, find_packages
setup(
name='template-singer-source',
description='Singer source implementation template',
author='Airbyte',
author_email='contact@airbyte.io',
packages=find_packages(),
package_data={
'': ['*.json']
},
install_requires=[
'tap-exchangeratesapi==0.1.1',
'base_singer',
'airbyte_protocol'
]
)

1
airbyte-integrations/template/singer-source/template_singer_source/__init__.py Normal file
View File

@@ -0,0 +1 @@
from .source import *

28
airbyte-integrations/template/singer-source/template_singer_source/source.py Normal file
View File

@@ -0,0 +1,28 @@
import urllib.request
from typing import Generator
from airbyte_protocol import AirbyteCatalog
from airbyte_protocol import AirbyteCheckResponse
from airbyte_protocol import AirbyteMessage
from airbyte_protocol import Source
from base_singer import SingerHelper
class TemplateSingerSource(Source):
def __init__(self):
pass
def check(self, logger, config_container) -> AirbyteCheckResponse:
code = urllib.request.urlopen("https://api.exchangeratesapi.io/").getcode()
logger.info(f"Ping response code: {code}")
return AirbyteCheckResponse(code == 200, {})
def discover(self, logger, config_container) -> AirbyteCatalog:
cmd = "tap-exchangeratesapi | grep '\"type\": \"SCHEMA\"' | head -1 | jq -c '{\"streams\":[{\"stream\": .stream, \"schema\": .schema}]}'"
catalogs = SingerHelper.get_catalogs(logger, cmd)
return catalogs.airbyte_catalog
def read(self, logger, config_container, catalog_path, state=None) -> Generator[AirbyteMessage, None, None]:
config_option = f"--config {config_container.rendered_config_path}"
state_option = f"--state {state}" if state else ""
return SingerHelper.read(logger, f"tap-exchangeratesapi {config_option} {state_option}")

21
airbyte-integrations/template/singer-source/template_singer_source/spec.json Normal file
View File

@@ -0,0 +1,21 @@
{
"documentationUrl": "https://docs.airbyte.io/integrations/sources/exchangeratesapi-io",
"connectionSpecification": {
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Template Singer Source Spec",
"type": "object",
"required": ["start_date", "base"],
"additionalProperties": false,
"properties": {
"start_date": {
"type": "string",
"description": "Start getting data from that date.",
"examples": ["YYYY-MM-DD"]
},
"base": {
"type": "string",
"description": "ISO reference currency. See <a href=\"https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html\">here</a>."
}
}
}
}

2
build.gradle
View File

@@ -2,7 +2,7 @@ plugins {
id 'base'
id 'java'
id 'pmd'
id 'com.diffplug.spotless' version '5.4.0'
id 'com.diffplug.spotless' version '5.6.1'
// id 'de.aaschmid.cpd' version '3.1'
}