134 lines
5.7 KiB
Python
134 lines
5.7 KiB
Python
#
|
|
# MIT License
|
|
#
|
|
# Copyright (c) 2020 Airbyte
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in all
|
|
# copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
#
|
|
|
|
|
|
import argparse
|
|
import importlib
|
|
import os.path
|
|
import sys
|
|
import tempfile
|
|
from typing import Iterable, List
|
|
|
|
from airbyte_cdk.logger import AirbyteLogger
|
|
from airbyte_cdk.models import AirbyteMessage, Status, Type
|
|
from airbyte_cdk.sources import Source
|
|
|
|
logger = AirbyteLogger()
|
|
|
|
|
|
class AirbyteEntrypoint(object):
|
|
def __init__(self, source: Source):
|
|
self.source = source
|
|
|
|
def parse_args(self, args: List[str]) -> argparse.Namespace:
|
|
# set up parent parsers
|
|
parent_parser = argparse.ArgumentParser(add_help=False)
|
|
main_parser = argparse.ArgumentParser()
|
|
subparsers = main_parser.add_subparsers(title="commands", dest="command")
|
|
|
|
# spec
|
|
subparsers.add_parser("spec", help="outputs the json configuration specification", parents=[parent_parser])
|
|
|
|
# check
|
|
check_parser = subparsers.add_parser("check", help="checks the config can be used to connect", parents=[parent_parser])
|
|
required_check_parser = check_parser.add_argument_group("required named arguments")
|
|
required_check_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file")
|
|
|
|
# discover
|
|
discover_parser = subparsers.add_parser(
|
|
"discover", help="outputs a catalog describing the source's schema", parents=[parent_parser]
|
|
)
|
|
required_discover_parser = discover_parser.add_argument_group("required named arguments")
|
|
required_discover_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file")
|
|
|
|
# read
|
|
read_parser = subparsers.add_parser("read", help="reads the source and outputs messages to STDOUT", parents=[parent_parser])
|
|
|
|
read_parser.add_argument("--state", type=str, required=False, help="path to the json-encoded state file")
|
|
required_read_parser = read_parser.add_argument_group("required named arguments")
|
|
required_read_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file")
|
|
required_read_parser.add_argument(
|
|
"--catalog", type=str, required=True, help="path to the catalog used to determine which data to read"
|
|
)
|
|
|
|
return main_parser.parse_args(args)
|
|
|
|
def run(self, parsed_args: argparse.Namespace) -> Iterable[str]:
|
|
cmd = parsed_args.command
|
|
if not cmd:
|
|
raise Exception("No command passed")
|
|
|
|
# todo: add try catch for exceptions with different exit codes
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
if cmd == "spec":
|
|
message = AirbyteMessage(type=Type.SPEC, spec=self.source.spec(logger))
|
|
yield message.json(exclude_unset=True)
|
|
else:
|
|
raw_config = self.source.read_config(parsed_args.config)
|
|
config = self.source.configure(raw_config, temp_dir)
|
|
|
|
if cmd == "check":
|
|
check_result = self.source.check(logger, config)
|
|
if check_result.status == Status.SUCCEEDED:
|
|
logger.info("Check succeeded")
|
|
else:
|
|
logger.error("Check failed")
|
|
|
|
output_message = AirbyteMessage(type=Type.CONNECTION_STATUS, connectionStatus=check_result).json(exclude_unset=True)
|
|
yield output_message
|
|
elif cmd == "discover":
|
|
catalog = self.source.discover(logger, config)
|
|
yield AirbyteMessage(type=Type.CATALOG, catalog=catalog).json(exclude_unset=True)
|
|
elif cmd == "read":
|
|
config_catalog = self.source.read_catalog(parsed_args.catalog)
|
|
state = self.source.read_state(parsed_args.state)
|
|
generator = self.source.read(logger, config, config_catalog, state)
|
|
for message in generator:
|
|
yield message.json(exclude_unset=True)
|
|
else:
|
|
raise Exception("Unexpected command " + cmd)
|
|
|
|
|
|
def launch(source: Source, args: List[str]):
|
|
source_entrypoint = AirbyteEntrypoint(source)
|
|
parsed_args = source_entrypoint.parse_args(args)
|
|
for message in source_entrypoint.run(parsed_args):
|
|
print(message)
|
|
|
|
|
|
def main():
|
|
impl_module = os.environ.get("AIRBYTE_IMPL_MODULE", Source.__module__)
|
|
impl_class = os.environ.get("AIRBYTE_IMPL_PATH", Source.__name__)
|
|
module = importlib.import_module(impl_module)
|
|
impl = getattr(module, impl_class)
|
|
|
|
# set up and run entrypoint
|
|
source = impl()
|
|
|
|
if not isinstance(source, Source):
|
|
raise Exception("Source implementation provided does not implement Source class!")
|
|
|
|
launch(source, sys.argv[1:])
|