Airbyte 8278 all in one static code checker to be run locally as well as during ci pipelines (#8873)

airbyte-8278 All-in-one static code checker to be run locally as well as during CI pipelines

.editorconfig

@@ -0,0 +1,140 @@
+root = true
+
+[*]
+charset = utf-8
+indent_size = 4
+indent_style = space
+insert_final_newline = true
+max_line_length = 140
+tab_width = 4
+ij_continuation_indent_size = 8
+ij_formatter_off_tag = @formatter:off
+ij_formatter_on_tag = @formatter:on
+ij_formatter_tags_enabled = false
+ij_smart_tabs = false
+ij_visual_guides = none
+ij_wrap_on_typing = false
+
+[{*.bash,*.sh,*.zsh}]
+indent_size = 2
+tab_width = 2
+ij_shell_binary_ops_start_line = false
+ij_shell_keep_column_alignment_padding = false
+ij_shell_minify_program = false
+ij_shell_redirect_followed_by_space = false
+ij_shell_switch_cases_indented = false
+ij_shell_use_unix_line_separator = true
+
+[{*.har,*.jsb2,*.jsb3,*.json,.babelrc,.eslintrc,.stylelintrc,bowerrc,jest.config}]
+indent_size = 2
+ij_json_keep_blank_lines_in_code = 0
+ij_json_keep_indents_on_empty_lines = false
+ij_json_keep_line_breaks = true
+ij_json_space_after_colon = true
+ij_json_space_after_comma = true
+ij_json_space_before_colon = true
+ij_json_space_before_comma = false
+ij_json_spaces_within_braces = false
+ij_json_spaces_within_brackets = false
+ij_json_wrap_long_lines = false
+
+[{*.markdown,*.md}]
+ij_markdown_force_one_space_after_blockquote_symbol = true
+ij_markdown_force_one_space_after_header_symbol = true
+ij_markdown_force_one_space_after_list_bullet = true
+ij_markdown_force_one_space_between_words = true
+ij_markdown_keep_indents_on_empty_lines = false
+ij_markdown_max_lines_around_block_elements = 1
+ij_markdown_max_lines_around_header = 1
+ij_markdown_max_lines_between_paragraphs = 1
+ij_markdown_min_lines_around_block_elements = 1
+ij_markdown_min_lines_around_header = 1
+ij_markdown_min_lines_between_paragraphs = 1
+
+[{*.py,*.pyw,Tiltfile}]
+ij_python_align_collections_and_comprehensions = true
+ij_python_align_multiline_imports = true
+ij_python_align_multiline_parameters = true
+ij_python_align_multiline_parameters_in_calls = true
+ij_python_blank_line_at_file_end = true
+ij_python_blank_lines_after_imports = 1
+ij_python_blank_lines_after_local_imports = 0
+ij_python_blank_lines_around_class = 1
+ij_python_blank_lines_around_method = 1
+ij_python_blank_lines_around_top_level_classes_functions = 2
+ij_python_blank_lines_before_first_method = 0
+ij_python_call_parameters_new_line_after_left_paren = false
+ij_python_call_parameters_right_paren_on_new_line = false
+ij_python_call_parameters_wrap = normal
+ij_python_dict_alignment = 0
+ij_python_dict_new_line_after_left_brace = false
+ij_python_dict_new_line_before_right_brace = false
+ij_python_dict_wrapping = 1
+ij_python_from_import_new_line_after_left_parenthesis = false
+ij_python_from_import_new_line_before_right_parenthesis = false
+ij_python_from_import_parentheses_force_if_multiline = false
+ij_python_from_import_trailing_comma_if_multiline = false
+ij_python_from_import_wrapping = 1
+ij_python_hang_closing_brackets = false
+ij_python_keep_blank_lines_in_code = 1
+ij_python_keep_blank_lines_in_declarations = 1
+ij_python_keep_indents_on_empty_lines = false
+ij_python_keep_line_breaks = true
+ij_python_method_parameters_new_line_after_left_paren = false
+ij_python_method_parameters_right_paren_on_new_line = false
+ij_python_method_parameters_wrap = normal
+ij_python_new_line_after_colon = false
+ij_python_new_line_after_colon_multi_clause = true
+ij_python_optimize_imports_always_split_from_imports = false
+ij_python_optimize_imports_case_insensitive_order = false
+ij_python_optimize_imports_join_from_imports_with_same_source = false
+ij_python_optimize_imports_sort_by_type_first = true
+ij_python_optimize_imports_sort_imports = true
+ij_python_optimize_imports_sort_names_in_from_imports = false
+ij_python_space_after_comma = true
+ij_python_space_after_number_sign = true
+ij_python_space_after_py_colon = true
+ij_python_space_before_backslash = true
+ij_python_space_before_comma = false
+ij_python_space_before_for_semicolon = false
+ij_python_space_before_lbracket = false
+ij_python_space_before_method_call_parentheses = false
+ij_python_space_before_method_parentheses = false
+ij_python_space_before_number_sign = true
+ij_python_space_before_py_colon = false
+ij_python_space_within_empty_method_call_parentheses = false
+ij_python_space_within_empty_method_parentheses = false
+ij_python_spaces_around_additive_operators = true
+ij_python_spaces_around_assignment_operators = true
+ij_python_spaces_around_bitwise_operators = true
+ij_python_spaces_around_eq_in_keyword_argument = false
+ij_python_spaces_around_eq_in_named_parameter = false
+ij_python_spaces_around_equality_operators = true
+ij_python_spaces_around_multiplicative_operators = true
+ij_python_spaces_around_power_operator = true
+ij_python_spaces_around_relational_operators = true
+ij_python_spaces_around_shift_operators = true
+ij_python_spaces_within_braces = false
+ij_python_spaces_within_brackets = false
+ij_python_spaces_within_method_call_parentheses = false
+ij_python_spaces_within_method_parentheses = false
+ij_python_use_continuation_indent_for_arguments = false
+ij_python_use_continuation_indent_for_collection_and_comprehensions = false
+ij_python_use_continuation_indent_for_parameters = true
+ij_python_wrap_long_lines = false
+
+[{*.toml,Cargo.lock,Cargo.toml.orig,Gopkg.lock,Pipfile,poetry.lock}]
+ij_toml_keep_indents_on_empty_lines = false
+
+[{*.yaml,*.yml}]
+indent_size = 2
+ij_yaml_align_values_properties = do_not_align
+ij_yaml_autoinsert_sequence_marker = true
+ij_yaml_block_mapping_on_new_line = false
+ij_yaml_indent_sequence_value = true
+ij_yaml_keep_indents_on_empty_lines = false
+ij_yaml_keep_line_breaks = true
+ij_yaml_sequence_on_new_line = false
+ij_yaml_space_before_colon = false
+ij_yaml_spaces_within_braces = true
+ij_yaml_spaces_within_brackets = true

.gitignore

@@ -10,6 +10,7 @@ data
 .classpath
 .project
 .settings
+**/gmon.out
 
 # Logs
 acceptance_tests_logs/

.pre-commit-config.yaml

@@ -1,3 +1,6 @@
+default_language_version:
+  python: python3.7
+
 repos:
   - repo: https://github.com/johann-petrak/licenseheaders.git
     rev: v0.8.8
@@ -6,19 +9,19 @@ repos:
         args: ["--tmpl=LICENSE_SHORT", "--ext=py", "-f"]
 
   - repo: https://github.com/ambv/black
-    rev: 21.10b0
+    rev: 21.11b1
     hooks:
       - id: black
-        args: ["--line-length=140"]
 
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.6.4
+    rev: 5.10.1
     hooks:
       - id: isort
-        args: ["--settings-path=tools/python/.isort.cfg"]
+        args: ["--dont-follow-links", "--jobs=-1"]
+        additional_dependencies: ["colorama"]
 
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.3.2
+    rev: v2.5.0
     hooks:
       - id: prettier
         types_or: [yaml, json]
@@ -29,17 +32,17 @@ repos:
               destination_specs.yaml
           ).?$
 
-  - repo: https://gitlab.com/pycqa/flake8
+  - repo: https://github.com/csachs/pyproject-flake8
-    rev: 3.8.4
+    rev: v0.0.1a2.post1
     hooks:
-      - id: flake8
+      - id: pyproject-flake8
-        args: ["--config=tools/python/.flake8"]
+        additional_dependencies: ["mccabe"]
+        alias: flake8
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.812
+    rev: v0.910-1
     hooks:
       - id: mypy
-        args: ["--config-file=tools/python/.mypy.ini"]
 
   - repo: local
     hooks:

airbyte-integrations/bases/base-python/setup.py

@@ -14,14 +14,14 @@ setuptools.setup(
     packages=setuptools.find_packages(),
     package_data={"": ["models/yaml/*.yaml"]},
     install_requires=[
-        "PyYAML==5.4",
-        "pydantic==1.6.*",
         "airbyte-protocol",
-        "jsonschema==3.2.0",
-        "requests",
         "backoff",
-        "pytest",
+        "jsonschema==3.2.0",
         "pendulum",
+        "pydantic==1.6.*",
+        "pytest",
+        "PyYAML==5.4",
+        "requests",
     ],
     entry_points={
         "console_scripts": ["base-python=base_python.entrypoint:main"],

airbyte-integrations/connectors/requirements.txt

@@ -0,0 +1,2 @@
+invoke~=1.6.0
+virtualenv~=20.10.0

airbyte-integrations/connectors/tasks.py

@@ -0,0 +1,245 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+import os
+import shutil
+import tempfile
+from glob import glob
+from multiprocessing import Pool
+from typing import Any, Dict, Iterable, List, Set
+
+import virtualenv
+from invoke import Context, Exit, task
+
+CONNECTORS_DIR: str = os.path.abspath(os.path.curdir)
+
+ROOT_DIR = os.path.dirname(os.path.dirname(CONNECTORS_DIR))
+CONFIG_FILE: str = os.path.join(ROOT_DIR, "pyproject.toml")
+
+# TODO: Get it from a single place with `pre-commit` (or make pre-commit to use these tasks)
+TOOLS_VERSIONS: Dict[str, str] = {
+    "black": "21.12b0",
+    "colorama": "0.4.4",
+    "coverage": "6.2",
+    "flake": "0.0.1a2",
+    "isort": "5.10.1",
+    "mccabe": "0.6.1",
+    "mypy": "0.910",
+}
+
+
+TASK_COMMANDS: Dict[str, List[str]] = {
+    "black": [
+        f"pip install black~={TOOLS_VERSIONS['black']}",
+        f"XDG_CACHE_HOME={os.devnull} black -v {{check_option}} {{source_path}}/.",
+    ],
+    "coverage": [
+        "pip install .",
+        f"pip install coverage[toml]~={TOOLS_VERSIONS['coverage']}",
+        f"coverage report --rcfile={CONFIG_FILE}",
+    ],
+    "flake": [
+        f"pip install mccabe~={TOOLS_VERSIONS['mccabe']}",
+        f"pip install pyproject-flake8~={TOOLS_VERSIONS['flake']}",
+        "pflake8 -v {source_path}",
+    ],
+    "isort": [
+        f"pip install colorama~={TOOLS_VERSIONS['colorama']}",
+        f"pip install isort~={TOOLS_VERSIONS['isort']}",
+        "isort -v {check_option} {source_path}/.",
+    ],
+    "mypy": [
+        "pip install .",
+        f"pip install mypy~={TOOLS_VERSIONS['mypy']}",
+        f"mypy {{source_path}} --config-file={CONFIG_FILE}",
+    ],
+    "test": [
+        f"cp -rf {os.path.join(CONNECTORS_DIR, os.pardir, 'bases', 'source-acceptance-test')} {{venv}}/",
+        "pip install build",
+        f"python -m build {os.path.join('{venv}', 'source-acceptance-test')}",
+        f"pip install {os.path.join('{venv}', 'source-acceptance-test', 'dist', 'source_acceptance_test-*.whl')}",
+        "pip install .",
+        "pip install .[tests]",
+        "pip install pytest-cov",
+        "pytest -v --cov={source_path} --cov-report xml unit_tests",
+    ],
+}
+
+
+###########################################################################################################################################
+# HELPER FUNCTIONS
+###########################################################################################################################################
+
+
+def get_connectors_names() -> Set[str]:
+    cur_dir = os.path.abspath(os.curdir)
+    os.chdir(CONNECTORS_DIR)
+    names = set()
+    for name in glob("source-*"):
+        if os.path.exists(os.path.join(name, "setup.py")):
+            if not name.endswith("-singer"):  # There are some problems with those. The optimal way is to wait until it's replaced by CDK.
+                names.add(name.split("source-", 1)[1].rstrip())
+
+    os.chdir(cur_dir)
+    return names
+
+
+CONNECTORS_NAMES = get_connectors_names()
+
+
+def _run_single_connector_task(args: Iterable) -> int:
+    """
+    Wrapper for unpack task arguments.
+    """
+    return _run_task(*args)
+
+
+def _run_task(ctx: Context, connector_string: str, task_name: str, multi_envs: bool = True, **kwargs: Any) -> int:
+    """
+    Run task in its own environment.
+    """
+    if multi_envs:
+        source_path = f"source_{connector_string.replace('-', '_')}"
+        os.chdir(os.path.join(CONNECTORS_DIR, f"source-{connector_string}"))
+
+    else:
+        source_path = connector_string
+
+    venv_name = tempfile.mkdtemp(dir=os.curdir)
+    virtualenv.cli_run([venv_name])
+    activator = os.path.join(os.path.abspath(venv_name), "bin", "activate")
+
+    commands = []
+
+    commands.extend([cmd.format(source_path=source_path, venv=venv_name, **kwargs) for cmd in TASK_COMMANDS[task_name]])
+
+    exit_code: int = 0
+
+    try:
+        with ctx.prefix(f"source {activator}"):
+            for command in commands:
+                result = ctx.run(command, warn=True)
+                if result.return_code:
+                    exit_code = 1
+                    break
+    finally:
+        shutil.rmtree(venv_name, ignore_errors=True)
+
+    return exit_code
+
+
+def apply_task_for_connectors(ctx: Context, connectors_names: str, task_name: str, multi_envs: bool = False, **kwargs: Any) -> None:
+    """
+    Run task commands for every connector or for once for a set of connectors, depending on task needs (`multi_envs` param).
+    If `multi_envs == True` task for every connector runs in its own subprocess.
+    """
+    # TODO: Separate outputs to avoid a mess.
+
+    connectors = connectors_names.split(",") if connectors_names else CONNECTORS_NAMES
+    connectors = set(connectors) & CONNECTORS_NAMES
+
+    exit_code: int = 0
+
+    if multi_envs:
+        print(f"Running {task_name} for the following connectors: {connectors}")
+
+        task_args = [(ctx, connector, task_name) for connector in connectors]
+        with Pool() as pool:
+            for result in pool.imap_unordered(_run_single_connector_task, task_args):
+                if result:
+                    exit_code = 1
+
+    else:
+        source_path = " ".join([f"{os.path.join(CONNECTORS_DIR, f'source-{connector}')}" for connector in connectors])
+        exit_code = _run_task(ctx, source_path, task_name, multi_envs=False, **kwargs)
+
+    raise Exit(code=exit_code)
+
+
+###########################################################################################################################################
+# TASKS
+###########################################################################################################################################
+
+_arg_help_connectors = (
+    "Comma-separated connectors' names without 'source-' prefix (ex.: -c github,google-ads,s3). "
+    "The default is a list of all found connectors excluding the ones with `-singer` suffix."
+)
+
+
+@task(help={"connectors": _arg_help_connectors})
+def all_checks(ctx, connectors=None):  # type: ignore[no-untyped-def]
+    """
+    Run following checks one by one with default parameters: black, flake, isort, mypy, test, coverage.
+    Zero exit code indicates about successful passing of all checks.
+    Terminate on the first non-zero exit code.
+    """
+    black(ctx, connectors=connectors)
+    flake(ctx, connectors=connectors)
+    isort(ctx, connectors=connectors)
+    mypy(ctx, connectors=connectors)
+    coverage(ctx, connectors=connectors)
+
+
+@task(help={"connectors": _arg_help_connectors, "write": "Write changes into the files (runs 'black' without '--check' option)"})
+def black(ctx, connectors=None, write=False):  # type: ignore[no-untyped-def]
+    """
+    Run 'black' checks for one or more given connector(s) code.
+    Zero exit code indicates about successful passing of all checks.
+    """
+    check_option: str = "" if write else " --check"
+    apply_task_for_connectors(ctx, connectors, "black", check_option=check_option)
+
+
+@task(help={"connectors": _arg_help_connectors})
+def flake(ctx, connectors=None):  # type: ignore[no-untyped-def]
+    """
+    Run 'flake8' checks for one or more given connector(s) code.
+    Zero exit code indicates about successful passing of all checks.
+    """
+    apply_task_for_connectors(ctx, connectors, "flake")
+
+
+@task(help={"connectors": _arg_help_connectors, "write": "Write changes into the files (runs 'isort' without '--check' option)"})
+def isort(ctx, connectors=None, write=False):  # type: ignore[no-untyped-def]
+    """
+    Run 'isort' checks for one or more given connector(s) code.
+    Zero exit code indicates about successful passing of all checks.
+    """
+    check_option: str = "" if write else " --check"
+    apply_task_for_connectors(ctx, connectors, "isort", check_option=check_option)
+
+
+@task(help={"connectors": _arg_help_connectors})
+def mypy(ctx, connectors=None):  # type: ignore[no-untyped-def]
+    """
+    Run MyPy checks for one or more given connector(s) code.
+    A virtual environment is being created for every one.
+    Zero exit code indicates about successful passing of all checks.
+    """
+    apply_task_for_connectors(ctx, connectors, "mypy", multi_envs=True)
+
+
+@task(help={"connectors": _arg_help_connectors})
+def test(ctx, connectors=None):  # type: ignore[no-untyped-def]
+    """
+    Run unittests for one or more given connector(s).
+    A virtual environment is being created for every one.
+    Zero exit code indicates about successful passing of all tests.
+    """
+    apply_task_for_connectors(ctx, connectors, "test", multi_envs=True)
+
+
+@task(help={"connectors": _arg_help_connectors})
+def coverage(ctx, connectors=None):  # type: ignore[no-untyped-def]
+    """
+    Check test coverage of code for one or more given connector(s).
+    A virtual environment is being created for every one.
+    "test" command is being run before this one.
+    Zero exit code indicates about enough coverage level.
+    """
+    try:
+        test(ctx, connectors=connectors)
+    except Exit as e:
+        if e.code:
+            raise
+        apply_task_for_connectors(ctx, connectors, "coverage", multi_envs=True)

pyproject.toml

@@ -0,0 +1,48 @@
+[tool.black]
+line-length = 140
+target-version = ["py37"]
+
+[tool.coverage.report]
+fail_under = 100
+skip_empty = true
+sort = "-cover"
+
+[tool.flake8]
+extend-exclude = ".venv"
+max-complexity = 10
+max-line-length = 140
+
+[tool.isort]
+profile = "black"
+color_output = true
+skip_gitignore = true
+
+[tool.mypy]
+platform = "linux"
+
+# Strictness
+allow_redefinition = true
+disallow_incomplete_defs = true
+disallow_untyped_defs = true
+no_implicit_reexport = true
+no_strict_optional = true
+strict_equality = true
+
+# Output
+pretty = true
+show_column_numbers = true
+show_error_codes = true
+show_error_context = true
+
+# Warnings
+warn_redundant_casts = true
+warn_return_any = true
+warn_unreachable = true
+warn_unused_ignores = true
+
+[[tool.mypy.overrides]]
+module = [
+    "integration_tests.*",
+    "unit_tests.*",
+]
+allow_incomplete_defs = true