jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-19 18:14:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

CI: apply pre-commit format fix from #49806 (#49852)

Browse Source
This commit is contained in:
Aaron ("AJ") Steers
2024-12-18 14:05:43 -08:00
committed by GitHub
parent c7aeec0120
commit 83ecbe0fc3
1555 changed files with 7448 additions and 5802 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
airbyte-ci/connectors/base_images/base_images/commands.py
View File

@@ -10,9 +10,10 @@ import anyio
import dagger
import inquirer # type: ignore
import semver
from base_images import bases, console, consts, errors, hacks, publish, utils, version_registry
from jinja2 import Environment, FileSystemLoader
from base_images import bases, console, consts, errors, hacks, publish, utils, version_registry
async def _generate_docs(dagger_client: dagger.Client):
"""This function will generate the README.md file from the templates/README.md.j2 template.

4
airbyte-ci/connectors/base_images/base_images/consts.py
View File

@@ -2,10 +2,10 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#
"""This module declares constants used by the base_images module.
"""
"""This module declares constants used by the base_images module."""
import dagger
REMOTE_REGISTRY = "docker.io"
PLATFORMS_WE_PUBLISH_FOR = (dagger.Platform("linux/amd64"), dagger.Platform("linux/arm64"))

3
airbyte-ci/connectors/base_images/base_images/errors.py
View File

@@ -2,8 +2,7 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
#
"""This module contains the exceptions used by the base_images module.
"""
"""This module contains the exceptions used by the base_images module."""
from typing import Union

1
airbyte-ci/connectors/base_images/base_images/hacks.py
View File

@@ -4,6 +4,7 @@
import dagger
# If we perform addition dagger operations on the container, we need to make sure that a mapping exists for the new field name.
DAGGER_FIELD_NAME_TO_DOCKERFILE_INSTRUCTION = {
"from": lambda field: f'FROM {field.args.get("address")}',

7
airbyte-ci/connectors/base_images/base_images/java/bases.py
View File

@@ -6,6 +6,7 @@ from __future__ import annotations
from typing import Callable, Final
import dagger
from base_images import bases, published_image
from base_images import sanity_checks as base_sanity_checks
from base_images.python import sanity_checks as python_sanity_checks
@@ -22,9 +23,9 @@ class AirbyteJavaConnectorBaseImage(bases.AirbyteConnectorBaseImage):
DD_AGENT_JAR_URL: Final[str] = "https://dtdg.co/latest-java-tracer"
BASE_SCRIPT_URL = "https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base/base.sh"
JAVA_BASE_SCRIPT_URL: Final[
str
] = "https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base-java/javabase.sh"
JAVA_BASE_SCRIPT_URL: Final[str] = (
"https://raw.githubusercontent.com/airbytehq/airbyte/6d8a3a2bc4f4ca79f10164447a90fdce5c9ad6f9/airbyte-integrations/bases/base-java/javabase.sh"
)
def get_container(self, platform: dagger.Platform) -> dagger.Container:
"""Returns the container used to build the base image for java connectors

1
airbyte-ci/connectors/base_images/base_images/publish.py
View File

@@ -4,6 +4,7 @@
import dagger
from base_images import bases, consts, published_image

1
airbyte-ci/connectors/base_images/base_images/python/bases.py
View File

@@ -6,6 +6,7 @@ from __future__ import annotations
from typing import Callable, Final
import dagger
from base_images import bases, published_image
from base_images import sanity_checks as base_sanity_checks
from base_images.python import sanity_checks as python_sanity_checks

1
airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py
View File

@@ -3,6 +3,7 @@
#
import dagger
from base_images import errors
from base_images import sanity_checks as base_sanity_checks

1
airbyte-ci/connectors/base_images/base_images/root_images.py
View File

@@ -4,6 +4,7 @@
from .published_image import PublishedImage
PYTHON_3_9_18 = PublishedImage(
registry="docker.io",
repository="python",

1
airbyte-ci/connectors/base_images/base_images/sanity_checks.py
View File

@@ -6,6 +6,7 @@ import re
from typing import Optional
import dagger
from base_images import errors

2
airbyte-ci/connectors/base_images/base_images/utils/docker.py
View File

@@ -9,6 +9,7 @@ import uuid
from typing import List, Tuple
import dagger
from base_images import console, published_image
@@ -31,7 +32,6 @@ def get_credentials() -> Tuple[str, str]:
class CraneClient:
CRANE_IMAGE_ADDRESS = "gcr.io/go-containerregistry/crane/debug:c195f151efe3369874c72662cd69ad43ee485128@sha256:94f61956845714bea3b788445454ae4827f49a90dcd9dac28255c4cccb6220ad"
def __init__(self, dagger_client: dagger.Client, docker_credentials: Tuple[str, str], cache_ttl_seconds: int = 0):

2
airbyte-ci/connectors/base_images/base_images/version_registry.py
View File

@@ -11,6 +11,7 @@ from typing import Dict, List, Mapping, Optional, Tuple, Type
import dagger
import semver
from base_images import consts, published_image
from base_images.bases import AirbyteConnectorBaseImage
from base_images.java.bases import AirbyteJavaConnectorBaseImage
@@ -18,6 +19,7 @@ from base_images.python.bases import AirbyteManifestOnlyConnectorBaseImage, Airb
from base_images.utils import docker
from connector_ops.utils import ConnectorLanguage # type: ignore
MANAGED_BASE_IMAGES = [AirbytePythonConnectorBaseImage, AirbyteJavaConnectorBaseImage]

1
airbyte-ci/connectors/ci_credentials/ci_credentials/main.py
View File

@@ -11,6 +11,7 @@ from common_utils import Logger
from . import SecretsManager
logger = Logger()
ENV_GCP_GSM_CREDENTIALS = "GCP_GSM_CREDENTIALS"

1
airbyte-ci/connectors/ci_credentials/ci_credentials/models.py
View File

@@ -8,6 +8,7 @@ from __future__ import ( # Used to evaluate type hints at runtime, a NameError:
from dataclasses import dataclass
DEFAULT_SECRET_FILE = "config"

1
airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py
View File

@@ -17,6 +17,7 @@ from common_utils import GoogleApi, Logger
from .models import DEFAULT_SECRET_FILE, RemoteSecret, Secret
DEFAULT_SECRET_FILE_WITH_EXT = DEFAULT_SECRET_FILE + ".json"
GSM_SCOPES = ("https://www.googleapis.com/auth/cloud-platform",)

1
airbyte-ci/connectors/common_utils/common_utils/google_api.py
View File

@@ -11,6 +11,7 @@ import requests
from .logger import Logger
TOKEN_TTL = 3600

2
airbyte-ci/connectors/connector_ops/connector_ops/required_reviewer_checks.py
View File

@@ -5,8 +5,10 @@
from typing import Dict, List, Optional, Set, Tuple, Union
import yaml
from connector_ops import utils
# The breaking change reviewers is still in active use.
BREAKING_CHANGE_REVIEWERS = {"breaking-change-reviewers"}
CERTIFIED_MANIFEST_ONLY_CONNECTOR_REVIEWERS = {"dev-python"}

1
airbyte-ci/connectors/connector_ops/connector_ops/utils.py
View File

@@ -22,6 +22,7 @@ from pydash.objects import get
from rich.console import Console
from simpleeval import simple_eval
console = Console()
DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master")

1
airbyte-ci/connectors/connectors_insights/src/connectors_insights/cli.py
View File

@@ -12,6 +12,7 @@ import asyncer
import dagger
from anyio import Semaphore
from connector_ops.utils import Connector # type: ignore
from connectors_insights.insights import generate_insights_for_connector
from connectors_insights.result_backends import GCSBucket, LocalDir
from connectors_insights.utils import gcs_uri_to_bucket_key, get_all_connectors_in_directory, remove_strict_encrypt_suffix

1
airbyte-ci/connectors/connectors_insights/src/connectors_insights/insights.py
View File

@@ -9,6 +9,7 @@ import re
from typing import TYPE_CHECKING
import requests
from connectors_insights.hacks import get_ci_on_master_report
from connectors_insights.models import ConnectorInsights
from connectors_insights.pylint import get_pylint_output

1
airbyte-ci/connectors/connectors_insights/src/connectors_insights/pylint.py
View File

@@ -7,6 +7,7 @@ from pathlib import Path
from typing import TYPE_CHECKING
from connector_ops.utils import ConnectorLanguage # type: ignore
from connectors_insights.utils import never_fail_exec
if TYPE_CHECKING:

2
airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/assets.py
View File

@@ -6,6 +6,7 @@ import xml.etree.ElementTree as ET
from typing import TYPE_CHECKING
from connector_ops.utils import Connector # type: ignore
from connectors_qa.models import Check, CheckCategory, CheckResult
if TYPE_CHECKING:
@@ -27,7 +28,6 @@ class CheckConnectorIconIsAvailable(AssetsCheck):
requires_metadata = False
def _check_is_valid_svg(self, icon_path: Path) -> Tuple[bool, str | None]:
try:
# Ensure the file has an .svg extension
if not icon_path.suffix.lower() == ".svg":

3
airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/documentation/documentation.py
View File

@@ -7,9 +7,10 @@ from typing import List
import requests # type: ignore
from connector_ops.utils import Connector, ConnectorLanguage # type: ignore
from connectors_qa.models import Check, CheckCategory, CheckResult
from pydash.objects import get # type: ignore
from connectors_qa.models import Check, CheckCategory, CheckResult
from .helpers import (
generate_description,
prepare_changelog_to_compare,

3
airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py
View File

@@ -5,9 +5,10 @@ from datetime import datetime, timedelta
import toml
from connector_ops.utils import Connector, ConnectorLanguage # type: ignore
from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load # type: ignore
from connectors_qa import consts
from connectors_qa.models import Check, CheckCategory, CheckResult
from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load # type: ignore
class MetadataCheck(Check):

3
airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/packaging.py
View File

@@ -3,9 +3,10 @@
import semver
import toml
from connector_ops.utils import Connector, ConnectorLanguage # type: ignore
from pydash.objects import get # type: ignore
from connectors_qa import consts
from connectors_qa.models import Check, CheckCategory, CheckResult
from pydash.objects import get # type: ignore
class PackagingCheck(Check):

3
airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/security.py
View File

@@ -4,9 +4,10 @@ from pathlib import Path
from typing import Iterable, Optional, Set, Tuple
from connector_ops.utils import Connector, ConnectorLanguage # type: ignore
from pydash.objects import get # type: ignore
from connectors_qa import consts
from connectors_qa.models import Check, CheckCategory, CheckResult
from pydash.objects import get # type: ignore
class SecurityCheck(Check):

3
airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/testing.py
View File

@@ -2,9 +2,10 @@
from connector_ops.utils import Connector # type: ignore
from connectors_qa.models import Check, CheckCategory, CheckResult
from pydash.collections import find # type: ignore
from connectors_qa.models import Check, CheckCategory, CheckResult
class TestingCheck(Check):
category = CheckCategory.TESTING

3
airbyte-ci/connectors/connectors_qa/src/connectors_qa/cli.py
View File

@@ -6,11 +6,12 @@ from typing import Dict, Iterable, List
import asyncclick as click
import asyncer
from connector_ops.utils import Connector # type: ignore
from jinja2 import Environment, PackageLoader, select_autoescape
from connectors_qa.checks import ENABLED_CHECKS
from connectors_qa.consts import CONNECTORS_QA_DOC_TEMPLATE_NAME
from connectors_qa.models import Check, CheckCategory, CheckResult, CheckStatus, Report
from connectors_qa.utils import get_all_connectors_in_directory, remove_strict_encrypt_suffix
from jinja2 import Environment, PackageLoader, select_autoescape
# HELPERS

7
airbyte-ci/connectors/connectors_qa/src/connectors_qa/models.py
View File

@@ -11,6 +11,7 @@ from pathlib import Path
from typing import Dict, List, Optional
from connector_ops.utils import Connector, ConnectorLanguage # type: ignore
from connectors_qa import consts
ALL_LANGUAGES = [
@@ -289,9 +290,9 @@ class Report:
" ", "_"
)
connectors_report[connector_technical_name]["badge_text"] = badge_text
connectors_report[connector_technical_name][
"badge_url"
] = f"{self.image_shield_root_url}/{badge_name}-{badge_text}-{connectors_report[connector_technical_name]['badge_color']}"
connectors_report[connector_technical_name]["badge_url"] = (
f"{self.image_shield_root_url}/{badge_name}-{badge_text}-{connectors_report[connector_technical_name]['badge_color']}"
)
return json.dumps(
{
"generation_timestamp": datetime.utcnow().isoformat(),

1
airbyte-ci/connectors/connectors_qa/src/connectors_qa/utils.py
View File

@@ -4,6 +4,7 @@ from pathlib import Path
from typing import Set
from connector_ops.utils import Connector # type: ignore
from connectors_qa import consts

9
airbyte-ci/connectors/connectors_qa/tests/integration_tests/test_documentation.py
View File

@@ -6,23 +6,28 @@ from pathlib import Path
import git
import pytest
from asyncclick.testing import CliRunner
from connectors_qa.cli import generate_documentation
DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO = Path("docs/contributing-to-airbyte/resources/qa-checks.md")
@pytest.fixture
def airbyte_repo():
return git.Repo(search_parent_directories=True)
@pytest.mark.asyncio
async def test_generated_qa_checks_documentation_is_up_to_date(airbyte_repo, tmp_path):
# Arrange
current_doc = (airbyte_repo.working_dir / DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO).read_text()
newly_generated_doc_path = tmp_path / "qa-checks.md"
# Act
await CliRunner().invoke(generate_documentation, [str(tmp_path / "qa-checks.md")], catch_exceptions=False)
# Assert
suggested_command = f"connectors-qa generate-documentation {DOCUMENTATION_FILE_PATH_IN_AIRBYTE_REPO}"
assert newly_generated_doc_path.read_text() == current_doc, f"The generated documentation is not up to date. Please run `{suggested_command}` and commit the changes."
assert (
newly_generated_doc_path.read_text() == current_doc
), f"The generated documentation is not up to date. Please run `{suggested_command}` and commit the changes."

1
airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_assets.py
View File

@@ -42,7 +42,6 @@ class TestCheckConnectorIconIsAvailable:
assert result.status == CheckStatus.FAILED
assert result.message == "Icon file is not a SVG file"
def test_fail_when_icon_file_is_not_valid_svg(self, tmp_path, mocker):
# Arrange
connector = mocker.MagicMock()

97
airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_documentation.py
View File

@@ -196,7 +196,6 @@ class TestCheckDocumentationExists:
class TestCheckDocumentationContent:
def test_fail_when_documentation_file_path_does_not_exists(self, mocker, tmp_path):
# Arrange
connector = mocker.Mock(
@@ -204,7 +203,7 @@ class TestCheckDocumentationContent:
ab_internal_sl=300,
language="python",
connector_type="source",
documentation_file_path=tmp_path / "not_existing_documentation.md"
documentation_file_path=tmp_path / "not_existing_documentation.md",
)
# Act
@@ -217,11 +216,7 @@ class TestCheckDocumentationContent:
def test_fail_when_documentation_file_path_is_none(self, mocker):
# Arrange
connector = mocker.Mock(
technical_name="test-connector",
ab_internal_sl=300,
language="python",
connector_type="source",
documentation_file_path=None
technical_name="test-connector", ab_internal_sl=300, language="python", connector_type="source", documentation_file_path=None
)
# Act
@@ -263,34 +258,28 @@ class TestCheckDocumentationContent:
assert "Actual Heading: 'For Airbyte Cloud:'. Expected Heading: 'Setup guide'" in result.message
def test_fail_when_documentation_file_not_have_all_required_fields_in_prerequisites_section_content(
self,
connector_with_invalid_documentation
self, connector_with_invalid_documentation
):
# Act
result = documentation.CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec()._run(
connector_with_invalid_documentation
)
result = documentation.CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec()._run(connector_with_invalid_documentation)
# Assert
assert result.status == CheckStatus.FAILED
assert "Missing descriptions for required spec fields: github repositories" in result.message
def test_fail_when_documentation_file_has_invalid_source_section_content(
self,
connector_with_invalid_documentation
):
def test_fail_when_documentation_file_has_invalid_source_section_content(self, connector_with_invalid_documentation):
# Act
result = documentation.CheckSourceSectionContent()._run(connector_with_invalid_documentation)
# Assert
assert result.status == CheckStatus.FAILED
assert "Connector GitHub section content does not follow standard template:" in result.message
assert "+ This page contains the setup guide and reference information for the [GitHub]({docs_link}) source connector." in result.message
assert (
"+ This page contains the setup guide and reference information for the [GitHub]({docs_link}) source connector."
in result.message
)
def test_fail_when_documentation_file_has_invalid_for_airbyte_cloud_section_content(
self,
connector_with_invalid_documentation
):
def test_fail_when_documentation_file_has_invalid_for_airbyte_cloud_section_content(self, connector_with_invalid_documentation):
# Act
result = documentation.CheckForAirbyteCloudSectionContent()._run(connector_with_invalid_documentation)
@@ -299,10 +288,7 @@ class TestCheckDocumentationContent:
assert "Connector For Airbyte Cloud: section content does not follow standard template:" in result.message
assert "+ 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account." in result.message
def test_fail_when_documentation_file_has_invalid_for_airbyte_open_section_content(
self,
connector_with_invalid_documentation
):
def test_fail_when_documentation_file_has_invalid_for_airbyte_open_section_content(self, connector_with_invalid_documentation):
# Act
result = documentation.CheckForAirbyteOpenSectionContent()._run(connector_with_invalid_documentation)
@@ -311,23 +297,19 @@ class TestCheckDocumentationContent:
assert "Connector For Airbyte Open Source: section content does not follow standard template" in result.message
assert "+ 1. Navigate to the Airbyte Open Source dashboard." in result.message
def test_fail_when_documentation_file_has_invalid_supported_sync_modes_section_content(
self,
connector_with_invalid_documentation
):
def test_fail_when_documentation_file_has_invalid_supported_sync_modes_section_content(self, connector_with_invalid_documentation):
# Act
result = documentation.CheckSupportedSyncModesSectionContent()._run(connector_with_invalid_documentation)
# Assert
assert result.status == CheckStatus.FAILED
assert "Connector Supported sync modes section content does not follow standard template:" in result.message
assert ("+ The GitHub source connector supports the following"
" [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes):") in result.message
assert (
"+ The GitHub source connector supports the following"
" [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes):"
) in result.message
def test_fail_when_documentation_file_has_invalid_tutorials_section_content(
self,
connector_with_invalid_documentation
):
def test_fail_when_documentation_file_has_invalid_tutorials_section_content(self, connector_with_invalid_documentation):
# Act
result = documentation.CheckTutorialsSectionContent()._run(connector_with_invalid_documentation)
@@ -336,10 +318,7 @@ class TestCheckDocumentationContent:
assert "Connector Tutorials section content does not follow standard template:" in result.message
assert "+ Now that you have set up the GitHub source connector, check out the following GitHub tutorials:" in result.message
def test_fail_when_documentation_file_has_invalid_changelog_section_content(
self,
connector_with_invalid_documentation
):
def test_fail_when_documentation_file_has_invalid_changelog_section_content(self, connector_with_invalid_documentation):
# Act
result = documentation.CheckChangelogSectionContent()._run(connector_with_invalid_documentation)
@@ -356,10 +335,7 @@ class TestCheckDocumentationContent:
assert result.status == CheckStatus.PASSED
assert result.message == "Documentation guidelines are followed"
def test_pass_when_documentation_file_has_correct_prerequisites_section_content(
self,
connector_with_correct_documentation
):
def test_pass_when_documentation_file_has_correct_prerequisites_section_content(self, connector_with_correct_documentation):
# Act
result = documentation.CheckPrerequisitesSectionDescribesRequiredFieldsFromSpec()._run(connector_with_correct_documentation)
@@ -367,10 +343,7 @@ class TestCheckDocumentationContent:
assert result.status == CheckStatus.PASSED
assert "All required fields from spec are present in the connector documentation" in result.message
def test_pass_when_documentation_file_has_correct_source_section_content(
self,
connector_with_correct_documentation
):
def test_pass_when_documentation_file_has_correct_source_section_content(self, connector_with_correct_documentation):
# Act
result = documentation.CheckSourceSectionContent()._run(connector_with_correct_documentation)
@@ -378,10 +351,7 @@ class TestCheckDocumentationContent:
assert result.status == CheckStatus.PASSED
assert "Documentation guidelines are followed" in result.message
def test_pass_when_documentation_file_has_correct_for_airbyte_cloud_section_content(
self,
connector_with_correct_documentation
):
def test_pass_when_documentation_file_has_correct_for_airbyte_cloud_section_content(self, connector_with_correct_documentation):
# Act
result = documentation.CheckForAirbyteCloudSectionContent()._run(connector_with_correct_documentation)
@@ -389,10 +359,7 @@ class TestCheckDocumentationContent:
assert result.status == CheckStatus.PASSED
assert "Documentation guidelines are followed" in result.message
def test_pass_when_documentation_file_has_correct_for_airbyte_open_section_content(
self,
connector_with_correct_documentation
):
def test_pass_when_documentation_file_has_correct_for_airbyte_open_section_content(self, connector_with_correct_documentation):
# Act
result = documentation.CheckForAirbyteOpenSectionContent()._run(connector_with_correct_documentation)
@@ -400,10 +367,7 @@ class TestCheckDocumentationContent:
assert result.status == CheckStatus.PASSED
assert "Documentation guidelines are followed" in result.message
def test_pass_when_documentation_file_has_correct_supported_sync_modes_section_content(
self,
connector_with_correct_documentation
):
def test_pass_when_documentation_file_has_correct_supported_sync_modes_section_content(self, connector_with_correct_documentation):
# Act
result = documentation.CheckSupportedSyncModesSectionContent()._run(connector_with_correct_documentation)
@@ -411,10 +375,7 @@ class TestCheckDocumentationContent:
assert result.status == CheckStatus.PASSED
assert "Documentation guidelines are followed" in result.message
def test_pass_when_documentation_file_has_correct_tutorials_section_content(
self,
connector_with_correct_documentation
):
def test_pass_when_documentation_file_has_correct_tutorials_section_content(self, connector_with_correct_documentation):
# Act
result = documentation.CheckTutorialsSectionContent()._run(connector_with_correct_documentation)
@@ -422,10 +383,7 @@ class TestCheckDocumentationContent:
assert result.status == CheckStatus.PASSED
assert "Documentation guidelines are followed" in result.message
def test_pass_when_documentation_file_has_correct_headers_order(
self,
connector_with_correct_documentation
):
def test_pass_when_documentation_file_has_correct_headers_order(self, connector_with_correct_documentation):
# Act
result = documentation.CheckDocumentationHeadersOrder()._run(connector_with_correct_documentation)
@@ -433,10 +391,7 @@ class TestCheckDocumentationContent:
assert result.status == CheckStatus.PASSED
assert "Documentation guidelines are followed" in result.message
def test_pass_when_documentation_file_has_correct_changelog_section_content(
self,
connector_with_correct_documentation
):
def test_pass_when_documentation_file_has_correct_changelog_section_content(self, connector_with_correct_documentation):
# Act
result = documentation.CheckChangelogSectionContent()._run(connector_with_correct_documentation)

1
airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py
View File

@@ -4,6 +4,7 @@ from __future__ import annotations
import os
import pytest
from connectors_qa import consts
from connectors_qa.checks import metadata
from connectors_qa.models import CheckStatus

1
airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_packaging.py
View File

@@ -133,6 +133,7 @@ class TestCheckPublishToPyPiIsDeclared:
assert result.status == CheckStatus.PASSED
assert "PyPi publishing is declared" in result.message
class TestCheckConnectorLicense:
def test_fail_when_license_is_missing(self, mocker):
# Arrange

37
airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_testing.py
View File

@@ -2,6 +2,7 @@
import pytest
from connector_ops.utils import ConnectorLanguage
from connectors_qa.checks import testing
from connectors_qa.models import CheckStatus
@@ -56,9 +57,7 @@ METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS = {
"connectorTestSuitesOptions": [
{
"suite": testing.AcceptanceTestsEnabledCheck.test_suite_name,
"testSecrets": {
"testSecret": "test"
},
"testSecrets": {"testSecret": "test"},
},
{
"suite": "unit",
@@ -71,10 +70,10 @@ THRESHOLD_USAGE_VALUES = ["high", "medium"]
OTHER_USAGE_VALUES = ["low", "none", "unknown", None, ""]
DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES = [
METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS,
METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NONE_SECRETS,
METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_EMPTY_SECRETS,
METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NO_SECRETS,
METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS,
METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NONE_SECRETS,
METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_EMPTY_SECRETS,
METADATA_CASE_WITH_ACCEPTANCE_TEST_SUITE_OPTIONS_NO_SECRETS,
]
DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES = [
@@ -89,21 +88,9 @@ class TestAcceptanceTestsEnabledCheck:
@pytest.mark.parametrize(
"cases_to_test, usage_values_to_test, expected_result",
[
(
DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES + DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES,
OTHER_USAGE_VALUES,
CheckStatus.SKIPPED
),
(
DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES,
THRESHOLD_USAGE_VALUES,
CheckStatus.PASSED
),
(
DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES,
THRESHOLD_USAGE_VALUES,
CheckStatus.FAILED
)
(DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES + DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, OTHER_USAGE_VALUES, CheckStatus.SKIPPED),
(DYNAMIC_ACCEPTANCE_TESTS_ENABLED_CASES, THRESHOLD_USAGE_VALUES, CheckStatus.PASSED),
(DYNAMIC_ACCEPTANCE_TESTS_DISABLED_CASES, THRESHOLD_USAGE_VALUES, CheckStatus.FAILED),
],
)
def test_check_always_passes_when_usage_threshold_is_not_met(self, mocker, cases_to_test, usage_values_to_test, expected_result):
@@ -115,11 +102,13 @@ class TestAcceptanceTestsEnabledCheck:
metadata=metadata_case,
language=ConnectorLanguage.PYTHON,
connector_type="source",
ab_internal_sl=100
ab_internal_sl=100,
)
# Act
result = testing.AcceptanceTestsEnabledCheck().run(connector)
# Assert
assert result.status == expected_result, f"Usage value: {usage_value}, metadata case: {metadata_case}, expected result: {expected_result}"
assert (
result.status == expected_result
), f"Usage value: {usage_value}, metadata case: {metadata_case}, expected result: {expected_result}"

1
airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_models.py
View File

@@ -1,6 +1,7 @@
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
from connector_ops.utils import ConnectorLanguage
from connectors_qa import consts
from connectors_qa.checks import ENABLED_CHECKS
from connectors_qa.models import CheckStatus

139
airbyte-ci/connectors/erd/src/erd/dbml_assembler.py
View File

@@ -4,11 +4,22 @@ from pathlib import Path
from typing import List, Set, Union
import yaml
from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver
from airbyte_protocol.models import AirbyteCatalog, AirbyteStream # type: ignore # missing library stubs or py.typed marker
from erd.relationships import Relationships
from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import (
ManifestReferenceResolver,
)
from airbyte_protocol.models import ( # type: ignore # missing library stubs or py.typed marker
AirbyteCatalog,
AirbyteStream,
)
from pydbml import Database # type: ignore # missing library stubs or py.typed marker
from pydbml.classes import Column, Index, Reference, Table # type: ignore # missing library stubs or py.typed marker
from pydbml.classes import ( # type: ignore # missing library stubs or py.typed marker
Column,
Index,
Reference,
Table,
)
from erd.relationships import Relationships
class Source:
@@ -25,33 +36,67 @@ class Source:
manifest_static_streams = set()
if self._has_manifest():
with open(self._get_manifest_path()) as manifest_file:
resolved_manifest = ManifestReferenceResolver().preprocess_manifest(yaml.safe_load(manifest_file))
resolved_manifest = ManifestReferenceResolver().preprocess_manifest(
yaml.safe_load(manifest_file)
)
for stream in resolved_manifest["streams"]:
if "schema_loader" not in stream:
# stream is assumed to have `DefaultSchemaLoader` which will show in the schemas folder so we can skip
continue
if stream["schema_loader"]["type"] == "InlineSchemaLoader":
name = stream["name"] if "name" in stream else stream.get("$parameters").get("name", None)
name = (
stream["name"]
if "name" in stream
else stream.get("$parameters").get("name", None)
)
if not name:
print(f"Could not retrieve name for this stream: {stream}")
continue
manifest_static_streams.add(stream["name"] if "name" in stream else stream.get("$parameters").get("name", None))
manifest_static_streams.add(
stream["name"]
if "name" in stream
else stream.get("$parameters").get("name", None)
)
return stream_name not in manifest_static_streams | self._get_streams_from_schemas_folder()
return (
stream_name
not in manifest_static_streams | self._get_streams_from_schemas_folder()
)
def _get_streams_from_schemas_folder(self) -> Set[str]:
schemas_folder = self._source_folder / self._source_technical_name.replace("-", "_") / "schemas"
return {p.name.replace(".json", "") for p in schemas_folder.iterdir() if p.is_file()} if schemas_folder.exists() else set()
schemas_folder = (
self._source_folder
/ self._source_technical_name.replace("-", "_")
/ "schemas"
)
return (
{
p.name.replace(".json", "")
for p in schemas_folder.iterdir()
if p.is_file()
}
if schemas_folder.exists()
else set()
)
def _get_manifest_path(self) -> Path:
return self._source_folder / self._source_technical_name.replace("-", "_") / "manifest.yaml"
return (
self._source_folder
/ self._source_technical_name.replace("-", "_")
/ "manifest.yaml"
)
def _has_manifest(self) -> bool:
return self._get_manifest_path().exists()
class DbmlAssembler:
def assemble(self, source: Source, discovered_catalog: AirbyteCatalog, relationships: Relationships) -> Database:
def assemble(
self,
source: Source,
discovered_catalog: AirbyteCatalog,
relationships: Relationships,
) -> Database:
database = Database()
for stream in discovered_catalog.streams:
if source.is_dynamic(stream.name):
@@ -66,7 +111,9 @@ class DbmlAssembler:
def _create_table(self, stream: AirbyteStream) -> Table:
dbml_table = Table(stream.name)
for property_name, property_information in stream.json_schema.get("properties").items():
for property_name, property_information in stream.json_schema.get(
"properties"
).items():
try:
dbml_table.add_column(
Column(
@@ -79,12 +126,20 @@ class DbmlAssembler:
print(f"Ignoring field {property_name}: {exception}")
continue
if stream.source_defined_primary_key and len(stream.source_defined_primary_key) > 1:
if (
stream.source_defined_primary_key
and len(stream.source_defined_primary_key) > 1
):
if any(map(lambda key: len(key) != 1, stream.source_defined_primary_key)):
raise ValueError(f"Does not support nested key as part of primary key `{stream.source_defined_primary_key}`")
raise ValueError(
f"Does not support nested key as part of primary key `{stream.source_defined_primary_key}`"
)
composite_key_columns = [
column for key in stream.source_defined_primary_key for column in dbml_table.columns if column.name in key
column
for key in stream.source_defined_primary_key
for column in dbml_table.columns
if column.name in key
]
if len(composite_key_columns) < len(stream.source_defined_primary_key):
raise ValueError("Unexpected error: missing PK column from dbml table")
@@ -97,11 +152,15 @@ class DbmlAssembler:
)
return dbml_table
def _add_references(self, source: Source, database: Database, relationships: Relationships) -> None:
def _add_references(
self, source: Source, database: Database, relationships: Relationships
) -> None:
for stream in relationships["streams"]:
for column_name, relationship in stream["relations"].items():
if source.is_dynamic(stream["name"]):
print(f"Skipping relationship as stream {stream['name']} from relationship is dynamic")
print(
f"Skipping relationship as stream {stream['name']} from relationship is dynamic"
)
continue
try:
@@ -109,18 +168,26 @@ class DbmlAssembler:
".", 1
) # we support the field names having dots but not stream name hence we split on the first dot only
except ValueError as exception:
raise ValueError(f"Could not handle relationship {relationship}") from exception
raise ValueError(
f"Could not handle relationship {relationship}"
) from exception
if source.is_dynamic(target_table_name):
print(f"Skipping relationship as target stream {target_table_name} is dynamic")
print(
f"Skipping relationship as target stream {target_table_name} is dynamic"
)
continue
try:
database.add_reference(
Reference(
type="<>", # we don't have the information of which relationship type it is so we assume many-to-many for now
col1=self._get_column(database, stream["name"], column_name),
col2=self._get_column(database, target_table_name, target_column_name),
col1=self._get_column(
database, stream["name"], column_name
),
col2=self._get_column(
database, target_table_name, target_column_name
),
)
)
except ValueError as exception:
@@ -136,24 +203,38 @@ class DbmlAssembler:
# show this in DBML
types.remove("null")
if len(types) != 1:
raise ValueError(f"Expected only one type apart from `null` but got {len(types)}: {property_type}")
raise ValueError(
f"Expected only one type apart from `null` but got {len(types)}: {property_type}"
)
return types[0]
def _is_pk(self, stream: AirbyteStream, property_name: str) -> bool:
return stream.source_defined_primary_key == [[property_name]]
def _get_column(self, database: Database, table_name: str, column_name: str) -> Column:
matching_tables = list(filter(lambda dbml_table: dbml_table.name == table_name, database.tables))
def _get_column(
self, database: Database, table_name: str, column_name: str
) -> Column:
matching_tables = list(
filter(lambda dbml_table: dbml_table.name == table_name, database.tables)
)
if len(matching_tables) == 0:
raise ValueError(f"Could not find table {table_name}")
elif len(matching_tables) > 1:
raise ValueError(f"Unexpected error: many tables found with name {table_name}")
raise ValueError(
f"Unexpected error: many tables found with name {table_name}"
)
table: Table = matching_tables[0]
matching_columns = list(filter(lambda column: column.name == column_name, table.columns))
matching_columns = list(
filter(lambda column: column.name == column_name, table.columns)
)
if len(matching_columns) == 0:
raise ValueError(f"Could not find column {column_name} in table {table_name}. Columns are: {table.columns}")
raise ValueError(
f"Could not find column {column_name} in table {table_name}. Columns are: {table.columns}"
)
elif len(matching_columns) > 1:
raise ValueError(f"Unexpected error: many columns found with name {column_name} for table {table_name}")
raise ValueError(
f"Unexpected error: many columns found with name {column_name} for table {table_name}"
)
return matching_columns[0]

40
airbyte-ci/connectors/erd/src/erd/erd_service.py
View File

@@ -7,11 +7,16 @@ from typing import Any
import dpath
import google.generativeai as genai # type: ignore # missing library stubs or py.typed marker
from airbyte_protocol.models import AirbyteCatalog # type: ignore # missing library stubs or py.typed marker
from airbyte_protocol.models import (
AirbyteCatalog, # type: ignore # missing library stubs or py.typed marker
)
from markdown_it import MarkdownIt
from pydbml.renderer.dbml.default import (
DefaultDBMLRenderer, # type: ignore # missing library stubs or py.typed marker
)
from erd.dbml_assembler import DbmlAssembler, Source
from erd.relationships import Relationships, RelationshipsMerger
from markdown_it import MarkdownIt
from pydbml.renderer.dbml.default import DefaultDBMLRenderer # type: ignore # missing library stubs or py.typed marker
class ErdService:
@@ -21,12 +26,18 @@ class ErdService:
self._model = genai.GenerativeModel("gemini-1.5-flash")
if not self._discovered_catalog_path.exists():
raise ValueError(f"Could not find discovered catalog at path {self._discovered_catalog_path}")
raise ValueError(
f"Could not find discovered catalog at path {self._discovered_catalog_path}"
)
def generate_estimated_relationships(self) -> None:
normalized_catalog = self._normalize_schema_catalog(self._get_catalog())
estimated_relationships = self._get_relations_from_gemini(source_name=self._source_path.name, catalog=normalized_catalog)
with open(self._estimated_relationships_file, "w") as estimated_relationship_file:
estimated_relationships = self._get_relations_from_gemini(
source_name=self._source_path.name, catalog=normalized_catalog
)
with open(
self._estimated_relationships_file, "w"
) as estimated_relationship_file:
json.dump(estimated_relationships, estimated_relationship_file, indent=4)
def write_dbml_file(self) -> None:
@@ -34,7 +45,8 @@ class ErdService:
Source(self._source_path, self._source_technical_name),
self._get_catalog(),
RelationshipsMerger().merge(
self._get_relationships(self._estimated_relationships_file), self._get_relationships(self._confirmed_relationships_file)
self._get_relationships(self._estimated_relationships_file),
self._get_relationships(self._confirmed_relationships_file),
),
)
@@ -53,13 +65,19 @@ class ErdService:
to_rem = dpath.search(
stream["json_schema"]["properties"],
["**"],
afilter=lambda x: isinstance(x, dict) and ("array" in str(x.get("type", "")) or "object" in str(x.get("type", ""))),
afilter=lambda x: isinstance(x, dict)
and (
"array" in str(x.get("type", ""))
or "object" in str(x.get("type", ""))
),
)
for key in to_rem:
stream["json_schema"]["properties"].pop(key)
return streams # type: ignore # as this comes from an AirbyteCatalog dump, the format should be fine
def _get_relations_from_gemini(self, source_name: str, catalog: dict[str, Any]) -> Relationships:
def _get_relations_from_gemini(
self, source_name: str, catalog: dict[str, Any]
) -> Relationships:
"""
:param source_name:
@@ -74,9 +92,7 @@ You are working on the {source_name} API service.
The current JSON Schema format is as follows:
{current_schema}, where "streams" has a list of streams, which represents database tables, and list of properties in each, which in turn, represent DB columns. Streams presented in list are the only available ones.
Generate and add a `foreign_key` with reference for each field in top level of properties that is helpful in understanding what the data represents and how are streams related to each other. Pay attention to fields ends with '_id'.
""".format(
source_name=source_name, current_schema=catalog
)
""".format(source_name=source_name, current_schema=catalog)
task = """
Please provide answer in the following format:
{streams: [{"name": "<stream_name>", "relations": {"<foreign_key>": "<ref_table.column_name>"} }]}

37
airbyte-ci/connectors/erd/src/erd/relationships.py
View File

@@ -16,16 +16,28 @@ Relationships = TypedDict("Relationships", {"streams": List[Relationship]})
class RelationshipsMerger:
def merge(self, estimated_relationships: Relationships, confirmed_relationships: Relationships) -> Relationships:
def merge(
self,
estimated_relationships: Relationships,
confirmed_relationships: Relationships,
) -> Relationships:
streams = []
for estimated_stream in estimated_relationships["streams"]:
confirmed_relationships_for_stream = self._get_stream(confirmed_relationships, estimated_stream["name"])
confirmed_relationships_for_stream = self._get_stream(
confirmed_relationships, estimated_stream["name"]
)
if confirmed_relationships_for_stream:
streams.append(self._merge_for_stream(estimated_stream, confirmed_relationships_for_stream)) # type: ignore # at this point, we know confirmed_relationships_for_stream is not None
streams.append(
self._merge_for_stream(
estimated_stream, confirmed_relationships_for_stream
)
) # type: ignore # at this point, we know confirmed_relationships_for_stream is not None
else:
streams.append(estimated_stream)
already_processed_streams = set(map(lambda relationship: relationship["name"], streams))
already_processed_streams = set(
map(lambda relationship: relationship["name"], streams)
)
for confirmed_stream in confirmed_relationships["streams"]:
if confirmed_stream["name"] not in already_processed_streams:
streams.append(
@@ -36,13 +48,20 @@ class RelationshipsMerger:
)
return {"streams": streams}
def _merge_for_stream(self, estimated: Relationship, confirmed: Relationship) -> Relationship:
def _merge_for_stream(
self, estimated: Relationship, confirmed: Relationship
) -> Relationship:
relations = copy.deepcopy(confirmed.get("relations", {}))
# get estimated but filter out false positives
for field, target in estimated.get("relations", {}).items():
false_positives = confirmed["false_positives"] if "false_positives" in confirmed else {}
if field not in relations and (field not in false_positives or false_positives.get(field, None) != target): # type: ignore # at this point, false_positives should not be None
false_positives = (
confirmed["false_positives"] if "false_positives" in confirmed else {}
)
if field not in relations and (
field not in false_positives
or false_positives.get(field, None) != target
): # type: ignore # at this point, false_positives should not be None
relations[field] = target
return {
@@ -50,7 +69,9 @@ class RelationshipsMerger:
"relations": relations,
}
def _get_stream(self, relationships: Relationships, stream_name: str) -> Optional[Relationship]:
def _get_stream(
self, relationships: Relationships, stream_name: str
) -> Optional[Relationship]:
for stream in relationships["streams"]:
if stream.get("name", None) == stream_name:
return stream

5
airbyte-ci/connectors/erd/tests/test_dbml_assembler.py
View File

@@ -4,6 +4,7 @@ from unittest import TestCase
from unittest.mock import Mock
from airbyte_protocol.models import AirbyteCatalog, AirbyteStream, SyncMode
from erd.dbml_assembler import DbmlAssembler, Source
from tests.builder import RelationshipBuilder
@@ -18,7 +19,9 @@ class RelationshipsMergerTest(TestCase):
def test_given_no_streams_then_database_is_empty(self) -> None:
dbml = self._assembler.assemble(
self._source, AirbyteCatalog(streams=[]), {"streams": [RelationshipBuilder(_A_STREAM_NAME).build()]}
self._source,
AirbyteCatalog(streams=[]),
{"streams": [RelationshipBuilder(_A_STREAM_NAME).build()]},
)
assert not dbml.tables

76
airbyte-ci/connectors/erd/tests/test_relationships.py
View File

@@ -17,32 +17,72 @@ class RelationshipsMergerTest(TestCase):
self._merger = RelationshipsMerger()
def test_given_no_confirmed_then_return_estimation(self) -> None:
estimated: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]}
estimated: Relationships = {
"streams": [
RelationshipBuilder(_A_STREAM_NAME)
.with_relationship(_A_COLUMN, _A_TARGET)
.build()
]
}
confirmed: Relationships = {"streams": []}
merged = self._merger.merge(estimated, confirmed)
assert merged == estimated
def test_given_confirmed_as_false_positive_then_remove_from_estimation(self) -> None:
estimated: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]}
confirmed: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_false_positive(_A_COLUMN, _A_TARGET).build()]}
def test_given_confirmed_as_false_positive_then_remove_from_estimation(
self,
) -> None:
estimated: Relationships = {
"streams": [
RelationshipBuilder(_A_STREAM_NAME)
.with_relationship(_A_COLUMN, _A_TARGET)
.build()
]
}
confirmed: Relationships = {
"streams": [
RelationshipBuilder(_A_STREAM_NAME)
.with_false_positive(_A_COLUMN, _A_TARGET)
.build()
]
}
merged = self._merger.merge(estimated, confirmed)
assert merged == {"streams": [{"name": "a_stream_name", "relations": {}}]}
def test_given_no_estimated_but_confirmed_then_return_confirmed_without_false_positives(self) -> None:
def test_given_no_estimated_but_confirmed_then_return_confirmed_without_false_positives(
self,
) -> None:
estimated: Relationships = {"streams": []}
confirmed: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]}
confirmed: Relationships = {
"streams": [
RelationshipBuilder(_A_STREAM_NAME)
.with_relationship(_A_COLUMN, _A_TARGET)
.build()
]
}
merged = self._merger.merge(estimated, confirmed)
assert merged == confirmed
def test_given_different_columns_then_return_both(self) -> None:
estimated: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]}
confirmed: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_ANOTHER_COLUMN, _A_TARGET).build()]}
estimated: Relationships = {
"streams": [
RelationshipBuilder(_A_STREAM_NAME)
.with_relationship(_A_COLUMN, _A_TARGET)
.build()
]
}
confirmed: Relationships = {
"streams": [
RelationshipBuilder(_A_STREAM_NAME)
.with_relationship(_ANOTHER_COLUMN, _A_TARGET)
.build()
]
}
merged = self._merger.merge(estimated, confirmed)
@@ -58,9 +98,23 @@ class RelationshipsMergerTest(TestCase):
]
}
def test_given_same_column_but_different_value_then_prioritize_confirmed(self) -> None:
estimated: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _A_TARGET).build()]}
confirmed: Relationships = {"streams": [RelationshipBuilder(_A_STREAM_NAME).with_relationship(_A_COLUMN, _ANOTHER_TARGET).build()]}
def test_given_same_column_but_different_value_then_prioritize_confirmed(
self,
) -> None:
estimated: Relationships = {
"streams": [
RelationshipBuilder(_A_STREAM_NAME)
.with_relationship(_A_COLUMN, _A_TARGET)
.build()
]
}
confirmed: Relationships = {
"streams": [
RelationshipBuilder(_A_STREAM_NAME)
.with_relationship(_A_COLUMN, _ANOTHER_TARGET)
.build()
]
}
merged = self._merger.merge(estimated, confirmed)

3
airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/base_backend.py
View File

@@ -13,5 +13,4 @@ class BaseBackend(ABC):
"""
@abstractmethod
def write(self, airbyte_messages: Iterable[AirbyteMessage]) -> None:
...
def write(self, airbyte_messages: Iterable[AirbyteMessage]) -> None: ...

1
airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/duckdb_backend.py
View File

@@ -9,6 +9,7 @@ from typing import Optional
import duckdb
from airbyte_protocol.models import AirbyteMessage # type: ignore
from live_tests.commons.backends.file_backend import FileBackend

7
airbyte-ci/connectors/live-tests/src/live_tests/commons/backends/file_backend.py
View File

@@ -10,6 +10,7 @@ from typing import Any, TextIO
from airbyte_protocol.models import AirbyteMessage # type: ignore
from airbyte_protocol.models import Type as AirbyteMessageType
from cachetools import LRUCache, cached
from live_tests.commons.backends.base_backend import BaseBackend
from live_tests.commons.utils import sanitize_stream_name
@@ -123,7 +124,11 @@ class FileBackend(BaseBackend):
stream_file_path_data_only = self.record_per_stream_directory / f"{sanitize_stream_name(stream_name)}_data_only.jsonl"
self.record_per_stream_paths[stream_name] = stream_file_path
self.record_per_stream_paths_data_only[stream_name] = stream_file_path_data_only
return (self.RELATIVE_RECORDS_PATH, str(stream_file_path), str(stream_file_path_data_only),), (
return (
self.RELATIVE_RECORDS_PATH,
str(stream_file_path),
str(stream_file_path_data_only),
), (
message.json(sort_keys=True),
message.json(sort_keys=True),
json.dumps(message.record.data, sort_keys=True),

1
airbyte-ci/connectors/live-tests/src/live_tests/commons/connection_objects_retrieval.py
View File

@@ -10,6 +10,7 @@ from typing import Dict, Optional, Set
import rich
from connection_retriever import ConnectionObject, retrieve_objects # type: ignore
from connection_retriever.errors import NotPermittedError # type: ignore
from live_tests.commons.models import ConnectionSubset
from live_tests.commons.utils import build_connection_url

1
airbyte-ci/connectors/live-tests/src/live_tests/commons/connector_runner.py
View File

@@ -15,6 +15,7 @@ from pathlib import Path
import anyio
import asyncer
import dagger
from live_tests.commons import errors
from live_tests.commons.models import Command, ExecutionInputs, ExecutionResult
from live_tests.commons.proxy import Proxy

21
airbyte-ci/connectors/live-tests/src/live_tests/commons/models.py
View File

@@ -2,6 +2,7 @@
from __future__ import annotations
import _collections_abc
import json
import logging
import tempfile
@@ -13,17 +14,21 @@ from functools import cache
from pathlib import Path
from typing import Any, Dict, List, Optional
import _collections_abc
import dagger
import requests
from airbyte_protocol.models import AirbyteCatalog # type: ignore
from airbyte_protocol.models import AirbyteMessage # type: ignore
from airbyte_protocol.models import AirbyteStateMessage # type: ignore
from airbyte_protocol.models import AirbyteStreamStatusTraceMessage # type: ignore
from airbyte_protocol.models import ConfiguredAirbyteCatalog # type: ignore
from airbyte_protocol.models import TraceType # type: ignore
from airbyte_protocol.models import (
AirbyteCatalog, # type: ignore
AirbyteMessage, # type: ignore
AirbyteStateMessage, # type: ignore
AirbyteStreamStatusTraceMessage, # type: ignore
ConfiguredAirbyteCatalog, # type: ignore
TraceType, # type: ignore
)
from airbyte_protocol.models import Type as AirbyteMessageType
from genson import SchemaBuilder # type: ignore
from mitmproxy import http
from pydantic import ValidationError
from live_tests.commons.backends import DuckDbBackend, FileBackend
from live_tests.commons.secret_access import get_airbyte_api_key
from live_tests.commons.utils import (
@@ -33,8 +38,6 @@ from live_tests.commons.utils import (
sanitize_stream_name,
sort_dict_keys,
)
from mitmproxy import http
from pydantic import ValidationError
class UserDict(_collections_abc.MutableMapping): # type: ignore

3
airbyte-ci/connectors/live-tests/src/live_tests/conftest.py
View File

@@ -15,6 +15,8 @@ import pytest
from airbyte_protocol.models import AirbyteCatalog, AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification # type: ignore
from connection_retriever.audit_logging import get_user_email # type: ignore
from connection_retriever.retrieval import ConnectionNotFoundError, NotPermittedError, get_current_docker_image_tag # type: ignore
from rich.prompt import Confirm, Prompt
from live_tests import stash_keys
from live_tests.commons.connection_objects_retrieval import ConnectionObject, InvalidConnectionError, get_connection_objects
from live_tests.commons.connector_runner import ConnectorRunner, Proxy
@@ -35,7 +37,6 @@ from live_tests.commons.segment_tracking import track_usage
from live_tests.commons.utils import build_connection_url, clean_up_artifacts
from live_tests.report import Report, ReportState
from live_tests.utils import get_catalog, get_spec
from rich.prompt import Confirm, Prompt
if TYPE_CHECKING:
from _pytest.config import Config

1
airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_check.py
View File

@@ -5,6 +5,7 @@ from collections.abc import Callable
import pytest
from airbyte_protocol.models import Status, Type # type: ignore
from live_tests.commons.models import ExecutionResult
from live_tests.consts import MAX_LINES_IN_REPORT
from live_tests.utils import fail_test_on_failing_execution_results, is_successful_check, tail_file

1
airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_discover.py
View File

@@ -7,6 +7,7 @@ from collections.abc import Callable, Iterable
import pytest
from _pytest.fixtures import SubRequest
from airbyte_protocol.models import AirbyteCatalog, AirbyteStream, Type # type: ignore
from live_tests.commons.models import ExecutionResult
from live_tests.utils import fail_test_on_failing_execution_results, get_and_write_diff, get_catalog

1
airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_read.py
View File

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Optional
import pytest
from airbyte_protocol.models import AirbyteMessage # type: ignore
from deepdiff import DeepDiff # type: ignore
from live_tests.commons.models import ExecutionResult
from live_tests.utils import fail_test_on_failing_execution_results, get_and_write_diff, get_test_logger, write_string_to_test_artifact

1
airbyte-ci/connectors/live-tests/src/live_tests/regression_tests/test_spec.py
View File

@@ -5,6 +5,7 @@ from collections.abc import Callable
import pytest
from airbyte_protocol.models import Type # type: ignore
from live_tests.commons.models import ExecutionResult
from live_tests.utils import fail_test_on_failing_execution_results

2
airbyte-ci/connectors/live-tests/src/live_tests/report.py
View File

@@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Any, Optional
import requests
import yaml
from jinja2 import Environment, PackageLoader, select_autoescape
from live_tests import stash_keys
from live_tests.consts import MAX_LINES_IN_REPORT
@@ -21,6 +22,7 @@ if TYPE_CHECKING:
import pytest
from _pytest.config import Config
from airbyte_protocol.models import SyncMode, Type # type: ignore
from live_tests.commons.models import Command, ExecutionResult

1
airbyte-ci/connectors/live-tests/src/live_tests/stash_keys.py
View File

@@ -4,6 +4,7 @@ from __future__ import annotations
from pathlib import Path
import pytest
from live_tests.commons.evaluation_modes import TestEvaluationMode
from live_tests.commons.models import ConnectionObjects, ConnectionSubset
from live_tests.report import Report

5
airbyte-ci/connectors/live-tests/src/live_tests/utils.py
View File

@@ -11,11 +11,12 @@ import docker # type: ignore
import pytest
from airbyte_protocol.models import AirbyteCatalog, AirbyteMessage, ConnectorSpecification, Status, Type # type: ignore
from deepdiff import DeepDiff # type: ignore
from mitmproxy import http, io # type: ignore
from mitmproxy.addons.savehar import SaveHar # type: ignore
from live_tests import stash_keys
from live_tests.commons.models import ExecutionResult
from live_tests.consts import MAX_LINES_IN_REPORT
from mitmproxy import http, io # type: ignore
from mitmproxy.addons.savehar import SaveHar # type: ignore
if TYPE_CHECKING:
from _pytest.fixtures import SubRequest

1
airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_check.py
View File

@@ -7,6 +7,7 @@ from typing import Callable
import pytest
from airbyte_protocol.models import Type
from live_tests.commons.models import ExecutionResult
from live_tests.consts import MAX_LINES_IN_REPORT
from live_tests.utils import fail_test_on_failing_execution_results, is_successful_check, tail_file

1
airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_discover.py
View File

@@ -9,6 +9,7 @@ import dpath.util
import jsonschema
import pytest
from airbyte_protocol.models import AirbyteCatalog
from live_tests.commons.models import ExecutionResult
from live_tests.utils import fail_test_on_failing_execution_results, find_all_values_for_key_in_schema

1
airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_read.py
View File

@@ -16,6 +16,7 @@ from airbyte_protocol.models import (
AirbyteStreamStatusTraceMessage,
ConfiguredAirbyteCatalog,
)
from live_tests.commons.json_schema_helper import conforms_to_schema
from live_tests.commons.models import ExecutionResult
from live_tests.utils import fail_test_on_failing_execution_results, get_test_logger

1
airbyte-ci/connectors/live-tests/src/live_tests/validation_tests/test_spec.py
View File

@@ -9,6 +9,7 @@ import dpath.util
import jsonschema
import pytest
from airbyte_protocol.models import ConnectorSpecification
from live_tests.commons.json_schema_helper import JsonSchemaHelper, get_expected_schema_structure, get_paths_in_connector_config
from live_tests.commons.models import ExecutionResult, SecretDict
from live_tests.utils import fail_test_on_failing_execution_results, find_all_values_for_key_in_schema, get_test_logger

1
airbyte-ci/connectors/live-tests/tests/backends/test_file_backend.py
View File

@@ -13,6 +13,7 @@ from airbyte_protocol.models import (
Status,
)
from airbyte_protocol.models import Type as AirbyteMessageType
from live_tests.commons.backends import FileBackend

3
airbyte-ci/connectors/live-tests/tests/test_json_schema_helper.py
View File

@@ -16,6 +16,8 @@ from airbyte_protocol.models import (
SyncMode,
Type,
)
from pydantic import BaseModel
from live_tests.commons.json_schema_helper import (
ComparableType,
JsonSchemaHelper,
@@ -23,7 +25,6 @@ from live_tests.commons.json_schema_helper import (
get_expected_schema_structure,
get_object_structure,
)
from pydantic import BaseModel
def records_with_state(records, state, stream_mapping, state_cursor_paths) -> Iterable[Tuple[Any, Any, Any]]:

3
airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py
View File

@@ -5,6 +5,8 @@
import pathlib
import click
from pydantic import ValidationError
from metadata_service.constants import METADATA_FILE_NAME
from metadata_service.gcs_upload import (
MetadataDeleteInfo,
@@ -14,7 +16,6 @@ from metadata_service.gcs_upload import (
upload_metadata_to_gcs,
)
from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load
from pydantic import ValidationError
def log_metadata_upload_info(metadata_upload_info: MetadataUploadInfo):

5
airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py
View File

@@ -16,6 +16,9 @@ import requests
import yaml
from google.cloud import storage
from google.oauth2 import service_account
from pydash import set_
from pydash.objects import get
from metadata_service.constants import (
COMPONENTS_PY_FILE_NAME,
COMPONENTS_ZIP_FILE_NAME,
@@ -34,8 +37,6 @@ from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import Conn
from metadata_service.models.generated.GitInfo import GitInfo
from metadata_service.models.transform import to_json_sanitized_dict
from metadata_service.validators.metadata_validator import POST_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load
from pydash import set_
from pydash.objects import get
# 🧩 TYPES

6
airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py
View File

@@ -8,11 +8,12 @@ from typing import Callable, List, Optional, Tuple, Union
import semver
import yaml
from metadata_service.docker_hub import get_latest_version_on_dockerhub, is_image_on_docker_hub
from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0
from pydantic import ValidationError
from pydash.objects import get
from metadata_service.docker_hub import get_latest_version_on_dockerhub, is_image_on_docker_hub
from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0
@dataclass(frozen=True)
class ValidatorOptions:
@@ -247,7 +248,6 @@ def validate_rc_suffix_and_rollout_configuration(
if docker_image_tag is None:
return False, "The dockerImageTag field is not set."
try:
is_major_release_candidate_version = check_is_major_release_candidate_version(docker_image_tag)
is_dev_version = check_is_dev_version(docker_image_tag)
is_rc_version = check_is_release_candidate_version(docker_image_tag)

6
airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py
View File

@@ -6,11 +6,12 @@ import pathlib
import pytest
from click.testing import CliRunner
from pydantic import BaseModel, ValidationError, error_wrappers
from test_gcs_upload import stub_is_image_on_docker_hub
from metadata_service import commands
from metadata_service.gcs_upload import MetadataUploadInfo, UploadedFile
from metadata_service.validators.metadata_validator import ValidatorOptions, validate_docker_image_tag_is_not_decremented
from pydantic import BaseModel, ValidationError, error_wrappers
from test_gcs_upload import stub_is_image_on_docker_hub
NOT_TEST_VALIDATORS = [
# Not testing validate_docker_image_tag_is_not_decremented as its tested independently in test_validators
@@ -19,6 +20,7 @@ NOT_TEST_VALIDATORS = [
PATCHED_VALIDATORS = [v for v in commands.PRE_UPLOAD_VALIDATORS if v not in NOT_TEST_VALIDATORS]
# TEST VALIDATE COMMAND
def test_valid_metadata_yaml_files(mocker, valid_metadata_yaml_files, tmp_path):
runner = CliRunner()

1
airbyte-ci/connectors/metadata_service/lib/tests/test_docker_hub.py
View File

@@ -6,6 +6,7 @@
import warnings
import pytest
from metadata_service import docker_hub

3
airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py
View File

@@ -7,6 +7,8 @@ from typing import Optional
import pytest
import yaml
from pydash.objects import get
from metadata_service import gcs_upload
from metadata_service.constants import (
COMPONENTS_PY_FILE_NAME,
@@ -19,7 +21,6 @@ from metadata_service.constants import (
from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0
from metadata_service.models.transform import to_json_sanitized_dict
from metadata_service.validators.metadata_validator import ValidatorOptions
from pydash.objects import get
MOCK_VERSIONS_THAT_DO_NOT_EXIST = ["99.99.99", "0.0.0"]
MISSING_SHA = "MISSINGSHA"

9
airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py
View File

@@ -5,15 +5,16 @@
from unittest.mock import patch
import pytest
from metadata_service.spec_cache import CachedSpec, Registries, SpecCache, get_docker_info_from_spec_cache_path
@pytest.fixture
def mock_spec_cache():
with patch("google.cloud.storage.Client.create_anonymous_client") as MockClient, patch(
"google.cloud.storage.Client.bucket"
) as MockBucket:
with (
patch("google.cloud.storage.Client.create_anonymous_client") as MockClient,
patch("google.cloud.storage.Client.bucket") as MockBucket,
):
# Create stub mock client and bucket
MockClient.return_value
MockBucket.return_value

1
airbyte-ci/connectors/metadata_service/lib/tests/test_transform.py
View File

@@ -5,6 +5,7 @@
import pathlib
import yaml
from metadata_service.models import transform
from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0

1
airbyte-ci/connectors/metadata_service/lib/tests/test_validators/test_metadata_validators.py
View File

@@ -4,6 +4,7 @@ import pytest
import requests
import semver
import yaml
from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0
from metadata_service.validators import metadata_validator

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_metrics.py
View File

@@ -11,6 +11,7 @@ from dagster import OpExecutionContext, asset
from google.cloud import storage
from orchestrator.logging import sentry
GROUP_NAME = "connector_metrics"

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py
View File

@@ -22,6 +22,7 @@ from orchestrator.templates.render import (
)
from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe
T = TypeVar("T")
GROUP_NAME = "connector_test_report"

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/github.py
View File

@@ -17,6 +17,7 @@ from orchestrator.models.metadata import LatestMetadataEntry, MetadataDefinition
from orchestrator.ops.slack import send_slack_message
from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe
GROUP_NAME = "github"
TOOLING_TEAM_SLACK_TEAM_ID = "S077R8636CV"
# We give 6 hours for the metadata to be updated

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py
View File

@@ -16,6 +16,7 @@ from orchestrator.logging import sentry
from orchestrator.models.metadata import LatestMetadataEntry, MetadataDefinition, PartialMetadataDefinition
from orchestrator.utils.object_helpers import are_values_equal, merge_values
GROUP_NAME = "metadata"

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py
View File

@@ -20,6 +20,7 @@ from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLif
from orchestrator.utils.object_helpers import default_none_to_dict
from pydash.objects import set_with
PolymorphicRegistryEntry = Union[ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition]
GROUP_NAME = "registry"

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py
View File

@@ -31,6 +31,7 @@ from orchestrator.utils.object_helpers import CaseInsensitveKeys, deep_copy_para
from pydantic import BaseModel, ValidationError
from pydash.objects import get, set_with
GROUP_NAME = "registry_entry"
# TYPES

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py
View File

@@ -22,6 +22,7 @@ from orchestrator.templates.render import (
)
from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe
GROUP_NAME = "registry_reports"
OSS_SUFFIX = "_oss"

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/slack.py
View File

@@ -8,6 +8,7 @@ import pandas as pd
from dagster import AutoMaterializePolicy, FreshnessPolicy, OpExecutionContext, Output, asset
from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe
GROUP_NAME = "slack"
USER_REQUEST_CHUNK_SIZE = 2000

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/specs_secrets_mask.py
View File

@@ -11,6 +11,7 @@ from dagster import MetadataValue, OpExecutionContext, Output, asset
from metadata_service.models.generated.ConnectorRegistryV0 import ConnectorRegistryV0
from orchestrator.logging import sentry
GROUP_NAME = "specs_secrets_mask"
# HELPERS

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py
View File

@@ -5,6 +5,7 @@
import os
from typing import Optional
DEFAULT_ASSET_URL = "https://storage.googleapis.com"
VALID_REGISTRIES = ["oss", "cloud"]

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/fetcher/connector_cdk_version.py
View File

@@ -7,6 +7,7 @@ from typing import Optional
import requests
from orchestrator.models.metadata import LatestMetadataEntry
GROUP_NAME = "connector_cdk_versions"
BASE_URL = "https://storage.googleapis.com/dev-airbyte-cloud-connector-metadata-service/"

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py
View File

@@ -10,6 +10,7 @@ from metadata_service.constants import METADATA_FILE_NAME, METADATA_FOLDER
from metadata_service.models.generated.ConnectorRegistryDestinationDefinition import ConnectorRegistryDestinationDefinition
from metadata_service.models.generated.ConnectorRegistrySourceDefinition import ConnectorRegistrySourceDefinition
PolymorphicRegistryEntry = Union[ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition]

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/connector_test_report.py
View File

@@ -4,6 +4,7 @@
from dagster import AssetSelection, define_asset_job
nightly_reports_inclusive = AssetSelection.keys("generate_nightly_report").upstream()
generate_nightly_reports = define_asset_job(name="generate_nightly_reports", selection=nightly_reports_inclusive)

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/metadata.py
View File

@@ -4,6 +4,7 @@
from dagster import AssetSelection, define_asset_job
stale_gcs_latest_metadata_file_inclusive = AssetSelection.keys("stale_gcs_latest_metadata_file").upstream()
generate_stale_gcs_latest_metadata_file = define_asset_job(
name="generate_stale_metadata_report", selection=stale_gcs_latest_metadata_file_inclusive

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py
View File

@@ -7,6 +7,7 @@ from orchestrator.assets import metadata, registry, registry_entry, specs_secret
from orchestrator.config import HIGH_QUEUE_PRIORITY, MAX_METADATA_PARTITION_RUN_REQUEST
from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLifecycle, PublishConnectorLifecycleStage, StageStatus
oss_registry_inclusive = AssetSelection.keys("persisted_oss_registry", "specs_secrets_mask_yaml").upstream()
generate_oss_registry = define_asset_job(name="generate_oss_registry", selection=oss_registry_inclusive)

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py
View File

@@ -8,6 +8,7 @@ import os
import sentry_sdk
from dagster import AssetExecutionContext, OpExecutionContext, SensorEvaluationContext, get_dagster_logger
sentry_logger = get_dagster_logger("sentry")

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/ci_report.py
View File

@@ -6,6 +6,7 @@ from typing import List, Optional
from pydantic import BaseModel, Extra
# TODO (ben): When the pipeline project is brought into the airbyte-ci folder
# we should update these models to import their twin models from the pipeline project

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/file_managers/local_file_manager.py
View File

@@ -13,6 +13,7 @@ from dagster._core.storage.file_manager import LocalFileHandle, LocalFileManager
from dagster._utils import mkdir_p
from typing_extensions import TypeAlias
IOStream: TypeAlias = Union[TextIO, BinaryIO]

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/github.py
View File

@@ -6,6 +6,7 @@ from datetime import datetime
from dagster import DefaultSensorStatus, RunRequest, SensorDefinition, SensorEvaluationContext, SkipReason, build_resources, sensor
# e.g. 2023-06-02T17:42:36Z
EXPECTED_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ"

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py
View File

@@ -12,6 +12,7 @@ import pandas as pd
from jinja2 import Environment, PackageLoader
from orchestrator.utils.object_helpers import deep_copy_params
# 🔗 HTML Renderers

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py
View File

@@ -8,6 +8,7 @@ from typing import List, Optional
import pandas as pd
from dagster import MetadataValue, Output
OutputDataFrame = Output[pd.DataFrame]
CURSOR_SEPARATOR = ":"

1
airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py
View File

@@ -9,6 +9,7 @@ from typing import TypeVar
import mergedeep
from deepdiff import DeepDiff
T = TypeVar("T")

1
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py
View File

@@ -6,6 +6,7 @@ from typing import List
import asyncclick as click
import dagger
from pipelines import main_logger
from pipelines.airbyte_ci.connectors.build_image.steps import run_connector_build_pipeline
from pipelines.airbyte_ci.connectors.context import ConnectorContext

1
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py
View File

@@ -11,6 +11,7 @@ from base_images.bases import AirbyteConnectorBaseImage # type: ignore
from click import UsageError
from connector_ops.utils import Connector # type: ignore
from dagger import Container, ExecError, Platform, QueryError
from pipelines.airbyte_ci.connectors.context import ConnectorContext
from pipelines.helpers.utils import export_container_to_tarball, sh_dash_c
from pipelines.models.steps import Step, StepResult, StepStatus

1
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py
View File

@@ -3,6 +3,7 @@
#
from dagger import Container, Directory, File, Platform, QueryError
from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase
from pipelines.airbyte_ci.connectors.context import ConnectorContext
from pipelines.airbyte_ci.steps.gradle import GradleTask

3
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/manifest_only_connectors.py
View File

@@ -6,13 +6,14 @@
from typing import Any
from dagger import Container, Platform
from pydash.objects import get # type: ignore
from pipelines.airbyte_ci.connectors.build_image.steps import build_customization
from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase
from pipelines.airbyte_ci.connectors.context import ConnectorContext
from pipelines.consts import COMPONENTS_FILE_PATH, MANIFEST_FILE_PATH
from pipelines.dagger.actions.python.common import apply_python_development_overrides
from pipelines.models.steps import StepResult
from pydash.objects import get # type: ignore
class BuildConnectorImages(BuildConnectorImagesBase):

1
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py
View File

@@ -3,6 +3,7 @@
#
from dagger import Platform
from pipelines.airbyte_ci.connectors.context import ConnectorContext
from pipelines.dagger.actions.connector import normalization
from pipelines.models.steps import Step, StepResult, StepStatus

1
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py
View File

@@ -6,6 +6,7 @@
from typing import Any
from dagger import Container, Platform
from pipelines.airbyte_ci.connectors.build_image.steps import build_customization
from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase
from pipelines.airbyte_ci.connectors.context import ConnectorContext

1
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py
View File

@@ -6,6 +6,7 @@ from typing import Optional
import asyncclick as click
import semver
from pipelines.airbyte_ci.connectors.bump_version.pipeline import run_connector_version_bump_pipeline
from pipelines.airbyte_ci.connectors.context import ConnectorContext
from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines

Some files were not shown because too many files have changed in this diff Show More