diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index b36ba80e843..aead5f00f06 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -89,9 +89,70 @@ jobs: # - run: | # echo '${{ toJSON(needs) }}' + format: + needs: changes + runs-on: ubuntu-latest + # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. + if: needs.changes.outputs.any_change == 'true' || (always() && github.ref == 'refs/heads/master') + name: "Apply All Formatting Rules" + timeout-minutes: 20 + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + + - name: Cache Build Artifacts + uses: ./.github/actions/cache-build-artifacts + with: + cache-key: ${{ secrets.CACHE_VERSION }}-format + + - uses: actions/setup-java@v3 + with: + distribution: "zulu" + java-version: "17" + + - uses: actions/setup-python@v4 + with: + python-version: "3.9" + + - name: Set up CI Gradle Properties + run: | + mkdir -p ~/.gradle/ + cat > ~/.gradle/gradle.properties < str: docker_username = os.environ.get("DOCKER_HUB_USERNAME") diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py index 7e0e922be17..74de80534ce 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py @@ -1,25 +1,23 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from pathlib import Path -from typing import Tuple, Optional -from pydash.objects import get import base64 import hashlib import json import os -import yaml +from dataclasses import dataclass +from pathlib import Path +from typing import Optional, Tuple +import yaml from google.cloud import storage from google.oauth2 import service_account - -from metadata_service.constants import METADATA_FILE_NAME, METADATA_FOLDER, ICON_FILE_NAME -from metadata_service.validators.metadata_validator import POST_UPLOAD_VALIDATORS, validate_and_load, ValidatorOptions -from metadata_service.models.transform import to_json_sanitized_dict +from metadata_service.constants import ICON_FILE_NAME, METADATA_FILE_NAME, METADATA_FOLDER from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 - -from dataclasses import dataclass +from metadata_service.models.transform import to_json_sanitized_dict +from metadata_service.validators.metadata_validator import POST_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load +from pydash.objects import get @dataclass(frozen=True) diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ActorDefinitionResourceRequirements.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ActorDefinitionResourceRequirements.py index 1f6e484eef7..e3ec41f010b 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ActorDefinitionResourceRequirements.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ActorDefinitionResourceRequirements.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: ActorDefinitionResourceRequirements.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AirbyteInternal.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AirbyteInternal.py index c7026652131..9714260f267 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AirbyteInternal.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AirbyteInternal.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: AirbyteInternal.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AllowedHosts.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AllowedHosts.py index ce4534f3adc..c7401961b29 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AllowedHosts.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AllowedHosts.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: AllowedHosts.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py index 24a58a4ea62..4721e9f15f2 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: ConnectorMetadataDefinitionV0.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py index cc151e2e769..5b7e71acb9f 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: ConnectorRegistryDestinationDefinition.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py index c11d1190f78..7836ffe5ed4 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: ConnectorRegistrySourceDefinition.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py index ae743c919b9..290b76e527e 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: ConnectorRegistryV0.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py index fd6e275f0d0..493f56fcb0c 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: ConnectorReleases.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/JobType.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/JobType.py index aef4f7ad5f9..860a77ac7c2 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/JobType.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/JobType.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: JobType.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/NormalizationDestinationDefinitionConfig.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/NormalizationDestinationDefinitionConfig.py index 00a642bfaeb..0e719498056 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/NormalizationDestinationDefinitionConfig.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/NormalizationDestinationDefinitionConfig.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: NormalizationDestinationDefinitionConfig.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/RegistryOverrides.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/RegistryOverrides.py index eb6908bc65b..1af53d9865b 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/RegistryOverrides.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/RegistryOverrides.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: RegistryOverrides.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ReleaseStage.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ReleaseStage.py index cb7c9b909b0..59587282973 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ReleaseStage.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ReleaseStage.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: ReleaseStage.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ResourceRequirements.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ResourceRequirements.py index abc7e6173d0..ad2ebf0da9a 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ResourceRequirements.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ResourceRequirements.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: ResourceRequirements.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SuggestedStreams.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SuggestedStreams.py index 9a3d7cdf401..58944134697 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SuggestedStreams.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SuggestedStreams.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: SuggestedStreams.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SupportLevel.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SupportLevel.py index 4a0f7d77c87..9367e03ddbd 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SupportLevel.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SupportLevel.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + # generated by datamodel-codegen: # filename: SupportLevel.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/transform.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/transform.py index e327ec1702c..971e3e76008 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/transform.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/transform.py @@ -1,4 +1,9 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import json + from pydantic import BaseModel diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/spec_cache.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/spec_cache.py index 14bf3660995..51df94c9fa7 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/spec_cache.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/spec_cache.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import json from dataclasses import dataclass from typing import List diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py index fc31d199aee..bc8840eb019 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py @@ -1,12 +1,16 @@ -import semver -import pathlib -import yaml +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import pathlib from dataclasses import dataclass -from pydantic import ValidationError -from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 -from typing import Optional, Tuple, Union, List, Callable +from typing import Callable, List, Optional, Tuple, Union + +import semver +import yaml from metadata_service.docker_hub import is_image_on_docker_hub +from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 +from pydantic import ValidationError from pydash.objects import get diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/conftest.py b/airbyte-ci/connectors/metadata_service/lib/tests/conftest.py index 8f08325d8a8..4384ae46857 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/conftest.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/conftest.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + pytest_plugins = [ "tests.fixtures", ] diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/__init__.py b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/__init__.py index 5819d0f6141..b6f56dc8733 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/__init__.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/__init__.py @@ -1,6 +1,7 @@ -import pytest import os -from typing import List, Callable +from typing import Callable, List + +import pytest def list_all_paths_in_fixture_directory(folder_name: str) -> List[str]: diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py index 58e89a231d3..ddd44b4ab6f 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py @@ -1,9 +1,10 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import pytest + import pathlib +import pytest from click.testing import CliRunner from metadata_service import commands from metadata_service.gcs_upload import MetadataUploadInfo diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py index a3cc6d7e7c6..a4213787176 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py @@ -1,18 +1,18 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +import json import pathlib import pytest -import json import yaml -from pydash.objects import get - from metadata_service import gcs_upload -from metadata_service.validators.metadata_validator import ValidatorOptions -from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 from metadata_service.constants import METADATA_FILE_NAME +from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 from metadata_service.models.transform import to_json_sanitized_dict +from metadata_service.validators.metadata_validator import ValidatorOptions +from pydash.objects import get # Version exists by default, but "666" is bad! (6.0.0 too since breaking changes regex tho) MOCK_VERSIONS_THAT_DO_NOT_EXIST = ["6.6.6", "6.0.0"] diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_transform.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_transform.py index 8222e5e12e8..5fb7bdf41ac 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_transform.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_transform.py @@ -1,11 +1,12 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import pathlib -import yaml -from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 +import pathlib + +import yaml from metadata_service.models import transform +from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 def get_all_dict_key_paths(dict_to_traverse, key_path=""): diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/__init__.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/__init__.py index 55fd5d3d4ac..81805b44a77 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/__init__.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/__init__.py @@ -1,55 +1,43 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from dagster import Definitions, ScheduleDefinition, EnvVar, load_assets_from_modules +from dagster import Definitions, EnvVar, ScheduleDefinition, load_assets_from_modules from dagster_slack import SlackResource - +from metadata_service.constants import METADATA_FILE_NAME, METADATA_FOLDER +from orchestrator.assets import connector_test_report, github, metadata, registry, registry_entry, registry_report, specs_secrets_mask +from orchestrator.config import ( + CI_MASTER_TEST_OUTPUT_REGEX, + CI_TEST_REPORT_PREFIX, + CONNECTOR_REPO_NAME, + CONNECTORS_PATH, + HIGH_QUEUE_PRIORITY, + NIGHTLY_COMPLETE_REPORT_FILE_NAME, + NIGHTLY_FOLDER, + NIGHTLY_GHA_WORKFLOW_ID, + NIGHTLY_INDIVIDUAL_TEST_REPORT_FILE_NAME, + REGISTRIES_FOLDER, + REPORT_FOLDER, +) +from orchestrator.jobs.connector_test_report import generate_connector_test_summary_reports, generate_nightly_reports +from orchestrator.jobs.metadata import generate_stale_gcs_latest_metadata_file +from orchestrator.jobs.registry import ( + add_new_metadata_partitions, + generate_cloud_registry, + generate_oss_registry, + generate_registry_entry, + generate_registry_reports, +) +from orchestrator.logging.sentry import setup_dagster_sentry from orchestrator.resources.gcp import gcp_gcs_client, gcs_directory_blobs, gcs_file_blob, gcs_file_manager from orchestrator.resources.github import ( github_client, github_connector_repo, github_connectors_directory, - github_workflow_runs, github_connectors_metadata_files, + github_workflow_runs, ) - -from orchestrator.assets import ( - connector_test_report, - github, - specs_secrets_mask, - registry, - registry_report, - registry_entry, - metadata, -) - -from orchestrator.jobs.registry import ( - generate_registry_reports, - generate_oss_registry, - generate_cloud_registry, - generate_registry_entry, - add_new_metadata_partitions, -) -from orchestrator.jobs.connector_test_report import generate_nightly_reports, generate_connector_test_summary_reports -from orchestrator.jobs.metadata import generate_stale_gcs_latest_metadata_file -from orchestrator.sensors.registry import registry_updated_sensor from orchestrator.sensors.gcs import new_gcs_blobs_sensor -from orchestrator.logging.sentry import setup_dagster_sentry - -from orchestrator.config import ( - REPORT_FOLDER, - REGISTRIES_FOLDER, - CONNECTORS_PATH, - CONNECTOR_REPO_NAME, - NIGHTLY_FOLDER, - NIGHTLY_COMPLETE_REPORT_FILE_NAME, - NIGHTLY_INDIVIDUAL_TEST_REPORT_FILE_NAME, - NIGHTLY_GHA_WORKFLOW_ID, - CI_TEST_REPORT_PREFIX, - CI_MASTER_TEST_OUTPUT_REGEX, - HIGH_QUEUE_PRIORITY, -) -from metadata_service.constants import METADATA_FILE_NAME, METADATA_FOLDER +from orchestrator.sensors.registry import registry_updated_sensor ASSETS = load_assets_from_modules( [ diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py index c65311f34d9..6d2ca249f07 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/connector_test_report.py @@ -1,27 +1,26 @@ -import pandas as pd +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import json import os import re from datetime import datetime - -from dagster import Output, asset, OpExecutionContext, MetadataValue -from google.cloud import storage from typing import List, Type, TypeVar -from orchestrator.ops.slack import send_slack_message +import pandas as pd +from dagster import MetadataValue, OpExecutionContext, Output, asset +from google.cloud import storage +from orchestrator.config import CONNECTOR_TEST_SUMMARY_FOLDER, NIGHTLY_COMPLETE_REPORT_FILE_NAME +from orchestrator.logging import sentry from orchestrator.models.ci_report import ConnectorNightlyReport, ConnectorPipelineReport -from orchestrator.config import ( - NIGHTLY_COMPLETE_REPORT_FILE_NAME, - CONNECTOR_TEST_SUMMARY_FOLDER, -) +from orchestrator.ops.slack import send_slack_message from orchestrator.templates.render import ( render_connector_nightly_report_md, - render_connector_test_summary_html, render_connector_test_badge, + render_connector_test_summary_html, ) from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe -from orchestrator.logging import sentry - T = TypeVar("T") diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/github.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/github.py index 23c4d0fe1d9..1d4049d5a8b 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/github.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/github.py @@ -1,18 +1,20 @@ -import pandas as pd -import hashlib +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import base64 -import dateutil import datetime -import humanize +import hashlib import os -from dagster import Output, asset, OpExecutionContext +import dateutil +import humanize +import pandas as pd +from dagster import OpExecutionContext, Output, asset from github import Repository - +from orchestrator.logging import sentry from orchestrator.ops.slack import send_slack_message from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe -from orchestrator.logging import sentry - GROUP_NAME = "github" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py index 40c04fee27e..5321c11d20f 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/metadata.py @@ -1,17 +1,20 @@ -import pandas as pd -import numpy as np +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import os from typing import List -from dagster import Output, asset, OpExecutionContext + +import numpy as np +import pandas as pd import yaml - +from dagster import OpExecutionContext, Output, asset +from metadata_service.constants import ICON_FILE_NAME, METADATA_FILE_NAME from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 -from metadata_service.constants import METADATA_FILE_NAME, ICON_FILE_NAME - -from orchestrator.utils.object_helpers import are_values_equal, merge_values -from orchestrator.models.metadata import PartialMetadataDefinition, MetadataDefinition, LatestMetadataEntry from orchestrator.config import get_public_url_for_gcs_file from orchestrator.logging import sentry +from orchestrator.models.metadata import LatestMetadataEntry, MetadataDefinition, PartialMetadataDefinition +from orchestrator.utils.object_helpers import are_values_equal, merge_values GROUP_NAME = "metadata" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py index cd392af82f3..7013069464f 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry.py @@ -1,22 +1,19 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import json -import sentry_sdk -from google.cloud import storage - -from dagster import asset, OpExecutionContext, MetadataValue, Output -from dagster_gcp.gcs.file_manager import GCSFileManager, GCSFileHandle - -from metadata_service.models.generated.ConnectorRegistryV0 import ConnectorRegistryV0 -from metadata_service.models.transform import to_json_sanitized_dict - -from orchestrator.assets.registry_entry import read_registry_entry_blob -from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLifecycle, PublishConnectorLifecycleStage, StageStatus -from orchestrator.logging import sentry - from typing import List +import sentry_sdk +from dagster import MetadataValue, OpExecutionContext, Output, asset +from dagster_gcp.gcs.file_manager import GCSFileHandle, GCSFileManager +from google.cloud import storage +from metadata_service.models.generated.ConnectorRegistryV0 import ConnectorRegistryV0 +from metadata_service.models.transform import to_json_sanitized_dict +from orchestrator.assets.registry_entry import read_registry_entry_blob +from orchestrator.logging import sentry +from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLifecycle, PublishConnectorLifecycleStage, StageStatus GROUP_NAME = "registry" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py index b6285169131..c00aa7e8e65 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py @@ -1,31 +1,31 @@ -import yaml -import json -import pandas as pd -import os +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import copy -import sentry_sdk - -from pydantic import ValidationError -from google.cloud import storage -from dagster_gcp.gcs.file_manager import GCSFileManager, GCSFileHandle -from dagster import DynamicPartitionsDefinition, asset, OpExecutionContext, Output, MetadataValue, AutoMaterializePolicy -from pydash.objects import get - -from metadata_service.spec_cache import get_cached_spec, list_cached_specs -from metadata_service.models.generated.ConnectorRegistrySourceDefinition import ConnectorRegistrySourceDefinition -from metadata_service.models.generated.ConnectorRegistryDestinationDefinition import ConnectorRegistryDestinationDefinition -from metadata_service.constants import METADATA_FILE_NAME, ICON_FILE_NAME - -from orchestrator.utils.object_helpers import deep_copy_params -from orchestrator.utils.dagster_helpers import OutputDataFrame -from orchestrator.models.metadata import MetadataDefinition, LatestMetadataEntry -from orchestrator.config import get_public_url_for_gcs_file, VALID_REGISTRIES, MAX_METADATA_PARTITION_RUN_REQUEST -from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLifecycle, PublishConnectorLifecycleStage, StageStatus -from orchestrator.logging import sentry +import json +import os +from typing import List, Optional, Tuple, Union import orchestrator.hacks as HACKS - -from typing import List, Optional, Tuple, Union +import pandas as pd +import sentry_sdk +import yaml +from dagster import AutoMaterializePolicy, DynamicPartitionsDefinition, MetadataValue, OpExecutionContext, Output, asset +from dagster_gcp.gcs.file_manager import GCSFileHandle, GCSFileManager +from google.cloud import storage +from metadata_service.constants import ICON_FILE_NAME, METADATA_FILE_NAME +from metadata_service.models.generated.ConnectorRegistryDestinationDefinition import ConnectorRegistryDestinationDefinition +from metadata_service.models.generated.ConnectorRegistrySourceDefinition import ConnectorRegistrySourceDefinition +from metadata_service.spec_cache import get_cached_spec, list_cached_specs +from orchestrator.config import MAX_METADATA_PARTITION_RUN_REQUEST, VALID_REGISTRIES, get_public_url_for_gcs_file +from orchestrator.logging import sentry +from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLifecycle, PublishConnectorLifecycleStage, StageStatus +from orchestrator.models.metadata import LatestMetadataEntry, MetadataDefinition +from orchestrator.utils.dagster_helpers import OutputDataFrame +from orchestrator.utils.object_helpers import deep_copy_params +from pydantic import ValidationError +from pydash.objects import get PolymorphicRegistryEntry = Union[ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition] TaggedRegistryEntry = Tuple[str, PolymorphicRegistryEntry] diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py index 07601cd4e3a..8895b963eff 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py @@ -1,22 +1,26 @@ -import sentry_sdk -import pandas as pd -from dagster import MetadataValue, Output, asset -from typing import List -from orchestrator.templates.render import ( - render_connector_registry_locations_html, - dataframe_to_table_html, - simple_link_html, - icon_image_html, - test_badge_html, - internal_level_html, - ColumnInfo, -) -from orchestrator.config import CONNECTOR_REPO_NAME, CONNECTOR_TEST_SUMMARY_FOLDER, REPORT_FOLDER, get_public_metadata_service_url -from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe -from orchestrator.logging import sentry +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# -from metadata_service.models.transform import to_json_sanitized_dict +from typing import List + +import pandas as pd +import sentry_sdk +from dagster import MetadataValue, Output, asset from metadata_service.models.generated.ConnectorRegistryV0 import ConnectorRegistryV0 +from metadata_service.models.transform import to_json_sanitized_dict +from orchestrator.config import CONNECTOR_REPO_NAME, CONNECTOR_TEST_SUMMARY_FOLDER, REPORT_FOLDER, get_public_metadata_service_url +from orchestrator.logging import sentry +from orchestrator.templates.render import ( + ColumnInfo, + dataframe_to_table_html, + icon_image_html, + internal_level_html, + render_connector_registry_locations_html, + simple_link_html, + test_badge_html, +) +from orchestrator.utils.dagster_helpers import OutputDataFrame, output_dataframe GROUP_NAME = "registry_reports" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/specs_secrets_mask.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/specs_secrets_mask.py index d4218473b92..a82cbc5df35 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/specs_secrets_mask.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/specs_secrets_mask.py @@ -1,14 +1,13 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from typing import List, Set import dpath.util -import yaml import sentry_sdk - -from dagster import MetadataValue, Output, asset, OpExecutionContext - +import yaml +from dagster import MetadataValue, OpExecutionContext, Output, asset from metadata_service.models.generated.ConnectorRegistryV0 import ConnectorRegistryV0 from orchestrator.logging import sentry diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py index f45f834a570..874dec4f9ad 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import os from typing import Optional diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py index 24799dbec4a..3cce26dd2a1 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py @@ -1,14 +1,17 @@ -from dagster import get_dagster_logger -from dagster_gcp.gcs.file_manager import GCSFileManager, GCSFileHandle - -from orchestrator.models.metadata import LatestMetadataEntry -from metadata_service.constants import METADATA_FILE_NAME -from metadata_service.gcs_upload import get_metadata_remote_file_path -from metadata_service.models.generated.ConnectorRegistrySourceDefinition import ConnectorRegistrySourceDefinition -from metadata_service.models.generated.ConnectorRegistryDestinationDefinition import ConnectorRegistryDestinationDefinition +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# from typing import Union +from dagster import get_dagster_logger +from dagster_gcp.gcs.file_manager import GCSFileHandle, GCSFileManager +from metadata_service.constants import METADATA_FILE_NAME +from metadata_service.gcs_upload import get_metadata_remote_file_path +from metadata_service.models.generated.ConnectorRegistryDestinationDefinition import ConnectorRegistryDestinationDefinition +from metadata_service.models.generated.ConnectorRegistrySourceDefinition import ConnectorRegistrySourceDefinition +from orchestrator.models.metadata import LatestMetadataEntry + PolymorphicRegistryEntry = Union[ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition] diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/connector_test_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/connector_test_report.py index 759963ba664..4528a6d8c31 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/connector_test_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/connector_test_report.py @@ -1,4 +1,8 @@ -from dagster import define_asset_job, AssetSelection +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dagster import AssetSelection, define_asset_job nightly_reports_inclusive = AssetSelection.keys("generate_nightly_report").upstream() generate_nightly_reports = define_asset_job(name="generate_nightly_reports", selection=nightly_reports_inclusive) diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/metadata.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/metadata.py index 575c133816c..01ce46dace7 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/metadata.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/metadata.py @@ -1,4 +1,8 @@ -from dagster import define_asset_job, AssetSelection +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dagster import AssetSelection, define_asset_job stale_gcs_latest_metadata_file_inclusive = AssetSelection.keys("stale_gcs_latest_metadata_file").upstream() generate_stale_gcs_latest_metadata_file = define_asset_job( diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py index a9140fe2214..c24d4513bb0 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/jobs/registry.py @@ -1,6 +1,10 @@ -from dagster import define_asset_job, AssetSelection, job, SkipReason, op +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dagster import AssetSelection, SkipReason, define_asset_job, job, op from orchestrator.assets import registry_entry -from orchestrator.config import MAX_METADATA_PARTITION_RUN_REQUEST, HIGH_QUEUE_PRIORITY +from orchestrator.config import HIGH_QUEUE_PRIORITY, MAX_METADATA_PARTITION_RUN_REQUEST from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLifecycle, PublishConnectorLifecycleStage, StageStatus oss_registry_inclusive = AssetSelection.keys("persisted_oss_registry", "specs_secrets_mask_yaml").upstream() diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/publish_connector_lifecycle.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/publish_connector_lifecycle.py index 3ec1d64cf7e..494badd6ae5 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/publish_connector_lifecycle.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/publish_connector_lifecycle.py @@ -1,6 +1,10 @@ -import os +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import os from enum import Enum + from dagster import OpExecutionContext from orchestrator.ops.slack import send_slack_message diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py index 27c63643e7d..e65d9fd691e 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/logging/sentry.py @@ -1,8 +1,12 @@ -import os -import sentry_sdk -import functools +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# -from dagster import OpExecutionContext, SensorEvaluationContext, AssetExecutionContext, get_dagster_logger +import functools +import os + +import sentry_sdk +from dagster import AssetExecutionContext, OpExecutionContext, SensorEvaluationContext, get_dagster_logger sentry_logger = get_dagster_logger("sentry") diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/ci_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/ci_report.py index 3c523158244..a8e791abd5b 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/ci_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/ci_report.py @@ -1,4 +1,9 @@ -from typing import Optional, List +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List, Optional + from pydantic import BaseModel, Extra # TODO (ben): When the pipeline project is brought into the airbyte-ci folder diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/metadata.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/metadata.py index e38cd1f63a0..4e5ac4b3829 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/metadata.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/metadata.py @@ -1,7 +1,12 @@ -from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 -from pydantic import ValidationError, BaseModel +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + from dataclasses import dataclass -from typing import Tuple, Any, Optional +from typing import Any, Optional, Tuple + +from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 +from pydantic import BaseModel, ValidationError class PydanticDelayValidationMixin: diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/ops/slack.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/ops/slack.py index 9dc220b7776..07f65955e38 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/ops/slack.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/ops/slack.py @@ -1,8 +1,12 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import os -from dagster import op, OpExecutionContext -from slack_sdk import WebhookClient +from dagster import OpExecutionContext, op from dagster_slack import SlackResource +from slack_sdk import WebhookClient def chunk_messages(report): diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/file_managers/local_file_manager.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/file_managers/local_file_manager.py index 6b23a44a0bf..78477fe5863 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/file_managers/local_file_manager.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/file_managers/local_file_manager.py @@ -1,13 +1,16 @@ -import os -import uuid -import shutil +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import io - -from dagster._core.storage.file_manager import LocalFileManager, LocalFileHandle, check_file_like_obj -import dagster._check as check -from dagster._utils import mkdir_p - +import os +import shutil +import uuid from typing import BinaryIO, Optional, TextIO, Union + +import dagster._check as check +from dagster._core.storage.file_manager import LocalFileHandle, LocalFileManager, check_file_like_obj +from dagster._utils import mkdir_p from typing_extensions import TypeAlias IOStream: TypeAlias = Union[TextIO, BinaryIO] diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/gcp.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/gcp.py index 7e4a6191bf0..c75a783a484 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/gcp.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/gcp.py @@ -1,18 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import json import re -from google.cloud import storage -from google.oauth2 import service_account - -from dagster import StringSource, InitResourceContext, Noneable, resource -from dagster_gcp.gcs.file_manager import GCSFileManager, GCSFileHandle - import uuid from typing import Optional -import dagster._check as check -from dagster._core.storage.file_manager import ( - check_file_like_obj, -) +import dagster._check as check +from dagster import InitResourceContext, Noneable, StringSource, resource +from dagster._core.storage.file_manager import check_file_like_obj +from dagster_gcp.gcs.file_manager import GCSFileHandle, GCSFileManager +from google.cloud import storage +from google.oauth2 import service_account from orchestrator.config import get_public_url_for_gcs_file diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/github.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/github.py index 49dcbe36f60..146e00e98b0 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/github.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/github.py @@ -1,11 +1,15 @@ -from typing import List -from dagster import StringSource, InitResourceContext, resource -from github import Github, Repository, ContentFile, GitTreeElement -from datetime import datetime, timedelta -from dateutil.parser import parse +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# -from orchestrator.config import CONNECTORS_PATH +from datetime import datetime, timedelta +from typing import List + +from dagster import InitResourceContext, StringSource, resource +from dateutil.parser import parse +from github import ContentFile, Github, GitTreeElement, Repository from metadata_service.constants import METADATA_FILE_NAME +from orchestrator.config import CONNECTORS_PATH def _valid_metadata_file_path(path: str) -> bool: diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/local.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/local.py index 883e99866c8..70fa3bd6d73 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/local.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/resources/local.py @@ -1,7 +1,13 @@ -from dagster import StringSource, InitResourceContext, resource, Field -from .file_managers.local_file_manager import SimpleLocalFileManager +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import os +from dagster import Field, InitResourceContext, StringSource, resource + +from .file_managers.local_file_manager import SimpleLocalFileManager + @resource( config_schema={ diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/gcs.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/gcs.py index a8eb2f8fa6c..b6fda58d291 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/gcs.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/gcs.py @@ -1,15 +1,19 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + from dagster import ( - sensor, + DefaultSensorStatus, RunRequest, - SkipReason, SensorDefinition, SensorEvaluationContext, - build_resources, - DefaultSensorStatus, SensorResult, + SkipReason, + build_resources, + sensor, ) -from orchestrator.utils.dagster_helpers import string_array_to_hash from orchestrator.logging import sentry +from orchestrator.utils.dagster_helpers import string_array_to_hash def new_gcs_blobs_sensor( diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/github.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/github.py index 427b05724c5..cc3a3d42f40 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/github.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/github.py @@ -1,5 +1,10 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + from datetime import datetime -from dagster import sensor, RunRequest, SkipReason, SensorDefinition, SensorEvaluationContext, build_resources, DefaultSensorStatus + +from dagster import DefaultSensorStatus, RunRequest, SensorDefinition, SensorEvaluationContext, SkipReason, build_resources, sensor # e.g. 2023-06-02T17:42:36Z EXPECTED_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/registry.py index 1e3df0cff73..c06fabb36cc 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/sensors/registry.py @@ -1,5 +1,8 @@ -from dagster import sensor, RunRequest, SkipReason, SensorDefinition, SensorEvaluationContext, build_resources, DefaultSensorStatus +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from dagster import DefaultSensorStatus, RunRequest, SensorDefinition, SensorEvaluationContext, SkipReason, build_resources, sensor from orchestrator.utils.dagster_helpers import string_array_to_hash diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py index 131277558c9..3153a86f395 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py @@ -1,13 +1,16 @@ -import pandas as pd -import urllib.parse -import json +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# -from jinja2 import Environment, PackageLoader -from typing import List, Optional, Callable, Any +import json +import urllib.parse from dataclasses import dataclass from datetime import timedelta -from orchestrator.utils.object_helpers import deep_copy_params +from typing import Any, Callable, List, Optional +import pandas as pd +from jinja2 import Environment, PackageLoader +from orchestrator.utils.object_helpers import deep_copy_params # 🔗 HTML Renderers diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py index 69e4cb200f6..3ccc4fa0841 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py @@ -1,7 +1,12 @@ -from dagster import MetadataValue, Output -import pandas as pd +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import hashlib -from typing import Optional, List +from typing import List, Optional + +import pandas as pd +from dagster import MetadataValue, Output OutputDataFrame = Output[pd.DataFrame] CURSOR_SEPARATOR = ":" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py index 7589ae0f8a1..ebb42a5c4a3 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py @@ -1,8 +1,13 @@ -import mergedeep -import json -from deepdiff import DeepDiff -from typing import TypeVar +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import copy +import json +from typing import TypeVar + +import mergedeep +from deepdiff import DeepDiff T = TypeVar("T") diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/conftest.py b/airbyte-ci/connectors/metadata_service/orchestrator/tests/conftest.py index 8f08325d8a8..4384ae46857 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/conftest.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/conftest.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + pytest_plugins = [ "tests.fixtures", ] diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/__init__.py b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/__init__.py index 1f0a6a24a01..57104ac0b2e 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/__init__.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/__init__.py @@ -1,7 +1,8 @@ import json -import pytest import os +import pytest + @pytest.fixture def oss_registry_dict(): diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_dagster_helpers.py b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_dagster_helpers.py index 13d92550f51..83ec00b89cc 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_dagster_helpers.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_dagster_helpers.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + from orchestrator.utils.dagster_helpers import string_array_to_hash diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_debug.py b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_debug.py index bf46d532109..9450c05b74a 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_debug.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_debug.py @@ -1,13 +1,16 @@ -from dagster import build_op_context +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# -from orchestrator.resources.gcp import gcp_gcs_client, gcs_directory_blobs, gcs_file_manager -from orchestrator.assets.connector_test_report import generate_nightly_report, persist_connectors_test_summary_files -from orchestrator.assets.registry_entry import registry_entry, metadata_entry -from orchestrator.assets.registry import persisted_oss_registry -from orchestrator.assets.github import github_metadata_file_md5s, stale_gcs_latest_metadata_file -from orchestrator.config import NIGHTLY_INDIVIDUAL_TEST_REPORT_FILE_NAME, NIGHTLY_FOLDER, NIGHTLY_COMPLETE_REPORT_FILE_NAME, REPORT_FOLDER -from orchestrator import REGISTRY_ENTRY_RESOURCE_TREE, GITHUB_RESOURCE_TREE, METADATA_RESOURCE_TREE +from dagster import build_op_context from metadata_service.constants import METADATA_FILE_NAME, METADATA_FOLDER +from orchestrator import GITHUB_RESOURCE_TREE, METADATA_RESOURCE_TREE, REGISTRY_ENTRY_RESOURCE_TREE +from orchestrator.assets.connector_test_report import generate_nightly_report, persist_connectors_test_summary_files +from orchestrator.assets.github import github_metadata_file_md5s, stale_gcs_latest_metadata_file +from orchestrator.assets.registry import persisted_oss_registry +from orchestrator.assets.registry_entry import metadata_entry, registry_entry +from orchestrator.config import NIGHTLY_COMPLETE_REPORT_FILE_NAME, NIGHTLY_FOLDER, NIGHTLY_INDIVIDUAL_TEST_REPORT_FILE_NAME, REPORT_FOLDER +from orchestrator.resources.gcp import gcp_gcs_client, gcs_directory_blobs, gcs_file_manager def debug_nightly_report(): diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py index 37a71482e90..b5ed2d349d8 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py @@ -1,30 +1,32 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest import mock +from uuid import UUID + import pytest import yaml -from unittest import mock - -from uuid import UUID -from pydantic import ValidationError from google.cloud import storage - -from metadata_service.models.generated.ConnectorRegistryV0 import ConnectorRegistryV0 -from metadata_service.models.generated.ConnectorRegistrySourceDefinition import ConnectorRegistrySourceDefinition from metadata_service.models.generated.ConnectorRegistryDestinationDefinition import ConnectorRegistryDestinationDefinition - +from metadata_service.models.generated.ConnectorRegistrySourceDefinition import ConnectorRegistrySourceDefinition +from metadata_service.models.generated.ConnectorRegistryV0 import ConnectorRegistryV0 from orchestrator.assets.registry_entry import ( - metadata_to_registry_entry, get_connector_type_from_registry_entry, get_registry_status_lists, + metadata_to_registry_entry, safe_parse_metadata_definition, ) from orchestrator.assets.registry_report import ( - all_sources_dataframe, all_destinations_dataframe, - oss_destinations_dataframe, + all_sources_dataframe, cloud_destinations_dataframe, - oss_sources_dataframe, cloud_sources_dataframe, + oss_destinations_dataframe, + oss_sources_dataframe, ) -from orchestrator.models.metadata import MetadataDefinition, LatestMetadataEntry +from orchestrator.models.metadata import LatestMetadataEntry, MetadataDefinition +from pydantic import ValidationError VALID_METADATA_DICT = { "metadataSpecVersion": "1.0", diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_specs_secrets_mask.py b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_specs_secrets_mask.py index 6c5529cb509..c75a4e788ab 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_specs_secrets_mask.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_specs_secrets_mask.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import pytest from orchestrator.assets.specs_secrets_mask import get_secrets_properties_from_registry_entry diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py b/airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py index b5f2fb96147..3024cf8378c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py +++ b/airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + """This module groups functions to interact with remote storage services like S3 or GCS.""" import uuid diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/actions/secrets.py index dde74f68644..985ca064b5b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/actions/secrets.py +++ b/airbyte-ci/connectors/pipelines/pipelines/actions/secrets.py @@ -8,13 +8,13 @@ from __future__ import annotations import datetime from typing import TYPE_CHECKING +from dagger import Secret from pipelines.actions import environments from pipelines.utils import get_file_contents, get_secret_host_variable -from dagger import Secret if TYPE_CHECKING: - from pipelines.contexts import ConnectorContext from dagger import Container + from pipelines.contexts import ConnectorContext async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container) -> list[str]: diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/builds/__init__.py index c2a7823c391..3d92eabf623 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/builds/__init__.py +++ b/airbyte-ci/connectors/pipelines/pipelines/builds/__init__.py @@ -8,11 +8,11 @@ from __future__ import annotations import platform import anyio +from connector_ops.utils import ConnectorLanguage +from dagger import Platform from pipelines.bases import ConnectorReport, StepResult from pipelines.builds import common, java_connectors, python_connectors from pipelines.contexts import ConnectorContext -from connector_ops.utils import ConnectorLanguage -from dagger import Platform class NoBuildStepForLanguageError(Exception): diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/common.py b/airbyte-ci/connectors/pipelines/pipelines/builds/common.py index dc03a5c951b..d8de2bffc4a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/builds/common.py +++ b/airbyte-ci/connectors/pipelines/pipelines/builds/common.py @@ -1,15 +1,16 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from abc import ABC from typing import Tuple import docker +from dagger import Container, Platform from pipelines.bases import Step, StepResult, StepStatus from pipelines.consts import BUILD_PLATFORMS from pipelines.contexts import ConnectorContext from pipelines.utils import export_container_to_tarball -from dagger import Container, Platform class BuildConnectorImageBase(Step, ABC): diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/builds/java_connectors.py index 9227b9cfc2e..ffd6ff1e8bd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/builds/java_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/builds/java_connectors.py @@ -3,13 +3,9 @@ # from dagger import ExecError, File, QueryError - from pipelines.actions import environments from pipelines.bases import StepResult, StepStatus -from pipelines.builds.common import ( - BuildConnectorImageBase, - BuildConnectorImageForAllPlatformsBase, -) +from pipelines.builds.common import BuildConnectorImageBase, BuildConnectorImageForAllPlatformsBase from pipelines.contexts import ConnectorContext from pipelines.gradle import GradleTask diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/normalization.py b/airbyte-ci/connectors/pipelines/pipelines/builds/normalization.py index 50db75e86c6..3494086eee8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/builds/normalization.py +++ b/airbyte-ci/connectors/pipelines/pipelines/builds/normalization.py @@ -2,10 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from dagger import Platform from pipelines.actions import environments from pipelines.bases import Step, StepResult, StepStatus from pipelines.contexts import ConnectorContext -from dagger import Platform # TODO this class could be deleted diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py index d7316d5f2ae..1fe82c244cf 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py @@ -11,7 +11,7 @@ from typing import List, Set, Tuple import anyio import click -from connector_ops.utils import ConnectorLanguage, console, get_all_connectors_in_repo, SupportLevelEnum +from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, console, get_all_connectors_in_repo from pipelines import main_logger from pipelines.bases import ConnectorWithModifiedFiles from pipelines.builds import run_connector_build_pipeline @@ -21,11 +21,7 @@ from pipelines.github import update_global_commit_status_check_for_tests from pipelines.pipelines.connectors import run_connectors_pipelines from pipelines.publish import reorder_contexts, run_connector_publish_pipeline from pipelines.tests import run_connector_test_pipeline -from pipelines.utils import ( - DaggerPipelineCommand, - get_connector_modified_files, - get_modified_connectors, -) +from pipelines.utils import DaggerPipelineCommand, get_connector_modified_files, get_modified_connectors # HELPERS @@ -181,10 +177,7 @@ def connectors( @click.option( "--code-tests-only", is_flag=True, - help=( - "Only execute code tests. " - "Metadata checks, QA, and acceptance tests will be skipped." - ), + help=("Only execute code tests. " "Metadata checks, QA, and acceptance tests will be skipped."), default=False, type=bool, ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/metadata.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/metadata.py index 50b2d2077be..a9988db89fe 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/metadata.py +++ b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/metadata.py @@ -12,12 +12,7 @@ from pipelines.pipelines.metadata import ( run_metadata_upload_pipeline, run_metadata_validation_pipeline, ) -from pipelines.utils import ( - DaggerPipelineCommand, - get_all_metadata_files, - get_expected_metadata_files, - get_modified_metadata_files, -) +from pipelines.utils import DaggerPipelineCommand, get_all_metadata_files, get_expected_metadata_files, get_modified_metadata_files # MAIN GROUP diff --git a/airbyte-ci/connectors/pipelines/pipelines/contexts.py b/airbyte-ci/connectors/pipelines/pipelines/contexts.py index 7fd8f333b4f..897006696f9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/contexts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/contexts.py @@ -17,7 +17,6 @@ from anyio import Path from asyncer import asyncify from dagger import Client, Directory, Secret from github import PullRequest - from pipelines import hacks from pipelines.actions import secrets from pipelines.bases import CIContext, ConnectorReport, ConnectorWithModifiedFiles, Report diff --git a/airbyte-ci/connectors/pipelines/pipelines/format/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/format/__init__.py index 0adfc81294e..730874bd6b9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/format/__init__.py +++ b/airbyte-ci/connectors/pipelines/pipelines/format/__init__.py @@ -10,13 +10,13 @@ from typing import List, Optional import anyio import dagger +from connector_ops.utils import ConnectorLanguage from pipelines.actions import environments from pipelines.bases import ConnectorReport, Step, StepResult, StepStatus from pipelines.contexts import ConnectorContext from pipelines.format import java_connectors, python_connectors from pipelines.git import GitPushChanges from pipelines.pipelines.connectors import run_report_complete_pipeline -from connector_ops.utils import ConnectorLanguage class NoFormatStepForLanguageError(Exception): diff --git a/airbyte-ci/connectors/pipelines/pipelines/git.py b/airbyte-ci/connectors/pipelines/pipelines/git.py index 675131cc8b7..acf23c2e8ee 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/git.py +++ b/airbyte-ci/connectors/pipelines/pipelines/git.py @@ -2,10 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from dagger import Client, Directory, Secret from pipelines.actions import environments from pipelines.bases import Step, StepResult from pipelines.github import AIRBYTE_GITHUB_REPO -from dagger import Client, Directory, Secret class GitPushChanges(Step): diff --git a/airbyte-ci/connectors/pipelines/pipelines/github.py b/airbyte-ci/connectors/pipelines/pipelines/github.py index e28134f00ed..fd6bb7e4753 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/github.py +++ b/airbyte-ci/connectors/pipelines/pipelines/github.py @@ -9,9 +9,9 @@ from __future__ import annotations import os from typing import TYPE_CHECKING, Optional +from connector_ops.utils import console from pipelines import main_logger from pipelines.bases import CIContext -from connector_ops.utils import console if TYPE_CHECKING: from logging import Logger diff --git a/airbyte-ci/connectors/pipelines/pipelines/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/gradle.py index 1b22992c32b..c1dbf283d7a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/gradle.py +++ b/airbyte-ci/connectors/pipelines/pipelines/gradle.py @@ -8,7 +8,6 @@ from abc import ABC from typing import ClassVar, Tuple from dagger import CacheVolume, Container, Directory, QueryError - from pipelines import consts from pipelines.actions import environments from pipelines.bases import Step, StepResult @@ -29,10 +28,7 @@ class GradleTask(Step, ABC): gradle_task_name: ClassVar gradle_task_options: Tuple[str, ...] = () - def __init__( - self, context: PipelineContext, - with_java_cdk_snapshot: bool = True - ) -> None: + def __init__(self, context: PipelineContext, with_java_cdk_snapshot: bool = True) -> None: super().__init__(context) self.with_java_cdk_snapshot = with_java_cdk_snapshot @@ -95,9 +91,7 @@ class GradleTask(Step, ABC): .with_(environments.mounted_connector_secrets(self.context, f"{self.context.connector.code_directory}/secrets")) ) if self.with_java_cdk_snapshot: - connector_under_test = connector_under_test.with_exec( - ["./gradlew", ":airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded"] - ) + connector_under_test = connector_under_test.with_exec(["./gradlew", ":airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded"]) connector_under_test = connector_under_test.with_exec(self._get_gradle_command()) results = await self.get_step_result(connector_under_test) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/steps.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/steps.py index 6616e8877bf..c2456122778 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/steps.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/steps.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + """The actions package is made to declare reusable pipeline components.""" from __future__ import annotations diff --git a/airbyte-ci/connectors/pipelines/pipelines/pipelines/connectors.py b/airbyte-ci/connectors/pipelines/pipelines/pipelines/connectors.py index 78f3a87e5a9..5acb57b9558 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/pipelines/connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/pipelines/connectors.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + """This module groups the functions to run full pipelines for connector testing.""" import sys @@ -9,12 +10,12 @@ from typing import Callable, List, Optional import anyio import dagger +from connector_ops.utils import ConnectorLanguage +from dagger import Config from pipelines.actions import environments from pipelines.bases import NoOpStep, Report, StepResult, StepStatus from pipelines.contexts import ConnectorContext, ContextState from pipelines.utils import create_and_open_file -from connector_ops.utils import ConnectorLanguage -from dagger import Config GITHUB_GLOBAL_CONTEXT = "[POC please ignore] Connectors CI" GITHUB_GLOBAL_DESCRIPTION = "Running connectors tests" diff --git a/airbyte-ci/connectors/pipelines/pipelines/pipelines/metadata.py b/airbyte-ci/connectors/pipelines/pipelines/pipelines/metadata.py index ad97646fb07..d5288ef05ba 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/pipelines/metadata.py +++ b/airbyte-ci/connectors/pipelines/pipelines/pipelines/metadata.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import functools import uuid from pathlib import Path diff --git a/airbyte-ci/connectors/pipelines/pipelines/publish.py b/airbyte-ci/connectors/pipelines/pipelines/publish.py index d5e48eb9875..2948cc9ef86 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/publish.py +++ b/airbyte-ci/connectors/pipelines/pipelines/publish.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import json import uuid from typing import List, Tuple diff --git a/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py b/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py index d1bd0b652d4..da36bb015eb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import importlib.metadata import os diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/tests/__init__.py index 2272c006e13..0d3d26c27e1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/tests/__init__.py +++ b/airbyte-ci/connectors/pipelines/pipelines/tests/__init__.py @@ -9,16 +9,11 @@ from typing import List import anyio import asyncer from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage - from pipelines.bases import ConnectorReport, StepResult from pipelines.contexts import ConnectorContext from pipelines.pipelines.metadata import MetadataValidation from pipelines.tests import java_connectors, python_connectors -from pipelines.tests.common import ( - QaChecks, - VersionFollowsSemverCheck, - VersionIncrementCheck, -) +from pipelines.tests.common import QaChecks, VersionFollowsSemverCheck, VersionIncrementCheck LANGUAGE_MAPPING = { "run_all_tests": { diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py index a56bb400b8a..052547ac46e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py +++ b/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py @@ -8,14 +8,10 @@ from typing import List, Optional import anyio from dagger import File, QueryError - from pipelines.actions import environments, secrets from pipelines.bases import StepResult, StepStatus from pipelines.builds import LOCAL_BUILD_PLATFORM -from pipelines.builds.java_connectors import ( - BuildConnectorDistributionTar, - BuildConnectorImage, -) +from pipelines.builds.java_connectors import BuildConnectorDistributionTar, BuildConnectorImage from pipelines.builds.normalization import BuildOrPullNormalization from pipelines.contexts import ConnectorContext from pipelines.gradle import GradleTask diff --git a/airbyte-ci/connectors/pipelines/tests/conftest.py b/airbyte-ci/connectors/pipelines/tests/conftest.py index 188fbd44de2..47cfb0fc195 100644 --- a/airbyte-ci/connectors/pipelines/tests/conftest.py +++ b/airbyte-ci/connectors/pipelines/tests/conftest.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import os import sys from pathlib import Path diff --git a/airbyte-ci/connectors/pipelines/tests/test_bases.py b/airbyte-ci/connectors/pipelines/tests/test_bases.py index 53b8cee4bdb..e1f1ebee2f7 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_bases.py +++ b/airbyte-ci/connectors/pipelines/tests/test_bases.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from datetime import timedelta import anyio diff --git a/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py index 42c3f351751..0f9cda290e2 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from typing import Callable import pytest diff --git a/airbyte-ci/connectors/pipelines/tests/test_environments.py b/airbyte-ci/connectors/pipelines/tests/test_environments.py index c941b304579..db48914fdeb 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_environments.py +++ b/airbyte-ci/connectors/pipelines/tests/test_environments.py @@ -1,3 +1,5 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_gradle.py b/airbyte-ci/connectors/pipelines/tests/test_gradle.py index f6623121ae8..e45027c860d 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_gradle.py +++ b/airbyte-ci/connectors/pipelines/tests/test_gradle.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from pathlib import Path import pytest diff --git a/airbyte-ci/connectors/pipelines/tests/test_publish.py b/airbyte-ci/connectors/pipelines/tests/test_publish.py index 03af1317564..9bcf38a9bfc 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_publish.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import json import random from typing import List diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py index 50b1921e2f0..0bdcd5158c9 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import datetime import pathlib import time diff --git a/airbyte-ci/connectors/pipelines/tests/test_utils.py b/airbyte-ci/connectors/pipelines/tests/test_utils.py index 4816356cbee..58b9c9fd78c 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_utils.py +++ b/airbyte-ci/connectors/pipelines/tests/test_utils.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from pathlib import Path from unittest import mock diff --git a/airbyte-ci/connectors/pipelines/tests/utils.py b/airbyte-ci/connectors/pipelines/tests/utils.py index fd3a832c706..fc1c4ae5d6e 100644 --- a/airbyte-ci/connectors/pipelines/tests/utils.py +++ b/airbyte-ci/connectors/pipelines/tests/utils.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import random from connector_ops.utils import Connector, ConnectorLanguage, get_all_connectors_in_repo diff --git a/airbyte-ci/connectors/qa-engine/qa_engine/constants.py b/airbyte-ci/connectors/qa-engine/qa_engine/constants.py index 832e5706c2a..92d98ae6172 100644 --- a/airbyte-ci/connectors/qa-engine/qa_engine/constants.py +++ b/airbyte-ci/connectors/qa-engine/qa_engine/constants.py @@ -1,6 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import os CONNECTOR_TEST_SUMMARY_URL = "https://connectors.airbyte.com/files/generated_reports/test_summary" diff --git a/airbyte-ci/connectors/qa-engine/qa_engine/models.py b/airbyte-ci/connectors/qa-engine/qa_engine/models.py index 230b403295d..c5c009a7b08 100644 --- a/airbyte-ci/connectors/qa-engine/qa_engine/models.py +++ b/airbyte-ci/connectors/qa-engine/qa_engine/models.py @@ -5,10 +5,9 @@ from datetime import datetime from typing import List -from pydantic import BaseModel, Field from connector_ops.utils import ConnectorTypeEnum, SupportLevelEnum - +from pydantic import BaseModel, Field PUBLIC_FIELD = Field(..., is_public=True) PRIVATE_FIELD = Field(..., is_public=False) diff --git a/airbyte-ci/connectors/qa-engine/tests/conftest.py b/airbyte-ci/connectors/qa-engine/tests/conftest.py index 49edde917f7..e68c067c61f 100644 --- a/airbyte-ci/connectors/qa-engine/tests/conftest.py +++ b/airbyte-ci/connectors/qa-engine/tests/conftest.py @@ -4,10 +4,10 @@ from datetime import datetime + import pandas as pd import pytest - -from qa_engine.constants import OSS_CATALOG_URL, CLOUD_CATALOG_URL +from qa_engine.constants import CLOUD_CATALOG_URL, OSS_CATALOG_URL from qa_engine.inputs import fetch_remote_catalog diff --git a/airbyte-ci/connectors/qa-engine/tests/test_enrichments.py b/airbyte-ci/connectors/qa-engine/tests/test_enrichments.py index 52a8c4354d3..73ae7fde8c4 100644 --- a/airbyte-ci/connectors/qa-engine/tests/test_enrichments.py +++ b/airbyte-ci/connectors/qa-engine/tests/test_enrichments.py @@ -7,7 +7,6 @@ import re import pandas as pd import pytest - from qa_engine import enrichments diff --git a/airbyte-ci/connectors/qa-engine/tests/test_main.py b/airbyte-ci/connectors/qa-engine/tests/test_main.py index 692e80a8c98..2f532774c7f 100644 --- a/airbyte-ci/connectors/qa-engine/tests/test_main.py +++ b/airbyte-ci/connectors/qa-engine/tests/test_main.py @@ -4,8 +4,8 @@ import pytest -from qa_engine import main from click.testing import CliRunner +from qa_engine import main @pytest.mark.parametrize("create_prs", [False, True]) diff --git a/airbyte-ci/connectors/qa-engine/tests/test_validations.py b/airbyte-ci/connectors/qa-engine/tests/test_validations.py index 1725553747f..d31271f2067 100644 --- a/airbyte-ci/connectors/qa-engine/tests/test_validations.py +++ b/airbyte-ci/connectors/qa-engine/tests/test_validations.py @@ -3,11 +3,11 @@ # +from unittest.mock import MagicMock, call + import pandas as pd import pytest -from unittest.mock import MagicMock, call import requests - from qa_engine import enrichments, inputs, models, validations diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py index 23c9506b30b..b0fb44c41fd 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py @@ -1068,7 +1068,7 @@ class TestBasicRead(BaseTest): cleaned_actual = cleaned_actual or actual complete_diff = "\n".join( - diff_dicts(cleaned_actual if not extra_records else cleaned_actual[:len(expected)], expected, use_markup=False) + diff_dicts(cleaned_actual if not extra_records else cleaned_actual[: len(expected)], expected, use_markup=False) ) for r1, r2 in zip(expected, cleaned_actual): if r1 is None: diff --git a/airbyte-integrations/connectors/source-babelforce/metadata.yaml b/airbyte-integrations/connectors/source-babelforce/metadata.yaml index 79c5ae71d67..9425eae3c84 100644 --- a/airbyte-integrations/connectors/source-babelforce/metadata.yaml +++ b/airbyte-integrations/connectors/source-babelforce/metadata.yaml @@ -18,7 +18,6 @@ data: name: Babelforce releaseDate: 2022-05-09 releaseStage: alpha - supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/babelforce tags: - language:lowcode diff --git a/airbyte-integrations/connectors/source-babelforce/source_babelforce/manifest.yaml b/airbyte-integrations/connectors/source-babelforce/source_babelforce/manifest.yaml index fd20d1b6feb..40b0785582b 100644 --- a/airbyte-integrations/connectors/source-babelforce/source_babelforce/manifest.yaml +++ b/airbyte-integrations/connectors/source-babelforce/source_babelforce/manifest.yaml @@ -20,8 +20,8 @@ streams: http_method: GET request_parameters: {} request_headers: - X-Auth-Access-ID: '{{ config[''access_key_id''] }}' - X-Auth-Access-Token: '{{ config[''access_token''] }}' + X-Auth-Access-ID: "{{ config['access_key_id'] }}" + X-Auth-Access-Token: "{{ config['access_token'] }}" authenticator: type: NoAuth request_body_json: {} @@ -44,18 +44,18 @@ streams: pagination_strategy: type: CursorPagination page_size: 100 - cursor_value: '{{ response[''pagination''][''current''] + 1 }}' - stop_condition: '{{ ''current'' not in response[''pagination''] }}' + cursor_value: "{{ response['pagination']['current'] + 1 }}" + stop_condition: "{{ 'current' not in response['pagination'] }}" incremental_sync: type: DatetimeBasedCursor cursor_field: dateCreated cursor_datetime_formats: - - '%s' - datetime_format: '%s' + - "%s" + datetime_format: "%s" start_datetime: type: MinMaxDatetime - datetime: '{{ config[''date_created_from''] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ config['date_created_from'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" start_time_option: inject_into: request_parameter field_name: filters.dateCreated.from @@ -66,8 +66,8 @@ streams: type: RequestOption end_datetime: type: MinMaxDatetime - datetime: '{{ config[''date_created_to''] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ config['date_created_to'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" spec: documentation_url: https://example.org diff --git a/airbyte-integrations/connectors/source-commercetools/metadata.yaml b/airbyte-integrations/connectors/source-commercetools/metadata.yaml index f9b417590f9..dcbb88d3faf 100644 --- a/airbyte-integrations/connectors/source-commercetools/metadata.yaml +++ b/airbyte-integrations/connectors/source-commercetools/metadata.yaml @@ -18,7 +18,6 @@ data: license: MIT name: Commercetools releaseStage: alpha - supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/commercetools tags: - language:lowcode diff --git a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml index 18abaaa6778..3afe5254441 100644 --- a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml +++ b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml @@ -19,7 +19,6 @@ data: name: Dockerhub releaseDate: 2022-05-20 releaseStage: alpha - supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/dockerhub tags: - language:lowcode diff --git a/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/manifest.yaml b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/manifest.yaml index b9f14fa7013..936cd50cda9 100644 --- a/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/manifest.yaml +++ b/airbyte-integrations/connectors/source-dockerhub/source_dockerhub/manifest.yaml @@ -25,7 +25,7 @@ streams: type: HttpRequester url_base: >- https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/alpine:pull/ - path: '' + path: "" authenticator: type: NoAuth http_method: GET diff --git a/airbyte-integrations/connectors/source-exchange-rates/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/configured_catalog.json index 87445b1949e..5add0beeec9 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-exchange-rates/integration_tests/configured_catalog.json @@ -6,10 +6,10 @@ "json_schema": { "type": "object", "properties": { - "success":{ + "success": { "type": ["null", "boolean"] }, - "timestamp":{ + "timestamp": { "type": ["null", "integer"] }, "base": { diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/manifest.yaml b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/manifest.yaml index d2b01e13404..82f5825f366 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/manifest.yaml +++ b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/manifest.yaml @@ -33,16 +33,16 @@ streams: type: NoPagination incremental_sync: type: DatetimeBasedCursor - datetime_format: '%Y-%m-%d' + datetime_format: "%Y-%m-%d" cursor_field: date start_datetime: type: MinMaxDatetime - datetime: '{{ config[''start_date''] }}' - datetime_format: '%Y-%m-%d' + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%d" end_datetime: type: MinMaxDatetime - datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" metadata: autoImportSchema: exchange_rates: true diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/schemas/exchange_rates.json b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/schemas/exchange_rates.json index 98a6f1ba286..e68948f2b05 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/schemas/exchange_rates.json +++ b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/schemas/exchange_rates.json @@ -4,10 +4,10 @@ "additionalProperties": true, "required": ["base", "date", "rates"], "properties": { - "success":{ + "success": { "type": ["null", "boolean"] }, - "timestamp":{ + "timestamp": { "type": ["null", "integer"] }, "base": { diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/manifest.yaml b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/manifest.yaml index 6ee2cd9b71e..aec61695eca 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/manifest.yaml +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/manifest.yaml @@ -35,7 +35,7 @@ definitions: requester: type: HttpRequester url_base: https://{{ config['company'] }}.hellobaton.com/api/ - path: '{{ parameters.path }}' + path: "{{ parameters.path }}" http_method: GET request_parameters: {} request_headers: {} diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json index 77f7d8e30ed..1c56c2670dc 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/activity.json @@ -3,18 +3,18 @@ "type": "object", "additionalProperties": true, "properties": { - "id": {"type": ["integer", "null"]}, - "_self": {"type": ["string", "null"]}, - "type": {"type": ["string", "null"]}, - "group": {"type": ["string", "null"]}, - "parent": {"type": ["string", "null"]}, - "child": {"type": ["string", "null"]}, - "actor": {"type": ["string", "null"]}, - "project": {"type": ["string", "null"]}, - "parent_type": {"type": ["string", "null"]}, - "child_type": {"type": ["string", "null"]}, - "meta": {"type": ["object", "null"]}, - "created": {"type": ["string", "null"], "format": "date-time"}, - "modified": {"type": ["string", "null"], "format": "date-time"} + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "type": { "type": ["string", "null"] }, + "group": { "type": ["string", "null"] }, + "parent": { "type": ["string", "null"] }, + "child": { "type": ["string", "null"] }, + "actor": { "type": ["string", "null"] }, + "project": { "type": ["string", "null"] }, + "parent_type": { "type": ["string", "null"] }, + "child_type": { "type": ["string", "null"] }, + "meta": { "type": ["object", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json index febd8f8796d..dcd834dc03c 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/companies.json @@ -3,11 +3,11 @@ "type": "object", "additionalProperties": true, "properties": { - "id": {"type": ["integer", "null"]}, - "_self": {"type": ["string", "null"]}, - "name": {"type": ["string", "null"]}, - "type": {"type": ["string", "null"]}, - "created": {"type": ["string", "null"], "format": "date-time"}, - "modified": {"type": ["string", "null"], "format": "date-time"} + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "name": { "type": ["string", "null"] }, + "type": { "type": ["string", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json index 39e1fe42498..1e5a6479cff 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/milestones.json @@ -3,42 +3,42 @@ "type": "object", "additionalProperties": true, "properties": { - "id": {"type": ["integer", "null"]}, - "_self": {"type": ["string", "null"]}, - "title": {"type": ["string", "null"]}, - "description": {"type": ["string", "null"]}, - "project": {"type": ["string", "null"]}, - "task_list": {"type": ["string", "null"]}, + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "title": { "type": ["string", "null"] }, + "description": { "type": ["string", "null"] }, + "project": { "type": ["string", "null"] }, + "task_list": { "type": ["string", "null"] }, "phase": { "type": ["object", "null"], "properties": { - "id": {"type": ["integer", "null"]}, - "_self": {"type": ["string", "null"]}, - "name": {"type": ["string", "null"]}, - "created": {"type": ["string", "null"], "format": "date-time"}, - "modified": {"type": ["string", "null"], "format": "date-time"} + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "name": { "type": ["string", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } }, - "deadline_fixed": {"type": ["boolean", "null"]}, - "deadline_datetime": {"type": ["string", "null"], "format": "date-time"}, + "deadline_fixed": { "type": ["boolean", "null"] }, + "deadline_datetime": { "type": ["string", "null"], "format": "date-time" }, "risk_profiles": { "type": ["array", "null"], "items": { "type": ["object", "null"], "properties": { - "id": {"type": ["integer", "null"]}, - "risk_level": {"type": ["string", "null"]}, - "formula": {"type": ["string", "null"]}, - "over_run": {"type": ["integer", "null"]} + "id": { "type": ["integer", "null"] }, + "risk_level": { "type": ["string", "null"] }, + "formula": { "type": ["string", "null"] }, + "over_run": { "type": ["integer", "null"] } } } }, - "start_datetime": {"type": ["string", "null"]}, - "finish_datetime": {"type": ["string", "null"], "format": "date-time"}, - "created_from": {"type": ["string", "null"]}, - "duration": {"type": ["integer", "null"]}, - "feedback_list": {"type": ["string", "null"]}, - "created": {"type": ["string", "null"], "format": "date-time"}, - "modified": {"type": ["string", "null"], "format": "date-time"} + "start_datetime": { "type": ["string", "null"] }, + "finish_datetime": { "type": ["string", "null"], "format": "date-time" }, + "created_from": { "type": ["string", "null"] }, + "duration": { "type": ["integer", "null"] }, + "feedback_list": { "type": ["string", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json index a392a8717e4..e7ab1ae41df 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/phases.json @@ -3,11 +3,11 @@ "type": "object", "additionalProperties": true, "properties": { - "id": {"type": ["integer", "null"]}, - "_self": {"type": ["string", "null"]}, - "name": {"type": ["string", "null"]}, - "order": {"type": ["integer", "null"]}, - "created": {"type": ["string", "null"], "format": "date-time"}, - "modified": {"type": ["string", "null"], "format": "date-time"} + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "name": { "type": ["string", "null"] }, + "order": { "type": ["integer", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json index d5b3df0ebb0..5aacde35356 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/project_attachments.json @@ -3,16 +3,16 @@ "type": "object", "additionalProperties": true, "properties": { - "id": {"type": ["integer", "null"]}, - "_self": {"type": ["string", "null"]}, - "project": {"type": ["string", "null"]}, - "url": {"type": ["string", "null"]}, - "label": {"type": ["string", "null"]}, - "created_by": {"type": ["string", "null"]}, - "type": {"type": ["string", "null"]}, - "is_sow": {"type": ["boolean", "null"]}, - "original_filename": {"type": ["string", "null"]}, - "created": {"type": ["string", "null"], "format": "date-time"}, - "modified": {"type": ["string", "null"], "format": "date-time"} + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "project": { "type": ["string", "null"] }, + "url": { "type": ["string", "null"] }, + "label": { "type": ["string", "null"] }, + "created_by": { "type": ["string", "null"] }, + "type": { "type": ["string", "null"] }, + "is_sow": { "type": ["boolean", "null"] }, + "original_filename": { "type": ["string", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json index 41d8122843b..c36ca3de4a0 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/projects.json @@ -3,63 +3,66 @@ "type": "object", "additionalProperties": true, "properties": { - "_self": {"type": ["string", "null"]}, - "annual_contract_value": {"type": ["string", "null"]}, - "archived": {"type": ["boolean", "null"]}, - "attachment_list": {"type": ["string", "null"]}, - "client_systems": {"type": ["string", "null"]}, + "_self": { "type": ["string", "null"] }, + "annual_contract_value": { "type": ["string", "null"] }, + "archived": { "type": ["boolean", "null"] }, + "attachment_list": { "type": ["string", "null"] }, + "client_systems": { "type": ["string", "null"] }, "companies": { "type": ["array", "null"], "items": { "type": ["string", "null"] } }, - "completed_datetime": {"type": ["string", "null"], "format": "date-time"}, - "cost": {"type": ["integer", "null"]}, - "created": {"type": ["string", "null"], "format": "date-time"}, - "created_from": {"type": ["string", "null"]}, - "created_from_template": {"type": ["string", "null"]}, - "creator": {"type": ["string", "null"]}, - "custom_field_list": { "type": ["string", "null"]}, - "deadline_datetime": {"type": ["string", "null"], "format": "date-time"}, - "estimated_duration": {"type": ["integer", "null"]}, - "id": {"type": ["integer", "null"]}, - "implementation_budget": {"type": ["string", "null"]}, - "milestone_feedback_list": { "type": ["string", "null"]}, - "milestone_list": { "type": ["string", "null"]}, - "modified": {"type": ["string", "null"], "format": "date-time"}, + "completed_datetime": { "type": ["string", "null"], "format": "date-time" }, + "cost": { "type": ["integer", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "created_from": { "type": ["string", "null"] }, + "created_from_template": { "type": ["string", "null"] }, + "creator": { "type": ["string", "null"] }, + "custom_field_list": { "type": ["string", "null"] }, + "deadline_datetime": { "type": ["string", "null"], "format": "date-time" }, + "estimated_duration": { "type": ["integer", "null"] }, + "id": { "type": ["integer", "null"] }, + "implementation_budget": { "type": ["string", "null"] }, + "milestone_feedback_list": { "type": ["string", "null"] }, + "milestone_list": { "type": ["string", "null"] }, + "modified": { "type": ["string", "null"], "format": "date-time" }, "phase": { "type": ["object", "null"], "properties": { - "id": {"type": ["integer", "null"]}, - "_self": {"type": ["string", "null"]}, - "name": {"type": ["string", "null"]}, - "order": {"type": ["integer", "null"]}, - "created": {"type": ["string", "null"], "format": "date-time"}, - "modified": {"type": ["string", "null"], "format": "date-time"} + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "name": { "type": ["string", "null"] }, + "order": { "type": ["integer", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } }, - "project_phases_list": {"type": ["string", "null"]}, + "project_phases_list": { "type": ["string", "null"] }, "project_user_list": { "type": ["string", "null"] }, "risk_profiles": { "type": ["array", "null"], "items": { "type": ["object", "null"], "properties": { - "id": {"type": ["integer", "null"]}, - "risk_score": {"type": ["number", "null"]}, - "level": {"type": ["string", "null"]}, - "variance": {"type": ["integer", "null"]}, - "formula": {"type": ["string", "null"]}, - "projected_golive_datetime": {"type": ["string", "null"], "format": "date-time"} + "id": { "type": ["integer", "null"] }, + "risk_score": { "type": ["number", "null"] }, + "level": { "type": ["string", "null"] }, + "variance": { "type": ["integer", "null"] }, + "formula": { "type": ["string", "null"] }, + "projected_golive_datetime": { + "type": ["string", "null"], + "format": "date-time" + } } } }, - "start_datetime": {"type": ["string", "null"], "format": "date-time"}, - "started_datetime": {"type": ["string", "null"], "format": "date-time"}, - "status": {"type": ["string", "null"]}, - "task_list": {"type": ["string", "null"]}, - "time_entry_list": {"type": ["string", "null"]}, - "title": {"type": ["string", "null"]} + "start_datetime": { "type": ["string", "null"], "format": "date-time" }, + "started_datetime": { "type": ["string", "null"], "format": "date-time" }, + "status": { "type": ["string", "null"] }, + "task_list": { "type": ["string", "null"] }, + "time_entry_list": { "type": ["string", "null"] }, + "title": { "type": ["string", "null"] } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json index 345d50439f3..0c41e829e19 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/task_attachments.json @@ -3,19 +3,19 @@ "type": "object", "additionalProperties": true, "properties": { - "id": {"type": ["integer", "null"]}, - "_self": {"type": ["string", "null"]}, - "task": {"type": ["string", "null"]}, - "url": {"type": ["string", "null"]}, - "type": {"type": ["string", "null"]}, - "label": {"type": ["string", "null"]}, - "deliverable": {"type": ["boolean", "null"]}, - "requires_approval": {"type": ["boolean", "null"]}, - "approved": {"type": ["boolean", "null"]}, - "revision_task": {"type": ["string", "null"]}, - "original_filename": {"type": ["string", "null"]}, - "created_by": {"type": ["string", "null"]}, - "created": {"type": ["string", "null"]}, - "modified": {"type": ["string", "null"]} + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "task": { "type": ["string", "null"] }, + "url": { "type": ["string", "null"] }, + "type": { "type": ["string", "null"] }, + "label": { "type": ["string", "null"] }, + "deliverable": { "type": ["boolean", "null"] }, + "requires_approval": { "type": ["boolean", "null"] }, + "approved": { "type": ["boolean", "null"] }, + "revision_task": { "type": ["string", "null"] }, + "original_filename": { "type": ["string", "null"] }, + "created_by": { "type": ["string", "null"] }, + "created": { "type": ["string", "null"] }, + "modified": { "type": ["string", "null"] } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json index 8563ba277bb..b2ff5e9c16d 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/tasks.json @@ -3,49 +3,55 @@ "type": "object", "additionalProperties": true, "properties": { - "id": {"type": ["integer", "null"]}, - "_self": {"type": ["string", "null"]}, - "title": {"type": ["string", "null"]}, - "description": {"type": ["string", "null"]}, - "project": {"type": ["string", "null"]}, - "status": {"type": ["string", "null"]}, + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "title": { "type": ["string", "null"] }, + "description": { "type": ["string", "null"] }, + "project": { "type": ["string", "null"] }, + "status": { "type": ["string", "null"] }, "dependencies": { "type": ["array", "null"], "items": { "type": ["string", "null"] } }, - "start_datetime": {"type": ["string", "null"], "format": "date-time"}, - "due_datetime": {"type": ["string", "null"], "format": "date-time"}, - "started_datetime": {"type": ["string", "null"], "format": "date-time"}, - "finished_datetime": {"type": ["string", "null"], "format": "date-time"}, - "started_overridden_datetime": {"type": ["string", "null"], "format": "date-time"}, - "finished_overridden_datetime": {"type": ["string", "null"], "format": "date-time"}, - "estimated_duration": {"type": ["integer", "null"]}, - "milestone": {"type": ["string", "null"]}, - "created_by": {"type": ["string", "null"]}, - "assigned_to": {"type": ["string", "null"]}, - "created_from": {"type": ["string", "null"]}, + "start_datetime": { "type": ["string", "null"], "format": "date-time" }, + "due_datetime": { "type": ["string", "null"], "format": "date-time" }, + "started_datetime": { "type": ["string", "null"], "format": "date-time" }, + "finished_datetime": { "type": ["string", "null"], "format": "date-time" }, + "started_overridden_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "finished_overridden_datetime": { + "type": ["string", "null"], + "format": "date-time" + }, + "estimated_duration": { "type": ["integer", "null"] }, + "milestone": { "type": ["string", "null"] }, + "created_by": { "type": ["string", "null"] }, + "assigned_to": { "type": ["string", "null"] }, + "created_from": { "type": ["string", "null"] }, "risk_profiles": { "type": ["array", "null"], "items": { "type": ["object", "null"], "properties": { - "id": {"type": ["integer", "null"]}, - "risk_level": {"type": ["string", "null"]}, - "formula": {"type": ["string", "null"]}, - "over_run": {"type": ["integer", "null"]}, - "task_variance": {"type": ["integer", "null"]}, - "cool_down": {"type": ["integer", "null"]}, - "reason": {"type": ["integer", "null"]}, - "duration": {"type": ["integer", "null"]}, - "estimated_duration": {"type": ["integer", "null"]} + "id": { "type": ["integer", "null"] }, + "risk_level": { "type": ["string", "null"] }, + "formula": { "type": ["string", "null"] }, + "over_run": { "type": ["integer", "null"] }, + "task_variance": { "type": ["integer", "null"] }, + "cool_down": { "type": ["integer", "null"] }, + "reason": { "type": ["integer", "null"] }, + "duration": { "type": ["integer", "null"] }, + "estimated_duration": { "type": ["integer", "null"] } } } }, - "time_entry_list": {"type": ["string", "null"]}, - "attachment_list": {"type": ["string", "null"]}, - "created": {"type": ["string", "null"], "format": "date-time"}, - "modified": {"type": ["string", "null"], "format": "date-time"} + "time_entry_list": { "type": ["string", "null"] }, + "attachment_list": { "type": ["string", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json index ebd8b40c57e..e364f7ddb2b 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/templates.json @@ -3,32 +3,32 @@ "type": "object", "additionalProperties": true, "properties": { - "id": {"type": ["integer", "null"]}, - "_self": {"type": ["string", "null"]}, - "title": {"type": ["string", "null"]}, - "status": {"type": ["string", "null"]}, - "cost": {"type": ["string", "null"]}, - "annual_contract_value": {"type": ["string", "null"]}, - "implementation_budget": {"type": ["string", "null"]}, - "estimated_duration": {"type": ["integer", "null"]}, - "created_from_template": {"type": ["string", "null"]}, - "created_from": {"type": ["string", "null"]}, - "start_datetime": {"type": ["string", "null"], "format": "date-time"}, - "started_datetime": {"type": ["string", "null"], "format": "date-time"}, - "deadline_datetime": {"type": ["string", "null"], "format": "date-time"}, - "completed_datetime": {"type": ["string", "null"], "format": "date-time"}, - "client_systems": {"type": ["string", "null"]}, + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "title": { "type": ["string", "null"] }, + "status": { "type": ["string", "null"] }, + "cost": { "type": ["string", "null"] }, + "annual_contract_value": { "type": ["string", "null"] }, + "implementation_budget": { "type": ["string", "null"] }, + "estimated_duration": { "type": ["integer", "null"] }, + "created_from_template": { "type": ["string", "null"] }, + "created_from": { "type": ["string", "null"] }, + "start_datetime": { "type": ["string", "null"], "format": "date-time" }, + "started_datetime": { "type": ["string", "null"], "format": "date-time" }, + "deadline_datetime": { "type": ["string", "null"], "format": "date-time" }, + "completed_datetime": { "type": ["string", "null"], "format": "date-time" }, + "client_systems": { "type": ["string", "null"] }, "phase": { "type": ["object", "null"], "properties": { - "id": {"type": ["integer", "null"]}, - "_self": {"type": ["string", "null"]}, - "order": {"type": ["integer", "null"]}, - "created": {"type": ["string", "null"], "format": "date-time"}, - "modified": {"type": ["string", "null"], "format": "date-time"} + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "order": { "type": ["integer", "null"] }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } }, - "creator": {"type": ["string", "null"]}, - "task_list": {"type": ["string", "null"]} + "creator": { "type": ["string", "null"] }, + "task_list": { "type": ["string", "null"] } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json index ecaa1e1d0ef..b75a95128fc 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/time_entries.json @@ -3,25 +3,25 @@ "type": "object", "additionalProperties": true, "properties": { - "id": {"type": ["integer", "null"]}, - "_self": {"type": ["string", "null"]}, - "user": {"type": ["string", "null"]}, - "created_by": {"type": ["string", "null"]}, - "project": {"type": ["string", "null"]}, - "task": {"type": ["string", "null"]}, + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "user": { "type": ["string", "null"] }, + "created_by": { "type": ["string", "null"] }, + "project": { "type": ["string", "null"] }, + "task": { "type": ["string", "null"] }, "rate": { "type": ["object", "null"], "properties": { - "id": {"type": ["integer", "null"]}, - "hourly_rate": {"type": ["string", "null"]} + "id": { "type": ["integer", "null"] }, + "hourly_rate": { "type": ["string", "null"] } } }, - "started_at": {"type": ["string", "null"], "format": "date-time"}, - "ended_at": {"type": ["string", "null"], "format": "date-time"}, - "reference_date": {"type": ["string", "null"], "format": "date-time"}, - "billable": {"type": ["boolean", "null"]}, - "calculated_duration": {"type": ["integer", "null"]}, - "inputted_duration": {"type": ["integer", "null"]}, - "notes": {"type": ["string", "null"]} + "started_at": { "type": ["string", "null"], "format": "date-time" }, + "ended_at": { "type": ["string", "null"], "format": "date-time" }, + "reference_date": { "type": ["string", "null"], "format": "date-time" }, + "billable": { "type": ["boolean", "null"] }, + "calculated_duration": { "type": ["integer", "null"] }, + "inputted_duration": { "type": ["integer", "null"] }, + "notes": { "type": ["string", "null"] } } } diff --git a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json index c6ed2da4f35..4ce6c1eed02 100644 --- a/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json +++ b/airbyte-integrations/connectors/source-hellobaton/source_hellobaton/schemas/users.json @@ -3,20 +3,20 @@ "type": "object", "additionalProperties": true, "properties": { - "id": {"type": ["integer", "null"]}, - "_self": {"type": ["string", "null"]}, - "first_name": {"type": ["string", "null"]}, - "last_name": {"type": ["string", "null"]}, - "capacity": {"type": ["integer", "null"]}, - "department": {"type": ["object", "null"]}, - "email": {"type": ["string", "null"]}, - "account_type": {"type": ["string", "null"]}, - "job_title": {"type": ["string", "null"]}, - "company": {"type": ["string", "null"]}, - "avatar_url": {"type": ["string", "null"]}, - "created_by": {"type": ["string", "null"]}, - "signed_up_at": {"type": ["string", "null"], "format": "date-time"}, - "created": {"type": ["string", "null"], "format": "date-time"}, - "modified": {"type": ["string", "null"], "format": "date-time"} + "id": { "type": ["integer", "null"] }, + "_self": { "type": ["string", "null"] }, + "first_name": { "type": ["string", "null"] }, + "last_name": { "type": ["string", "null"] }, + "capacity": { "type": ["integer", "null"] }, + "department": { "type": ["object", "null"] }, + "email": { "type": ["string", "null"] }, + "account_type": { "type": ["string", "null"] }, + "job_title": { "type": ["string", "null"] }, + "company": { "type": ["string", "null"] }, + "avatar_url": { "type": ["string", "null"] }, + "created_by": { "type": ["string", "null"] }, + "signed_up_at": { "type": ["string", "null"], "format": "date-time" }, + "created": { "type": ["string", "null"], "format": "date-time" }, + "modified": { "type": ["string", "null"], "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-stock-ticker-api-tutorial/source.py b/airbyte-integrations/connectors/source-stock-ticker-api-tutorial/source.py index c9ec489513c..f03a5373120 100644 --- a/airbyte-integrations/connectors/source-stock-ticker-api-tutorial/source.py +++ b/airbyte-integrations/connectors/source-stock-ticker-api-tutorial/source.py @@ -22,13 +22,14 @@ import argparse # helps parse commandline arguments -import json -import sys -import os -import requests import datetime -from datetime import date -from datetime import timedelta +import json +import os +import sys +from datetime import date, timedelta + +import requests + def read(config, catalog): # Assert required configuration was provided @@ -52,7 +53,7 @@ def read(config, catalog): sys.exit(1) # If we've made it this far, all the configuration is good and we can pull the last 7 days of market data - response = _call_api(ticker=config["stock_ticker"], token = config["api_key"]) + response = _call_api(ticker=config["stock_ticker"], token=config["api_key"]) if response.status_code != 200: # In a real scenario we'd handle this error better :) log_error("Failure occurred when calling Polygon.io API") @@ -62,7 +63,11 @@ def read(config, catalog): # We want to output them one by one as AirbyteMessages results = response.json()["results"] for result in results: - data = {"date": date.fromtimestamp(result["t"]/1000).isoformat(), "stock_ticker": config["stock_ticker"], "price": result["c"]} + data = { + "date": date.fromtimestamp(result["t"] / 1000).isoformat(), + "stock_ticker": config["stock_ticker"], + "price": result["c"], + } record = {"stream": "stock_prices", "data": data, "emitted_at": int(datetime.datetime.now().timestamp()) * 1000} output_message = {"type": "RECORD", "record": record} print(json.dumps(output_message)) @@ -115,23 +120,15 @@ def log_error(error_message): def discover(): catalog = { - "streams": [{ - "name": "stock_prices", - "supported_sync_modes": ["full_refresh"], - "json_schema": { - "properties": { - "date": { - "type": "string" - }, - "price": { - "type": "number" - }, - "stock_ticker": { - "type": "string" - } - } + "streams": [ + { + "name": "stock_prices", + "supported_sync_modes": ["full_refresh"], + "json_schema": { + "properties": {"date": {"type": "string"}, "price": {"type": "number"}, "stock_ticker": {"type": "string"}} + }, } - }] + ] } airbyte_message = {"type": "CATALOG", "catalog": catalog} print(json.dumps(airbyte_message)) @@ -179,9 +176,7 @@ def run(args): read_parser.add_argument("--state", type=str, required=False, help="path to the json-encoded state file") required_read_parser = read_parser.add_argument_group("required named arguments") required_read_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file") - required_read_parser.add_argument( - "--catalog", type=str, required=True, help="path to the catalog used to determine which data to read" - ) + required_read_parser.add_argument("--catalog", type=str, required=True, help="path to the catalog used to determine which data to read") parsed_args = main_parser.parse_args(args) command = parsed_args.command diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml index ceb4dfb3a55..003390c81a7 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/metadata.yaml @@ -18,7 +18,6 @@ data: name: Zendesk Sunshine releaseDate: 2021-07-08 releaseStage: alpha - supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-sunshine tags: - language:low-code diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml index 071a8296800..9e0053abbf6 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/source_zendesk_sunshine/manifest.yaml @@ -91,10 +91,10 @@ definitions: request_body_json: query: _type: - $eq: '{{ stream_partition.type }}' + $eq: "{{ stream_partition.type }}" sort_by: _updated_at asc _updated_at: - start: '{{ stream_interval.start_time.strftime(''%Y-%m-%d %H:%M:%s.%f'')[:-3] }}' + start: "{{ stream_interval.start_time.strftime('%Y-%m-%d %H:%M:%s.%f')[:-3] }}" record_selector: $ref: "#/definitions/selector" paginator: @@ -111,16 +111,16 @@ definitions: type: DatetimeBasedCursor cursor_field: updated_at cursor_datetime_formats: - - '%Y-%m-%dT%H:%M:%S.%f%z' - datetime_format: '%Y-%m-%dT%H:%M:%S.%f%z' + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" start_datetime: type: MinMaxDatetime - datetime: '{{ config[''start_date''] }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: type: MinMaxDatetime - datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' - datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" object_type_policies_stream: type: DeclarativeStream name: object_type_policies @@ -147,7 +147,7 @@ definitions: fields: - path: - object_type - value: '{{ stream_partition.type }}' + value: "{{ stream_partition.type }}" relationship_records_stream: type: DeclarativeStream name: relationship_records @@ -158,7 +158,7 @@ definitions: $ref: "#/definitions/requester" path: relationships/records request_parameters: - type: '{{ stream_partition.type }}' + type: "{{ stream_partition.type }}" record_selector: $ref: "#/definitions/selector" paginator: @@ -203,7 +203,7 @@ spec: description: The date from which you'd like to replicate data for Zendesk Sunshine API, in the format YYYY-MM-DDT00:00:00Z. pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ examples: - - '2021-01-01T00:00:00Z' + - "2021-01-01T00:00:00Z" order: 1 credentials: title: Authorization Method @@ -221,7 +221,7 @@ spec: type: string const: oauth2.0 enum: - - oauth2.0 + - oauth2.0 default: oauth2.0 order: 0 client_id: @@ -256,7 +256,8 @@ spec: api_token: type: string title: API Token - description: API Token. See the docs + description: + API Token. See the docs for information on how to generate this key. airbyte_secret: true email: diff --git a/airbyte-integrations/connectors/tasks.py b/airbyte-integrations/connectors/tasks.py index 3323183fedc..5463f1f1680 100644 --- a/airbyte-integrations/connectors/tasks.py +++ b/airbyte-integrations/connectors/tasks.py @@ -1,6 +1,7 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import os import shutil import tempfile diff --git a/build.gradle b/build.gradle index 0158371abf6..3a94a394ec5 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,6 @@ import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage import com.github.spotbugs.snom.SpotBugsTask +import ru.vyarus.gradle.plugin.python.task.PythonTask // The buildscript block defines dependencies in order for .gradle file evaluation. // This is separate from application dependencies. @@ -38,6 +39,7 @@ plugins { id 'distribution' id 'version-catalog' id 'maven-publish' + id 'ru.vyarus.use-python' } apply from: "$rootDir/publish-repositories.gradle" @@ -555,16 +557,6 @@ subprojects { subproj -> check.dependsOn 'jacocoTestCoverageVerification' } -task('generate') { - dependsOn subprojects.collect { it.getTasksByName('generateComponentManifestClassFiles', true) } - dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) } -} - -task('format') { - dependsOn generate - dependsOn spotlessApply - dependsOn subprojects.collect { it.getTasksByName('airbytePythonFormat', true) } -} // add licenses for python projects. subprojects { @@ -600,6 +592,78 @@ subprojects { licenseTask.dependsOn generateManifestFilesTask } } +} + +python { + envPath = '.venv' + minPythonVersion = '3.9' + scope = 'VIRTUALENV' + installVirtualenv = true + pip 'pip:21.3.1' + // https://github.com/csachs/pyproject-flake8/issues/13 + pip 'flake8:4.0.1' + // flake8 doesn't support pyproject.toml files + // and thus there is the wrapper "pyproject-flake8" for this + pip 'pyproject-flake8:0.0.1a2' + pip 'black:22.3.0' + pip 'mypy:1.4.1' + pip 'isort:5.6.4' + pip 'coverage[toml]:6.3.1' +} + +task('generate') { + dependsOn subprojects.collect { it.getTasksByName('generateComponentManifestClassFiles', true) } + dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) } +} + +license { + header rootProject.file("LICENSE_SHORT") +} + +task licensePythonGlobalFormat(type: com.hierynomus.gradle.license.tasks.LicenseFormat) { + header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT')) + source = fileTree(dir: rootProject.rootDir) + .include("**/*.py") + .exclude("**/.venv/**") + .exclude("**/build/**") + .exclude("**/node_modules/**") + .exclude("**/airbyte_api_client/**") + .exclude("**/__init__.py") + .exclude("airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py") + .exclude("airbyte-integrations/connectors/source-stock-ticker-api-tutorial/source.py") + .exclude("resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py") + .exclude("tools/git_hooks/tests/test_spec_linter.py") + .exclude("tools/schema_generator/schema_generator/infer_schemas.py") + strictCheck = true + dependsOn generate +} + +task isortGlobalFormat(type: PythonTask) { + module = "isort" + command = "--settings-file=${rootProject.file('pyproject.toml').absolutePath} ./" + dependsOn generate +} + +task blackGlobalFormat(type: PythonTask) { + module = "black" + // the line length should match .isort.cfg + command = "--config ${rootProject.file('pyproject.toml').absolutePath} ./" + dependsOn generate +} + +task('format') { + dependsOn generate + dependsOn spotlessApply + dependsOn licensePythonGlobalFormat + dependsOn isortGlobalFormat + dependsOn blackGlobalFormat + tasks.findByName('spotlessApply').mustRunAfter 'generate' + tasks.findByName('licensePythonGlobalFormat').mustRunAfter 'generate' + tasks.findByName('isortGlobalFormat').mustRunAfter 'licensePythonGlobalFormat' + tasks.findByName('blackGlobalFormat').mustRunAfter 'isortGlobalFormat' +} + +subprojects { task listAllDependencies(type: DependencyReportTask) {} } diff --git a/resources/examples/airflow/dags/dag_airbyte_example.py b/resources/examples/airflow/dags/dag_airbyte_example.py index 9af59791a5f..9bd2f8e2332 100644 --- a/resources/examples/airflow/dags/dag_airbyte_example.py +++ b/resources/examples/airflow/dags/dag_airbyte_example.py @@ -1,21 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + from airflow import DAG -from airflow.utils.dates import days_ago -from airflow.providers.airbyte.operators.airbyte import AirbyteTriggerSyncOperator from airflow.models import Variable +from airflow.providers.airbyte.operators.airbyte import AirbyteTriggerSyncOperator +from airflow.utils.dates import days_ago airbyte_connection_id = Variable.get("AIRBYTE_CONNECTION_ID") -with DAG(dag_id='trigger_airbyte_job_example', - default_args={'owner': 'airflow'}, - schedule_interval='@daily', - start_date=days_ago(1) - ) as dag: +with DAG( + dag_id="trigger_airbyte_job_example", default_args={"owner": "airflow"}, schedule_interval="@daily", start_date=days_ago(1) +) as dag: example_sync = AirbyteTriggerSyncOperator( - task_id='airbyte_example', - airbyte_conn_id='airbyte_example', + task_id="airbyte_example", + airbyte_conn_id="airbyte_example", connection_id=airbyte_connection_id, asynchronous=False, timeout=3600, - wait_seconds=3 - ) \ No newline at end of file + wait_seconds=3, + ) diff --git a/resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py b/resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py index a3ca8b1ec6c..623bcdd5bcf 100644 --- a/resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py +++ b/resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py @@ -38,9 +38,7 @@ def get_env_variable(var_name, default=None): if default is not None: return default else: - error_msg = "The environment variable {} was missing, abort...".format( - var_name - ) + error_msg = "The environment variable {} was missing, abort...".format(var_name) raise EnvironmentError(error_msg) @@ -106,8 +104,6 @@ try: import superset_config_docker from superset_config_docker import * # noqa - logger.info( - f"Loaded your Docker configuration at " f"[{superset_config_docker.__file__}]" - ) + logger.info(f"Loaded your Docker configuration at " f"[{superset_config_docker.__file__}]") except ImportError: logger.info("Using default Docker config...") diff --git a/tools/bin/cleanup-workflow-runs.py b/tools/bin/cleanup-workflow-runs.py index 993d67e662b..d2ef9a38cbb 100644 --- a/tools/bin/cleanup-workflow-runs.py +++ b/tools/bin/cleanup-workflow-runs.py @@ -1,14 +1,19 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import argparse import os -import subprocess import re +import subprocess from datetime import datetime, timedelta + from github import Github DAYS_TO_KEEP_ORPHANED_JOBS = 90 -''' +""" This script is intended to be run in conjuction with identify-dormant-workflows.py to keep GH actions clean. @@ -23,14 +28,21 @@ We need to clean up the runs because even if a workflow is deleted, the runs lin We don't want to delete workflow runs newer than 90 days on GH actions, even if the workflow doesn't exist. it's possible that people might test things off the master branch and we don't want to delete their recent runs -''' +""" # Initiate the parser parser = argparse.ArgumentParser() # Add long and short argument parser.add_argument("--pat", "-p", help="Set github personal access token") -parser.add_argument("--delete", "-d", action='store', nargs='*', help="By default, the script will only print runs that will be deleted. Pass --delete to actually delete them") +parser.add_argument( + "--delete", + "-d", + action="store", + nargs="*", + help="By default, the script will only print runs that will be deleted. Pass --delete to actually delete them", +) + def main(): # Read arguments from the command line @@ -43,19 +55,19 @@ def main(): if args.pat: token = args.pat else: - token = os.getenv('GITHUB_TOKEN') + token = os.getenv("GITHUB_TOKEN") if not token: raise Exception("Github personal access token not provided via args and not available in GITHUB_TOKEN variable") g = Github(token) - git_url = subprocess.run(["git", "config", "--get", "remote.origin.url"], check=True, capture_output=True) + git_url = subprocess.run(["git", "config", "--get", "remote.origin.url"], check=True, capture_output=True) # will match both forms (git and https url) of github e.g. # git@github.com:airbytehq/airbyte.git # https://github.com/airbytehq/airbyte.git - git_url_regex = re.compile(r'(?:git@|https://)github\.com[:/](.*?)(\.git|$)') + git_url_regex = re.compile(r"(?:git@|https://)github\.com[:/](.*?)(\.git|$)") re_match = git_url_regex.match(git_url.stdout.decode("utf-8")) repo = g.get_repo(re_match.group(1)) @@ -64,22 +76,24 @@ def main(): runs_to_delete = [] for workflow in workflows: - if not os.path.exists(workflow.path): # it's not in the current branch + if not os.path.exists(workflow.path): # it's not in the current branch runs = workflow.get_runs() for run in runs: - if run.updated_at > datetime.now() - timedelta(days=DAYS_TO_KEEP_ORPHANED_JOBS): - break # don't clean up if it has a run newer than 90 days + if run.updated_at > datetime.now() - timedelta(days=DAYS_TO_KEEP_ORPHANED_JOBS): + break # don't clean up if it has a run newer than 90 days if args.delete is not None: print("Deleting run id " + str(run.id)) - run._requester.requestJson("DELETE", run.url) # normally we would use run.delete() but even though it's been merged it's not yet in pypi: https://github.com/PyGithub/PyGithub/pull/2078 + run._requester.requestJson( + "DELETE", run.url + ) # normally we would use run.delete() but even though it's been merged it's not yet in pypi: https://github.com/PyGithub/PyGithub/pull/2078 else: runs_to_delete.append((workflow.name, run.id, run.created_at.strftime("%m/%d/%Y, %H:%M:%S"))) - + if args.delete is None: print("[DRY RUN] A total of " + str(len(runs_to_delete)) + " runs would be deleted: ") for run in runs_to_delete: print(run) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/tools/bin/identify-dormant-workflows.py b/tools/bin/identify-dormant-workflows.py index 64a7b65ab88..5dfa954c605 100644 --- a/tools/bin/identify-dormant-workflows.py +++ b/tools/bin/identify-dormant-workflows.py @@ -1,9 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import argparse -import os -import subprocess -import re import json +import os +import re +import subprocess from datetime import datetime, timedelta + from github import Github from slack_sdk import WebClient from slack_sdk.errors import SlackApiError @@ -12,7 +17,7 @@ DAYS_TO_KEEP_ORPHANED_JOBS = 90 SLACK_CHANNEL_FOR_NOTIFICATIONS = "infra-alerts" -''' +""" This script is intended to be run in conjuction with cleanup-workflow-runs.py to keep GH actions clean. @@ -27,7 +32,7 @@ We need to clean up the runs because even if a workflow is deleted, the runs lin We don't want to delete workflow runs newer than 90 days on GH actions, even if the workflow doesn't exist. it's possible that people might test things off the master branch and we don't want to delete their recent runs -''' +""" # Initiate the parser parser = argparse.ArgumentParser() @@ -36,11 +41,11 @@ parser = argparse.ArgumentParser() parser.add_argument("--pat", "-p", help="Set github personal access token") parser.add_argument("--sat", "-s", help="Set slack api token. Optional. If not passed, will just print to console") + def main(): # Read arguments from the command line args = parser.parse_args() - # Check for user supplied PAT. If not supplied, assume we are running in actions # and pull from environment gh_token = None @@ -49,25 +54,24 @@ def main(): if args.pat: gh_token = args.pat else: - gh_token = os.getenv('GITHUB_TOKEN') + gh_token = os.getenv("GITHUB_TOKEN") if not gh_token: raise Exception("Github personal access token not provided via args and not available in GITHUB_TOKEN variable") - + if args.sat: slack_token = args.sat else: - slack_token = os.getenv('SLACK_TOKEN') + slack_token = os.getenv("SLACK_TOKEN") g = Github(gh_token) - git_url = subprocess.run(["git", "config", "--get", "remote.origin.url"], check=True, capture_output=True) + git_url = subprocess.run(["git", "config", "--get", "remote.origin.url"], check=True, capture_output=True) # will match both forms (git and https url) of github e.g. # git@github.com:airbytehq/airbyte.git # https://github.com/airbytehq/airbyte.git - - git_url_regex = re.compile(r'(?:git@|https://)github\.com[:/](.*?)(\.git|$)') + git_url_regex = re.compile(r"(?:git@|https://)github\.com[:/](.*?)(\.git|$)") re_match = git_url_regex.match(git_url.stdout.decode("utf-8")) repo_name = re_match.group(1) @@ -82,19 +86,28 @@ def main(): for run in runs: # check and see if a workflow exists but is not actively being triggered/run if os.path.exists(workflow.path) and run.updated_at < datetime.now() - timedelta(days=DAYS_TO_KEEP_ORPHANED_JOBS): - message = "The Github Workflow '" + workflow.name + "' exists in " + repo_name + " but has no run newer than 90 days old. URL: " + workflow.html_url - print(message) + message = ( + "The Github Workflow '" + + workflow.name + + "' exists in " + + repo_name + + " but has no run newer than 90 days old. URL: " + + workflow.html_url + ) + print(message) if slack_token: print("Sending Slack notification...") client = WebClient(slack_token) - try: response = client.chat_postMessage(channel = SLACK_CHANNEL_FOR_NOTIFICATIONS, text = message) - except SlackApiError as e: - print(e, '\n\n') + try: + response = client.chat_postMessage(channel=SLACK_CHANNEL_FOR_NOTIFICATIONS, text=message) + except SlackApiError as e: + print(e, "\n\n") raise Exception("Error calling the Slack API") break -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/tools/bin/prep_test_results_for_gcs.py b/tools/bin/prep_test_results_for_gcs.py index d1f015ea942..d044a45bebc 100644 --- a/tools/bin/prep_test_results_for_gcs.py +++ b/tools/bin/prep_test_results_for_gcs.py @@ -1,24 +1,28 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import argparse import json import os - -''' +""" This script is intended to be run in conjuction with https://github.com/EnricoMi/publish-unit-test-result-action to upload trimmed test results from the output to a GCS bucket for further analysis. The script takes as input the filename of the json output by the aforementioned action, trims it, and writes it out in jsonl format with ".jsonl" filename -''' +""" # Initiate the parser parser = argparse.ArgumentParser() # Add long and short argument parser.add_argument("--json", "-j", help="Path to the result json output by https://github.com/EnricoMi/publish-unit-test-result-action") -parser.add_argument("--runid", "-r", help="Run id of the action") # this can be derived from checks api, but it's easier to derive it here -parser.add_argument("--jobid", "-c", help="Job id of the action") # this can be derived from checks api, but it's easier to derive it here +parser.add_argument("--runid", "-r", help="Run id of the action") # this can be derived from checks api, but it's easier to derive it here +parser.add_argument("--jobid", "-c", help="Job id of the action") # this can be derived from checks api, but it's easier to derive it here + def main(): # Read arguments from the command line @@ -27,32 +31,32 @@ def main(): f = open(args.json) d = json.load(f) out = [] - + check_run_id = int(d["check_url"].split("/")[-1]) - for elem in d['cases']: - for conclusion in ('success', 'failure', 'skipped'): - if conclusion not in elem['states']: + for elem in d["cases"]: + for conclusion in ("success", "failure", "skipped"): + if conclusion not in elem["states"]: continue - for i in range(len(elem['states'][conclusion])): + for i in range(len(elem["states"][conclusion])): output = { - "test_name": elem['states'][conclusion][i]['test_name'], - "class_name": elem['states'][conclusion][i]['class_name'], - "result_file": elem['states'][conclusion][i]['result_file'], - "time": elem['states'][conclusion][i]['time'], + "test_name": elem["states"][conclusion][i]["test_name"], + "class_name": elem["states"][conclusion][i]["class_name"], + "result_file": elem["states"][conclusion][i]["result_file"], + "time": elem["states"][conclusion][i]["time"], "state": conclusion, "check_run_id": check_run_id, "workflow_run_id": args.runid, "job_id": args.jobid, - "repo": "airbytehq/airbyte" + "repo": "airbytehq/airbyte", } out.append(output) - with open(args.json + "l", 'w') as f: + with open(args.json + "l", "w") as f: for o in out: json.dump(o, f) - f.write('\n') + f.write("\n") -if __name__ == '__main__': - main() \ No newline at end of file +if __name__ == "__main__": + main() diff --git a/tools/bin/update_intellij_venv.py b/tools/bin/update_intellij_venv.py index 36b955c9386..dfb259ee925 100644 --- a/tools/bin/update_intellij_venv.py +++ b/tools/bin/update_intellij_venv.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import argparse import os import subprocess @@ -31,13 +35,12 @@ def add_venv_to_xml_root(module: str, module_full_path: str, xml_root): print(f"{environment_name} already exists. Skipping...") return - jdk_node = ET.SubElement(table, 'jdk', {"version": "2"}) + jdk_node = ET.SubElement(table, "jdk", {"version": "2"}) ET.SubElement(jdk_node, "name", {"value": environment_name}) ET.SubElement(jdk_node, "type", {"value": "Python SDK"}) ET.SubElement(jdk_node, "version", {"value": f"{python_version}"}) - ET.SubElement(jdk_node, "homePath", - {"value": f"{module_full_path}/.venv/bin/python"}) + ET.SubElement(jdk_node, "homePath", {"value": f"{module_full_path}/.venv/bin/python"}) roots = ET.SubElement(jdk_node, "roots") annotationsPath = ET.SubElement(roots, "annotationsPath") @@ -46,10 +49,7 @@ def add_venv_to_xml_root(module: str, module_full_path: str, xml_root): classPath = ET.SubElement(roots, "classPath") classPathRoot = ET.SubElement(classPath, "root", {"type": "composite"}) - ET.SubElement(classPathRoot, "root", {"url": - f"file://{path_to_lib}{python_version}/site-packages", - "type": "simple" - }) + ET.SubElement(classPathRoot, "root", {"url": f"file://{path_to_lib}{python_version}/site-packages", "type": "simple"}) def get_output_path(input_path, output_path): @@ -72,7 +72,8 @@ def get_input_path(input_from_args, version, home_directory): intellij_version_to_update = intellij_versions[0] else: raise RuntimeError( - f"Please select which version of Intellij to update with the `{INTELLIJ_VERSION_FLAG}` flag. Options are: {intellij_versions}") + f"Please select which version of Intellij to update with the `{INTELLIJ_VERSION_FLAG}` flag. Options are: {intellij_versions}" + ) return f"{path_to_intellij_settings}{intellij_version_to_update}/options/jdk.table.xml" @@ -91,12 +92,12 @@ def get_default_airbyte_path(): def create_parser(): parser = argparse.ArgumentParser(description="Prepare Python virtual environments for Python connectors") actions_group = parser.add_argument_group("actions") - actions_group.add_argument("--install-venv", action="store_true", - help="Create virtual environment and install the module's dependencies") + actions_group.add_argument( + "--install-venv", action="store_true", help="Create virtual environment and install the module's dependencies" + ) actions_group.add_argument("--update-intellij", action="store_true", help="Add interpreter to IntelliJ's list of known interpreters") - parser.add_argument("-airbyte", default=get_default_airbyte_path(), - help="Path to Airbyte root directory") + parser.add_argument("-airbyte", default=get_default_airbyte_path(), help="Path to Airbyte root directory") modules_group = parser.add_mutually_exclusive_group(required=True) modules_group.add_argument("-modules", nargs="?", help="Comma separated list of modules to add (eg source-strava,source-stripe)") @@ -150,7 +151,7 @@ if __name__ == "__main__": input_path = get_input_path(args.input, args.intellij_version, home_directory) output_path = get_output_path(input_path, args.output) - with open(input_path, 'r') as f: + with open(input_path, "r") as f: root = ET.fromstring(f.read()) for module in modules: @@ -173,7 +174,6 @@ def setup_module(): if "pytest" in sys.argv[0]: import unittest - class TestNoneTypeError(unittest.TestCase): def test_output_is_input_if_not_set(self): input_path = "/input_path" @@ -187,8 +187,7 @@ if "pytest" in sys.argv[0]: @unittest.mock.patch("os.walk") def test_input_is_selected(self, mock_os): - os.walk.return_value = iter( - (("./test1", ["consentOptions", "IdeaIC2021.3", "PyCharmCE2021.3"], []),)) + os.walk.return_value = iter((("./test1", ["consentOptions", "IdeaIC2021.3", "PyCharmCE2021.3"], []),)) os.getenv.return_value = "{HOME}" input_from_args = None version = "IdeaIC2021.3" @@ -197,8 +196,7 @@ if "pytest" in sys.argv[0]: @unittest.mock.patch("os.walk") def test_input_single_intellij_version(self, mock_os): - os.walk.return_value = iter( - (("./test1", ["consentOptions", "IdeaIC2021.3"], []),)) + os.walk.return_value = iter((("./test1", ["consentOptions", "IdeaIC2021.3"], []),)) input_from_args = None version = None @@ -207,8 +205,7 @@ if "pytest" in sys.argv[0]: @unittest.mock.patch("os.walk") def test_input_multiple_intellij_versions(self, mock_os): - os.walk.return_value = iter( - (('./test1', ['consentOptions', 'IdeaIC2021.3', "PyCharmCE2021.3"], []),)) + os.walk.return_value = iter((("./test1", ["consentOptions", "IdeaIC2021.3", "PyCharmCE2021.3"], []),)) input_from_args = None version = None diff --git a/tools/git_hooks/spec_linter.py b/tools/git_hooks/spec_linter.py index d572362f465..7fe9dfac897 100644 --- a/tools/git_hooks/spec_linter.py +++ b/tools/git_hooks/spec_linter.py @@ -1,6 +1,7 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + """ This script is responsible for connectors spec.json file validation. diff --git a/tools/schema_generator/schema_generator/configure_catalog.py b/tools/schema_generator/schema_generator/configure_catalog.py index bf770016654..2604f6af9fe 100644 --- a/tools/schema_generator/schema_generator/configure_catalog.py +++ b/tools/schema_generator/schema_generator/configure_catalog.py @@ -1,7 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import json import os -from airbyte_cdk.models import ConfiguredAirbyteCatalog, AirbyteMessage, ConfiguredAirbyteStream, DestinationSyncMode +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode def configure_catalog(): diff --git a/tools/schema_generator/schema_generator/main.py b/tools/schema_generator/schema_generator/main.py index ed0e4bda634..bd1cb14ae81 100644 --- a/tools/schema_generator/schema_generator/main.py +++ b/tools/schema_generator/schema_generator/main.py @@ -1,12 +1,12 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import argparse import sys -from .infer_schemas import infer_schemas from .configure_catalog import configure_catalog +from .infer_schemas import infer_schemas def main(): diff --git a/tools/schema_generator/tests/test_schema_generator.py b/tools/schema_generator/tests/test_schema_generator.py index 4be44e7651b..590b80f1342 100644 --- a/tools/schema_generator/tests/test_schema_generator.py +++ b/tools/schema_generator/tests/test_schema_generator.py @@ -1,17 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + import io +import json +import os import sys import tempfile -import os -import json import pytest - -from schema_generator.configure_catalog import configure_catalog -from schema_generator.infer_schemas import infer_schemas - from airbyte_cdk.models import ( - AirbyteMessage, AirbyteCatalog, + AirbyteMessage, AirbyteRecordMessage, AirbyteStream, ConfiguredAirbyteCatalog, @@ -20,6 +20,8 @@ from airbyte_cdk.models import ( SyncMode, Type, ) +from schema_generator.configure_catalog import configure_catalog +from schema_generator.infer_schemas import infer_schemas def test_configure_catalog(): diff --git a/tools/schema_generator/tests/tests.py b/tools/schema_generator/tests/tests.py index 033c7cc21fc..14a47ae4d4f 100644 --- a/tools/schema_generator/tests/tests.py +++ b/tools/schema_generator/tests/tests.py @@ -1,3 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + from airbyte_cdk.models import ( AirbyteCatalog, AirbyteConnectionStatus,