jprdonnelly/airbyte
1
0
Fork 0
mirror of synced 2025-12-19 18:14:56 -05:00
Code Issues Packages Projects Releases Wiki Activity

[airbyte-ci] Format using a poe task (#38043)

Browse Source
This commit is contained in:
Natik Gadzhi
2024-05-08 08:41:41 -07:00
committed by GitHub
parent 8d914d9ffe
commit 7309360af6
10 changed files with 134 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
airbyte-cdk/python/airbyte_cdk/connector_builder/connector_builder_handler.py
View File

@@ -54,7 +54,11 @@ def create_source(config: Mapping[str, Any], limits: TestReadLimits) -> Manifest
def read_stream(
source: DeclarativeSource, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, state: List[AirbyteStateMessage], limits: TestReadLimits
source: DeclarativeSource,
config: Mapping[str, Any],
configured_catalog: ConfiguredAirbyteCatalog,
state: List[AirbyteStateMessage],
limits: TestReadLimits,
) -> AirbyteMessage:
try:
handler = MessageGrouper(limits.max_pages_per_slice, limits.max_slices, limits.max_records)

22
airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py
View File

@@ -120,7 +120,8 @@ class MessageGrouper:
raise ValueError(f"Unknown message group type: {type(message_group)}")
try:
configured_stream = configured_catalog.streams[0] # The connector builder currently only supports reading from a single stream at a time
# The connector builder currently only supports reading from a single stream at a time
configured_stream = configured_catalog.streams[0]
schema = schema_inferrer.get_stream_schema(configured_stream.stream.name)
except SchemaValidationException as exception:
for validation_error in exception.validation_errors:
@@ -183,7 +184,11 @@ class MessageGrouper:
and message.type == MessageType.LOG
and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX)
):
yield StreamReadSlices(pages=current_slice_pages, slice_descriptor=current_slice_descriptor, state=[latest_state_message] if latest_state_message else [])
yield StreamReadSlices(
pages=current_slice_pages,
slice_descriptor=current_slice_descriptor,
state=[latest_state_message] if latest_state_message else [],
)
current_slice_descriptor = self._parse_slice_description(message.log.message)
current_slice_pages = []
at_least_one_page_in_group = False
@@ -230,7 +235,11 @@ class MessageGrouper:
else:
if current_page_request or current_page_response or current_page_records:
self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records)
yield StreamReadSlices(pages=current_slice_pages, slice_descriptor=current_slice_descriptor, state=[latest_state_message] if latest_state_message else [])
yield StreamReadSlices(
pages=current_slice_pages,
slice_descriptor=current_slice_descriptor,
state=[latest_state_message] if latest_state_message else [],
)
@staticmethod
def _need_to_close_page(at_least_one_page_in_group: bool, message: AirbyteMessage, json_message: Optional[Dict[str, Any]]) -> bool:
@@ -281,8 +290,11 @@ class MessageGrouper:
current_page_records.clear()
def _read_stream(
self, source: DeclarativeSource, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog,
state: List[AirbyteStateMessage]
self,
source: DeclarativeSource,
config: Mapping[str, Any],
configured_catalog: ConfiguredAirbyteCatalog,
state: List[AirbyteStateMessage],
) -> Iterator[AirbyteMessage]:
# the generator can raise an exception
# iterate over the generated messages. if next raise an exception, catch it and yield it as an AirbyteLogMessage

6
airbyte-cdk/python/airbyte_cdk/test/catalog_builder.py
View File

@@ -54,7 +54,11 @@ class CatalogBuilder:
# to avoid a breaking change, `name` needs to stay in the API but this can be either a name or a builder
name_or_builder = name
builder = name_or_builder if isinstance(name_or_builder, ConfiguredAirbyteStreamBuilder) else ConfiguredAirbyteStreamBuilder().with_name(name_or_builder).with_sync_mode(sync_mode)
builder = (
name_or_builder
if isinstance(name_or_builder, ConfiguredAirbyteStreamBuilder)
else ConfiguredAirbyteStreamBuilder().with_name(name_or_builder).with_sync_mode(sync_mode)
)
self._streams.append(builder)
return self

15
airbyte-cdk/python/airbyte_cdk/test/mock_http/response_builder.py
View File

@@ -138,10 +138,7 @@ class RecordBuilder:
class HttpResponseBuilder:
def __init__(
self,
template: Dict[str, Any],
records_path: Union[FieldPath, NestedPath],
pagination_strategy: Optional[PaginationStrategy]
self, template: Dict[str, Any], records_path: Union[FieldPath, NestedPath], pagination_strategy: Optional[PaginationStrategy]
):
self._response = template
self._records: List[RecordBuilder] = []
@@ -198,16 +195,16 @@ def create_record_builder(
try:
record_template = records_path.extract(response_template)[0]
if not record_template:
raise ValueError(f"Could not extract any record from template at path `{records_path}`. "
f"Please fix the template to provide a record sample or fix `records_path`.")
raise ValueError(
f"Could not extract any record from template at path `{records_path}`. "
f"Please fix the template to provide a record sample or fix `records_path`."
)
return RecordBuilder(record_template, record_id_path, record_cursor_path)
except (IndexError, KeyError):
raise ValueError(f"Error while extracting records at path `{records_path}` from response template `{response_template}`")
def create_response_builder(
response_template: Dict[str, Any],
records_path: Union[FieldPath, NestedPath],
pagination_strategy: Optional[PaginationStrategy] = None
response_template: Dict[str, Any], records_path: Union[FieldPath, NestedPath], pagination_strategy: Optional[PaginationStrategy] = None
) -> HttpResponseBuilder:
return HttpResponseBuilder(response_template, records_path, pagination_strategy)

12
airbyte-cdk/python/airbyte_cdk/test/state_builder.py
View File

@@ -10,15 +10,9 @@ class StateBuilder:
self._state: List[AirbyteStateMessage] = []
def with_stream_state(self, stream_name: str, state: Any) -> "StateBuilder":
self._state.append(AirbyteStateMessage.parse_obj({
"type": "STREAM",
"stream": {
"stream_state": state,
"stream_descriptor": {
"name": stream_name
}
}
}))
self._state.append(
AirbyteStateMessage.parse_obj({"type": "STREAM", "stream": {"stream_state": state, "stream_descriptor": {"name": stream_name}}})
)
return self
def build(self) -> List[AirbyteStateMessage]:

2
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/configuration.py
View File

@@ -51,6 +51,6 @@ FORMATTERS_CONFIGURATIONS: List[FormatConfiguration] = [
Formatter.PYTHON,
["**/*.py"],
format_python_container,
["poetry run isort --settings-file pyproject.toml .", "poetry run black --config pyproject.toml ."],
["poetry run poe format"],
),
]

1
airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py
View File

@@ -35,7 +35,6 @@ DEFAULT_FORMAT_IGNORE_LIST = [
"**/pnpm-lock.yaml", # This file is generated and should not be formatted
"**/normalization_test_output",
"**/source-amplitude/unit_tests/api_data/zipped.json", # Zipped file presents as non-UTF-8 making spotless sad
"**/tools/git_hooks/tests/test_spec_linter.py",
"airbyte-cdk/python/airbyte_cdk/sources/declarative/models/**", # These files are generated and should not be formatted
"airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/**", # These files are generated and should not be formatted
"**/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/**/invalid", # This is a test directory with invalid and sometimes unformatted code

1
airbyte-integrations/connectors/source-zendesk-chat/build_customization.py
View File

@@ -9,7 +9,6 @@ if TYPE_CHECKING:
from dagger import Container
async def pre_connector_install(base_image_container: Container) -> Container:
"""This function will run before the connector installation.
We set these environment variable to match what was originally in the Dockerfile.

76
poetry.lock generated
View File

@@ -97,6 +97,17 @@ files = [
{file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
]
[[package]]
name = "pastel"
version = "0.2.1"
description = "Bring colors to your terminal."
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
files = [
{file = "pastel-0.2.1-py2.py3-none-any.whl", hash = "sha256:4349225fcdf6c2bb34d483e523475de5bb04a5c10ef711263452cb37d7dd4364"},
{file = "pastel-0.2.1.tar.gz", hash = "sha256:e6581ac04e973cac858828c6202c1e1e81fee1dc7de7683f3e1ffe0bfd8a573d"},
]
[[package]]
name = "pathspec"
version = "0.12.1"
@@ -110,43 +121,62 @@ files = [
[[package]]
name = "platformdirs"
version = "4.2.0"
description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"."
version = "4.2.1"
description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`."
optional = false
python-versions = ">=3.8"
files = [
{file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"},
{file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"},
{file = "platformdirs-4.2.1-py3-none-any.whl", hash = "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1"},
{file = "platformdirs-4.2.1.tar.gz", hash = "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf"},
]
[package.extras]
docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"]
test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"]
type = ["mypy (>=1.8)"]
[[package]]
name = "poethepoet"
version = "0.26.1"
description = "A task runner that works well with poetry."
optional = false
python-versions = ">=3.8"
files = [
{file = "poethepoet-0.26.1-py3-none-any.whl", hash = "sha256:aa43b443fec5d17d7e76771cccd484e5285805301721a74f059c483ad3276edd"},
{file = "poethepoet-0.26.1.tar.gz", hash = "sha256:aaad8541f6072617a60bcff2562d00779b58b353bd0f1847b06d8d0f2b6dc192"},
]
[package.dependencies]
pastel = ">=0.2.1,<0.3.0"
tomli = ">=1.2.2"
[package.extras]
poetry-plugin = ["poetry (>=1.0,<2.0)"]
[[package]]
name = "ruff"
version = "0.4.1"
version = "0.4.3"
description = "An extremely fast Python linter and code formatter, written in Rust."
optional = false
python-versions = ">=3.7"
files = [
{file = "ruff-0.4.1-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2d9ef6231e3fbdc0b8c72404a1a0c46fd0dcea84efca83beb4681c318ea6a953"},
{file = "ruff-0.4.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9485f54a7189e6f7433e0058cf8581bee45c31a25cd69009d2a040d1bd4bfaef"},
{file = "ruff-0.4.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2921ac03ce1383e360e8a95442ffb0d757a6a7ddd9a5be68561a671e0e5807e"},
{file = "ruff-0.4.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eec8d185fe193ad053eda3a6be23069e0c8ba8c5d20bc5ace6e3b9e37d246d3f"},
{file = "ruff-0.4.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:baa27d9d72a94574d250f42b7640b3bd2edc4c58ac8ac2778a8c82374bb27984"},
{file = "ruff-0.4.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:f1ee41580bff1a651339eb3337c20c12f4037f6110a36ae4a2d864c52e5ef954"},
{file = "ruff-0.4.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0926cefb57fc5fced629603fbd1a23d458b25418681d96823992ba975f050c2b"},
{file = "ruff-0.4.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c6e37f2e3cd74496a74af9a4fa67b547ab3ca137688c484749189bf3a686ceb"},
{file = "ruff-0.4.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efd703a5975ac1998c2cc5e9494e13b28f31e66c616b0a76e206de2562e0843c"},
{file = "ruff-0.4.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:b92f03b4aa9fa23e1799b40f15f8b95cdc418782a567d6c43def65e1bbb7f1cf"},
{file = "ruff-0.4.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1c859f294f8633889e7d77de228b203eb0e9a03071b72b5989d89a0cf98ee262"},
{file = "ruff-0.4.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:b34510141e393519a47f2d7b8216fec747ea1f2c81e85f076e9f2910588d4b64"},
{file = "ruff-0.4.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:6e68d248ed688b9d69fd4d18737edcbb79c98b251bba5a2b031ce2470224bdf9"},
{file = "ruff-0.4.1-py3-none-win32.whl", hash = "sha256:b90506f3d6d1f41f43f9b7b5ff845aeefabed6d2494307bc7b178360a8805252"},
{file = "ruff-0.4.1-py3-none-win_amd64.whl", hash = "sha256:c7d391e5936af5c9e252743d767c564670dc3889aff460d35c518ee76e4b26d7"},
{file = "ruff-0.4.1-py3-none-win_arm64.whl", hash = "sha256:a1eaf03d87e6a7cd5e661d36d8c6e874693cb9bc3049d110bc9a97b350680c43"},
{file = "ruff-0.4.1.tar.gz", hash = "sha256:d592116cdbb65f8b1b7e2a2b48297eb865f6bdc20641879aa9d7b9c11d86db79"},
{file = "ruff-0.4.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b70800c290f14ae6fcbb41bbe201cf62dfca024d124a1f373e76371a007454ce"},
{file = "ruff-0.4.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:08a0d6a22918ab2552ace96adeaca308833873a4d7d1d587bb1d37bae8728eb3"},
{file = "ruff-0.4.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eba1f14df3c758dd7de5b55fbae7e1c8af238597961e5fb628f3de446c3c40c5"},
{file = "ruff-0.4.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:819fb06d535cc76dfddbfe8d3068ff602ddeb40e3eacbc90e0d1272bb8d97113"},
{file = "ruff-0.4.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0bfc9e955e6dc6359eb6f82ea150c4f4e82b660e5b58d9a20a0e42ec3bb6342b"},
{file = "ruff-0.4.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:510a67d232d2ebe983fddea324dbf9d69b71c4d2dfeb8a862f4a127536dd4cfb"},
{file = "ruff-0.4.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dc9ff11cd9a092ee7680a56d21f302bdda14327772cd870d806610a3503d001f"},
{file = "ruff-0.4.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29efff25bf9ee685c2c8390563a5b5c006a3fee5230d28ea39f4f75f9d0b6f2f"},
{file = "ruff-0.4.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18b00e0bcccf0fc8d7186ed21e311dffd19761cb632241a6e4fe4477cc80ef6e"},
{file = "ruff-0.4.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:262f5635e2c74d80b7507fbc2fac28fe0d4fef26373bbc62039526f7722bca1b"},
{file = "ruff-0.4.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7363691198719c26459e08cc17c6a3dac6f592e9ea3d2fa772f4e561b5fe82a3"},
{file = "ruff-0.4.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:eeb039f8428fcb6725bb63cbae92ad67b0559e68b5d80f840f11914afd8ddf7f"},
{file = "ruff-0.4.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:927b11c1e4d0727ce1a729eace61cee88a334623ec424c0b1c8fe3e5f9d3c865"},
{file = "ruff-0.4.3-py3-none-win32.whl", hash = "sha256:25cacda2155778beb0d064e0ec5a3944dcca9c12715f7c4634fd9d93ac33fd30"},
{file = "ruff-0.4.3-py3-none-win_amd64.whl", hash = "sha256:7a1c3a450bc6539ef00da6c819fb1b76b6b065dec585f91456e7c0d6a0bbc725"},
{file = "ruff-0.4.3-py3-none-win_arm64.whl", hash = "sha256:71ca5f8ccf1121b95a59649482470c5601c60a416bf189d553955b0338e34614"},
{file = "ruff-0.4.3.tar.gz", hash = "sha256:ff0a3ef2e3c4b6d133fbedcf9586abfbe38d076041f2dc18ffb2c7e0485d5a07"},
]
[[package]]
@@ -163,4 +193,4 @@ files = [
[metadata]
lock-version = "2.0"
python-versions = "~3.10"
content-hash = "e37fb350d576f394c80e29753ec32ce94ba0b322ba7ea01af4deedbed51e039d"
content-hash = "86b7578e744e8b71526d947edba4c42a687b4aade96dde24ec0dbc1c3b245eb0"

69
pyproject.toml
View File

@@ -11,11 +11,51 @@ python = "~3.10"
isort = "5.6.4"
black = "~22.3.0"
ruff = "^0.4"
poethepoet = "^0.26.1"
[tool.poe.tasks]
isort = { cmd = "poetry run isort --settings-file pyproject.toml ." }
black = { cmd = "poetry run black --config pyproject.toml ." }
format = { sequence = [
"isort",
"black",
], help = "Format Python code in the repository. This command is invoked in airbyte-ci format." }
[tool.black]
line-length = 140
target-version = ["py310"]
extend-exclude = "(build|integration_tests|unit_tests|generated)"
extend-exclude = """
/(
build
| integration_tests
| unit_tests
| generated
| airbyte-cdk/python/airbyte_cdk/sources/declarative/models
| invalid
| non_formatted_code
)/
"""
[tool.flake8]
extend-exclude = [
"*/lib/*/site-packages",
".venv",
"build",
"models",
".eggs",
"**/__init__.py",
"**/generated/*",
"**/declarative/models/*",
]
max-complexity = 20
max-line-length = 140
extend-ignore = [
"E203", # whitespace before ':' (conflicts with Black)
"E231", # Bad trailing comma (conflicts with Black)
"E501", # line too long (conflicts with Black)
"W503", # line break before binary operator (conflicts with Black)
"F811", # TODO: ella fix after pflake8 version update
]
[tool.coverage.report]
fail_under = 0
@@ -30,30 +70,6 @@ omit = [
"**/generated/*",
]
[tool.flake8]
extend-exclude = [
"*/lib/*/site-packages",
".venv",
"build",
"models",
".eggs",
"airbyte-cdk/python/airbyte_cdk/models/__init__.py",
"airbyte-cdk/python/airbyte_cdk/sources/declarative/models/__init__.py",
".tox",
"airbyte_api_client",
"**/generated/*",
]
max-complexity = 20
max-line-length = 140
extend-ignore = [
"E203", # whitespace before ':' (conflicts with Black)
"E231", # Bad trailing comma (conflicts with Black)
"E501", # line too long (conflicts with Black)
"W503", # line break before binary operator (conflicts with Black)
"F811", # TODO: ella fix after pflake8 version update
]
# TODO: This will be removed in favor of the section below.
[tool.isort]
profile = "black"
@@ -65,6 +81,9 @@ include_trailing_comma = true
force_grid_wrap = 0
use_parentheses = true
skip_glob = [
"airbyte-cdk/python/airbyte_cdk/sources/declarative/models/**",
"**/invalid/**",
"**/non_formatted_code/**",
"**/connector_builder/generated/**",
# TODO: Remove this after we move to Ruff. Ruff is mono-repo-aware and
# correctly handles first-party imports in subdirectories.