diff --git a/.github/workflows/community_ci.yml b/.github/workflows/community_ci.yml
index 791a2a67345..2aa16dce58a 100644
--- a/.github/workflows/community_ci.yml
+++ b/.github/workflows/community_ci.yml
@@ -37,45 +37,6 @@ jobs:
echo "The fork has changes in protected paths. This is not allowed."
exit 1
- format_check:
- # IMPORTANT: This name must match the require check name on the branch protection settings
- name: "Check for formatting errors"
- if: github.event.pull_request.head.repo.fork == true
- environment: community-ci-auto
- runs-on: community-tooling-test-small
- needs: fail_on_protected_path_changes
- timeout-minutes: 30
- env:
- MAIN_BRANCH_NAME: "master"
- steps:
- # This checkouts a fork which can contain untrusted code
- # It's deemed safe as the formatter are not executing any checked out code
- - name: Checkout fork
- uses: actions/checkout@v4
- with:
- repository: ${{ github.event.pull_request.head.repo.full_name }}
- ref: ${{ github.event.pull_request.head.sha }}
- fetch-depth: 1
-
- # This will sync the .github folder of the main repo with the fork
- # This allows us to use up to date actions and CI logic from the main repo
- - name: Pull .github folder and internal packages from main repository
- id: pull_github_folder
- run: |
- git remote add main https://github.com/airbytehq/airbyte.git
- git fetch main ${MAIN_BRANCH_NAME}
- git checkout main/${MAIN_BRANCH_NAME} -- .github
- git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
-
- - name: Run airbyte-ci format check all
- # This path refers to the fork .github folder.
- # We make sure its content is in sync with the main repo .github folder by pulling it in the previous step
- uses: ./.github/actions/run-airbyte-ci
- with:
- context: "pull_request"
- sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
- subcommand: "format check all"
- is_fork: "true"
connectors_early_ci:
name: Run connectors early CI on fork
if: github.event.pull_request.head.repo.fork == true
diff --git a/.github/workflows/format-fix-command.yml b/.github/workflows/format-fix-command.yml
index 2606ea1a900..331c4bc52ee 100644
--- a/.github/workflows/format-fix-command.yml
+++ b/.github/workflows/format-fix-command.yml
@@ -29,7 +29,7 @@ concurrency:
jobs:
format-fix:
- name: "Run airbyte-ci format fix all"
+ name: "Run pre-commit fix"
runs-on: ubuntu-24.04
steps:
- name: Get job variables
@@ -67,16 +67,21 @@ jobs:
[1]: ${{ steps.job-vars.outputs.run-url }}
- - name: Run airbyte-ci format fix all
- uses: ./.github/actions/run-airbyte-ci
- continue-on-error: true
+ # Compare the below to the `format_check.yml` workflow
+ - name: Setup Java
+ uses: actions/setup-java@v3
with:
- context: "manual"
- gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
- sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
- github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
- subcommand: "format fix all"
- dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_2 }}
+ distribution: "zulu"
+ java-version: "21"
+ - name: Setup Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.10"
+ cache: "pip"
+ - name: Run pre-commit
+ uses: pre-commit/action@v3.0.1
+ continue-on-error: true
+ id: format-fix
# This is helpful in the case that we change a previously committed generated file to be ignored by git.
- name: Remove any files that have been gitignored
diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml
index 4bb83c11831..4dcf3480ce9 100644
--- a/.github/workflows/format_check.yml
+++ b/.github/workflows/format_check.yml
@@ -10,53 +10,31 @@ on:
jobs:
format-check:
- # IMPORTANT: This name must match the require check name on the branch protection settings
name: "Check for formatting errors"
- # Do not run this job on forks
- # Forked PRs are handled by the community_ci.yml workflow
- if: github.event.pull_request.head.repo.fork != true
- runs-on: tooling-test-small
+ runs-on: ubuntu-24.04
steps:
- - name: Checkout Airbyte (with credentials)
- uses: actions/checkout@v4
+ - uses: actions/checkout@v4
+ - name: Setup Java
+ uses: actions/setup-java@v3
with:
- ref: ${{ github.head_ref }}
- token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
- fetch-depth: 1
- - name: Run airbyte-ci format check [MASTER]
- id: airbyte_ci_format_check_all_master
- if: github.ref == 'refs/heads/master'
- uses: ./.github/actions/run-airbyte-ci
- continue-on-error: true
+ distribution: "zulu"
+ java-version: "21"
+ - name: Setup Python
+ uses: actions/setup-python@v5
with:
- context: "master"
- sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
- subcommand: "format check all"
-
- - name: Run airbyte-ci format check [PULL REQUEST]
- id: airbyte_ci_format_check_all_pr
- if: github.event_name == 'pull_request'
- uses: ./.github/actions/run-airbyte-ci
- continue-on-error: false
+ python-version: "3.10"
+ cache: "pip"
+ - name: Run pre-commit
+ uses: pre-commit/action@v3.0.1
+ id: format-check
with:
- context: "pull_request"
- sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
- subcommand: "format check all"
- dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_2 }}
-
- - name: Run airbyte-ci format check [WORKFLOW DISPATCH]
- id: airbyte_ci_format_check_all_manual
- if: github.event_name == 'workflow_dispatch'
- uses: ./.github/actions/run-airbyte-ci
- continue-on-error: false
- with:
- context: "manual"
- sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
- subcommand: "format check all"
- dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_2 }}
+ extra_args: --all-files
- name: Match GitHub User to Slack User [MASTER]
- if: github.ref == 'refs/heads/master'
+ if: >
+ always() && steps.format-check.outcome == 'failure' &&
+ github.ref == 'refs/heads/master' &&
+ github.event.pull_request.head.repo.fork == false
id: match-github-to-slack-user
uses: ./.github/actions/match-github-to-slack-user
env:
@@ -64,7 +42,10 @@ jobs:
GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Format Failure on Master Slack Channel [MASTER]
- if: steps.airbyte_ci_format_check_all_master.outcome == 'failure' && github.ref == 'refs/heads/master'
+ if: >
+ always() && steps.format-check.outcome == 'failure' &&
+ github.ref == 'refs/heads/master' &&
+ github.event.pull_request.head.repo.fork == false
uses: abinoda/slack-action@master
env:
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4fd1a68cdc3..a394cc362a8 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,10 +1,59 @@
+exclude: |
+ (?x)(
+ # Python/system files
+ ^.*/__init__\.py$|
+ ^.*?/\.venv/.*$|
+ ^.*?/node_modules/.*$|
+
+ ^.*?/charts/.*$|
+ ^airbyte-integrations/bases/base-normalization/.*$|
+ ^.*?/normalization_test_output/.*$|
+
+ ^.*?/pnpm-lock\.yaml$|
+ ^.*?/source-amplitude/unit_tests/api_data/zipped\.json$|
+
+ # Generated/test files
+ ^airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/.*$|
+ ^.*?/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/.*/invalid/.*$|
+ ^airbyte-ci/connectors/metadata_service/lib/tests/fixtures/.*/invalid/.*$|
+ ^.*?/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/.*$|
+ ^airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/.*$|
+ ^.*?/airbyte-integrations/connectors/destination-.*/expected-spec\.json$
+ )
+
repos:
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.8.3
+ hooks:
+ # Run the linter.
+ - id: ruff
+ args:
+ - --fix
+ - --select=I
+
+ # Run the formatter.
+ - id: ruff-format
+
+ - repo: https://github.com/pre-commit/mirrors-prettier
+ rev: v3.0.3
+ hooks:
+ - id: prettier
+ types_or: [json, yaml]
+ additional_dependencies:
+ - prettier@3.0.3
+
- repo: local
hooks:
- - id: format-fix-all-on-push
- always_run: true
- entry: airbyte-ci --disable-update-check format fix all
+ - id: addlicense
+ name: Add license headers
+ entry: addlicense -c "Airbyte, Inc." -l apache -v -f LICENSE_SHORT
+ language: golang
+ additional_dependencies: [github.com/google/addlicense@v1.1.1]
+ files: \.(java|kt|py)$
+
+ - id: spotless
+ name: Format Java files with Spotless
+ entry: bash -c 'command -v mvn >/dev/null 2>&1 || { echo "Maven not installed, skipping spotless" >&2; exit 0; }; mvn -f spotless-maven-pom.xml spotless:apply'
language: system
- name: Run airbyte-ci format fix on git push (~30s)
+ files: \.(java|kt|gradle)$
pass_filenames: false
- stages: [push]
diff --git a/airbyte-ci/connectors/auto_merge/pyproject.toml b/airbyte-ci/connectors/auto_merge/pyproject.toml
index 545a036cb4c..e73e19f4c3e 100644
--- a/airbyte-ci/connectors/auto_merge/pyproject.toml
+++ b/airbyte-ci/connectors/auto_merge/pyproject.toml
@@ -20,6 +20,9 @@ ruff = "^0.4.3"
pytest = "^8.2.0"
pyinstrument = "^4.6.2"
+[tool.ruff]
+line-length = 140
+
[tool.ruff.lint]
select = [
"I" # isort
diff --git a/airbyte-ci/connectors/live-tests/pyproject.toml b/airbyte-ci/connectors/live-tests/pyproject.toml
index 63134614ea2..09e6e7d38d6 100644
--- a/airbyte-ci/connectors/live-tests/pyproject.toml
+++ b/airbyte-ci/connectors/live-tests/pyproject.toml
@@ -53,11 +53,13 @@ pandas-stubs = "^2.2.0.240218"
types-requests = "^2.31.0.20240311"
types-pyyaml = "^6.0.12.20240311"
-[tool.ruff.lint]
-select = ["I", "F"]
+[tool.ruff]
target-version = "py310"
line-length = 140
+[tool.ruff.lint]
+select = ["I", "F"]
+
[tool.ruff.lint.isort]
known-first-party = ["connection-retriever"]
diff --git a/airbyte-ci/connectors/pipelines/ruff.toml b/airbyte-ci/connectors/pipelines/ruff.toml
index 490ad78fac3..e153f58f693 100644
--- a/airbyte-ci/connectors/pipelines/ruff.toml
+++ b/airbyte-ci/connectors/pipelines/ruff.toml
@@ -1,15 +1,13 @@
target-version = "py310"
line-length = 140
-ignore = ["ANN101", "ANN002", "ANN003"]
-
[lint]
+ignore = ["ANN003"]
extend-select = [
"ANN", # flake8-annotations
]
-
[lint.pydocstyle]
convention = "google"
diff --git a/pyproject.toml b/pyproject.toml
index dc3ee94e6e6..7293758090c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -38,27 +38,6 @@ extend-exclude = """
)/
"""
-[tool.flake8]
-extend-exclude = [
- "*/lib/*/site-packages",
- ".venv",
- "build",
- "models",
- ".eggs",
- "**/__init__.py",
- "**/generated/*",
- "**/declarative/models/*",
-]
-max-complexity = 20
-max-line-length = 140
-extend-ignore = [
- "E203", # whitespace before ':' (conflicts with Black)
- "E231", # Bad trailing comma (conflicts with Black)
- "E501", # line too long (conflicts with Black)
- "W503", # line break before binary operator (conflicts with Black)
- "F811", # TODO: ella fix after pflake8 version update
-]
-
[tool.coverage.report]
fail_under = 0
skip_empty = true
@@ -94,78 +73,86 @@ skip_glob = [
"airbyte-integrations/connectors/destination-snowflake-cortex/**"
]
-[tool.ruff.pylint]
+[tool.ruff.lint.pylint]
max-args = 8 # Relaxed from default of 5
max-branches = 15 # Relaxed from default of 12
[tool.ruff]
target-version = "py310"
+line-length = 140
+extend-exclude = ["docs", "test", "tests"]
+
+[tool.ruff.lint]
+
select = [
- # For rules reference, see https://docs.astral.sh/ruff/rules/
- "A", # flake8-builtins
- "ANN", # flake8-annotations
- "ARG", # flake8-unused-arguments
- "ASYNC", # flake8-async
- "B", # flake8-bugbear
- "FBT", # flake8-boolean-trap
- "BLE", # Blind except
- "C4", # flake8-comprehensions
- "C90", # mccabe (complexity)
- "COM", # flake8-commas
- "CPY", # missing copyright notice
- # "D", # pydocstyle # TODO: Re-enable when adding docstrings
- "DTZ", # flake8-datetimez
- "E", # pycodestyle (errors)
- "ERA", # flake8-eradicate (commented out code)
- "EXE", # flake8-executable
- "F", # Pyflakes
- "FA", # flake8-future-annotations
- "FIX", # flake8-fixme
- "FLY", # flynt
- "FURB", # Refurb
- "I", # isort
- "ICN", # flake8-import-conventions
- "INP", # flake8-no-pep420
- "INT", # flake8-gettext
- "ISC", # flake8-implicit-str-concat
- "ICN", # flake8-import-conventions
- "LOG", # flake8-logging
- "N", # pep8-naming
- "PD", # pandas-vet
- "PERF", # Perflint
- "PIE", # flake8-pie
- "PGH", # pygrep-hooks
- "PL", # Pylint
- "PT", # flake8-pytest-style
- "PTH", # flake8-use-pathlib
- "PYI", # flake8-pyi
- "Q", # flake8-quotes
- "RET", # flake8-return
- "RSE", # flake8-raise
- "RUF", # Ruff-specific rules
- "SIM", # flake8-simplify
- "SLF", # flake8-self
- "SLOT", # flake8-slots
- "T10", # debugger calls
- # "T20", # flake8-print # TODO: Re-enable once we have logging
- "TCH", # flake8-type-checking
- "TD", # flake8-todos
- "TID", # flake8-tidy-imports
- "TRY", # tryceratops
- "TRY002", # Disallow raising vanilla Exception. Create or use a custom exception instead.
- "TRY003", # Disallow vanilla string passing. Prefer kwargs to the exception constructur.
- "UP", # pyupgrade
- "W", # pycodestyle (warnings)
- "YTT", # flake8-2020
+ "I", # isort replacement
]
+# TODO: Re-enable these once we have time to address them
+# select = [
+# # For rules reference, see https://docs.astral.sh/ruff/rules/
+# "A", # flake8-builtins
+# "ANN", # flake8-annotations
+# "ARG", # flake8-unused-arguments
+# "ASYNC", # flake8-async
+# "B", # flake8-bugbear
+# "FBT", # flake8-boolean-trap
+# "BLE", # Blind except
+# "C4", # flake8-comprehensions
+# "C90", # mccabe (complexity)
+# "COM", # flake8-commas
+# # "CPY", # missing copyright notice # Requires 'preview=true'
+# # "D", # pydocstyle # TODO: Re-enable when adding docstrings
+# "DTZ", # flake8-datetimez
+# "E", # pycodestyle (errors)
+# "ERA", # flake8-eradicate (commented out code)
+# "EXE", # flake8-executable
+# "F", # Pyflakes
+# "FA", # flake8-future-annotations
+# "FIX", # flake8-fixme
+# "FLY", # flynt
+# "FURB", # Refurb
+# "I", # isort
+# "ICN", # flake8-import-conventions
+# "INP", # flake8-no-pep420
+# "INT", # flake8-gettext
+# "ISC", # flake8-implicit-str-concat
+# "ICN", # flake8-import-conventions
+# "LOG", # flake8-logging
+# "N", # pep8-naming
+# "PD", # pandas-vet
+# "PERF", # Perflint
+# "PIE", # flake8-pie
+# "PGH", # pygrep-hooks
+# "PL", # Pylint
+# "PT", # flake8-pytest-style
+# "PTH", # flake8-use-pathlib
+# "PYI", # flake8-pyi
+# "Q", # flake8-quotes
+# "RET", # flake8-return
+# "RSE", # flake8-raise
+# "RUF", # Ruff-specific rules
+# "SIM", # flake8-simplify
+# "SLF", # flake8-self
+# "SLOT", # flake8-slots
+# "T10", # debugger calls
+# # "T20", # flake8-print # TODO: Re-enable once we have logging
+# "TCH", # flake8-type-checking
+# "TD", # flake8-todos
+# "TID", # flake8-tidy-imports
+# "TRY", # tryceratops
+# "TRY002", # Disallow raising vanilla Exception. Create or use a custom exception instead.
+# "TRY003", # Disallow vanilla string passing. Prefer kwargs to the exception constructur.
+# "UP", # pyupgrade
+# "W", # pycodestyle (warnings)
+# "YTT", # flake8-2020
+# ]
+
ignore = [
# For rules reference, see https://docs.astral.sh/ruff/rules/
# These we don't agree with or don't want to prioritize to enforce:
"ANN003", # kwargs missing type annotations
- "ANN101", # Type annotations for 'self' args
- "ANN102", # Type annotations for 'cls' args
"COM812", # Because it conflicts with ruff auto-format
"EM", # flake8-errmsgs (may reconsider later)
"DJ", # Django linting
@@ -177,7 +164,7 @@ ignore = [
"S", # flake8-bandit (noisy, security related)
"SIM910", # Allow "None" as second argument to Dict.get(). "Explicit is better than implicit."
"TD002", # Require author for TODOs
- "TRIO", # flake8-trio (opinionated, noisy)
+ "ASYNC1", # flake8-trio (opinionated, noisy)
"INP001", # Dir 'examples' is part of an implicit namespace package. Add an __init__.py.
# TODO: Consider re-enabling these before release:
@@ -196,11 +183,9 @@ unfixable = [
"T201", # print() calls (avoid silent loss of code / log messages)
]
-line-length = 140
-extend-exclude = ["docs", "test", "tests"]
dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
force-sort-within-sections = false
lines-after-imports = 2
known-first-party = [
@@ -211,8 +196,6 @@ known-first-party = [
"connector_ops",
"pipelines",
]
-known-local-folder = ["airbyte"]
-required-imports = ["from __future__ import annotations"]
known-third-party = []
section-order = [
"future",
@@ -222,16 +205,16 @@ section-order = [
"local-folder",
]
-[tool.ruff.mccabe]
+[tool.ruff.lint.mccabe]
max-complexity = 24
-[tool.ruff.pycodestyle]
+[tool.ruff.lint.pycodestyle]
ignore-overlong-task-comments = true
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
convention = "google"
-[tool.ruff.flake8-annotations]
+[tool.ruff.lint.flake8-annotations]
allow-star-arg-any = false
ignore-fully-untyped = false
diff --git a/spotless-maven-pom.xml b/spotless-maven-pom.xml
index 63de55c6766..0a24e538b56 100644
--- a/spotless-maven-pom.xml
+++ b/spotless-maven-pom.xml
@@ -26,6 +26,9 @@
**/*.java
+
+ **/non_formatted_code/*
+
4.21