feat(ci): use pre-commit instead of airbyte-ci format (#49806)

.github/workflows/community_ci.yml

@@ -37,45 +37,6 @@ jobs:
           echo "The fork has changes in protected paths. This is not allowed."
           exit 1
 
-  format_check:
-    # IMPORTANT: This name must match the require check name on the branch protection settings
-    name: "Check for formatting errors"
-    if: github.event.pull_request.head.repo.fork == true
-    environment: community-ci-auto
-    runs-on: community-tooling-test-small
-    needs: fail_on_protected_path_changes
-    timeout-minutes: 30
-    env:
-      MAIN_BRANCH_NAME: "master"
-    steps:
-      # This checkouts a fork which can contain untrusted code
-      # It's deemed safe as the formatter are not executing any checked out code
-      - name: Checkout fork
-        uses: actions/checkout@v4
-        with:
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 1
-
-      # This will sync the .github folder of the main repo with the fork
-      # This allows us to use up to date actions and CI logic from the main repo
-      - name: Pull .github folder and internal packages from main repository
-        id: pull_github_folder
-        run: |
-          git remote add main https://github.com/airbytehq/airbyte.git
-          git fetch main ${MAIN_BRANCH_NAME}
-          git checkout main/${MAIN_BRANCH_NAME} -- .github
-          git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
-
-      - name: Run airbyte-ci format check all
-        # This path refers to the fork .github folder.
-        # We make sure its content is in sync with the main repo .github folder by pulling it in the previous step
-        uses: ./.github/actions/run-airbyte-ci
-        with:
-          context: "pull_request"
-          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
-          subcommand: "format check all"
-          is_fork: "true"
   connectors_early_ci:
     name: Run connectors early CI on fork
     if: github.event.pull_request.head.repo.fork == true

.github/workflows/format-fix-command.yml

@@ -29,7 +29,7 @@ concurrency:
 
 jobs:
   format-fix:
-    name: "Run airbyte-ci format fix all"
+    name: "Run pre-commit fix"
     runs-on: ubuntu-24.04
     steps:
       - name: Get job variables
@@ -67,16 +67,21 @@ jobs:
 
             [1]: ${{ steps.job-vars.outputs.run-url }}
 
-      - name: Run airbyte-ci format fix all
+      # Compare the below to the `format_check.yml` workflow
-        uses: ./.github/actions/run-airbyte-ci
+      - name: Setup Java
-        continue-on-error: true
+        uses: actions/setup-java@v3
         with:
-          context: "manual"
+          distribution: "zulu"
-          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          java-version: "21"
-          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+      - name: Setup Python
-          github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
+        uses: actions/setup-python@v5
-          subcommand: "format fix all"
+        with:
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_2 }}
+          python-version: "3.10"
+          cache: "pip"
+      - name: Run pre-commit
+        uses: pre-commit/action@v3.0.1
+        continue-on-error: true
+        id: format-fix
 
       # This is helpful in the case that we change a previously committed generated file to be ignored by git.
       - name: Remove any files that have been gitignored

.github/workflows/format_check.yml

@@ -10,53 +10,31 @@ on:
 
 jobs:
   format-check:
-    # IMPORTANT: This name must match the require check name on the branch protection settings
     name: "Check for formatting errors"
-    # Do not run this job on forks
+    runs-on: ubuntu-24.04
-    # Forked PRs are handled by the community_ci.yml workflow
-    if: github.event.pull_request.head.repo.fork != true
-    runs-on: tooling-test-small
     steps:
-      - name: Checkout Airbyte (with credentials)
+      - uses: actions/checkout@v4
-        uses: actions/checkout@v4
+      - name: Setup Java
+        uses: actions/setup-java@v3
         with:
-          ref: ${{ github.head_ref }}
+          distribution: "zulu"
-          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          java-version: "21"
-          fetch-depth: 1
+      - name: Setup Python
-      - name: Run airbyte-ci format check [MASTER]
+        uses: actions/setup-python@v5
-        id: airbyte_ci_format_check_all_master
-        if: github.ref == 'refs/heads/master'
-        uses: ./.github/actions/run-airbyte-ci
-        continue-on-error: true
         with:
-          context: "master"
+          python-version: "3.10"
-          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          cache: "pip"
-          subcommand: "format check all"
+      - name: Run pre-commit
-
+        uses: pre-commit/action@v3.0.1
-      - name: Run airbyte-ci format check [PULL REQUEST]
+        id: format-check
-        id: airbyte_ci_format_check_all_pr
-        if: github.event_name == 'pull_request'
-        uses: ./.github/actions/run-airbyte-ci
-        continue-on-error: false
         with:
-          context: "pull_request"
+          extra_args: --all-files
-          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
-          subcommand: "format check all"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_2 }}
-
-      - name: Run airbyte-ci format check [WORKFLOW DISPATCH]
-        id: airbyte_ci_format_check_all_manual
-        if: github.event_name == 'workflow_dispatch'
-        uses: ./.github/actions/run-airbyte-ci
-        continue-on-error: false
-        with:
-          context: "manual"
-          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
-          subcommand: "format check all"
-          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_2 }}
 
       - name: Match GitHub User to Slack User [MASTER]
-        if: github.ref == 'refs/heads/master'
+        if: >
+          always() && steps.format-check.outcome == 'failure' &&
+          github.ref == 'refs/heads/master' &&
+          github.event.pull_request.head.repo.fork == false
         id: match-github-to-slack-user
         uses: ./.github/actions/match-github-to-slack-user
         env:
@@ -64,7 +42,10 @@ jobs:
           GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Format Failure on Master Slack Channel [MASTER]
-        if: steps.airbyte_ci_format_check_all_master.outcome == 'failure' && github.ref == 'refs/heads/master'
+        if: >
+          always() && steps.format-check.outcome == 'failure' &&
+          github.ref == 'refs/heads/master' &&
+          github.event.pull_request.head.repo.fork == false
         uses: abinoda/slack-action@master
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}

.pre-commit-config.yaml

@@ -1,10 +1,59 @@
+exclude: |
+  (?x)(
+    # Python/system files
+    ^.*/__init__\.py$|
+    ^.*?/\.venv/.*$|
+    ^.*?/node_modules/.*$|
+
+    ^.*?/charts/.*$|
+    ^airbyte-integrations/bases/base-normalization/.*$|
+    ^.*?/normalization_test_output/.*$|
+
+    ^.*?/pnpm-lock\.yaml$|
+    ^.*?/source-amplitude/unit_tests/api_data/zipped\.json$|
+
+    # Generated/test files
+    ^airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/.*$|
+    ^.*?/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/.*/invalid/.*$|
+    ^airbyte-ci/connectors/metadata_service/lib/tests/fixtures/.*/invalid/.*$|
+    ^.*?/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/.*$|
+    ^airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/.*$|
+    ^.*?/airbyte-integrations/connectors/destination-.*/expected-spec\.json$
+  )
+
 repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.3
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args:
+          - --fix
+          - --select=I
+
+      # Run the formatter.
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.0.3
+    hooks:
+      - id: prettier
+        types_or: [json, yaml]
+        additional_dependencies:
+          - prettier@3.0.3
+
   - repo: local
     hooks:
-      - id: format-fix-all-on-push
+      - id: addlicense
-        always_run: true
+        name: Add license headers
-        entry: airbyte-ci --disable-update-check format fix all
+        entry: addlicense -c "Airbyte, Inc." -l apache -v -f LICENSE_SHORT
+        language: golang
+        additional_dependencies: [github.com/google/addlicense@v1.1.1]
+        files: \.(java|kt|py)$
+
+      - id: spotless
+        name: Format Java files with Spotless
+        entry: bash -c 'command -v mvn >/dev/null 2>&1 || { echo "Maven not installed, skipping spotless" >&2; exit 0; }; mvn -f spotless-maven-pom.xml spotless:apply'
         language: system
-        name: Run airbyte-ci format fix on git push (~30s)
+        files: \.(java|kt|gradle)$
         pass_filenames: false
-        stages: [push]

airbyte-ci/connectors/auto_merge/pyproject.toml

@@ -20,6 +20,9 @@ ruff = "^0.4.3"
 pytest = "^8.2.0"
 pyinstrument = "^4.6.2"
 
+[tool.ruff]
+line-length = 140
+
 [tool.ruff.lint]
 select = [
     "I" # isort

airbyte-ci/connectors/live-tests/pyproject.toml

@@ -53,11 +53,13 @@ pandas-stubs = "^2.2.0.240218"
 types-requests = "^2.31.0.20240311"
 types-pyyaml = "^6.0.12.20240311"
 
-[tool.ruff.lint]
+[tool.ruff]
-select = ["I", "F"]
 target-version = "py310"
 line-length = 140
 
+[tool.ruff.lint]
+select = ["I", "F"]
+
 [tool.ruff.lint.isort]
 known-first-party = ["connection-retriever"]
 

airbyte-ci/connectors/pipelines/ruff.toml

@@ -1,15 +1,13 @@
 target-version = "py310"
 line-length = 140
 
-ignore = ["ANN101", "ANN002", "ANN003"]
-
 [lint]
 
+ignore = ["ANN003"]
 extend-select = [
     "ANN", # flake8-annotations
 ]
 
-
 [lint.pydocstyle]
 convention = "google"
 

pyproject.toml

@@ -38,27 +38,6 @@ extend-exclude = """
 )/
 """
 
-[tool.flake8]
-extend-exclude = [
-    "*/lib/*/site-packages",
-    ".venv",
-    "build",
-    "models",
-    ".eggs",
-    "**/__init__.py",
-    "**/generated/*",
-    "**/declarative/models/*",
-]
-max-complexity = 20
-max-line-length = 140
-extend-ignore = [
-    "E203", # whitespace before ':' (conflicts with Black)
-    "E231", # Bad trailing comma (conflicts with Black)
-    "E501", # line too long (conflicts with Black)
-    "W503", # line break before binary operator (conflicts with Black)
-    "F811", # TODO: ella fix after pflake8 version update
-]
-
 [tool.coverage.report]
 fail_under = 0
 skip_empty = true
@@ -94,78 +73,86 @@ skip_glob = [
     "airbyte-integrations/connectors/destination-snowflake-cortex/**"
 ]
 
-[tool.ruff.pylint]
+[tool.ruff.lint.pylint]
 max-args = 8      # Relaxed from default of 5
 max-branches = 15 # Relaxed from default of 12
 
 [tool.ruff]
 target-version = "py310"
+line-length = 140
+extend-exclude = ["docs", "test", "tests"]
+
+[tool.ruff.lint]
+
 select = [
-    # For rules reference, see https://docs.astral.sh/ruff/rules/
+    "I",    # isort replacement
-    "A",     # flake8-builtins
-    "ANN",   # flake8-annotations
-    "ARG",   # flake8-unused-arguments
-    "ASYNC", # flake8-async
-    "B",     # flake8-bugbear
-    "FBT",   # flake8-boolean-trap
-    "BLE",   # Blind except
-    "C4",    # flake8-comprehensions
-    "C90",   # mccabe (complexity)
-    "COM",   # flake8-commas
-    "CPY",   # missing copyright notice
-    # "D", # pydocstyle # TODO: Re-enable when adding docstrings
-    "DTZ",  # flake8-datetimez
-    "E",    # pycodestyle (errors)
-    "ERA",  # flake8-eradicate (commented out code)
-    "EXE",  # flake8-executable
-    "F",    # Pyflakes
-    "FA",   # flake8-future-annotations
-    "FIX",  # flake8-fixme
-    "FLY",  # flynt
-    "FURB", # Refurb
-    "I",    # isort
-    "ICN",  # flake8-import-conventions
-    "INP",  # flake8-no-pep420
-    "INT",  # flake8-gettext
-    "ISC",  # flake8-implicit-str-concat
-    "ICN",  # flake8-import-conventions
-    "LOG",  # flake8-logging
-    "N",    # pep8-naming
-    "PD",   # pandas-vet
-    "PERF", # Perflint
-    "PIE",  # flake8-pie
-    "PGH",  # pygrep-hooks
-    "PL",   # Pylint
-    "PT",   # flake8-pytest-style
-    "PTH",  # flake8-use-pathlib
-    "PYI",  # flake8-pyi
-    "Q",    # flake8-quotes
-    "RET",  # flake8-return
-    "RSE",  # flake8-raise
-    "RUF",  # Ruff-specific rules
-    "SIM",  # flake8-simplify
-    "SLF",  # flake8-self
-    "SLOT", # flake8-slots
-    "T10",  # debugger calls
-    # "T20", # flake8-print # TODO: Re-enable once we have logging
-    "TCH",    # flake8-type-checking
-    "TD",     # flake8-todos
-    "TID",    # flake8-tidy-imports
-    "TRY",    # tryceratops
-    "TRY002", # Disallow raising vanilla Exception. Create or use a custom exception instead.
-    "TRY003", # Disallow vanilla string passing. Prefer kwargs to the exception constructur.
-    "UP",     # pyupgrade
-    "W",      # pycodestyle (warnings)
-    "YTT",    # flake8-2020
 ]
 
+# TODO: Re-enable these once we have time to address them
+# select = [
+#     # For rules reference, see https://docs.astral.sh/ruff/rules/
+#     "A",     # flake8-builtins
+#     "ANN",   # flake8-annotations
+#     "ARG",   # flake8-unused-arguments
+#     "ASYNC", # flake8-async
+#     "B",     # flake8-bugbear
+#     "FBT",   # flake8-boolean-trap
+#     "BLE",   # Blind except
+#     "C4",    # flake8-comprehensions
+#     "C90",   # mccabe (complexity)
+#     "COM",   # flake8-commas
+#     # "CPY",   # missing copyright notice  # Requires 'preview=true'
+#     # "D", # pydocstyle # TODO: Re-enable when adding docstrings
+#     "DTZ",  # flake8-datetimez
+#     "E",    # pycodestyle (errors)
+#     "ERA",  # flake8-eradicate (commented out code)
+#     "EXE",  # flake8-executable
+#     "F",    # Pyflakes
+#     "FA",   # flake8-future-annotations
+#     "FIX",  # flake8-fixme
+#     "FLY",  # flynt
+#     "FURB", # Refurb
+#     "I",    # isort
+#     "ICN",  # flake8-import-conventions
+#     "INP",  # flake8-no-pep420
+#     "INT",  # flake8-gettext
+#     "ISC",  # flake8-implicit-str-concat
+#     "ICN",  # flake8-import-conventions
+#     "LOG",  # flake8-logging
+#     "N",    # pep8-naming
+#     "PD",   # pandas-vet
+#     "PERF", # Perflint
+#     "PIE",  # flake8-pie
+#     "PGH",  # pygrep-hooks
+#     "PL",   # Pylint
+#     "PT",   # flake8-pytest-style
+#     "PTH",  # flake8-use-pathlib
+#     "PYI",  # flake8-pyi
+#     "Q",    # flake8-quotes
+#     "RET",  # flake8-return
+#     "RSE",  # flake8-raise
+#     "RUF",  # Ruff-specific rules
+#     "SIM",  # flake8-simplify
+#     "SLF",  # flake8-self
+#     "SLOT", # flake8-slots
+#     "T10",  # debugger calls
+#     # "T20", # flake8-print # TODO: Re-enable once we have logging
+#     "TCH",    # flake8-type-checking
+#     "TD",     # flake8-todos
+#     "TID",    # flake8-tidy-imports
+#     "TRY",    # tryceratops
+#     "TRY002", # Disallow raising vanilla Exception. Create or use a custom exception instead.
+#     "TRY003", # Disallow vanilla string passing. Prefer kwargs to the exception constructur.
+#     "UP",     # pyupgrade
+#     "W",      # pycodestyle (warnings)
+#     "YTT",    # flake8-2020
+# ]
+
 ignore = [
     # For rules reference, see https://docs.astral.sh/ruff/rules/
 
     # These we don't agree with or don't want to prioritize to enforce:
     "ANN003",  # kwargs missing type annotations
-    "ANN101",  # Type annotations for 'self' args
-    "ANN102",  # Type annotations for 'cls' args
     "COM812",  # Because it conflicts with ruff auto-format
     "EM",      # flake8-errmsgs (may reconsider later)
     "DJ",      # Django linting
@@ -177,7 +164,7 @@ ignore = [
     "S",       # flake8-bandit (noisy, security related)
     "SIM910",  # Allow "None" as second argument to Dict.get(). "Explicit is better than implicit."
     "TD002",   # Require author for TODOs
-    "TRIO",    # flake8-trio (opinionated, noisy)
+    "ASYNC1",  # flake8-trio (opinionated, noisy)
     "INP001",  # Dir 'examples' is part of an implicit namespace package. Add an __init__.py.
 
     # TODO: Consider re-enabling these before release:
@@ -196,11 +183,9 @@ unfixable = [
     "T201",   # print() calls (avoid silent loss of code / log messages)
 ]
 
-line-length = 140
-extend-exclude = ["docs", "test", "tests"]
 dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 force-sort-within-sections = false
 lines-after-imports = 2
 known-first-party = [
@@ -211,8 +196,6 @@ known-first-party = [
     "connector_ops",
     "pipelines",
 ]
-known-local-folder = ["airbyte"]
-required-imports = ["from __future__ import annotations"]
 known-third-party = []
 section-order = [
     "future",
@@ -222,16 +205,16 @@ section-order = [
     "local-folder",
 ]
 
-[tool.ruff.mccabe]
+[tool.ruff.lint.mccabe]
 max-complexity = 24
 
-[tool.ruff.pycodestyle]
+[tool.ruff.lint.pycodestyle]
 ignore-overlong-task-comments = true
 
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "google"
 
-[tool.ruff.flake8-annotations]
+[tool.ruff.lint.flake8-annotations]
 allow-star-arg-any = false
 ignore-fully-untyped = false
 

spotless-maven-pom.xml

@@ -26,6 +26,9 @@
                             <includes>
                                 <include>**/*.java</include>
                             </includes>
+                            <excludes>
+                                <exclude>**/non_formatted_code/*</exclude>
+                            </excludes>
                             <importOrder />
                             <eclipse>
                                 <version>4.21</version>