feat: add '/build-connector-images' command and related workflows for pre-release docker image builds (#61347)

.github/pr-welcome-community.md

@@ -12,10 +12,12 @@ Thank you for your contribution from **{{ .repo_name }}**! We're excited to have
 ### PR Slash Commands
 
 As needed or by request, Airbyte Maintainers can execute the following slash commands on your PR:
+
 - `/format-fix` - Fixes most formatting issues.
 - `/bump-version` - Bumps connector versions.
 - `/run-connector-tests` - Runs connector tests.
 - `/run-cat-tests` - Runs CAT tests.
+- `/build-connector-images` - Builds and publishes a pre-release docker image for the modified connector(s).
 
 If you have any questions, feel free to ask in the PR comments or join our [Slack community](https://airbytehq.slack.com/).
 

.github/pr-welcome-internal.md

@@ -19,5 +19,6 @@ Airbyte Maintainers (that's you!) can execute the following slash commands on yo
 - `/format-fix` - Fixes most formatting issues.
 - `/bump-version` - Bumps connector versions.
 - `/run-cat-tests` - Runs legacy CAT tests (Connector Acceptance Tests)
+- `/build-connector-images` - Builds and publishes a pre-release docker image for the modified connector(s).
 
 [📝 _Edit this welcome message._](https://github.com/airbytehq/airbyte/blob/master/.github/pr-welcome-internal.md)

.github/workflows/build-connector-images-command.yml

@@ -0,0 +1,106 @@
+name: Connector Image Builds
+# This workflow is used to run connector image builds.
+# It can be triggered manually via slash command or workflow dispatch.
+
+on:
+  # # For testing (TODO: comment out the below before merging):
+  # pull_request:
+  #  types: [opened, synchronize, reopened]
+
+  workflow_dispatch:
+    inputs:
+      repo:
+        description: "The repository name"
+        required: false
+        type: string
+      gitref:
+        description: "The git reference (branch or tag)"
+        required: false
+        type: string
+      comment-id:
+        description: "The ID of the comment triggering the workflow"
+        required: false
+        type: number
+      pr:
+        description: "The pull request number, if applicable"
+        required: false
+        type: number
+
+jobs:
+  init-workflow:
+    name: "Initialize Workflow"
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Get job variables
+        id: job-vars
+        # Create the job run variables.
+        # We intentionally ignore `inputs.repo` because the check run will be run on the
+        # current repository, not the one with the commit being tested.
+        run: |
+          echo "run-url=https://github.com/${{ github.repository }}/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
+          echo "image-name=ghcr.io/airbytehq/${{ inputs.repo || github.repository }}:${{ inputs.gitref || github.ref_name }}" >> $GITHUB_OUTPUT
+          echo "pr-number=${{ github.event_name == 'pull_request' && github.event.pull_request.number || inputs.pr }}" >> $GITHUB_OUTPUT
+
+      - name: Append start comment
+        id: append-start-comment
+        uses: peter-evans/create-or-update-comment@v4
+        if: success()
+        with:
+          comment-id: ${{ inputs.comment-id }}
+          issue-number: ${{ steps.job-vars.outputs.pr-number }}
+          reactions: "+1"
+          body: |
+            > **Connector Image Build Started**
+            >
+            > - This workflow will build the connector image and run basic tests.
+            > - The connector image(s) will be pushed to the [GitHub Container Registry](https://github.com/orgs/airbytehq/packages).
+            >
+            > [Check job output.](${{ steps.job-vars.outputs.run-url }})
+
+      - name: Repo Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.repo || github.repository }}
+          ref: ${{ inputs.gitref || '' }}
+          # Use fetch-depth: 0 to ensure we can access the full commit history
+          fetch-depth: 0
+
+      - name: Generate Connector Matrix from Changes
+        id: generate-matrix
+        run: |
+          # Get the list of modified connectors
+          echo "connectors_matrix=$(./poe-tasks/get-modified-connectors.sh --json)" | tee -a $GITHUB_OUTPUT
+
+    outputs:
+      connectors-matrix: ${{ steps.generate-matrix.outputs.connectors_matrix }}
+      pr-number: ${{ steps.job-vars.outputs.pr-number }}
+      comment-id: ${{ steps.append-start-comment.outputs.comment-id }}
+
+  call-connector-ci-build:
+    name: "Call Connector CI Build"
+    uses: ./.github/workflows/connector-image-build.yml
+    needs: [init-workflow]
+    strategy:
+      matrix: ${{ fromJson(needs.init-workflow.outputs.connectors-matrix) }}
+      max-parallel: 5 # Limit number of parallel jobs
+      fail-fast: false # Don't stop on first failure
+    with:
+      repo: "${{ inputs.repo || github.repository }}"
+      gitref: "${{ inputs.gitref || '' }}"
+      comment-id: "${{ github.event_name == 'workflow_dispatch' && inputs['comment-id'] || '' }}"
+      pr: "${{ github.event_name == 'workflow_dispatch' && inputs.pr || '' }}"
+      connector: ${{ matrix.connector }}
+    secrets: inherit
+
+  post-end-comment:
+    name: "Post Completion Comment"
+    needs: [init-workflow, call-connector-ci-build]
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Append end comment
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ needs.init-workflow.outputs.comment-id }}
+          issue-number: ${{ needs.init-workflow.outputs.pr-number }}
+          body: |
+            > Connector Image Builds job completed. See logs for details.

.github/workflows/connector-image-build.yml

@@ -0,0 +1,129 @@
+name: Connector Image Build
+
+on:
+  # Available as a reusable workflow
+  # (https://docs.github.com/en/actions/sharing-automations/reusing-workflows)
+  workflow_call:
+    inputs:
+      repo:
+        type: string
+        required: false
+        description: "The repository name"
+      gitref:
+        type: string
+        required: false
+        description: "The git reference (branch or tag)"
+      comment-id:
+        type: string
+        required: false
+        description: "The ID of the comment triggering the workflow. Unused as of now."
+      pr:
+        type: string
+        required: false
+        description: "The pull request number, if applicable. Unused as of now."
+      connector:
+        type: string
+        required: true
+        description: "The name of the connector to build and test. If empty, this is a no-op."
+
+permissions:
+  contents: read
+  pull-requests: write
+  actions: write
+  checks: write
+  packages: write
+
+jobs:
+  build-connector-image:
+    runs-on: ubuntu-24.04
+    if: inputs.connector
+    environment:
+      name: ghcr.io/airbytehq
+      url: https://ghcr.io/airbytehq/${{ inputs.connector }}
+    steps:
+      - name: Checkout Current Branch
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.repo || github.event.pull_request.head.repo.full_name }}
+          ref: ${{ inputs.gitref || github.head_ref }}
+          fetch-depth: 1
+
+      - name: Install the latest version of uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Install Poe
+        run: |
+          # Install Poe so we can run the connector tasks:
+          uv tool install poethepoet
+
+      - name: Resolve Vars
+        id: vars
+        run: |
+          set -eu
+          # Detect connector root directory:
+          CONNECTOR_ROOT="airbyte-integrations/connectors/${{ inputs.connector }}"
+          echo "connector-root=$CONNECTOR_ROOT" | tee -a $GITHUB_OUTPUT
+          cd $CONNECTOR_ROOT
+          # Read connector language and base image:
+          echo "connector-language=$(poe -qq get-language)" | tee -a $GITHUB_OUTPUT
+          echo "connector-base-image=$(poe -qq get-base-image)" | tee -a $GITHUB_OUTPUT
+
+      # Java deps
+      - uses: actions/setup-java@v4
+        if: ${{ steps.vars.outputs.connector-language == 'java' }}
+        with:
+          distribution: zulu
+          java-version: 21
+          cache: gradle
+
+      - uses: gradle/actions/setup-gradle@v4
+        if: ${{ steps.vars.outputs.connector-language == 'java' }}
+        with:
+          cache-read-only: false
+          cache-write-only: false
+          add-job-summary: "on-failure"
+
+      - name: Build Connector Tarball
+        if: ${{ steps.vars.outputs.connector-language == 'java' }}
+        run: |
+          ./gradlew :airbyte-integrations:connectors:${{ inputs.connector }}:distTar
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Connector Image
+        id: build-connector-image
+        uses: docker/build-push-action@v5
+        with:
+          context: airbyte-integrations/connectors/${{ inputs.connector }}
+          file: docker-images/Dockerfile.${{ steps.vars.outputs.connector-language }}-connector
+          platforms: linux/amd64,linux/arm64
+          tags: |
+            ghcr.io/airbytehq/${{ inputs.connector }}:draft-pr-${{ github.event.pull_request.number }}
+            ghcr.io/airbytehq/${{ inputs.connector }}:draft-pr-${{ github.event.pull_request.number }}-build${{ github.run_number }}
+          build-args: |
+            BASE_IMAGE=${{ steps.vars.outputs.connector-base-image }}
+            CONNECTOR_NAME=${{ inputs.connector }}
+          push: true
+
+      - name: Run `spec` Image Test
+        run: |
+          docker run \
+            --rm \
+            ghcr.io/airbytehq/${{ inputs.connector }}:draft-pr-${{ github.event.pull_request.number }} \
+            spec
+
+      - name: Run ${{ inputs.connector }} Image Vulnerability Scan
+        uses: anchore/scan-action@v6
+        with:
+          image: "ghcr.io/airbytehq/${{ inputs.connector }}:draft-pr-${{ github.event.pull_request.number }}"
+          output-format: "table"
+          severity-cutoff: high
+          fail-build: false

.github/workflows/slash-commands.yml

@@ -28,6 +28,7 @@ jobs:
             bump-bulk-cdk-and-release-connectors
             bump-cdk-version-and-merge
             bump-version
+            build-connector-images
             connector-performance
             format-fix
             poe

docs/platform/connector-development/local-connector-development.md

@@ -194,6 +194,7 @@ Maintainers can execute any of the following connector admin commands upon reque
 - `/format-fix` - Fixes any formatting issues.
 - `/run-connector-tests` - Run the connector tests for any modified connectors.
 - `/run-cat-tests` - Run the legacy connector acceptance tests (CAT) for any modified connectors. This is helpful if the connector has poor test coverage overall.
+- `/build-connector-images` - Builds and publishes a pre-release docker image for the modified connector(s).
 - `/poe` - Run a Poe task.
 
 When working on PRs from forks, maintainers can apply `/format-fix` to help expedite formatting fixes, and `/run-connector-tests` if the fork does not have sufficient secrets bootstrapping or other permissions needed to fully test the connector changes.

docusaurus/platform_versioned_docs/version-1.6/connector-development/local-connector-development.md

@@ -194,6 +194,7 @@ Maintainers can execute any of the following connector admin commands upon reque
 - `/format-fix` - Fixes any formatting issues.
 - `/run-connector-tests` - Run the connector tests for any modified connectors.
 - `/run-cat-tests` - Run the legacy connector acceptance tests (CAT) for any modified connectors. This is helpful if the connector has poor test coverage overall.
+- `/build-connector-images` - Builds and publishes a pre-release docker image for the modified connector(s).
 - `/poe` - Run a Poe task.
 
 When working on PRs from forks, maintainers can apply `/format-fix` to help expedite formatting fixes, and `/run-connector-tests` if the fork does not have sufficient secrets bootstrapping or other permissions needed to fully test the connector changes.

poe-tasks/gradle-connector-tasks.toml

@@ -27,7 +27,6 @@
 
 [tasks]
 
-get-language = "echo 'java'" # Use with -qq to get just the language name
 get-connector-name = 'basename "$PWD"' # Use with -qq to get just the connector name
 fetch-secrets = "airbyte-cdk secrets fetch"
 
@@ -71,6 +70,16 @@ args = [
   { name = "task_and_args", positional = true, multiple = true, help = "Gradle task name and its arguments" }
 ]
 
+# Generic tasks (same across all connector types)
+
+[tasks.get-language]
+cmd = """yq eval '.data.tags[] | select(test("^language:")) | sub("language:","")' $PWD/metadata.yaml"""
+help = "Get the language of the connector from its metadata.yaml file. Use with -qq to get just the language name."
+
+[tasks.get-base-image]
+cmd = """yq eval '.data.connectorBuildOptions.baseImage' $PWD/metadata.yaml"""
+help = "Get the base image of the connector from its metadata.yaml file."
+
 [tasks.run-cat-tests]
 shell = "airbyte-ci connectors --name=`poe -qq get-connector-name` test --only-step=acceptance"
 help = "Run the legacy Airbyte-CI acceptance tests (CAT tests)."

poe-tasks/manifest-only-connector-tasks.toml

@@ -10,7 +10,6 @@
 
 [tasks]
 
-get-language = "echo 'manifest-only'" # Use with -qq to get just the language name
 get-connector-name = 'basename "$PWD"' # Use with -qq to get just the connector name
 fetch-secrets = "airbyte-cdk secrets fetch ${POE_PWD}"
 
@@ -59,6 +58,16 @@ else
 fi
 '''
 
+# Generic tasks (same across all connector types)
+
+[tasks.get-language]
+cmd = """yq eval '.data.tags[] | select(test("^language:")) | sub("language:","")' $PWD/metadata.yaml"""
+help = "Get the language of the connector from its metadata.yaml file. Use with -qq to get just the language name."
+
+[tasks.get-base-image]
+cmd = """yq eval '.data.connectorBuildOptions.baseImage' $PWD/metadata.yaml"""
+help = "Get the base image of the connector from its metadata.yaml file."
+
 [tasks.run-cat-tests]
 shell = "airbyte-ci connectors --name=`poe -qq get-connector-name` test --only-step=acceptance"
 help = "Run the legacy Airbyte-CI acceptance tests (CAT tests)."

poe-tasks/poetry-connector-tasks.toml

@@ -16,7 +16,6 @@
 
 [tool.poe.tasks]
 
-get-language = "echo 'python'" # Use with -qq to get just the language name
 get-connector-name = 'basename "$PWD"' # Use with -qq to get just the connector name
 fetch-secrets = "airbyte-cdk secrets fetch"
 
@@ -126,6 +125,16 @@ args = [
 ]
 help = "Pin to a specific branch of the CDK."
 
+# Generic tasks (same across all connector types)
+
+[tool.poe.tasks.get-language]
+cmd = """yq eval '.data.tags[] | select(test("^language:")) | sub("language:","")' $PWD/metadata.yaml"""
+help = "Get the language of the connector from its metadata.yaml file. Use with -qq to get just the language name."
+
+[tool.poe.tasks.get-base-image]
+cmd = """yq eval '.data.connectorBuildOptions.baseImage' $PWD/metadata.yaml"""
+help = "Get the base image of the connector from its metadata.yaml file."
+
 [tool.poe.tasks.run-cat-tests]
 shell = "airbyte-ci connectors --name=`poe -qq get-connector-name` test --only-step=acceptance"
 help = "Run the legacy Airbyte-CI acceptance tests (CAT tests)."

poe_tasks.toml

@@ -66,3 +66,10 @@ args = [
   { name = "destination", positional = true, help = "Name of the destination (e.g., 'motherduck')" },
   { name = "task_and_args", positional = true, multiple = true, help = "Task name and its arguments" }
 ]
+
+[tasks.dummy-change]
+help = "A task to add a dummy change to the specified connector."
+shell = "echo '# Dummy change (revert-me)' >> ${POE_ROOT}/airbyte-integrations/connectors/${connector}/metadata.yaml"
+args = [
+  { name = "connector", positional = true, help = "Name of the connector to add a dummy change to (e.g., 'source-hardcoded-records')" },
+]