chore: connector publish workflow runs as matrix (#64498)

.github/workflows/publish_connectors.yml

@@ -32,9 +32,41 @@ on:
         description: "URL to the airbyte-ci binary to use for the action. If not provided, the action will use the latest release of airbyte-ci."
         default: "https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci"
 jobs:
+  list_connectors:
+    name: List connectors to publish
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2 # Required so we can conduct a diff from the previous commit to understand what connectors have changed.
+      - name: List connectors [manual]
+        id: list-connectors-manual
+        if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
+        shell: bash
+        # We need to run an actual script to parse the `--name` args and filter for JVM connectors.
+        run: ./poe-tasks/parse-connector-name-args.sh ${{ inputs.connectors-options }} | tee -a $GITHUB_OUTPUT
+      - name: List connectors [On merge to master]
+        id: list-connectors-master
+        if: github.event_name == 'push'
+        shell: bash
+        # get-modified-connectors.sh already prints things in the right format, so just use that output directly
+        run: |
+          echo connectors-to-publish=$(./poe-tasks/get-modified-connectors.sh --prev-commit --json) | tee -a $GITHUB_OUTPUT
+          echo connectors-to-publish-jvm=$(./poe-tasks/get-modified-connectors.sh --prev-commit --java --json) | tee -a $GITHUB_OUTPUT
+    outputs:
+      # Exactly one of the manual/master steps will run, so just OR them together.
+      connectors-to-publish: ${{ steps.list-connectors-manual.outputs.connectors-to-publish || steps.list-connectors-master.outputs.connectors-to-publish }}
+      connectors-to-publish-jvm: ${{ steps.list-connectors-manual.outputs.connectors-to-publish-jvm || steps.list-connectors-master.outputs.connectors-to-publish-jvm }}
   publish_connectors:
     name: Publish connectors
+    needs: [list_connectors]
     runs-on: ubuntu-24.04
+    strategy:
+      matrix: ${{ fromJson(needs.list_connectors.outputs.connectors-to-publish) }}
+      max-parallel: 5
+      # Allow all jobs to run, even if one fails
+      fail-fast: false
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v4
@@ -74,22 +106,22 @@ jobs:
       - name: Install metadata_service
         run: poetry install --directory airbyte-ci/connectors/metadata_service/lib
 
-      # If you modify this step, remember to also modify validate-connector-metadata-manual.
-      - name: Validate connector metadata [On merge to master]
-        id: validate-connector-metadata-master
-        if: github.event_name == 'push'
+      - name: Validate connector metadata
+        id: validate-connector-metadata
         shell: bash
-        run: ./poe-tasks/get-modified-connectors.sh --prev-commit --json | ./poe-tasks/validate-connector-metadata.sh
+        run: ./poe-tasks/validate-connector-metadata.sh --name ${{ matrix.connector }}
 
       - name: Build and publish JVM connectors images [On merge to master]
         id: build-and-publish-JVM-connectors-images-master
-        if: github.event_name == 'push'
+        if: >
+          github.event_name == 'push' &&
+          contains(fromJson(needs.list_connectors.outputs.connectors-to-publish-jvm).connector, matrix.connector)
         shell: bash
-        run: ./poe-tasks/get-modified-connectors.sh --prev-commit --json  --java | ./poe-tasks/build-and-publish-java-connectors-with-tag.sh --main-release --publish
+        run: ./poe-tasks/build-and-publish-java-connectors-with-tag.sh --main-release --publish --name ${{ matrix.connector }}
 
-      - name: Publish modified connectors [On merge to master]
+      - name: Publish connectors [On merge to master]
         # JVM docker images are published beforehand so Airbyte-CI only does registry publishing.
-        id: publish-modified-connectors
+        id: publish-connectors-master
         if: github.event_name == 'push'
         uses: ./.github/actions/run-airbyte-ci
         with:
@@ -106,26 +138,21 @@ jobs:
           spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release"
+          subcommand: "connectors --name ${{ matrix.connector }} publish --main-release"
           python_registry_token: ${{ secrets.PYPI_TOKEN }}
           max_attempts: 2
           retry_wait_seconds: 600 # 10 minutes
 
-      # If you modify this step, remember to also modify validate-connector-metadata-master.
-      - name: Validate connector metadata [manual]
-        id: validate-connector-metadata-manual
-        if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
-        shell: bash
-        run: ./poe-tasks/validate-connector-metadata.sh ${{ inputs.publish-options }} ${{ inputs.connectors-options }}
-
       - name: Build and publish JVM connectors images [manual]
         id: build-and-publish-JVM-connectors-images-manual
-        if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
+        if: >
+          (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') &&
+          contains(fromJson(needs.list_connectors.outputs.connectors-to-publish-jvm).connector, matrix.connector)
         shell: bash
-        run: ./poe-tasks/build-and-publish-java-connectors-with-tag.sh  ${{ inputs.publish-options }} ${{ inputs.connectors-options }} --publish
+        run: ./poe-tasks/build-and-publish-java-connectors-with-tag.sh  ${{ inputs.publish-options }} --name ${{ matrix.connector }} --publish
 
       - name: Publish connectors [manual]
-        id: publish-connectors
+        id: publish-connectors-manual
         if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
         uses: ./.github/actions/run-airbyte-ci
         with:
@@ -142,7 +169,7 @@ jobs:
           spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
           s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
           s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          subcommand: "connectors ${{ inputs.connectors-options }} publish ${{ inputs.publish-options }}"
+          subcommand: "connectors --name ${{ matrix.connector }} publish ${{ inputs.publish-options }}"
           python_registry_token: ${{ secrets.PYPI_TOKEN }}
           airbyte_ci_binary_url: ${{ inputs.airbyte_ci_binary_url }}
           max_attempts: 2

poe-tasks/parse-connector-name-args.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+# This script takes any number of command-line arguments in the format `--name <connector_name>`
+# and prints some key-value pairs, formatted as Github output variables.
+# For example, `./parse-connector-name-args.sh --name source-faker --name destination-bigquery` will print:
+# connectors-to-publish={"connector":["source-faker","destination-bigquery"]}
+# connectors-to-publish-jvm={"connector":["destination-bigquery"]}
+# (assuming that source-faker is a non-JVM connector, and destination-bigquery is a JVM connector)
+
+source "${BASH_SOURCE%/*}/lib/util.sh"
+
+source "${BASH_SOURCE%/*}/lib/parse_args.sh"
+
+connectors_jvm=()
+for connector in "${connectors[@]}"
+do
+  meta="${CONNECTORS_DIR}/${connector}/metadata.yaml"
+  if ! test -f "$meta"; then
+    echo "Error: metadata.yaml not found for ${connector}" >&2
+    exit 1
+  fi
+
+  if grep -qE 'language:\s*java' "$meta"; then
+    connectors_jvm+=($connector)
+  fi
+done
+
+# `printf '%s\n' "${arr[@]}"` prints each array item on a new line.
+# `jq --raw-input .` reads each line as a string value, and writes that value back out as a JSON string (i.e. wrapped in double quotes).
+#   (this also handles JSON escaping, though hopefully that's not needed for any connector name...)
+# `jq --compact-output --slurp .` then reads each line as a JSON value and writes them back out as a JSON array.
+#   `--compact-output` makes jq minify the output, rather than prettyprinting it.
+#   `--slurp` makes jq parse each line into a JSON value, then combine them all into an array.
+# We then wrap the entire thing in a JSON object.
+connectors_output='{"connector":'$(printf '%s\n' "${connectors[@]}" | jq --raw-input . | jq --compact-output --slurp .)'}'
+connectors_jvm_output='{"connector":'$(printf '%s\n' "${connectors_jvm[@]}" | jq --raw-input . | jq --compact-output --slurp .)'}'
+
+echo "connectors-to-publish=$connectors_output"
+echo "connectors-to-publish-jvm=$connectors_jvm_output"