chore: DRY up connector publishing pipeline [part 1] (#65086)

.github/workflows/publish_connectors.yml

@@ -8,29 +8,22 @@ on:
       - "airbyte-integrations/connectors/**/metadata.yaml"
   workflow_call:
     inputs:
-      connectors-options:
+      connectors:
-        description: "Options to pass to the 'airbyte-ci connectors' command group."
+        description: "list of connectors to publish. This should be a string of the form --name=source-pokeapi --name=destination-postgres."
         default: "--name=source-pokeapi"
         type: string
       publish-options:
         description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. "
         default: "--pre-release"
         type: string
-      airbyte_ci_binary_url:
-        description: "URL to the airbyte-ci binary to use for the action. If not provided, the action will use the latest release of airbyte-ci."
-        default: "https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci"
-        type: string
   workflow_dispatch:
     inputs:
-      connectors-options:
+      connectors:
-        description: "Options to pass to the 'airbyte-ci connectors' command group."
+        description: "list of connectors to publish. This should be a string of the form --name=source-pokeapi --name=destination-postgres."
         default: "--name=source-pokeapi"
       publish-options:
         description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. "
         default: "--pre-release"
-      airbyte_ci_binary_url:
-        description: "URL to the airbyte-ci binary to use for the action. If not provided, the action will use the latest release of airbyte-ci."
-        default: "https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci"
 jobs:
   list_connectors:
     name: List connectors to publish
@@ -44,20 +37,17 @@ jobs:
         id: list-connectors-manual
         if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
         shell: bash
-        # We need to run an actual script to parse the `--name` args and filter for JVM connectors.
+        # When invoked manually, we run on the connectors specified in the input.
-        run: ./poe-tasks/parse-connector-name-args.sh ${{ inputs.connectors-options }} | tee -a $GITHUB_OUTPUT
+        run: echo connectors-to-publish=$(./poe-tasks/parse-connector-name-args.sh ${{ inputs.connectors }}) | tee -a $GITHUB_OUTPUT
       - name: List connectors [On merge to master]
         id: list-connectors-master
         if: github.event_name == 'push'
         shell: bash
-        # get-modified-connectors.sh already prints things in the right format, so just use that output directly
+        # When merging to master, we run on connectors that have changed since the previous commit.
-        run: |
+        run: echo connectors-to-publish=$(./poe-tasks/get-modified-connectors.sh --prev-commit --json) | tee -a $GITHUB_OUTPUT
-          echo connectors-to-publish=$(./poe-tasks/get-modified-connectors.sh --prev-commit --json) | tee -a $GITHUB_OUTPUT
-          echo connectors-to-publish-jvm=$(./poe-tasks/get-modified-connectors.sh --prev-commit --java --json) | tee -a $GITHUB_OUTPUT
     outputs:
       # Exactly one of the manual/master steps will run, so just OR them together.
       connectors-to-publish: ${{ steps.list-connectors-manual.outputs.connectors-to-publish || steps.list-connectors-master.outputs.connectors-to-publish }}
-      connectors-to-publish-jvm: ${{ steps.list-connectors-manual.outputs.connectors-to-publish-jvm || steps.list-connectors-master.outputs.connectors-to-publish-jvm }}
   publish_connectors:
     name: Publish connectors
     needs: [list_connectors]

poe-tasks/parse-connector-name-args.sh

@@ -1,28 +1,30 @@
 #!/usr/bin/env bash
 
 # This script takes any number of command-line arguments in the format `--name <connector_name>`
-# and prints some key-value pairs, formatted as Github output variables.
+# and prints some key-value pairs, formatted to be used as a GitHub output variable for matrix builds.
 # For example, `./parse-connector-name-args.sh --name source-faker --name destination-bigquery` will print:
-# connectors-to-publish={"connector":["source-faker","destination-bigquery"]}
+# {"connector":["source-faker","destination-bigquery"]}
-# connectors-to-publish-jvm={"connector":["destination-bigquery"]}
+connectors=()
-# (assuming that source-faker is a non-JVM connector, and destination-bigquery is a JVM connector)
 
-source "${BASH_SOURCE%/*}/lib/util.sh"
+while [[ $# -gt 0 ]]; do
-
+  case "$1" in
-source "${BASH_SOURCE%/*}/lib/parse_args.sh"
+    --name=*)
-
+      connectors+=("${1#*=}")
-connectors_jvm=()
+      shift
-for connector in "${connectors[@]}"
+      ;;
-do
+    --name)
-  meta="${CONNECTORS_DIR}/${connector}/metadata.yaml"
+      connectors+=("$2")
-  if ! test -f "$meta"; then
+      shift 2
-    echo "Error: metadata.yaml not found for ${connector}" >&2
+      ;;
-    exit 1
+    --*)
-  fi
+      echo "Error: Unknown flag $1" >&2
-
+      exit 1
-  if grep -qE 'language:\s*java' "$meta"; then
+      ;;
-    connectors_jvm+=($connector)
+    *)
-  fi
+      connectors+=("$1")
+      shift
+      ;;
+  esac
 done
 
 # `printf '%s\n' "${arr[@]}"` prints each array item on a new line.
@@ -33,7 +35,5 @@ done
 #   `--slurp` makes jq parse each line into a JSON value, then combine them all into an array.
 # We then wrap the entire thing in a JSON object.
 connectors_output='{"connector":'$(printf '%s\n' "${connectors[@]}" | jq --raw-input . | jq --compact-output --slurp .)'}'
-connectors_jvm_output='{"connector":'$(printf '%s\n' "${connectors_jvm[@]}" | jq --raw-input . | jq --compact-output --slurp .)'}'
 
-echo "connectors-to-publish=$connectors_output"
+echo "$connectors_output"
-echo "connectors-to-publish-jvm=$connectors_jvm_output"