Dbgold17/dry up connector publish pipeline (#65098)

.github/actions/connector-image-build-push/action.yml

@@ -12,6 +12,10 @@ inputs:
     description: "Docker registry"
     required: false
     default: "docker.io/airbyte"
+  release-type:
+    description: "Release type: 'pre-release' or 'main-release'. Determines tagging and latest push behavior."
+    required: false
+    default: "pre-release"
   tag-override:
     description: "Override the image tag (optional). If not provided, the tag will be derived from the connector metadata.yaml."
     required: false
@@ -118,17 +122,43 @@ runs:
           exit 1
         fi
 
-        # Resolve connector version to use for image tagging
+        # Get connector version from metadata
+        CONNECTOR_VERSION=$(poe -qq get-version)
+        if [[ -z "$CONNECTOR_VERSION" ]]; then
+          echo "❌ Docker tag not found in metadata.yaml"
+          exit 1
+        fi
+
+        # Check if this is a release candidate
+        IS_RELEASE_CANDIDATE=$(poe -qq get-version | grep -q 'rc' && echo "true" || echo "false")
+
+        # Determine the image tags to use 
         if [[ "${{ inputs.tag-override }}" ]]; then
           CONNECTOR_VERSION_TAG="${{ inputs.tag-override }}"
-          echo "🏷 Using provided tag: $CONNECTOR_VERSION_TAG"
+          echo "🏷 Using provided tag override: $CONNECTOR_VERSION_TAG"
+        elif [[ "${{ inputs.release-type }}" == "pre-release" ]]; then
+          hash=$(git rev-parse --short=10 HEAD)
+          CONNECTOR_VERSION_TAG="${CONNECTOR_VERSION}-dev.${hash}"
+          echo "🏷 Using pre-release tag: $CONNECTOR_VERSION_TAG"
         else
-          CONNECTOR_VERSION_TAG=$(poe -qq get-version)
+          CONNECTOR_VERSION_TAG="$CONNECTOR_VERSION"
-          if [[ -z "$CONNECTOR_VERSION_TAG" ]]; then
+          echo "🏷 Using main release tag: $CONNECTOR_VERSION_TAG"
-            echo "❌ Docker tag not found in metadata.yaml"
+        fi
-            exit 1
+
+        # Determine whether to push with latest tag
+        if [[ "${{ inputs.push-latest }}" == "true" || "${{ inputs.push-latest }}" == "false" ]]; then
+          # Use explicit input value
+          PUSH_LATEST="${{ inputs.push-latest }}"
+          echo "🏷 Using explicit push-latest setting: $PUSH_LATEST"
+        else
+          # Auto-determine based on release-type and release candidate status
+          if [[ "${{ inputs.release-type }}" == "pre-release" || "$IS_RELEASE_CANDIDATE" == "true" ]]; then
+            PUSH_LATEST="false"
+            echo "🏷 Skipping latest tag for pre-release or release candidate build"
+          else
+            PUSH_LATEST="true"
+            echo "🏷 Will push with latest tag for main release"
           fi
-          echo "🏷 Using tag from metadata.yaml: $CONNECTOR_VERSION_TAG"
         fi
 
         # Can't dry-run and force-publish at the same time
@@ -137,7 +167,7 @@ runs:
           exit 1
         fi
 
-        # Validate registry input
+        # Validate image registry input
         case "${{ inputs.registry }}" in
           "docker.io/airbyte")
             # Supported registries
@@ -157,7 +187,7 @@ runs:
 
         # Get all tags to use for image pushing
         DOCKER_TAGS="${FULL_IMAGE_NAME}"
-        if [[ "${{ inputs.push-latest }}" == "true" ]]; then
+        if [[ "$PUSH_LATEST" == "true" ]]; then
           DOCKER_TAGS="${DOCKER_TAGS},${{ inputs.registry }}/${CONNECTOR_NAME}:latest"
         fi
 
@@ -168,6 +198,8 @@ runs:
         echo "  Version Tag: $CONNECTOR_VERSION_TAG"
         echo "  Registry: ${{ inputs.registry }}"
         echo "  Full Image Name: $FULL_IMAGE_NAME"
+        echo "  Release Type: ${{ inputs.release-type }}"
+        echo "  Push Latest: $PUSH_LATEST"
         echo "  Dry Run: ${{ inputs.dry-run }}"
         echo "  Force Publish: ${{ inputs.force-publish }}" 
 
@@ -251,15 +283,12 @@ runs:
           DO_PUBLISH="true"
         fi
 
-        push_latest="${{ inputs.push-latest }}"
-
         # Output decision summary
         echo ""
         echo "📊 Build Decision Summary:"
         echo "  Image exists: $IMAGE_EXISTS"
         echo "  Will build: $DO_BUILD"
         echo "  Will publish: $DO_PUBLISH"
-        echo "  Will tag with latest: ${push_latest:-false}"
         echo "  Dry run mode: ${{ inputs.dry-run }}"
         echo "  Force publish: ${{ inputs.force-publish }}"
 

.github/workflows/bump-bulk-cdk-and-release-connectors-command.yml

@@ -64,7 +64,7 @@ jobs:
         connector_name: ${{ fromJson(needs.bump-cdk-version.outputs.modified_connectors_json) }}
     with:
       connectors: "--name=${{ matrix.connector_name }}"
-      publish-options: --main-release
+      release-type: main-release
     secrets: inherit
 
   update-changelog-and-merge:

.github/workflows/bump-cdk-version-and-merge-command.yml

@@ -139,7 +139,7 @@ jobs:
         connector_name: ${{ fromJson(needs.bump-cdk-version.outputs.modified_connectors) }}
     with:
       connectors: "--name=${{ matrix.connector_name }}"
-      publish-options: --pre-release
+      release-type: pre-release
     secrets: inherit
 
   update-changelog-and-merge:

.github/workflows/publish_connectors.yml

@@ -1,4 +1,4 @@
-name: Connector Ops CI - Publish Connectors
+name: Publish Connectors
 
 on:
   push:
@@ -9,51 +9,68 @@ on:
   workflow_call:
     inputs:
       connectors:
-        description: "list of connectors to publish. This should be a string of the form --name=source-pokeapi --name=destination-postgres."
+        description: "Connectors to Publish. This should be a string of the form: --name=source-pokeapi --name=destination-postgres"
         default: "--name=source-pokeapi"
         type: string
-      publish-options:
+      release-type:
-        description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. "
+        description: "Whether to publish as a pre-release (tagged for development) or as a main-release (production)"
-        default: "--pre-release"
+        default: pre-release
         type: string
   workflow_dispatch:
     inputs:
       connectors:
-        description: "list of connectors to publish. This should be a string of the form --name=source-pokeapi --name=destination-postgres."
+        description: "Connectors to Publish. This should be a string of the form: --name=source-pokeapi --name=destination-postgres"
         default: "--name=source-pokeapi"
-      publish-options:
+        type: string
-        description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. "
+      release-type:
-        default: "--pre-release"
+        description: "Whether to publish as a pre-release (tagged for development) or as a main-release (production)"
+        default: pre-release
+        type: choice
+        options:
+          - pre-release
+          - main-release
 jobs:
-  list_connectors:
+  publish_options:
-    name: List connectors to publish
+    name: Resolve options for connector publishing
     runs-on: ubuntu-24.04
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v4
         with:
           fetch-depth: 2 # Required so we can conduct a diff from the previous commit to understand what connectors have changed.
-      - name: List connectors [manual]
+      - name: List connectors to publish [manual]
         id: list-connectors-manual
         if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
         shell: bash
         # When invoked manually, we run on the connectors specified in the input.
         run: echo connectors-to-publish=$(./poe-tasks/parse-connector-name-args.sh ${{ inputs.connectors }}) | tee -a $GITHUB_OUTPUT
-      - name: List connectors [On merge to master]
+      - name: List connectors to publish [On merge to master]
         id: list-connectors-master
         if: github.event_name == 'push'
         shell: bash
         # When merging to master, we run on connectors that have changed since the previous commit.
         run: echo connectors-to-publish=$(./poe-tasks/get-modified-connectors.sh --prev-commit --json) | tee -a $GITHUB_OUTPUT
+      - name: Resolve release type
+        id: resolve-release-type
+        shell: bash
+        run: |
+          # If workflow is triggered on merge to master, do a main release.
+          # If workflow is triggered manually, use the input release type.
+          if [[ "${{ github.event_name }}" == "push" ]]; then
+            echo "release-type=main-release" | tee -a $GITHUB_OUTPUT
+          else
+            echo "release-type=${{ inputs.release-type }}" | tee -a $GITHUB_OUTPUT
+          fi
     outputs:
       # Exactly one of the manual/master steps will run, so just OR them together.
       connectors-to-publish: ${{ steps.list-connectors-manual.outputs.connectors-to-publish || steps.list-connectors-master.outputs.connectors-to-publish }}
+      release-type: ${{ steps.resolve-release-type.outputs.release-type }}
   publish_connectors:
     name: Publish connectors
-    needs: [list_connectors]
+    needs: [publish_options]
     runs-on: ubuntu-24.04
     strategy:
-      matrix: ${{ fromJson(needs.list_connectors.outputs.connectors-to-publish) }}
+      matrix: ${{ fromJson(needs.publish_options.outputs.connectors-to-publish) }}
       max-parallel: 5
       # Allow all jobs to run, even if one fails
       fail-fast: false
@@ -127,156 +144,59 @@ jobs:
         shell: bash
         run: ./poe-tasks/validate-connector-metadata.sh --name ${{ matrix.connector }}
 
-      - name: Publish to Python Registry [On merge to master]
+      - name: Publish to Python Registry
-        id: publish-python-registry-master
+        id: publish-python-registry
-        if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language == 'python'
+        if: steps.connector-metadata.outputs.connector-language == 'python'
         shell: bash
         run: |
-          ./poe-tasks/publish-python-registry.sh --name ${{ matrix.connector }}
+          ./poe-tasks/publish-python-registry.sh --name ${{ matrix.connector }} --release-type ${{ needs.publish_options.outputs.release-type }}
         env:
           PYTHON_REGISTRY_TOKEN: ${{ secrets.PYPI_TOKEN }}
-        continue-on-error: true # We allow this to fail because we are testing this step. TODO: remove this once we are sure it works.
 
-      - name: Publish to Python Registry [Manual]
+      - name: Upload Python Dependencies to GCS
-        id: publish-python-registry-manual
-        if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language == 'python'
-        shell: bash
-        run: |
-          # Determine if this is a pre-release
-          is_pre_release=$(echo "${{ inputs.publish-options }}" | grep -c -- '--pre-release' || true)
-          if [[ $is_pre_release -gt 0 ]]; then
-            PRE_RELEASE_FLAG="--pre-release"
-          else
-            PRE_RELEASE_FLAG=""
-          fi
-
-          ./poe-tasks/publish-python-registry.sh --name ${{ matrix.connector }} $PRE_RELEASE_FLAG
-        env:
-          PYTHON_REGISTRY_TOKEN: ${{ secrets.PYPI_TOKEN }}
-        continue-on-error: true # We allow this to fail because we are testing this step. TODO: remove this once we are sure it works.
-
-      - name: Upload Python Dependencies to GCS [On merge to master]
         id: upload-python-dependencies-master
-        if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language == 'python'
+        if: steps.connector-metadata.outputs.connector-language == 'python'
         shell: bash
         env:
           GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
         run: |
-          ./poe-tasks/upload-python-dependencies.sh --name ${{ matrix.connector }} --bucket prod-airbyte-cloud-connector-metadata-service
+          ./poe-tasks/upload-python-dependencies.sh \
-        continue-on-error: true # We allow this to fail because we are testing this step. TODO: remove this once we are sure it works.
+            --name ${{ matrix.connector }} \
+            --bucket prod-airbyte-cloud-connector-metadata-service \
+            --release-type ${{ needs.publish_options.outputs.release-type }}
 
-      - name: Upload Python Dependencies to GCS [Manual]
+      - name: Build and publish JVM connectors images
-        id: upload-python-dependencies-manual
+        id: build-and-publish-JVM-connectors-images
-        if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language == 'python'
+        if: steps.connector-metadata.outputs.connector-language == 'java'
-        shell: bash
-        env:
-          GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
-        run: |
-          # Determine if this is a pre-release
-          is_pre_release=$(echo "${{ inputs.publish-options }}" | grep -c -- '--pre-release' || true)
-          if [[ $is_pre_release -gt 0 ]]; then
-            PRE_RELEASE_FLAG="--pre-release"
-          else
-            PRE_RELEASE_FLAG=""
-          fi
-          ./poe-tasks/upload-python-dependencies.sh --name ${{ matrix.connector }} --bucket prod-airbyte-cloud-connector-metadata-service $PRE_RELEASE_FLAG
-        continue-on-error: true # We allow this to fail because we are testing this step. TODO: remove this once we are sure it works.
-
-      - name: Build and publish JVM connectors images [On merge to master]
-        id: build-and-publish-JVM-connectors-images-master
-        if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language == 'java'
         shell: bash
         run: |
-          ./poe-tasks/build-and-publish-java-connectors-with-tag.sh --main-release --publish --name ${{ matrix.connector }}
+          ./poe-tasks/build-and-publish-java-connectors-with-tag.sh --name ${{ matrix.connector }} --release-type ${{ needs.publish_options.outputs.release-type }} --publish
 
-      - name: Publish JVM connectors tar file [On merge to master]]
+      - name: Publish JVM connectors tar file
-        id: publish-JVM-connectors-tar-file-master
+        id: publish-JVM-connectors-tar-file
-        if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language == 'java'
+        if: steps.connector-metadata.outputs.connector-language == 'java'
         shell: bash
-        run: ./poe-tasks/upload-java-connector-tar-file.sh --name ${{ matrix.connector }} --main-release
+        run: ./poe-tasks/upload-java-connector-tar-file.sh --name ${{ matrix.connector }} --release-type ${{ needs.publish_options.outputs.release-type }}
         env:
           GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}
         # we allow it to fail because we are testing this step. We should remove this once we are sure it works.
         continue-on-error: true
 
-      - name: Determine build and publish options
+      - name: Build and publish Python and Manifest-Only connectors images
-        id: get-connector-options
+        id: build-and-publish-python-manifest-only-connectors-images
         if: steps.connector-metadata.outputs.connector-language != 'java'
-        shell: bash
-        run: |
-          # TODO: refactor this logic to be shared by all connector types
-          is_pre_release=$(echo ${{ inputs.publish-options }} | grep -c -- '--pre-release' || true)
-          is_release_candidate=$(echo ${{ steps.connector-metadata.outputs.connector-version }} | grep -c -- '-rc' || true)
-
-          # Don't tag with latest for pre-release or release candidate builds
-          if [[ $is_pre_release -gt 0 || $is_release_candidate -gt 0 ]]; then
-            echo "push-latest=false" >> $GITHUB_OUTPUT
-            echo "Skipping latest tag for pre-release or release candidate build."
-          else
-            echo "push-latest=true" >> $GITHUB_OUTPUT
-          fi
-
-          if [[ $is_pre_release -gt 0 ]]; then
-            hash=$(git rev-parse --short=10 HEAD)
-            tag_override="${{ steps.connector-metadata.outputs.connector-version }}-dev.${hash}"
-            echo "tag-override=${tag_override}" >> $GITHUB_OUTPUT
-            echo "Using a dev tag for a pre-release build: ${tag_override}"
-          fi
-
-      - name: Build and publish Python and Manifest-Only connectors images [On merge to master]
-        id: build-and-publish-python-manifest-only-connectors-images-master
-        if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language != 'java'
         uses: ./.github/actions/connector-image-build-push
         with:
           connector-name: ${{ matrix.connector }}
-          push-latest: ${{ steps.get-connector-options.outputs.push-latest }}
+          release-type: ${{ needs.publish_options.outputs.release-type }}
           dry-run: "false"
           docker-hub-username: ${{ secrets.DOCKER_HUB_USERNAME }}
           docker-hub-password: ${{ secrets.DOCKER_HUB_PASSWORD }}
 
-      - name: Upload connector metadata [On merge to master]
+      - name: Upload connector metadata
-        id: upload-connector-metadata-master
+        id: upload-connector-metadata
-        if: github.event_name == 'push'
         shell: bash
-        run: ./poe-tasks/upload-connector-metadata.sh --name ${{ matrix.connector }} --main-release
+        run: ./poe-tasks/upload-connector-metadata.sh --name ${{ matrix.connector }} --release-type ${{ needs.publish_options.outputs.release-type }}
-        env:
-          GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
-          SPEC_CACHE_GCS_CREDENTIALS: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
-          METADATA_SERVICE_GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
-
-      - name: Build and publish JVM connectors images [manual]
-        id: build-and-publish-JVM-connectors-images-manual
-        if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language == 'java'
-        shell: bash
-        run: ./poe-tasks/build-and-publish-java-connectors-with-tag.sh  ${{ inputs.publish-options }} --name ${{ matrix.connector }} --publish
-
-      - name: Publish JVM connectors tar file [manual]
-        id: publish-JVM-connectors-tar-file-manual
-        if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language == 'java'
-        shell: bash
-        run: ./poe-tasks/upload-java-connector-tar-file.sh --name ${{ matrix.connector }} ${{ inputs.publish-options }}
-        env:
-          GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}
-        # we allow it to fail because we are testing this step. We should remove this once we are sure it works.
-        continue-on-error: true
-
-      - name: Build and publish Python and Manifest-Only connectors images [manual]
-        id: build-and-publish-python-manifest-only-connectors-images-manual
-        if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language != 'java'
-        uses: ./.github/actions/connector-image-build-push
-        with:
-          connector-name: ${{ matrix.connector }}
-          push-latest: ${{ steps.get-connector-options.outputs.push-latest }}
-          tag-override: ${{ steps.get-connector-options.outputs.tag-override }}
-          dry-run: "false"
-          docker-hub-username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          docker-hub-password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-
-      - name: Upload connector metadata [Manual]
-        id: upload-connector-metadata-manual
-        if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
-        shell: bash
-        run: ./poe-tasks/upload-connector-metadata.sh --name ${{ matrix.connector }} ${{ inputs.publish-options }}
         env:
           GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
           SPEC_CACHE_GCS_CREDENTIALS: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}

poe-tasks/build-and-publish-java-connectors-with-tag.sh

@@ -1,16 +1,15 @@
 #!/usr/bin/env bash
 
 # This script builds and optionally publishes Java connector Docker images.
-# Usage: ./build-and-publish-java-connectors-with-tag.sh --name <name> [--pre-release] [--main-release] [--publish]
+# Usage: ./build-and-publish-java-connectors-with-tag.sh --name <name> --release-type [pre-release | main-release] [--publish]
 #
 # Flag descriptions:
-#   --main-release: Publishes images with the exact version from metadata.yaml.
+#   --name <name>:    Specifies the connector name (e.g., destination-bigquery).
-#                   Only publishes if the image does not exists on Dockerhub.
+#                    
-#                   Used for production releases on merge to master.
+#   --release-type:   Specifies the release type:
-#
+#                     - pre-release: Builds with a dev tag (version-dev.githash).
-#   --pre-release:  Publishes images with a dev tag (version-dev.githash).
+#                     - main-release: Builds with the exact version from metadata.yaml.
-#                   Only publishes if the image does not exists on Dockerhub.
+#                     Defaults to pre-release if not specified.
-#                   Used for development/testing purposes.
 #
 #   --publish:      Actually publishes the images. Without this flag, the script runs in dry-run mode
 #                   and only shows what would be published without actually publishing.
@@ -24,7 +23,7 @@
 #   ./build-and-publish-java-connectors-with-tag.sh --name=foo-conn
 #
 #   2) Mixed: positional + pre-release
-#   ./build-and-publish-java-connectors-with-tag.sh --pre-release foo-conn
+#   ./build-and-publish-java-connectors-with-tag.sh --release-type=pre-release foo-conn
 #
 #   3) Enable actual publishing (default is dry-run mode)
 #   ./build-and-publish-java-connectors-with-tag.sh --publish foo-conn

poe-tasks/lib/parse_args.sh

@@ -1,8 +1,8 @@
 # Parse the command-line args that we care about.
 # Scripts sourcing this script can be invoked as either:
-# ./foo.sh [--pre-release] [--main-release] [--publish] [--name=<source/destination-foo>]* [--name <source/destination-foo>]*
+# ./foo.sh [--release-type=<pre-release|main-release>] [--publish] [--name=<source/destination-foo>]* [--name <source/destination-foo>]*
 # Or, if invoked with no `--name` flags:
-# ./get-modified-connectors.sh --json | ./foo.sh [--pre-release] [--main-release] [--publish]
+# ./get-modified-connectors.sh --json | ./foo.sh [--release-type=<pre-release|main-release>] [--publish]
 publish_mode="pre-release"
 do_publish=false
 connectors=()
@@ -13,13 +13,13 @@ while [[ $# -gt 0 ]]; do
       sed -n '1,34p' "$0"
       exit 0
       ;;
-    --main-release)
+    --release-type=*)
-      publish_mode="main-release"
+      publish_mode="${1#*=}"
       shift
       ;;
-    --pre-release)
+    --release-type)
-      publish_mode="pre-release"
+      publish_mode="$2"
-      shift
+      shift 2
       ;;
     --publish)
       do_publish=true

poe-tasks/publish-python-registry.sh

@@ -16,9 +16,9 @@ Must be run from the root of the Airbyte repository with Poetry installed
 
 Options:
     -n, --name CONNECTOR_NAME     Connector name (required)
-    -t, --token TOKEN             PyPI token (required)
+    -t, --token TOKEN             PyPI token (optional, specify this or set PYTHON_REGISTRY_TOKEN environment variable)
     -v, --version VERSION         Override version (optional)
-    --pre-release                 Publish as a pre-release (uses a dev version derived from the current timestamp)
+    --release-type TYPE           Release type (optional): 'pre-release' or 'main-release' (default is 'pre-release')
     --test-registry               Use the test PyPI registry (default is production registry)
     -h, --help                    Show this help message
 
@@ -27,7 +27,7 @@ Environment Variables:
 
 Examples:
     $0 --name source-faker --token \$PYPI_TOKEN
-    $0 --name source-faker --token \$PYPI_TOKEN --pre-release
+    $0 --name source-faker --token \$PYPI_TOKEN --release-type main-release
 EOF
 }
 
@@ -54,7 +54,7 @@ TEST_REGISTRY_UPLOAD_URL="https://test.pypi.org/legacy/"
 TEST_REGISTRY_CHECK_URL="https://test.pypi.org/pypi"
 TEST_REGISTRY_PACKAGE_URL="https://test.pypi.org/project"
 
-PRE_RELEASE=false
+RELEASE_TYPE="pre-release"
 CONNECTOR_NAME=""
 PYPI_TOKEN=""
 VERSION_OVERRIDE=""
@@ -74,9 +74,9 @@ while [[ $# -gt 0 ]]; do
             VERSION_OVERRIDE="$2"
             shift 2
             ;;
-        --pre-release)
+        --release-type)
-            PRE_RELEASE=true
+            RELEASE_TYPE="$2"
-            shift
+            shift 2
             ;;
         --test-registry)
             REGISTRY_UPLOAD_URL="$TEST_REGISTRY_UPLOAD_URL"
@@ -104,6 +104,12 @@ if [[ -z "$CONNECTOR_NAME" ]]; then
     exit 1
 fi
 
+if [[ "$RELEASE_TYPE" != "pre-release" && "$RELEASE_TYPE" != "main-release" ]]; then
+    echo "Error: Invalid release type '$RELEASE_TYPE'. Valid options are 'pre-release' or 'main-release'." >&2
+    usage >&2
+    exit 1
+fi
+    
 # Use environment variables as fallback
 if [[ -z "$PYPI_TOKEN" && -n "${PYTHON_REGISTRY_TOKEN:-}" ]]; then
     PYPI_TOKEN="$PYTHON_REGISTRY_TOKEN"
@@ -150,7 +156,7 @@ BASE_VERSION=$(poe -qq get-version)
 # Determine version to use
 if [[ -n "$VERSION_OVERRIDE" ]]; then
     VERSION="$VERSION_OVERRIDE"
-elif [[ "$PRE_RELEASE" == "true" ]]; then
+elif [[ "$RELEASE_TYPE" == "pre-release" ]]; then
     # Add current timestamp for pre-release.
     # we can't use the git revision because not all python registries allow local version identifiers. 
     # Public version identifiers must conform to PEP 440 and only allow digits.
@@ -162,6 +168,7 @@ fi
 
 echo "Package name: $PACKAGE_NAME"
 echo "Version: $VERSION"
+echo "Release type: $RELEASE_TYPE"
 echo
 
 # Check if package already exists

poe-tasks/upload-connector-metadata.sh

@@ -2,7 +2,7 @@
 set -euo pipefail
 
 # Uploads the metadata (+SBOM+spec cache) to GCS.
-# Usage: ./poe-tasks/upload-connector-metadata.sh --name destination-bigquery [--pre-release] [--main-release]
+# Usage: ./poe-tasks/upload-connector-metadata.sh --name destination-bigquery --release-type <pre-release|main-release>
 # You must have three environment variables set (GCS_CREDENTIALS, METADATA_SERVICE_GCS_CREDENTIALS, SPEC_CACHE_GCS_CREDENTIALS),
 # each containing a JSON-formatted GCP service account key.
 # SPEC_CACHE_GCS_CREDENTIALS needs write access to `gs://$spec_cache_bucket/specs`.

poe-tasks/upload-java-connector-tar-file.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # Uploads the java tar for java connectors.
-# Usage: ./poe-tasks/upload-java-connector-tar-file.sh --name destination-bigquery [--pre-release] [--main-release]
+# Usage: ./poe-tasks/upload-java-connector-tar-file.sh --name destination-bigquery --release-type <pre-release|main-release>
 # You must have set the env var GCS_CREDENTIALS, which contains a JSON-formatted GCP service account key.
 # GCS_CREDENTIALS needs write access to `gs://$metadata_bucket/resources/java`.
 set -euo pipefail

poe-tasks/upload-python-dependencies.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 # Upload Python connector dependencies metadata to GCS
 # Extracted from airbyte-ci publish pipeline for GitHub Actions integration
 #
-# Usage: ./poe-tasks/upload-python-depdendencies.sh --name source-avri [--pre-release] [--main-release]
+# Usage: ./poe-tasks/upload-python-dependencies.sh --name source-avri --release-type [pre-release | main-release] --bucket my-bucket --connector-version 1.2.3
 #
 
 # source utility functions
@@ -23,7 +23,7 @@ Options:
     -n, --name CONNECTOR_NAME     Connector name (required)
     --bucket BUCKET_NAME          GCS bucket name (optional, defaults to dev bucket)
     --connector-version VERSION   Connector version (optional, default reads from metadata.yaml)
-    --pre-release                 Publish as a pre-release (uses a dev version derived from the git hash)
+    --release-type TYPE           Release type (optional): 'pre-release' or 'main-release' (default is 'pre-release')
     -h, --help                    Show this help message
 
 Environment Variables:
@@ -56,9 +56,9 @@ while [[ $# -gt 0 ]]; do
             VERSION="$2"
             shift 2
             ;;
-        --pre-release)
+        --release-type)
-            PRE_RELEASE=true
+            RELEASE_TYPE="$2"
-            shift
+            shift 2
             ;;
         -h|--help)
             usage
@@ -110,7 +110,7 @@ fi
 if [[ -z "$VERSION" ]]; then
     VERSION=$(poe -qq get-version)
 fi
-if [[ $PRE_RELEASE == true ]]; then
+if [[ $RELEASE_TYPE == "pre-release" ]]; then
     VERSION=$(generate_dev_tag "$VERSION")
 fi