Dbgold17/dry up connector publish pipeline (#65098)

2025-12-19 10:00:34 -05:00 · 2025-08-21 18:44:36 -07:00
parent 639cadf8de
commit 20ef661fcd
10 changed files with 137 additions and 182 deletions
--- a/.github/actions/connector-image-build-push/action.yml
+++ b/.github/actions/connector-image-build-push/action.yml
@@ -12,6 +12,10 @@ inputs:
    description: "Docker registry"
    required: false
    default: "docker.io/airbyte"
+  release-type:
+    description: "Release type: 'pre-release' or 'main-release'. Determines tagging and latest push behavior."
+    required: false
+    default: "pre-release"
  tag-override:
    description: "Override the image tag (optional). If not provided, the tag will be derived from the connector metadata.yaml."
    required: false
@@ -118,17 +122,43 @@ runs:
          exit 1
        fi

-        # Resolve connector version to use for image tagging
+        # Get connector version from metadata
+        CONNECTOR_VERSION=$(poe -qq get-version)
+        if [[ -z "$CONNECTOR_VERSION" ]]; then
+          echo "❌ Docker tag not found in metadata.yaml"
+          exit 1
+        fi
+
+        # Check if this is a release candidate
+        IS_RELEASE_CANDIDATE=$(poe -qq get-version | grep -q 'rc' && echo "true" || echo "false")
+
+        # Determine the image tags to use 
        if [[ "${{ inputs.tag-override }}" ]]; then
          CONNECTOR_VERSION_TAG="${{ inputs.tag-override }}"
-          echo "🏷 Using provided tag: $CONNECTOR_VERSION_TAG"
+          echo "🏷 Using provided tag override: $CONNECTOR_VERSION_TAG"
+        elif [[ "${{ inputs.release-type }}" == "pre-release" ]]; then
+          hash=$(git rev-parse --short=10 HEAD)
+          CONNECTOR_VERSION_TAG="${CONNECTOR_VERSION}-dev.${hash}"
+          echo "🏷 Using pre-release tag: $CONNECTOR_VERSION_TAG"
        else
-          CONNECTOR_VERSION_TAG=$(poe -qq get-version)
-          if [[ -z "$CONNECTOR_VERSION_TAG" ]]; then
-            echo "❌ Docker tag not found in metadata.yaml"
-            exit 1
+          CONNECTOR_VERSION_TAG="$CONNECTOR_VERSION"
+          echo "🏷 Using main release tag: $CONNECTOR_VERSION_TAG"
+        fi
+
+        # Determine whether to push with latest tag
+        if [[ "${{ inputs.push-latest }}" == "true" || "${{ inputs.push-latest }}" == "false" ]]; then
+          # Use explicit input value
+          PUSH_LATEST="${{ inputs.push-latest }}"
+          echo "🏷 Using explicit push-latest setting: $PUSH_LATEST"
+        else
+          # Auto-determine based on release-type and release candidate status
+          if [[ "${{ inputs.release-type }}" == "pre-release" || "$IS_RELEASE_CANDIDATE" == "true" ]]; then
+            PUSH_LATEST="false"
+            echo "🏷 Skipping latest tag for pre-release or release candidate build"
+          else
+            PUSH_LATEST="true"
+            echo "🏷 Will push with latest tag for main release"
          fi
-          echo "🏷 Using tag from metadata.yaml: $CONNECTOR_VERSION_TAG"
        fi

        # Can't dry-run and force-publish at the same time
@@ -137,7 +167,7 @@ runs:
          exit 1
        fi

-        # Validate registry input
+        # Validate image registry input
        case "${{ inputs.registry }}" in
          "docker.io/airbyte")
            # Supported registries
@@ -157,7 +187,7 @@ runs:

        # Get all tags to use for image pushing
        DOCKER_TAGS="${FULL_IMAGE_NAME}"
-        if [[ "${{ inputs.push-latest }}" == "true" ]]; then
+        if [[ "$PUSH_LATEST" == "true" ]]; then
          DOCKER_TAGS="${DOCKER_TAGS},${{ inputs.registry }}/${CONNECTOR_NAME}:latest"
        fi

@@ -168,6 +198,8 @@ runs:
        echo "  Version Tag: $CONNECTOR_VERSION_TAG"
        echo "  Registry: ${{ inputs.registry }}"
        echo "  Full Image Name: $FULL_IMAGE_NAME"
+        echo "  Release Type: ${{ inputs.release-type }}"
+        echo "  Push Latest: $PUSH_LATEST"
        echo "  Dry Run: ${{ inputs.dry-run }}"
        echo "  Force Publish: ${{ inputs.force-publish }}" 

@@ -251,15 +283,12 @@ runs:
          DO_PUBLISH="true"
        fi

-        push_latest="${{ inputs.push-latest }}"
-
        # Output decision summary
        echo ""
        echo "📊 Build Decision Summary:"
        echo "  Image exists: $IMAGE_EXISTS"
        echo "  Will build: $DO_BUILD"
        echo "  Will publish: $DO_PUBLISH"
-        echo "  Will tag with latest: ${push_latest:-false}"
        echo "  Dry run mode: ${{ inputs.dry-run }}"
        echo "  Force publish: ${{ inputs.force-publish }}"

--- a/.github/workflows/bump-bulk-cdk-and-release-connectors-command.yml
+++ b/.github/workflows/bump-bulk-cdk-and-release-connectors-command.yml
@@ -64,7 +64,7 @@ jobs:
        connector_name: ${{ fromJson(needs.bump-cdk-version.outputs.modified_connectors_json) }}
    with:
      connectors: "--name=${{ matrix.connector_name }}"
-      publish-options: --main-release
+      release-type: main-release
    secrets: inherit

  update-changelog-and-merge:
--- a/.github/workflows/bump-cdk-version-and-merge-command.yml
+++ b/.github/workflows/bump-cdk-version-and-merge-command.yml
@@ -139,7 +139,7 @@ jobs:
        connector_name: ${{ fromJson(needs.bump-cdk-version.outputs.modified_connectors) }}
    with:
      connectors: "--name=${{ matrix.connector_name }}"
-      publish-options: --pre-release
+      release-type: pre-release
    secrets: inherit

  update-changelog-and-merge:
--- a/.github/workflows/publish_connectors.yml
+++ b/.github/workflows/publish_connectors.yml
@@ -1,4 +1,4 @@
-name: Connector Ops CI - Publish Connectors
+name: Publish Connectors

 on:
  push:
@@ -9,51 +9,68 @@ on:
  workflow_call:
    inputs:
      connectors:
-        description: "list of connectors to publish. This should be a string of the form --name=source-pokeapi --name=destination-postgres."
+        description: "Connectors to Publish. This should be a string of the form: --name=source-pokeapi --name=destination-postgres"
        default: "--name=source-pokeapi"
        type: string
-      publish-options:
-        description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. "
-        default: "--pre-release"
+      release-type:
+        description: "Whether to publish as a pre-release (tagged for development) or as a main-release (production)"
+        default: pre-release
        type: string
  workflow_dispatch:
    inputs:
      connectors:
-        description: "list of connectors to publish. This should be a string of the form --name=source-pokeapi --name=destination-postgres."
+        description: "Connectors to Publish. This should be a string of the form: --name=source-pokeapi --name=destination-postgres"
        default: "--name=source-pokeapi"
-      publish-options:
-        description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. "
-        default: "--pre-release"
+        type: string
+      release-type:
+        description: "Whether to publish as a pre-release (tagged for development) or as a main-release (production)"
+        default: pre-release
+        type: choice
+        options:
+          - pre-release
+          - main-release
 jobs:
-  list_connectors:
-    name: List connectors to publish
+  publish_options:
+    name: Resolve options for connector publishing
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v4
        with:
          fetch-depth: 2 # Required so we can conduct a diff from the previous commit to understand what connectors have changed.
-      - name: List connectors [manual]
+      - name: List connectors to publish [manual]
        id: list-connectors-manual
        if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
        shell: bash
        # When invoked manually, we run on the connectors specified in the input.
        run: echo connectors-to-publish=$(./poe-tasks/parse-connector-name-args.sh ${{ inputs.connectors }}) | tee -a $GITHUB_OUTPUT
-      - name: List connectors [On merge to master]
+      - name: List connectors to publish [On merge to master]
        id: list-connectors-master
        if: github.event_name == 'push'
        shell: bash
        # When merging to master, we run on connectors that have changed since the previous commit.
        run: echo connectors-to-publish=$(./poe-tasks/get-modified-connectors.sh --prev-commit --json) | tee -a $GITHUB_OUTPUT
+      - name: Resolve release type
+        id: resolve-release-type
+        shell: bash
+        run: |
+          # If workflow is triggered on merge to master, do a main release.
+          # If workflow is triggered manually, use the input release type.
+          if [[ "${{ github.event_name }}" == "push" ]]; then
+            echo "release-type=main-release" | tee -a $GITHUB_OUTPUT
+          else
+            echo "release-type=${{ inputs.release-type }}" | tee -a $GITHUB_OUTPUT
+          fi
    outputs:
      # Exactly one of the manual/master steps will run, so just OR them together.
      connectors-to-publish: ${{ steps.list-connectors-manual.outputs.connectors-to-publish || steps.list-connectors-master.outputs.connectors-to-publish }}
+      release-type: ${{ steps.resolve-release-type.outputs.release-type }}
  publish_connectors:
    name: Publish connectors
-    needs: [list_connectors]
+    needs: [publish_options]
    runs-on: ubuntu-24.04
    strategy:
-      matrix: ${{ fromJson(needs.list_connectors.outputs.connectors-to-publish) }}
+      matrix: ${{ fromJson(needs.publish_options.outputs.connectors-to-publish) }}
      max-parallel: 5
      # Allow all jobs to run, even if one fails
      fail-fast: false
@@ -127,156 +144,59 @@ jobs:
        shell: bash
        run: ./poe-tasks/validate-connector-metadata.sh --name ${{ matrix.connector }}

-      - name: Publish to Python Registry [On merge to master]
-        id: publish-python-registry-master
-        if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language == 'python'
+      - name: Publish to Python Registry
+        id: publish-python-registry
+        if: steps.connector-metadata.outputs.connector-language == 'python'
        shell: bash
        run: |
-          ./poe-tasks/publish-python-registry.sh --name ${{ matrix.connector }}
+          ./poe-tasks/publish-python-registry.sh --name ${{ matrix.connector }} --release-type ${{ needs.publish_options.outputs.release-type }}
        env:
          PYTHON_REGISTRY_TOKEN: ${{ secrets.PYPI_TOKEN }}
-        continue-on-error: true # We allow this to fail because we are testing this step. TODO: remove this once we are sure it works.

-      - name: Publish to Python Registry [Manual]
-        id: publish-python-registry-manual
-        if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language == 'python'
-        shell: bash
-        run: |
-          # Determine if this is a pre-release
-          is_pre_release=$(echo "${{ inputs.publish-options }}" | grep -c -- '--pre-release' || true)
-          if [[ $is_pre_release -gt 0 ]]; then
-            PRE_RELEASE_FLAG="--pre-release"
-          else
-            PRE_RELEASE_FLAG=""
-          fi
-
-          ./poe-tasks/publish-python-registry.sh --name ${{ matrix.connector }} $PRE_RELEASE_FLAG
-        env:
-          PYTHON_REGISTRY_TOKEN: ${{ secrets.PYPI_TOKEN }}
-        continue-on-error: true # We allow this to fail because we are testing this step. TODO: remove this once we are sure it works.
-
-      - name: Upload Python Dependencies to GCS [On merge to master]
+      - name: Upload Python Dependencies to GCS
        id: upload-python-dependencies-master
-        if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language == 'python'
+        if: steps.connector-metadata.outputs.connector-language == 'python'
        shell: bash
        env:
          GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
        run: |
-          ./poe-tasks/upload-python-dependencies.sh --name ${{ matrix.connector }} --bucket prod-airbyte-cloud-connector-metadata-service
-        continue-on-error: true # We allow this to fail because we are testing this step. TODO: remove this once we are sure it works.
+          ./poe-tasks/upload-python-dependencies.sh \
+            --name ${{ matrix.connector }} \
+            --bucket prod-airbyte-cloud-connector-metadata-service \
+            --release-type ${{ needs.publish_options.outputs.release-type }}

-      - name: Upload Python Dependencies to GCS [Manual]
-        id: upload-python-dependencies-manual
-        if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language == 'python'
-        shell: bash
-        env:
-          GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
-        run: |
-          # Determine if this is a pre-release
-          is_pre_release=$(echo "${{ inputs.publish-options }}" | grep -c -- '--pre-release' || true)
-          if [[ $is_pre_release -gt 0 ]]; then
-            PRE_RELEASE_FLAG="--pre-release"
-          else
-            PRE_RELEASE_FLAG=""
-          fi
-          ./poe-tasks/upload-python-dependencies.sh --name ${{ matrix.connector }} --bucket prod-airbyte-cloud-connector-metadata-service $PRE_RELEASE_FLAG
-        continue-on-error: true # We allow this to fail because we are testing this step. TODO: remove this once we are sure it works.
-
-      - name: Build and publish JVM connectors images [On merge to master]
-        id: build-and-publish-JVM-connectors-images-master
-        if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language == 'java'
+      - name: Build and publish JVM connectors images
+        id: build-and-publish-JVM-connectors-images
+        if: steps.connector-metadata.outputs.connector-language == 'java'
        shell: bash
        run: |
-          ./poe-tasks/build-and-publish-java-connectors-with-tag.sh --main-release --publish --name ${{ matrix.connector }}
+          ./poe-tasks/build-and-publish-java-connectors-with-tag.sh --name ${{ matrix.connector }} --release-type ${{ needs.publish_options.outputs.release-type }} --publish

-      - name: Publish JVM connectors tar file [On merge to master]]
-        id: publish-JVM-connectors-tar-file-master
-        if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language == 'java'
+      - name: Publish JVM connectors tar file
+        id: publish-JVM-connectors-tar-file
+        if: steps.connector-metadata.outputs.connector-language == 'java'
        shell: bash
-        run: ./poe-tasks/upload-java-connector-tar-file.sh --name ${{ matrix.connector }} --main-release
+        run: ./poe-tasks/upload-java-connector-tar-file.sh --name ${{ matrix.connector }} --release-type ${{ needs.publish_options.outputs.release-type }}
        env:
          GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}
        # we allow it to fail because we are testing this step. We should remove this once we are sure it works.
        continue-on-error: true

-      - name: Determine build and publish options
-        id: get-connector-options
+      - name: Build and publish Python and Manifest-Only connectors images
+        id: build-and-publish-python-manifest-only-connectors-images
        if: steps.connector-metadata.outputs.connector-language != 'java'
-        shell: bash
-        run: |
-          # TODO: refactor this logic to be shared by all connector types
-          is_pre_release=$(echo ${{ inputs.publish-options }} | grep -c -- '--pre-release' || true)
-          is_release_candidate=$(echo ${{ steps.connector-metadata.outputs.connector-version }} | grep -c -- '-rc' || true)
-
-          # Don't tag with latest for pre-release or release candidate builds
-          if [[ $is_pre_release -gt 0 || $is_release_candidate -gt 0 ]]; then
-            echo "push-latest=false" >> $GITHUB_OUTPUT
-            echo "Skipping latest tag for pre-release or release candidate build."
-          else
-            echo "push-latest=true" >> $GITHUB_OUTPUT
-          fi
-
-          if [[ $is_pre_release -gt 0 ]]; then
-            hash=$(git rev-parse --short=10 HEAD)
-            tag_override="${{ steps.connector-metadata.outputs.connector-version }}-dev.${hash}"
-            echo "tag-override=${tag_override}" >> $GITHUB_OUTPUT
-            echo "Using a dev tag for a pre-release build: ${tag_override}"
-          fi
-
-      - name: Build and publish Python and Manifest-Only connectors images [On merge to master]
-        id: build-and-publish-python-manifest-only-connectors-images-master
-        if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language != 'java'
        uses: ./.github/actions/connector-image-build-push
        with:
          connector-name: ${{ matrix.connector }}
-          push-latest: ${{ steps.get-connector-options.outputs.push-latest }}
+          release-type: ${{ needs.publish_options.outputs.release-type }}
          dry-run: "false"
          docker-hub-username: ${{ secrets.DOCKER_HUB_USERNAME }}
          docker-hub-password: ${{ secrets.DOCKER_HUB_PASSWORD }}

-      - name: Upload connector metadata [On merge to master]
-        id: upload-connector-metadata-master
-        if: github.event_name == 'push'
+      - name: Upload connector metadata
+        id: upload-connector-metadata
        shell: bash
-        run: ./poe-tasks/upload-connector-metadata.sh --name ${{ matrix.connector }} --main-release
-        env:
-          GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
-          SPEC_CACHE_GCS_CREDENTIALS: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
-          METADATA_SERVICE_GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
-
-      - name: Build and publish JVM connectors images [manual]
-        id: build-and-publish-JVM-connectors-images-manual
-        if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language == 'java'
-        shell: bash
-        run: ./poe-tasks/build-and-publish-java-connectors-with-tag.sh  ${{ inputs.publish-options }} --name ${{ matrix.connector }} --publish
-
-      - name: Publish JVM connectors tar file [manual]
-        id: publish-JVM-connectors-tar-file-manual
-        if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language == 'java'
-        shell: bash
-        run: ./poe-tasks/upload-java-connector-tar-file.sh --name ${{ matrix.connector }} ${{ inputs.publish-options }}
-        env:
-          GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}
-        # we allow it to fail because we are testing this step. We should remove this once we are sure it works.
-        continue-on-error: true
-
-      - name: Build and publish Python and Manifest-Only connectors images [manual]
-        id: build-and-publish-python-manifest-only-connectors-images-manual
-        if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language != 'java'
-        uses: ./.github/actions/connector-image-build-push
-        with:
-          connector-name: ${{ matrix.connector }}
-          push-latest: ${{ steps.get-connector-options.outputs.push-latest }}
-          tag-override: ${{ steps.get-connector-options.outputs.tag-override }}
-          dry-run: "false"
-          docker-hub-username: ${{ secrets.DOCKER_HUB_USERNAME }}
-          docker-hub-password: ${{ secrets.DOCKER_HUB_PASSWORD }}
-
-      - name: Upload connector metadata [Manual]
-        id: upload-connector-metadata-manual
-        if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call'
-        shell: bash
-        run: ./poe-tasks/upload-connector-metadata.sh --name ${{ matrix.connector }} ${{ inputs.publish-options }}
+        run: ./poe-tasks/upload-connector-metadata.sh --name ${{ matrix.connector }} --release-type ${{ needs.publish_options.outputs.release-type }}
        env:
          GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
          SPEC_CACHE_GCS_CREDENTIALS: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
--- a/poe-tasks/build-and-publish-java-connectors-with-tag.sh
+++ b/poe-tasks/build-and-publish-java-connectors-with-tag.sh
@@ -1,16 +1,15 @@
 #!/usr/bin/env bash

 # This script builds and optionally publishes Java connector Docker images.
-# Usage: ./build-and-publish-java-connectors-with-tag.sh --name <name> [--pre-release] [--main-release] [--publish]
+# Usage: ./build-and-publish-java-connectors-with-tag.sh --name <name> --release-type [pre-release | main-release] [--publish]
 #
 # Flag descriptions:
-#   --main-release: Publishes images with the exact version from metadata.yaml.
-#                   Only publishes if the image does not exists on Dockerhub.
-#                   Used for production releases on merge to master.
-#
-#   --pre-release:  Publishes images with a dev tag (version-dev.githash).
-#                   Only publishes if the image does not exists on Dockerhub.
-#                   Used for development/testing purposes.
+#   --name <name>:    Specifies the connector name (e.g., destination-bigquery).
+#                    
+#   --release-type:   Specifies the release type:
+#                     - pre-release: Builds with a dev tag (version-dev.githash).
+#                     - main-release: Builds with the exact version from metadata.yaml.
+#                     Defaults to pre-release if not specified.
 #
 #   --publish:      Actually publishes the images. Without this flag, the script runs in dry-run mode
 #                   and only shows what would be published without actually publishing.
@@ -24,7 +23,7 @@
 #   ./build-and-publish-java-connectors-with-tag.sh --name=foo-conn
 #
 #   2) Mixed: positional + pre-release
-#   ./build-and-publish-java-connectors-with-tag.sh --pre-release foo-conn
+#   ./build-and-publish-java-connectors-with-tag.sh --release-type=pre-release foo-conn
 #
 #   3) Enable actual publishing (default is dry-run mode)
 #   ./build-and-publish-java-connectors-with-tag.sh --publish foo-conn
--- a/poe-tasks/lib/parse_args.sh
+++ b/poe-tasks/lib/parse_args.sh
@@ -1,8 +1,8 @@
 # Parse the command-line args that we care about.
 # Scripts sourcing this script can be invoked as either:
-# ./foo.sh [--pre-release] [--main-release] [--publish] [--name=<source/destination-foo>]* [--name <source/destination-foo>]*
+# ./foo.sh [--release-type=<pre-release|main-release>] [--publish] [--name=<source/destination-foo>]* [--name <source/destination-foo>]*
 # Or, if invoked with no `--name` flags:
-# ./get-modified-connectors.sh --json | ./foo.sh [--pre-release] [--main-release] [--publish]
+# ./get-modified-connectors.sh --json | ./foo.sh [--release-type=<pre-release|main-release>] [--publish]
 publish_mode="pre-release"
 do_publish=false
 connectors=()
@@ -13,13 +13,13 @@ while [[ $# -gt 0 ]]; do
      sed -n '1,34p' "$0"
      exit 0
      ;;
-    --main-release)
-      publish_mode="main-release"
+    --release-type=*)
+      publish_mode="${1#*=}"
      shift
      ;;
-    --pre-release)
-      publish_mode="pre-release"
-      shift
+    --release-type)
+      publish_mode="$2"
+      shift 2
      ;;
    --publish)
      do_publish=true
--- a/poe-tasks/publish-python-registry.sh
+++ b/poe-tasks/publish-python-registry.sh
@@ -16,9 +16,9 @@ Must be run from the root of the Airbyte repository with Poetry installed

 Options:
    -n, --name CONNECTOR_NAME     Connector name (required)
-    -t, --token TOKEN             PyPI token (required)
+    -t, --token TOKEN             PyPI token (optional, specify this or set PYTHON_REGISTRY_TOKEN environment variable)
    -v, --version VERSION         Override version (optional)
-    --pre-release                 Publish as a pre-release (uses a dev version derived from the current timestamp)
+    --release-type TYPE           Release type (optional): 'pre-release' or 'main-release' (default is 'pre-release')
    --test-registry               Use the test PyPI registry (default is production registry)
    -h, --help                    Show this help message

@@ -27,7 +27,7 @@ Environment Variables:

 Examples:
    $0 --name source-faker --token \$PYPI_TOKEN
-    $0 --name source-faker --token \$PYPI_TOKEN --pre-release
+    $0 --name source-faker --token \$PYPI_TOKEN --release-type main-release
 EOF
 }

@@ -54,7 +54,7 @@ TEST_REGISTRY_UPLOAD_URL="https://test.pypi.org/legacy/"
 TEST_REGISTRY_CHECK_URL="https://test.pypi.org/pypi"
 TEST_REGISTRY_PACKAGE_URL="https://test.pypi.org/project"

-PRE_RELEASE=false
+RELEASE_TYPE="pre-release"
 CONNECTOR_NAME=""
 PYPI_TOKEN=""
 VERSION_OVERRIDE=""
@@ -74,9 +74,9 @@ while [[ $# -gt 0 ]]; do
            VERSION_OVERRIDE="$2"
            shift 2
            ;;
-        --pre-release)
-            PRE_RELEASE=true
-            shift
+        --release-type)
+            RELEASE_TYPE="$2"
+            shift 2
            ;;
        --test-registry)
            REGISTRY_UPLOAD_URL="$TEST_REGISTRY_UPLOAD_URL"
@@ -104,6 +104,12 @@ if [[ -z "$CONNECTOR_NAME" ]]; then
    exit 1
 fi

+if [[ "$RELEASE_TYPE" != "pre-release" && "$RELEASE_TYPE" != "main-release" ]]; then
+    echo "Error: Invalid release type '$RELEASE_TYPE'. Valid options are 'pre-release' or 'main-release'." >&2
+    usage >&2
+    exit 1
+fi
+    
 # Use environment variables as fallback
 if [[ -z "$PYPI_TOKEN" && -n "${PYTHON_REGISTRY_TOKEN:-}" ]]; then
    PYPI_TOKEN="$PYTHON_REGISTRY_TOKEN"
@@ -150,7 +156,7 @@ BASE_VERSION=$(poe -qq get-version)
 # Determine version to use
 if [[ -n "$VERSION_OVERRIDE" ]]; then
    VERSION="$VERSION_OVERRIDE"
-elif [[ "$PRE_RELEASE" == "true" ]]; then
+elif [[ "$RELEASE_TYPE" == "pre-release" ]]; then
    # Add current timestamp for pre-release.
    # we can't use the git revision because not all python registries allow local version identifiers. 
    # Public version identifiers must conform to PEP 440 and only allow digits.
@@ -162,6 +168,7 @@ fi

 echo "Package name: $PACKAGE_NAME"
 echo "Version: $VERSION"
+echo "Release type: $RELEASE_TYPE"
 echo

 # Check if package already exists
--- a/poe-tasks/upload-connector-metadata.sh
+++ b/poe-tasks/upload-connector-metadata.sh
@@ -2,7 +2,7 @@
 set -euo pipefail

 # Uploads the metadata (+SBOM+spec cache) to GCS.
-# Usage: ./poe-tasks/upload-connector-metadata.sh --name destination-bigquery [--pre-release] [--main-release]
+# Usage: ./poe-tasks/upload-connector-metadata.sh --name destination-bigquery --release-type <pre-release|main-release>
 # You must have three environment variables set (GCS_CREDENTIALS, METADATA_SERVICE_GCS_CREDENTIALS, SPEC_CACHE_GCS_CREDENTIALS),
 # each containing a JSON-formatted GCP service account key.
 # SPEC_CACHE_GCS_CREDENTIALS needs write access to `gs://$spec_cache_bucket/specs`.
--- a/poe-tasks/upload-java-connector-tar-file.sh
+++ b/poe-tasks/upload-java-connector-tar-file.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash

 # Uploads the java tar for java connectors.
-# Usage: ./poe-tasks/upload-java-connector-tar-file.sh --name destination-bigquery [--pre-release] [--main-release]
+# Usage: ./poe-tasks/upload-java-connector-tar-file.sh --name destination-bigquery --release-type <pre-release|main-release>
 # You must have set the env var GCS_CREDENTIALS, which contains a JSON-formatted GCP service account key.
 # GCS_CREDENTIALS needs write access to `gs://$metadata_bucket/resources/java`.
 set -euo pipefail
--- a/poe-tasks/upload-python-dependencies.sh
+++ b/poe-tasks/upload-python-dependencies.sh
@@ -5,7 +5,7 @@ set -euo pipefail
 # Upload Python connector dependencies metadata to GCS
 # Extracted from airbyte-ci publish pipeline for GitHub Actions integration
 #
-# Usage: ./poe-tasks/upload-python-depdendencies.sh --name source-avri [--pre-release] [--main-release]
+# Usage: ./poe-tasks/upload-python-dependencies.sh --name source-avri --release-type [pre-release | main-release] --bucket my-bucket --connector-version 1.2.3
 #

 # source utility functions
@@ -23,7 +23,7 @@ Options:
    -n, --name CONNECTOR_NAME     Connector name (required)
    --bucket BUCKET_NAME          GCS bucket name (optional, defaults to dev bucket)
    --connector-version VERSION   Connector version (optional, default reads from metadata.yaml)
-    --pre-release                 Publish as a pre-release (uses a dev version derived from the git hash)
+    --release-type TYPE           Release type (optional): 'pre-release' or 'main-release' (default is 'pre-release')
    -h, --help                    Show this help message

 Environment Variables:
@@ -56,9 +56,9 @@ while [[ $# -gt 0 ]]; do
            VERSION="$2"
            shift 2
            ;;
-        --pre-release)
-            PRE_RELEASE=true
-            shift
+        --release-type)
+            RELEASE_TYPE="$2"
+            shift 2
            ;;
        -h|--help)
            usage
@@ -110,7 +110,7 @@ fi
 if [[ -z "$VERSION" ]]; then
    VERSION=$(poe -qq get-version)
 fi
-if [[ $PRE_RELEASE == true ]]; then
+if [[ $RELEASE_TYPE == "pre-release" ]]; then
    VERSION=$(generate_dev_tag "$VERSION")
 fi