diff --git a/.github/actions/connector-image-build-push/action.yml b/.github/actions/connector-image-build-push/action.yml index c4e221acce1..ab2c93d7277 100644 --- a/.github/actions/connector-image-build-push/action.yml +++ b/.github/actions/connector-image-build-push/action.yml @@ -12,6 +12,10 @@ inputs: description: "Docker registry" required: false default: "docker.io/airbyte" + release-type: + description: "Release type: 'pre-release' or 'main-release'. Determines tagging and latest push behavior." + required: false + default: "pre-release" tag-override: description: "Override the image tag (optional). If not provided, the tag will be derived from the connector metadata.yaml." required: false @@ -118,17 +122,43 @@ runs: exit 1 fi - # Resolve connector version to use for image tagging + # Get connector version from metadata + CONNECTOR_VERSION=$(poe -qq get-version) + if [[ -z "$CONNECTOR_VERSION" ]]; then + echo "❌ Docker tag not found in metadata.yaml" + exit 1 + fi + + # Check if this is a release candidate + IS_RELEASE_CANDIDATE=$(poe -qq get-version | grep -q 'rc' && echo "true" || echo "false") + + # Determine the image tags to use if [[ "${{ inputs.tag-override }}" ]]; then CONNECTOR_VERSION_TAG="${{ inputs.tag-override }}" - echo "🏷 Using provided tag: $CONNECTOR_VERSION_TAG" + echo "🏷 Using provided tag override: $CONNECTOR_VERSION_TAG" + elif [[ "${{ inputs.release-type }}" == "pre-release" ]]; then + hash=$(git rev-parse --short=10 HEAD) + CONNECTOR_VERSION_TAG="${CONNECTOR_VERSION}-dev.${hash}" + echo "🏷 Using pre-release tag: $CONNECTOR_VERSION_TAG" else - CONNECTOR_VERSION_TAG=$(poe -qq get-version) - if [[ -z "$CONNECTOR_VERSION_TAG" ]]; then - echo "❌ Docker tag not found in metadata.yaml" - exit 1 + CONNECTOR_VERSION_TAG="$CONNECTOR_VERSION" + echo "🏷 Using main release tag: $CONNECTOR_VERSION_TAG" + fi + + # Determine whether to push with latest tag + if [[ "${{ inputs.push-latest }}" == "true" || "${{ inputs.push-latest }}" == "false" ]]; then + # Use explicit input value + PUSH_LATEST="${{ inputs.push-latest }}" + echo "🏷 Using explicit push-latest setting: $PUSH_LATEST" + else + # Auto-determine based on release-type and release candidate status + if [[ "${{ inputs.release-type }}" == "pre-release" || "$IS_RELEASE_CANDIDATE" == "true" ]]; then + PUSH_LATEST="false" + echo "🏷 Skipping latest tag for pre-release or release candidate build" + else + PUSH_LATEST="true" + echo "🏷 Will push with latest tag for main release" fi - echo "🏷 Using tag from metadata.yaml: $CONNECTOR_VERSION_TAG" fi # Can't dry-run and force-publish at the same time @@ -137,7 +167,7 @@ runs: exit 1 fi - # Validate registry input + # Validate image registry input case "${{ inputs.registry }}" in "docker.io/airbyte") # Supported registries @@ -157,7 +187,7 @@ runs: # Get all tags to use for image pushing DOCKER_TAGS="${FULL_IMAGE_NAME}" - if [[ "${{ inputs.push-latest }}" == "true" ]]; then + if [[ "$PUSH_LATEST" == "true" ]]; then DOCKER_TAGS="${DOCKER_TAGS},${{ inputs.registry }}/${CONNECTOR_NAME}:latest" fi @@ -168,6 +198,8 @@ runs: echo " Version Tag: $CONNECTOR_VERSION_TAG" echo " Registry: ${{ inputs.registry }}" echo " Full Image Name: $FULL_IMAGE_NAME" + echo " Release Type: ${{ inputs.release-type }}" + echo " Push Latest: $PUSH_LATEST" echo " Dry Run: ${{ inputs.dry-run }}" echo " Force Publish: ${{ inputs.force-publish }}" @@ -251,15 +283,12 @@ runs: DO_PUBLISH="true" fi - push_latest="${{ inputs.push-latest }}" - # Output decision summary echo "" echo "📊 Build Decision Summary:" echo " Image exists: $IMAGE_EXISTS" echo " Will build: $DO_BUILD" echo " Will publish: $DO_PUBLISH" - echo " Will tag with latest: ${push_latest:-false}" echo " Dry run mode: ${{ inputs.dry-run }}" echo " Force publish: ${{ inputs.force-publish }}" diff --git a/.github/workflows/bump-bulk-cdk-and-release-connectors-command.yml b/.github/workflows/bump-bulk-cdk-and-release-connectors-command.yml index cda8f09178f..00fae7030a6 100644 --- a/.github/workflows/bump-bulk-cdk-and-release-connectors-command.yml +++ b/.github/workflows/bump-bulk-cdk-and-release-connectors-command.yml @@ -64,7 +64,7 @@ jobs: connector_name: ${{ fromJson(needs.bump-cdk-version.outputs.modified_connectors_json) }} with: connectors: "--name=${{ matrix.connector_name }}" - publish-options: --main-release + release-type: main-release secrets: inherit update-changelog-and-merge: diff --git a/.github/workflows/bump-cdk-version-and-merge-command.yml b/.github/workflows/bump-cdk-version-and-merge-command.yml index f825f185063..fddb0eaa6a0 100644 --- a/.github/workflows/bump-cdk-version-and-merge-command.yml +++ b/.github/workflows/bump-cdk-version-and-merge-command.yml @@ -139,7 +139,7 @@ jobs: connector_name: ${{ fromJson(needs.bump-cdk-version.outputs.modified_connectors) }} with: connectors: "--name=${{ matrix.connector_name }}" - publish-options: --pre-release + release-type: pre-release secrets: inherit update-changelog-and-merge: diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index 663543dd8ba..3eb29372ea0 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -1,4 +1,4 @@ -name: Connector Ops CI - Publish Connectors +name: Publish Connectors on: push: @@ -9,51 +9,68 @@ on: workflow_call: inputs: connectors: - description: "list of connectors to publish. This should be a string of the form --name=source-pokeapi --name=destination-postgres." + description: "Connectors to Publish. This should be a string of the form: --name=source-pokeapi --name=destination-postgres" default: "--name=source-pokeapi" type: string - publish-options: - description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. " - default: "--pre-release" + release-type: + description: "Whether to publish as a pre-release (tagged for development) or as a main-release (production)" + default: pre-release type: string workflow_dispatch: inputs: connectors: - description: "list of connectors to publish. This should be a string of the form --name=source-pokeapi --name=destination-postgres." + description: "Connectors to Publish. This should be a string of the form: --name=source-pokeapi --name=destination-postgres" default: "--name=source-pokeapi" - publish-options: - description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. " - default: "--pre-release" + type: string + release-type: + description: "Whether to publish as a pre-release (tagged for development) or as a main-release (production)" + default: pre-release + type: choice + options: + - pre-release + - main-release jobs: - list_connectors: - name: List connectors to publish + publish_options: + name: Resolve options for connector publishing runs-on: ubuntu-24.04 steps: - name: Checkout Airbyte uses: actions/checkout@v4 with: fetch-depth: 2 # Required so we can conduct a diff from the previous commit to understand what connectors have changed. - - name: List connectors [manual] + - name: List connectors to publish [manual] id: list-connectors-manual if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' shell: bash # When invoked manually, we run on the connectors specified in the input. run: echo connectors-to-publish=$(./poe-tasks/parse-connector-name-args.sh ${{ inputs.connectors }}) | tee -a $GITHUB_OUTPUT - - name: List connectors [On merge to master] + - name: List connectors to publish [On merge to master] id: list-connectors-master if: github.event_name == 'push' shell: bash # When merging to master, we run on connectors that have changed since the previous commit. run: echo connectors-to-publish=$(./poe-tasks/get-modified-connectors.sh --prev-commit --json) | tee -a $GITHUB_OUTPUT + - name: Resolve release type + id: resolve-release-type + shell: bash + run: | + # If workflow is triggered on merge to master, do a main release. + # If workflow is triggered manually, use the input release type. + if [[ "${{ github.event_name }}" == "push" ]]; then + echo "release-type=main-release" | tee -a $GITHUB_OUTPUT + else + echo "release-type=${{ inputs.release-type }}" | tee -a $GITHUB_OUTPUT + fi outputs: # Exactly one of the manual/master steps will run, so just OR them together. connectors-to-publish: ${{ steps.list-connectors-manual.outputs.connectors-to-publish || steps.list-connectors-master.outputs.connectors-to-publish }} + release-type: ${{ steps.resolve-release-type.outputs.release-type }} publish_connectors: name: Publish connectors - needs: [list_connectors] + needs: [publish_options] runs-on: ubuntu-24.04 strategy: - matrix: ${{ fromJson(needs.list_connectors.outputs.connectors-to-publish) }} + matrix: ${{ fromJson(needs.publish_options.outputs.connectors-to-publish) }} max-parallel: 5 # Allow all jobs to run, even if one fails fail-fast: false @@ -127,156 +144,59 @@ jobs: shell: bash run: ./poe-tasks/validate-connector-metadata.sh --name ${{ matrix.connector }} - - name: Publish to Python Registry [On merge to master] - id: publish-python-registry-master - if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language == 'python' + - name: Publish to Python Registry + id: publish-python-registry + if: steps.connector-metadata.outputs.connector-language == 'python' shell: bash run: | - ./poe-tasks/publish-python-registry.sh --name ${{ matrix.connector }} + ./poe-tasks/publish-python-registry.sh --name ${{ matrix.connector }} --release-type ${{ needs.publish_options.outputs.release-type }} env: PYTHON_REGISTRY_TOKEN: ${{ secrets.PYPI_TOKEN }} - continue-on-error: true # We allow this to fail because we are testing this step. TODO: remove this once we are sure it works. - - name: Publish to Python Registry [Manual] - id: publish-python-registry-manual - if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language == 'python' - shell: bash - run: | - # Determine if this is a pre-release - is_pre_release=$(echo "${{ inputs.publish-options }}" | grep -c -- '--pre-release' || true) - if [[ $is_pre_release -gt 0 ]]; then - PRE_RELEASE_FLAG="--pre-release" - else - PRE_RELEASE_FLAG="" - fi - - ./poe-tasks/publish-python-registry.sh --name ${{ matrix.connector }} $PRE_RELEASE_FLAG - env: - PYTHON_REGISTRY_TOKEN: ${{ secrets.PYPI_TOKEN }} - continue-on-error: true # We allow this to fail because we are testing this step. TODO: remove this once we are sure it works. - - - name: Upload Python Dependencies to GCS [On merge to master] + - name: Upload Python Dependencies to GCS id: upload-python-dependencies-master - if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language == 'python' + if: steps.connector-metadata.outputs.connector-language == 'python' shell: bash env: GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} run: | - ./poe-tasks/upload-python-dependencies.sh --name ${{ matrix.connector }} --bucket prod-airbyte-cloud-connector-metadata-service - continue-on-error: true # We allow this to fail because we are testing this step. TODO: remove this once we are sure it works. + ./poe-tasks/upload-python-dependencies.sh \ + --name ${{ matrix.connector }} \ + --bucket prod-airbyte-cloud-connector-metadata-service \ + --release-type ${{ needs.publish_options.outputs.release-type }} - - name: Upload Python Dependencies to GCS [Manual] - id: upload-python-dependencies-manual - if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language == 'python' - shell: bash - env: - GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} - run: | - # Determine if this is a pre-release - is_pre_release=$(echo "${{ inputs.publish-options }}" | grep -c -- '--pre-release' || true) - if [[ $is_pre_release -gt 0 ]]; then - PRE_RELEASE_FLAG="--pre-release" - else - PRE_RELEASE_FLAG="" - fi - ./poe-tasks/upload-python-dependencies.sh --name ${{ matrix.connector }} --bucket prod-airbyte-cloud-connector-metadata-service $PRE_RELEASE_FLAG - continue-on-error: true # We allow this to fail because we are testing this step. TODO: remove this once we are sure it works. - - - name: Build and publish JVM connectors images [On merge to master] - id: build-and-publish-JVM-connectors-images-master - if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language == 'java' + - name: Build and publish JVM connectors images + id: build-and-publish-JVM-connectors-images + if: steps.connector-metadata.outputs.connector-language == 'java' shell: bash run: | - ./poe-tasks/build-and-publish-java-connectors-with-tag.sh --main-release --publish --name ${{ matrix.connector }} + ./poe-tasks/build-and-publish-java-connectors-with-tag.sh --name ${{ matrix.connector }} --release-type ${{ needs.publish_options.outputs.release-type }} --publish - - name: Publish JVM connectors tar file [On merge to master]] - id: publish-JVM-connectors-tar-file-master - if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language == 'java' + - name: Publish JVM connectors tar file + id: publish-JVM-connectors-tar-file + if: steps.connector-metadata.outputs.connector-language == 'java' shell: bash - run: ./poe-tasks/upload-java-connector-tar-file.sh --name ${{ matrix.connector }} --main-release + run: ./poe-tasks/upload-java-connector-tar-file.sh --name ${{ matrix.connector }} --release-type ${{ needs.publish_options.outputs.release-type }} env: GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }} # we allow it to fail because we are testing this step. We should remove this once we are sure it works. continue-on-error: true - - name: Determine build and publish options - id: get-connector-options + - name: Build and publish Python and Manifest-Only connectors images + id: build-and-publish-python-manifest-only-connectors-images if: steps.connector-metadata.outputs.connector-language != 'java' - shell: bash - run: | - # TODO: refactor this logic to be shared by all connector types - is_pre_release=$(echo ${{ inputs.publish-options }} | grep -c -- '--pre-release' || true) - is_release_candidate=$(echo ${{ steps.connector-metadata.outputs.connector-version }} | grep -c -- '-rc' || true) - - # Don't tag with latest for pre-release or release candidate builds - if [[ $is_pre_release -gt 0 || $is_release_candidate -gt 0 ]]; then - echo "push-latest=false" >> $GITHUB_OUTPUT - echo "Skipping latest tag for pre-release or release candidate build." - else - echo "push-latest=true" >> $GITHUB_OUTPUT - fi - - if [[ $is_pre_release -gt 0 ]]; then - hash=$(git rev-parse --short=10 HEAD) - tag_override="${{ steps.connector-metadata.outputs.connector-version }}-dev.${hash}" - echo "tag-override=${tag_override}" >> $GITHUB_OUTPUT - echo "Using a dev tag for a pre-release build: ${tag_override}" - fi - - - name: Build and publish Python and Manifest-Only connectors images [On merge to master] - id: build-and-publish-python-manifest-only-connectors-images-master - if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language != 'java' uses: ./.github/actions/connector-image-build-push with: connector-name: ${{ matrix.connector }} - push-latest: ${{ steps.get-connector-options.outputs.push-latest }} + release-type: ${{ needs.publish_options.outputs.release-type }} dry-run: "false" docker-hub-username: ${{ secrets.DOCKER_HUB_USERNAME }} docker-hub-password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - name: Upload connector metadata [On merge to master] - id: upload-connector-metadata-master - if: github.event_name == 'push' + - name: Upload connector metadata + id: upload-connector-metadata shell: bash - run: ./poe-tasks/upload-connector-metadata.sh --name ${{ matrix.connector }} --main-release - env: - GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} - SPEC_CACHE_GCS_CREDENTIALS: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} - METADATA_SERVICE_GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} - - - name: Build and publish JVM connectors images [manual] - id: build-and-publish-JVM-connectors-images-manual - if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language == 'java' - shell: bash - run: ./poe-tasks/build-and-publish-java-connectors-with-tag.sh ${{ inputs.publish-options }} --name ${{ matrix.connector }} --publish - - - name: Publish JVM connectors tar file [manual] - id: publish-JVM-connectors-tar-file-manual - if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language == 'java' - shell: bash - run: ./poe-tasks/upload-java-connector-tar-file.sh --name ${{ matrix.connector }} ${{ inputs.publish-options }} - env: - GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }} - # we allow it to fail because we are testing this step. We should remove this once we are sure it works. - continue-on-error: true - - - name: Build and publish Python and Manifest-Only connectors images [manual] - id: build-and-publish-python-manifest-only-connectors-images-manual - if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language != 'java' - uses: ./.github/actions/connector-image-build-push - with: - connector-name: ${{ matrix.connector }} - push-latest: ${{ steps.get-connector-options.outputs.push-latest }} - tag-override: ${{ steps.get-connector-options.outputs.tag-override }} - dry-run: "false" - docker-hub-username: ${{ secrets.DOCKER_HUB_USERNAME }} - docker-hub-password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - - name: Upload connector metadata [Manual] - id: upload-connector-metadata-manual - if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' - shell: bash - run: ./poe-tasks/upload-connector-metadata.sh --name ${{ matrix.connector }} ${{ inputs.publish-options }} + run: ./poe-tasks/upload-connector-metadata.sh --name ${{ matrix.connector }} --release-type ${{ needs.publish_options.outputs.release-type }} env: GCS_CREDENTIALS: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} SPEC_CACHE_GCS_CREDENTIALS: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} diff --git a/poe-tasks/build-and-publish-java-connectors-with-tag.sh b/poe-tasks/build-and-publish-java-connectors-with-tag.sh index 1a4f4f18548..8b1b1ea1790 100755 --- a/poe-tasks/build-and-publish-java-connectors-with-tag.sh +++ b/poe-tasks/build-and-publish-java-connectors-with-tag.sh @@ -1,16 +1,15 @@ #!/usr/bin/env bash # This script builds and optionally publishes Java connector Docker images. -# Usage: ./build-and-publish-java-connectors-with-tag.sh --name [--pre-release] [--main-release] [--publish] +# Usage: ./build-and-publish-java-connectors-with-tag.sh --name --release-type [pre-release | main-release] [--publish] # # Flag descriptions: -# --main-release: Publishes images with the exact version from metadata.yaml. -# Only publishes if the image does not exists on Dockerhub. -# Used for production releases on merge to master. -# -# --pre-release: Publishes images with a dev tag (version-dev.githash). -# Only publishes if the image does not exists on Dockerhub. -# Used for development/testing purposes. +# --name : Specifies the connector name (e.g., destination-bigquery). +# +# --release-type: Specifies the release type: +# - pre-release: Builds with a dev tag (version-dev.githash). +# - main-release: Builds with the exact version from metadata.yaml. +# Defaults to pre-release if not specified. # # --publish: Actually publishes the images. Without this flag, the script runs in dry-run mode # and only shows what would be published without actually publishing. @@ -24,7 +23,7 @@ # ./build-and-publish-java-connectors-with-tag.sh --name=foo-conn # # 2) Mixed: positional + pre-release -# ./build-and-publish-java-connectors-with-tag.sh --pre-release foo-conn +# ./build-and-publish-java-connectors-with-tag.sh --release-type=pre-release foo-conn # # 3) Enable actual publishing (default is dry-run mode) # ./build-and-publish-java-connectors-with-tag.sh --publish foo-conn diff --git a/poe-tasks/lib/parse_args.sh b/poe-tasks/lib/parse_args.sh index c46da37b2d2..81de40955f4 100644 --- a/poe-tasks/lib/parse_args.sh +++ b/poe-tasks/lib/parse_args.sh @@ -1,8 +1,8 @@ # Parse the command-line args that we care about. # Scripts sourcing this script can be invoked as either: -# ./foo.sh [--pre-release] [--main-release] [--publish] [--name=]* [--name ]* +# ./foo.sh [--release-type=] [--publish] [--name=]* [--name ]* # Or, if invoked with no `--name` flags: -# ./get-modified-connectors.sh --json | ./foo.sh [--pre-release] [--main-release] [--publish] +# ./get-modified-connectors.sh --json | ./foo.sh [--release-type=] [--publish] publish_mode="pre-release" do_publish=false connectors=() @@ -13,13 +13,13 @@ while [[ $# -gt 0 ]]; do sed -n '1,34p' "$0" exit 0 ;; - --main-release) - publish_mode="main-release" + --release-type=*) + publish_mode="${1#*=}" shift ;; - --pre-release) - publish_mode="pre-release" - shift + --release-type) + publish_mode="$2" + shift 2 ;; --publish) do_publish=true diff --git a/poe-tasks/publish-python-registry.sh b/poe-tasks/publish-python-registry.sh index 8b2dcdef6fe..0e2c78aaaa0 100755 --- a/poe-tasks/publish-python-registry.sh +++ b/poe-tasks/publish-python-registry.sh @@ -16,9 +16,9 @@ Must be run from the root of the Airbyte repository with Poetry installed Options: -n, --name CONNECTOR_NAME Connector name (required) - -t, --token TOKEN PyPI token (required) + -t, --token TOKEN PyPI token (optional, specify this or set PYTHON_REGISTRY_TOKEN environment variable) -v, --version VERSION Override version (optional) - --pre-release Publish as a pre-release (uses a dev version derived from the current timestamp) + --release-type TYPE Release type (optional): 'pre-release' or 'main-release' (default is 'pre-release') --test-registry Use the test PyPI registry (default is production registry) -h, --help Show this help message @@ -27,7 +27,7 @@ Environment Variables: Examples: $0 --name source-faker --token \$PYPI_TOKEN - $0 --name source-faker --token \$PYPI_TOKEN --pre-release + $0 --name source-faker --token \$PYPI_TOKEN --release-type main-release EOF } @@ -54,7 +54,7 @@ TEST_REGISTRY_UPLOAD_URL="https://test.pypi.org/legacy/" TEST_REGISTRY_CHECK_URL="https://test.pypi.org/pypi" TEST_REGISTRY_PACKAGE_URL="https://test.pypi.org/project" -PRE_RELEASE=false +RELEASE_TYPE="pre-release" CONNECTOR_NAME="" PYPI_TOKEN="" VERSION_OVERRIDE="" @@ -74,9 +74,9 @@ while [[ $# -gt 0 ]]; do VERSION_OVERRIDE="$2" shift 2 ;; - --pre-release) - PRE_RELEASE=true - shift + --release-type) + RELEASE_TYPE="$2" + shift 2 ;; --test-registry) REGISTRY_UPLOAD_URL="$TEST_REGISTRY_UPLOAD_URL" @@ -104,6 +104,12 @@ if [[ -z "$CONNECTOR_NAME" ]]; then exit 1 fi +if [[ "$RELEASE_TYPE" != "pre-release" && "$RELEASE_TYPE" != "main-release" ]]; then + echo "Error: Invalid release type '$RELEASE_TYPE'. Valid options are 'pre-release' or 'main-release'." >&2 + usage >&2 + exit 1 +fi + # Use environment variables as fallback if [[ -z "$PYPI_TOKEN" && -n "${PYTHON_REGISTRY_TOKEN:-}" ]]; then PYPI_TOKEN="$PYTHON_REGISTRY_TOKEN" @@ -150,7 +156,7 @@ BASE_VERSION=$(poe -qq get-version) # Determine version to use if [[ -n "$VERSION_OVERRIDE" ]]; then VERSION="$VERSION_OVERRIDE" -elif [[ "$PRE_RELEASE" == "true" ]]; then +elif [[ "$RELEASE_TYPE" == "pre-release" ]]; then # Add current timestamp for pre-release. # we can't use the git revision because not all python registries allow local version identifiers. # Public version identifiers must conform to PEP 440 and only allow digits. @@ -162,6 +168,7 @@ fi echo "Package name: $PACKAGE_NAME" echo "Version: $VERSION" +echo "Release type: $RELEASE_TYPE" echo # Check if package already exists diff --git a/poe-tasks/upload-connector-metadata.sh b/poe-tasks/upload-connector-metadata.sh index 6782d802519..82d362a943d 100755 --- a/poe-tasks/upload-connector-metadata.sh +++ b/poe-tasks/upload-connector-metadata.sh @@ -2,7 +2,7 @@ set -euo pipefail # Uploads the metadata (+SBOM+spec cache) to GCS. -# Usage: ./poe-tasks/upload-connector-metadata.sh --name destination-bigquery [--pre-release] [--main-release] +# Usage: ./poe-tasks/upload-connector-metadata.sh --name destination-bigquery --release-type # You must have three environment variables set (GCS_CREDENTIALS, METADATA_SERVICE_GCS_CREDENTIALS, SPEC_CACHE_GCS_CREDENTIALS), # each containing a JSON-formatted GCP service account key. # SPEC_CACHE_GCS_CREDENTIALS needs write access to `gs://$spec_cache_bucket/specs`. diff --git a/poe-tasks/upload-java-connector-tar-file.sh b/poe-tasks/upload-java-connector-tar-file.sh index 9b7c7919f4e..627ff64b646 100755 --- a/poe-tasks/upload-java-connector-tar-file.sh +++ b/poe-tasks/upload-java-connector-tar-file.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Uploads the java tar for java connectors. -# Usage: ./poe-tasks/upload-java-connector-tar-file.sh --name destination-bigquery [--pre-release] [--main-release] +# Usage: ./poe-tasks/upload-java-connector-tar-file.sh --name destination-bigquery --release-type # You must have set the env var GCS_CREDENTIALS, which contains a JSON-formatted GCP service account key. # GCS_CREDENTIALS needs write access to `gs://$metadata_bucket/resources/java`. set -euo pipefail diff --git a/poe-tasks/upload-python-dependencies.sh b/poe-tasks/upload-python-dependencies.sh index 187904575f4..71a73e536e0 100755 --- a/poe-tasks/upload-python-dependencies.sh +++ b/poe-tasks/upload-python-dependencies.sh @@ -5,7 +5,7 @@ set -euo pipefail # Upload Python connector dependencies metadata to GCS # Extracted from airbyte-ci publish pipeline for GitHub Actions integration # -# Usage: ./poe-tasks/upload-python-depdendencies.sh --name source-avri [--pre-release] [--main-release] +# Usage: ./poe-tasks/upload-python-dependencies.sh --name source-avri --release-type [pre-release | main-release] --bucket my-bucket --connector-version 1.2.3 # # source utility functions @@ -23,7 +23,7 @@ Options: -n, --name CONNECTOR_NAME Connector name (required) --bucket BUCKET_NAME GCS bucket name (optional, defaults to dev bucket) --connector-version VERSION Connector version (optional, default reads from metadata.yaml) - --pre-release Publish as a pre-release (uses a dev version derived from the git hash) + --release-type TYPE Release type (optional): 'pre-release' or 'main-release' (default is 'pre-release') -h, --help Show this help message Environment Variables: @@ -56,9 +56,9 @@ while [[ $# -gt 0 ]]; do VERSION="$2" shift 2 ;; - --pre-release) - PRE_RELEASE=true - shift + --release-type) + RELEASE_TYPE="$2" + shift 2 ;; -h|--help) usage @@ -110,7 +110,7 @@ fi if [[ -z "$VERSION" ]]; then VERSION=$(poe -qq get-version) fi -if [[ $PRE_RELEASE == true ]]; then +if [[ $RELEASE_TYPE == "pre-release" ]]; then VERSION=$(generate_dev_tag "$VERSION") fi