From 7967d36e381a6d8807136b3c67d8c09048a1a01f Mon Sep 17 00:00:00 2001 From: David Gold <32782137+dbgold17@users.noreply.github.com> Date: Tue, 12 Aug 2025 09:23:58 -0700 Subject: [PATCH] ci: de-daggerize python and manifest-only image build and publish (#64542) --- .../connector-image-build-push/README.md | 125 +++++++ .../connector-image-build-push/action.yml | 326 ++++++++++++++++++ .github/workflows/publish_connectors.yml | 75 +++- poe-tasks/gradle-connector-tasks.toml | 8 +- poe-tasks/manifest-only-connector-tasks.toml | 8 +- poe-tasks/poetry-connector-tasks.toml | 10 +- 6 files changed, 538 insertions(+), 14 deletions(-) create mode 100644 .github/actions/connector-image-build-push/README.md create mode 100644 .github/actions/connector-image-build-push/action.yml diff --git a/.github/actions/connector-image-build-push/README.md b/.github/actions/connector-image-build-push/README.md new file mode 100644 index 00000000000..1a622dad151 --- /dev/null +++ b/.github/actions/connector-image-build-push/README.md @@ -0,0 +1,125 @@ +# Connector Image Build and Push Action + +This GitHub Action builds and pushes Docker images for Airbyte connectors (Python and manifest-only connectors). It was converted from the original `connector-image-build-push.yml` workflow to provide a reusable action that can be used across multiple workflows. + +## Features + +- **Multi-language Support**: Supports Python and manifest-only connectors +- **Smart Image Management**: Checks if images already exist before building/pushing +- **Multi-architecture Builds**: Builds images for both `linux/amd64` and `linux/arm64` +- **Security Scanning**: Includes vulnerability scanning with Anchore +- **Flexible Tagging**: Supports custom tags and automatic latest tag pushing +- **Dry Run Mode**: Build without pushing for testing purposes +- **Force Publishing**: Override existing images when needed + +## Usage + +### Basic Usage + +```yaml +- name: Build and Push Connector + uses: ./.github/actions/connector-image-build-push + with: + connector-name: 'source-faker' + docker-hub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_PASSWORD }} + dry-run: 'false' +``` + +### Advanced Usage + +```yaml +- name: Build and Push Connector with Custom Settings + uses: ./.github/actions/connector-image-build-push + with: + connector-name: 'source-postgres' + registry: 'docker.io/airbyte' + tag-override: 'v1.2.3' + push-latest: 'true' + force-publish: 'true' + docker-hub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_PASSWORD }} + dry-run: 'false' +``` + +### Dry Run for Testing + +```yaml +- name: Test Build Connector Image + uses: ./.github/actions/connector-image-build-push + with: + connector-name: 'destination-bigquery' + dry-run: 'true' +``` + +## Inputs + +| Input | Description | Required | Default | +|-------|-------------|----------|---------| +| `connector-name` | Connector name (e.g., source-faker, destination-postgres) | **Yes** | - | +| `registry` | Docker registry | No | `docker.io/airbyte` | +| `tag-override` | Override the image tag. If not provided, uses tag from metadata.yaml | No | `""` | +| `push-latest` | Also push image with 'latest' tag | No | `false` | +| `dry-run` | Build but don't push (for testing) | No | `true` | +| `force-publish` | Force publish even if image exists (CAUTION!) | No | `false` | +| `docker-hub-username` | Docker Hub username for authentication | Yes | - | +| `docker-hub-password` | Docker Hub password for authentication | Yes | - | + +## Outputs + +| Output | Description | +|--------|-------------| +| `connector-name` | The validated connector name | +| `connector-type` | The connector type (python or manifest-only) | +| `connector-dir` | The connector directory path | +| `base-image` | The base image from metadata.yaml | +| `connector-version-tag` | The version tag used for the image | +| `connector-image-name` | The full image name with tag | +| `docker-tags` | All Docker tags used for pushing | +| `image-exists` | Whether the image already exists on the registry | +| `build-executed` | Whether the build was executed | +| `push-executed` | Whether the push was executed | + +## How It Works + +### 1. Validation Phase +- Validates that the connector directory exists +- Checks for `metadata.yaml` file +- Ensures connector language is supported (Python or manifest-only) +- Extracts base image and version information +- Validates registry and input combinations + +### 2. Image Existence Check +- Queries Docker Hub API to check if the image already exists +- Uses exponential backoff retry logic for API reliability +- Makes build/push decisions based on existence and input flags + +### 3. Build Phase (if needed) +- Sets up Docker Buildx for multi-architecture builds +- First builds a local image for testing +- Runs a `spec` command test to validate the image +- Then builds the multi-architecture image for publishing + +### 4. Security Scan +- Runs Anchore vulnerability scanning on the built image +- Uses "high" severity cutoff but doesn't fail the build + +### 5. Push Phase (if enabled) +- Authenticates with Docker Hub (if credentials provided) +- Pushes the multi-architecture image with specified tags +- Can include both versioned and latest tags + +## Requirements + +### Repository Structure +- Connector must be located in `airbyte-integrations/connectors//` +- Must have a valid `metadata.yaml` file +- Must be a Python or manifest-only connector type + +### Dependencies +- The action installs `uv`, `poethepoet`, and `yq` automatically +- Requires appropriate Dockerfile templates in `docker-images/` +- Needs access to `poe` tasks for connector metadata extraction + +### Permissions +- `contents: read` - To checkout code diff --git a/.github/actions/connector-image-build-push/action.yml b/.github/actions/connector-image-build-push/action.yml new file mode 100644 index 00000000000..c4e221acce1 --- /dev/null +++ b/.github/actions/connector-image-build-push/action.yml @@ -0,0 +1,326 @@ +name: "Connector Image Build and Push" +description: "Builds and pushes connector images for Python and manifest-only connectors" +branding: + icon: "package" + color: "blue" + +inputs: + connector-name: + description: "Connector name (e.g., source-faker, destination-postgres)" + required: true + registry: + description: "Docker registry" + required: false + default: "docker.io/airbyte" + tag-override: + description: "Override the image tag (optional). If not provided, the tag will be derived from the connector metadata.yaml." + required: false + default: "" + push-latest: + description: "if pushing the image, to a remote registry, also push it with the latest tag" + required: false + default: "false" + dry-run: + description: "Dry run mode (build but do not push)" + required: false + default: "true" + force-publish: + description: "(CAUTION!) Enable to force publish (override) the image even if it already exists on the remote registry" + required: false + default: "false" + docker-hub-username: + description: "Docker Hub username for authentication" + required: true + docker-hub-password: + description: "Docker Hub password for authentication" + required: true + +outputs: + connector-name: + description: "The validated connector name" + value: ${{ steps.vars.outputs.connector-name }} + connector-type: + description: "The connector type (python or manifest-only)" + value: ${{ steps.vars.outputs.connector-type }} + connector-dir: + description: "The connector directory path" + value: ${{ steps.vars.outputs.connector-dir }} + base-image: + description: "The base image from metadata.yaml" + value: ${{ steps.vars.outputs.base-image }} + connector-version-tag: + description: "The version tag used for the image" + value: ${{ steps.vars.outputs.connector-version-tag }} + connector-image-name: + description: "The full image name with tag" + value: ${{ steps.vars.outputs.connector-image-name }} + docker-tags: + description: "All Docker tags to be used for pushing" + value: ${{ steps.vars.outputs.docker-tags }} + image-exists: + description: "Whether the image already exists on the registry" + value: ${{ steps.check-exists.outputs.image-exists }} + build-executed: + description: "Whether the build was executed" + value: ${{ steps.check-exists.outputs.do-build }} + push-executed: + description: "Whether the push was executed" + value: ${{ steps.check-exists.outputs.do-publish }} + +runs: + using: "composite" + steps: + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v6 + + - name: Install Poe and yq + shell: bash + run: | + uv tool install poethepoet + sudo snap install yq + + - name: Validate inputs and resolve variables + id: vars + shell: bash + run: | + set -euo pipefail + + CONNECTOR_NAME="${{ inputs.connector-name }}" + CONNECTOR_DIR="airbyte-integrations/connectors/${CONNECTOR_NAME}" + METADATA_FILE="${CONNECTOR_DIR}/metadata.yaml" + + # Check if connector directory exists + if [[ ! -d "$CONNECTOR_DIR" ]]; then + echo "❌ Connector directory not found: $CONNECTOR_DIR" + exit 1 + fi + + # Check if metadata.yaml exists + if [[ ! -f "$METADATA_FILE" ]]; then + echo "❌ metadata.yaml not found: $METADATA_FILE" + exit 1 + fi + + # Change to connector directory for poe commands + cd "$CONNECTOR_DIR" + + # Validate connector language (only Python and manifest-only supported for now) + CONNECTOR_LANGUAGE=$(poe -qq get-language) + if [[ "$CONNECTOR_LANGUAGE" != "python" && "$CONNECTOR_LANGUAGE" != "manifest-only" ]]; then + echo "❌ Unsupported connector language: $CONNECTOR_LANGUAGE" + exit 1 + fi + + # Get base image from metadata.yaml + BASE_IMAGE=$(poe -qq get-base-image) + if [[ -z "$BASE_IMAGE" ]]; then + echo "❌ Base image not found in metadata.yaml" + exit 1 + fi + + # Resolve connector version to use for image tagging + if [[ "${{ inputs.tag-override }}" ]]; then + CONNECTOR_VERSION_TAG="${{ inputs.tag-override }}" + echo "🏷 Using provided tag: $CONNECTOR_VERSION_TAG" + else + CONNECTOR_VERSION_TAG=$(poe -qq get-version) + if [[ -z "$CONNECTOR_VERSION_TAG" ]]; then + echo "❌ Docker tag not found in metadata.yaml" + exit 1 + fi + echo "🏷 Using tag from metadata.yaml: $CONNECTOR_VERSION_TAG" + fi + + # Can't dry-run and force-publish at the same time + if [[ "${{ inputs.dry-run }}" == "true" && "${{ inputs.force-publish }}" == "true" ]]; then + echo "❌ Cannot use dry-run and force-publish together" + exit 1 + fi + + # Validate registry input + case "${{ inputs.registry }}" in + "docker.io/airbyte") + # Supported registries + ;; + "ghcr.io/airbytehq") + # Unsupported for now + echo "ℹī¸ GitHub Container Registry is not supported yet, please use 'docker.io/airbyte' instead" + exit 1 + ;; + *) + echo "❌ Unsupported registry: ${{ inputs.registry }}" + exit 1 + ;; + esac + + FULL_IMAGE_NAME="${{ inputs.registry }}/${CONNECTOR_NAME}:${CONNECTOR_VERSION_TAG}" + + # Get all tags to use for image pushing + DOCKER_TAGS="${FULL_IMAGE_NAME}" + if [[ "${{ inputs.push-latest }}" == "true" ]]; then + DOCKER_TAGS="${DOCKER_TAGS},${{ inputs.registry }}/${CONNECTOR_NAME}:latest" + fi + + echo "📋 Build Details:" + echo " Connector: $CONNECTOR_NAME" + echo " Language: $CONNECTOR_LANGUAGE" + echo " Base Image: $BASE_IMAGE" + echo " Version Tag: $CONNECTOR_VERSION_TAG" + echo " Registry: ${{ inputs.registry }}" + echo " Full Image Name: $FULL_IMAGE_NAME" + echo " Dry Run: ${{ inputs.dry-run }}" + echo " Force Publish: ${{ inputs.force-publish }}" + + { + echo "connector-name=$CONNECTOR_NAME" + echo "connector-type=$CONNECTOR_LANGUAGE" + echo "connector-dir=$CONNECTOR_DIR" + echo "base-image=$BASE_IMAGE" + echo "connector-version-tag=$CONNECTOR_VERSION_TAG" + echo "connector-image-name=$FULL_IMAGE_NAME" + echo "docker-tags=$DOCKER_TAGS" + } >> $GITHUB_OUTPUT + + - name: Check if image already exists + id: check-exists + shell: bash + run: | + set -euo pipefail + + # Initialize decision variables + DO_BUILD="false" + DO_PUBLISH="false" + IMAGE_EXISTS="unknown" + + # Use Docker Hub API with retry and exponential backoff + URL="https://registry.hub.docker.com/v2/repositories/airbyte/${{ steps.vars.outputs.connector-name }}/tags/${{ steps.vars.outputs.connector-version-tag }}/" + STATUS="" + echo "🔍 Checking if image exists on Docker Hub: ${{ steps.vars.outputs.connector-name }}:${{ steps.vars.outputs.connector-version-tag }}" + + max_attempts=5 + for attempt in {1..$max_attempts}; do + echo " Attempt $attempt/$max_attempts..." + STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$URL") + case "$STATUS" in + "200"|"404") + break + ;; + *) + if [[ $attempt -lt $max_attempts ]]; then + delay=$((2 ** (attempt - 1))) + echo "⚠ī¸ Unexpected status $STATUS, retrying in $delay seconds..." + sleep $delay + else + echo "❌ Failed to check image after $max_attempts attempts. Last status: $STATUS" + exit 1 + fi + esac + done + + # Evaluate results + case "$STATUS" in + "200") + echo "ℹī¸ Image exists on Docker Hub" + IMAGE_EXISTS="true" + DO_BUILD="false" + DO_PUBLISH="false" + ;; + "404") + echo "✅ Image does not exist on Docker Hub" + IMAGE_EXISTS="false" + DO_BUILD="true" + DO_PUBLISH="true" + ;; + *) + echo "⚠ī¸ Unable to determine if image exists (HTTP $STATUS)... Aborting to be safe" + exit 1 + ;; + esac + + # If dry-run is enabled, set build and publish decisions accordingly + if [[ "${{ inputs.dry-run }}" == "true" ]]; then + echo "🔄 Dry run mode enabled - will build but not push" + DO_BUILD="true" + DO_PUBLISH="false" + fi + + # If force-publish is enabled, override decisions + if [[ "${{ inputs.force-publish }}" == "true" ]]; then + echo "🔄 Force publish enabled - will build and push regardless of existing image" + DO_BUILD="true" + DO_PUBLISH="true" + fi + + push_latest="${{ inputs.push-latest }}" + + # Output decision summary + echo "" + echo "📊 Build Decision Summary:" + echo " Image exists: $IMAGE_EXISTS" + echo " Will build: $DO_BUILD" + echo " Will publish: $DO_PUBLISH" + echo " Will tag with latest: ${push_latest:-false}" + echo " Dry run mode: ${{ inputs.dry-run }}" + echo " Force publish: ${{ inputs.force-publish }}" + + # Set outputs for subsequent steps + { + echo "image-exists=$IMAGE_EXISTS" + echo "do-build=$DO_BUILD" + echo "do-publish=$DO_PUBLISH" + } >> $GITHUB_OUTPUT + + - name: Login to Docker Hub + if: ${{ steps.check-exists.outputs.do-publish == 'true' && inputs.docker-hub-username != '' && inputs.docker-hub-password != '' }} + uses: docker/login-action@v3 + with: + username: ${{ inputs.docker-hub-username }} + password: ${{ inputs.docker-hub-password }} + + - name: Set up Docker Buildx + if: ${{ steps.check-exists.outputs.do-build == 'true' }} + uses: docker/setup-buildx-action@v3 + + - name: Build connector image for testing + if: ${{ steps.check-exists.outputs.do-build == 'true' }} + uses: docker/build-push-action@v5 + with: + context: ${{ steps.vars.outputs.connector-dir }} + file: docker-images/Dockerfile.${{ steps.vars.outputs.connector-type }}-connector + platforms: local + tags: ${{ steps.vars.outputs.docker-tags }} + build-args: | + BASE_IMAGE=${{ steps.vars.outputs.base-image }} + CONNECTOR_NAME=${{ steps.vars.outputs.connector-name }} + load: true + push: false + + - name: Run `spec` Image Test + if: ${{ steps.check-exists.outputs.do-build == 'true' }} + shell: bash + run: | + image_name="${{ steps.vars.outputs.connector-image-name }}" + echo "Running spec test for image: $image_name" + docker run --rm $image_name spec > /dev/null + + - name: Build and push connector image (multi-arch) + if: ${{ steps.check-exists.outputs.do-build == 'true' }} + uses: docker/build-push-action@v5 + with: + context: ${{ steps.vars.outputs.connector-dir }} + file: docker-images/Dockerfile.${{ steps.vars.outputs.connector-type }}-connector + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.vars.outputs.docker-tags }} + build-args: | + BASE_IMAGE=${{ steps.vars.outputs.base-image }} + CONNECTOR_NAME=${{ steps.vars.outputs.connector-name }} + push: ${{ steps.check-exists.outputs.do-publish == 'true' }} + + - name: Run Image Vulnerability Scan + if: ${{ steps.check-exists.outputs.do-build == 'true' }} + uses: anchore/scan-action@v6 + with: + image: "${{ steps.vars.outputs.connector-image-name }}" + output-format: "table" + severity-cutoff: high + fail-build: false diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index 3609d20fa36..c1215cf6d23 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -97,12 +97,29 @@ jobs: python-version: "3.11" check-latest: true update-environment: true + - name: Install and configure Poetry # v1 uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a with: version: 1.8.5 + - name: Install the latest version of uv + uses: astral-sh/setup-uv@v6 + + - name: Install Poe + run: | + # Install Poe so we can run the connector tasks: + uv tool install poethepoet + + - name: Get connector metadata + id: connector-metadata + working-directory: airbyte-integrations/connectors/${{ matrix.connector }} + run: | + set -euo pipefail + echo "connector-language=$(poe -qq get-language)" | tee -a $GITHUB_OUTPUT + echo "connector-version=$(poe -qq get-version)" | tee -a $GITHUB_OUTPUT + # We're intentionally not using the `google-github-actions/auth` action. # The upload-connector-metadata step runs a script which handles auth manually. # This is because we're writing files to multiple buckets, using different credentials @@ -122,11 +139,45 @@ jobs: - name: Build and publish JVM connectors images [On merge to master] id: build-and-publish-JVM-connectors-images-master - if: > - github.event_name == 'push' && - contains(fromJson(needs.list_connectors.outputs.connectors-to-publish-jvm).connector, matrix.connector) + if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language == 'java' shell: bash - run: ./poe-tasks/build-and-publish-java-connectors-with-tag.sh --main-release --publish --name ${{ matrix.connector }} + run: | + ./poe-tasks/build-and-publish-java-connectors-with-tag.sh --main-release --publish --name ${{ matrix.connector }} + + - name: Determine build and publish options + id: get-connector-options + if: steps.connector-metadata.outputs.connector-language != 'java' + shell: bash + run: | + # TODO: refactor this logic to be shared by all connector types + is_pre_release=$(echo ${{ inputs.publish-options }} | grep -c -- '--pre-release' || true) + is_release_candidate=$(echo ${{ steps.connector-metadata.outputs.connector-version }} | grep -c -- '-rc' || true) + + # Don't tag with latest for pre-release or release candidate builds + if [[ $is_pre_release -gt 0 || $is_release_candidate -gt 0 ]]; then + echo "push-latest=false" >> $GITHUB_OUTPUT + echo "Skipping latest tag for pre-release or release candidate build." + else + echo "push-latest=true" >> $GITHUB_OUTPUT + fi + + if [[ $is_pre_release -gt 0 ]]; then + hash=$(git rev-parse --short=10 HEAD) + tag_override="${{ steps.connector-metadata.outputs.connector-version }}-dev.${hash}" + echo "tag-override=${tag_override}" >> $GITHUB_OUTPUT + echo "Using a dev tag for a pre-release build: ${tag_override}" + fi + + - name: Build and publish Python and Manifest-Only connectors images [On merge to master] + id: build-and-publish-python-manifest-only-connectors-images-master + if: github.event_name == 'push' && steps.connector-metadata.outputs.connector-language != 'java' + uses: ./.github/actions/connector-image-build-push + with: + connector-name: ${{ matrix.connector }} + push-latest: ${{ steps.get-connector-options.outputs.push-latest }} + dry-run: "false" + docker-hub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_PASSWORD }} - name: Publish connectors [On merge to master] # JVM docker images are published beforehand so Airbyte-CI only does registry publishing. @@ -164,12 +215,22 @@ jobs: - name: Build and publish JVM connectors images [manual] id: build-and-publish-JVM-connectors-images-manual - if: > - (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && - contains(fromJson(needs.list_connectors.outputs.connectors-to-publish-jvm).connector, matrix.connector) + if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language == 'java' shell: bash run: ./poe-tasks/build-and-publish-java-connectors-with-tag.sh ${{ inputs.publish-options }} --name ${{ matrix.connector }} --publish + - name: Build and publish Python and Manifest-Only connectors images [manual] + id: build-and-publish-python-manifest-only-connectors-images-manual + if: (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call') && steps.connector-metadata.outputs.connector-language != 'java' + uses: ./.github/actions/connector-image-build-push + with: + connector-name: ${{ matrix.connector }} + push-latest: ${{ steps.get-connector-options.outputs.push-latest }} + tag-override: ${{ steps.get-connector-options.outputs.tag-override }} + dry-run: "false" + docker-hub-username: ${{ secrets.DOCKER_HUB_USERNAME }} + docker-hub-password: ${{ secrets.DOCKER_HUB_PASSWORD }} + - name: Publish connectors [manual] id: publish-connectors-manual if: github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' diff --git a/poe-tasks/gradle-connector-tasks.toml b/poe-tasks/gradle-connector-tasks.toml index 07f1dfc8a6e..bccd38ce9ae 100644 --- a/poe-tasks/gradle-connector-tasks.toml +++ b/poe-tasks/gradle-connector-tasks.toml @@ -73,13 +73,17 @@ args = [ # Generic tasks (same across all connector types) [tasks.get-language] -cmd = """yq eval '.data.tags[] | select(test("^language:")) | sub("language:","")' $PWD/metadata.yaml""" +cmd = """yq eval '.data.tags[] | select(test("^language:")) | sub("language:","")' ${POE_PWD}/metadata.yaml""" help = "Get the language of the connector from its metadata.yaml file. Use with -qq to get just the language name." [tasks.get-base-image] -cmd = """yq eval '.data.connectorBuildOptions.baseImage' $PWD/metadata.yaml""" +cmd = """yq eval '.data.connectorBuildOptions.baseImage' ${POE_PWD}/metadata.yaml""" help = "Get the base image of the connector from its metadata.yaml file." +[tasks.get-version] +cmd = """yq eval '.data.dockerImageTag' ${POE_PWD}/metadata.yaml""" +help = "Get the version of the connector from its metadata.yaml file." + [tasks.run-cat-tests] shell = "airbyte-ci connectors --name=`poe -qq get-connector-name` test --only-step=acceptance" help = "Run the legacy Airbyte-CI acceptance tests (CAT tests)." diff --git a/poe-tasks/manifest-only-connector-tasks.toml b/poe-tasks/manifest-only-connector-tasks.toml index ba2e9eeec77..4f57cdb26c0 100644 --- a/poe-tasks/manifest-only-connector-tasks.toml +++ b/poe-tasks/manifest-only-connector-tasks.toml @@ -62,13 +62,17 @@ fi # Generic tasks (same across all connector types) [tasks.get-language] -cmd = """yq eval '.data.tags[] | select(test("^language:")) | sub("language:","")' $PWD/metadata.yaml""" +cmd = """yq eval '.data.tags[] | select(test("^language:")) | sub("language:","")' ${POE_PWD}/metadata.yaml""" help = "Get the language of the connector from its metadata.yaml file. Use with -qq to get just the language name." [tasks.get-base-image] -cmd = """yq eval '.data.connectorBuildOptions.baseImage' $PWD/metadata.yaml""" +cmd = """yq eval '.data.connectorBuildOptions.baseImage' ${POE_PWD}/metadata.yaml""" help = "Get the base image of the connector from its metadata.yaml file." +[tasks.get-version] +cmd = """yq eval '.data.dockerImageTag' ${POE_PWD}/metadata.yaml""" +help = "Get the version of the connector from its metadata.yaml file." + [tasks.run-cat-tests] shell = "airbyte-ci connectors --name=`poe -qq get-connector-name` test --only-step=acceptance" help = "Run the legacy Airbyte-CI acceptance tests (CAT tests)." diff --git a/poe-tasks/poetry-connector-tasks.toml b/poe-tasks/poetry-connector-tasks.toml index 62e479bbf68..9e83d5fa192 100644 --- a/poe-tasks/poetry-connector-tasks.toml +++ b/poe-tasks/poetry-connector-tasks.toml @@ -128,13 +128,17 @@ help = "Pin to a specific branch of the CDK." # Generic tasks (same across all connector types) [tool.poe.tasks.get-language] -cmd = """yq eval '.data.tags[] | select(test("^language:")) | sub("language:","")' $PWD/metadata.yaml""" +cmd = """yq eval '.data.tags[] | select(test("^language:")) | sub("language:","")' ${POE_PWD}/metadata.yaml""" help = "Get the language of the connector from its metadata.yaml file. Use with -qq to get just the language name." [tool.poe.tasks.get-base-image] -cmd = """yq eval '.data.connectorBuildOptions.baseImage' $PWD/metadata.yaml""" +cmd = """yq eval '.data.connectorBuildOptions.baseImage' ${POE_PWD}/metadata.yaml""" help = "Get the base image of the connector from its metadata.yaml file." +[tool.poe.tasks.get-version] +cmd = """yq eval '.data.dockerImageTag' ${POE_PWD}/metadata.yaml""" +help = "Get the version of the connector from its metadata.yaml file." + [tool.poe.tasks.run-cat-tests] shell = "airbyte-ci connectors --name=`poe -qq get-connector-name` test --only-step=acceptance" -help = "Run the legacy Airbyte-CI acceptance tests (CAT tests)." +help = "Run the legacy Airbyte-CI acceptance tests (CAT tests)." \ No newline at end of file