From 396822378a00d92e1b359d070a0e2bbd9af808e5 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Jun 2025 02:45:12 +0000 Subject: [PATCH] fix(docker-images): align manifest-only Dockerfile with airbyte-ci build process (#61441) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: AJ Steers --- .../docker-connector-base-image-tests.yml | 70 +++++++++++++++++++ .../Dockerfile.manifest-only-connector | 17 ++++- ...rfile.manifest-only-connector.dockerignore | 1 + 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-connector-base-image-tests.yml b/.github/workflows/docker-connector-base-image-tests.yml index bb3729908c9..5d9e8ae362b 100644 --- a/.github/workflows/docker-connector-base-image-tests.yml +++ b/.github/workflows/docker-connector-base-image-tests.yml @@ -11,6 +11,7 @@ on: - docker-images/Dockerfile.java-connector-base - docker-images/Dockerfile.python-connector - docker-images/Dockerfile.python-connector-base + - docker-images/Dockerfile.manifest-only-connector concurrency: group: connector-base-images-${{ github.event.pull_request.number }} @@ -41,11 +42,14 @@ jobs: - 'docker-images/Dockerfile.python-connector' python-base: - 'docker-images/Dockerfile.python-connector-base' + manifest-only: + - 'docker-images/Dockerfile.manifest-only-connector' outputs: java-base: ${{ steps.changes.outputs.java-base }} python-base: ${{ steps.changes.outputs.python-base }} java-connector: ${{ steps.changes.outputs.java-connector }} python-connector: ${{ steps.changes.outputs.python-connector }} + manifest-only: ${{ steps.changes.outputs.manifest-only }} build-java-base-image: name: Build Java Base Image @@ -299,3 +303,69 @@ jobs: output-format: "table" severity-cutoff: critical # TODO: revert to 'high' fail-build: true + + test-manifest-only-connector-image-matrix: + runs-on: ubuntu-latest + needs: [detect-changes] + if: > + needs.detect-changes.outputs.manifest-only + strategy: + fail-fast: false + matrix: + connector: + - source-pokeapi + name: Build ${{ matrix.connector }} Manifest-Only Test Image + environment: + name: ghcr.io/airbytehq + url: https://ghcr.io/airbytehq/${{ matrix.connector }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Debug BUILD_ARGS + run: | + echo "BASE_IMAGE=docker.io/airbyte/source-declarative-manifest:latest" + echo "CONNECTOR_NAME=${{ matrix.connector }}" + + - name: Build Manifest-Only Connector Image + id: docker-build-manifest-only-connector + uses: docker/build-push-action@v5 + with: + context: airbyte-integrations/connectors/${{ matrix.connector }} + file: docker-images/Dockerfile.manifest-only-connector + platforms: linux/amd64,linux/arm64 + tags: | + ghcr.io/airbytehq/${{ matrix.connector }}:draft-pr-${{ github.event.pull_request.number }} + ghcr.io/airbytehq/${{ matrix.connector }}:draft-pr-${{ github.event.pull_request.number }}-build${{ github.run_number }} + build-args: | + BASE_IMAGE=docker.io/airbyte/source-declarative-manifest:latest + CONNECTOR_NAME=${{ matrix.connector }} + push: true + + - name: Run `spec` Image Test + run: | + docker run \ + --rm \ + ghcr.io/airbytehq/${{ matrix.connector }}:draft-pr-${{ github.event.pull_request.number }} \ + spec + + - name: Run ${{ matrix.connector }} Image Vulnerability Scan + uses: anchore/scan-action@v6 + with: + image: "ghcr.io/airbytehq/${{ matrix.connector }}:draft-pr-${{ github.event.pull_request.number }}" + output-format: "table" + severity-cutoff: high + # Don't fail build for manifest-only connectors since we don't control the base image here + fail-build: false diff --git a/docker-images/Dockerfile.manifest-only-connector b/docker-images/Dockerfile.manifest-only-connector index b33f95f9139..3058b1ed8e2 100644 --- a/docker-images/Dockerfile.manifest-only-connector +++ b/docker-images/Dockerfile.manifest-only-connector @@ -2,17 +2,30 @@ # check=skip=InvalidDefaultArgInFrom # Manifest-Only connector Dockerfile for Airbyte +# Updated to include manifest-only testing in CI workflow -# Manifest-Only connector images are built on top of SDM connector image -ARG BASE_IMAGE=airbyte/source-manifest-only:latest +# Manifest-Only connector images are built on top of source-declarative-manifest image +ARG BASE_IMAGE=docker.io/airbyte/source-declarative-manifest:latest FROM ${BASE_IMAGE} ARG CONNECTOR_SNAKE_NAME ARG CONNECTOR_NAME +ARG CONNECTOR_VERSION WORKDIR /airbyte/integration_code +# Create source_declarative_manifest directory first +RUN mkdir -p ./source_declarative_manifest + +# Copy all files to integration_code COPY . ./ +# Copy manifest.yaml to expected location (required for manifest-only connectors) +RUN cp manifest.yaml ./source_declarative_manifest/manifest.yaml + +# Copy components.py if it exists (optional) +RUN if [ -f components.py ]; then cp components.py ./source_declarative_manifest/components.py; fi + +# Set entrypoint explicitly as required for manifest-only connectors ENV AIRBYTE_ENTRYPOINT="python ./main.py" ENTRYPOINT ["python", "./main.py"] diff --git a/docker-images/Dockerfile.manifest-only-connector.dockerignore b/docker-images/Dockerfile.manifest-only-connector.dockerignore index d5ea75ceebc..7b6e3efa926 100644 --- a/docker-images/Dockerfile.manifest-only-connector.dockerignore +++ b/docker-images/Dockerfile.manifest-only-connector.dockerignore @@ -9,6 +9,7 @@ secrets/ !pyproject.toml !poetry.lock !poetry.toml +!manifest.yaml !components.py !requirements.txt !README.md