ci: run cdk compatibility test only on certified connectors (#67604)
## What The existing `CDK Connector Compatibility Test` workflow currently tests all connectors using the bulk CDK. However, we only want to test certified connectors. This PR modifies the workflow to run tests exclusively on certified connectors using the CDK. ## How Instead of getting the list of connectors to test from the diff generated by running a CDK upgrade task, we will get the list of certified connectors by reusing the script used in the auto bump workflow to get this information. ## How did I test this? Tested it in [this](https://github.com/airbytehq/airbyte/pull/67605) PR. More specifically, here's the PR check run https://github.com/airbytehq/airbyte/actions/runs/18410594109/job/52461620803?pr=67605 Note: it looks like the connectors we are testing didn't change. I'd guess it is possible that in the future we will have some none certified connectors using the cdk so the change in this PR still makes sense. ## Can this PR be safely reverted and rolled back? <!-- * If unsure, leave it blank. --> - [x] YES 💚 - [ ] NO ❌
This commit is contained in:
@@ -31,7 +31,7 @@ jobs:
|
||||
- name: List certified connectors
|
||||
id: list-connectors
|
||||
run: |
|
||||
json_array=$(tools/bin/bulk-cdk-auto-upgrade/list-connectors-to-upgrade.sh)
|
||||
json_array=$(tools/bin/bulk-cdk/get-certified-connectors.sh)
|
||||
echo "connectors=$json_array" >> $GITHUB_OUTPUT
|
||||
echo "Found bulk CDK connectors to upgrade: $json_array"
|
||||
|
||||
@@ -107,7 +107,7 @@ jobs:
|
||||
if: steps.check-changes.outputs.has_changes == 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
new_version=$(tools/bin/bulk-cdk-auto-upgrade/bump-connector-metadata.sh "${{ matrix.connector }}")
|
||||
new_version=$(tools/bin/bulk-cdk/auto-upgrade/bump-connector-metadata.sh "${{ matrix.connector }}")
|
||||
echo "Updated dockerImageTag to $new_version"
|
||||
echo "new_version=$new_version" >> $GITHUB_OUTPUT
|
||||
|
||||
@@ -141,7 +141,7 @@ jobs:
|
||||
if: steps.check-changes.outputs.has_changes == 'true' && inputs.repositories != 'airbyte-enterprise'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tools/bin/bulk-cdk-auto-upgrade/populate-connector-changelog.sh \
|
||||
tools/bin/bulk-cdk/auto-upgrade/populate-connector-changelog.sh \
|
||||
"${{ matrix.connector }}" \
|
||||
"${{ steps.bump-version.outputs.new_version }}" \
|
||||
"${{ steps.create-pr.outputs.pull-request-number }}" \
|
||||
|
||||
@@ -14,28 +14,16 @@ jobs:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- name: Checkout Airbyte
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4
|
||||
with:
|
||||
submodules: true # Needed for airbyte-enterprise connectors (no-op otherwise)
|
||||
fetch-depth: 0
|
||||
submodules: true # Needed for when airbyte-enterprise calls this workflow
|
||||
|
||||
- name: Setup Java
|
||||
uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00 # v4.7.1
|
||||
with:
|
||||
distribution: "zulu"
|
||||
java-version: "21"
|
||||
- name: Install yq
|
||||
run: sudo snap install yq
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@748248ddd2a24f49513d8f472f81c3a07d4d50e1 # v4.4.4
|
||||
with:
|
||||
gradle-version: "8.14"
|
||||
|
||||
- name: Trigger changes on all connectors using bulk cdk # this is just an easy way to get a list of all bulk cdk connectors
|
||||
run: ./gradlew upgradeCdk --cdkVersion=local
|
||||
|
||||
- name: Generate Connector Matrix from Changes
|
||||
- name: Generate Connector Matrix
|
||||
id: generate-matrix
|
||||
run: echo "connectors_matrix=$(./poe-tasks/get-modified-connectors.sh --json --local-cdk)" | tee -a $GITHUB_OUTPUT
|
||||
run: echo "connectors_matrix=$(tools/bin/bulk-cdk/get-certified-connectors.sh)" | tee -a $GITHUB_OUTPUT
|
||||
|
||||
outputs:
|
||||
connectors-matrix: ${{ steps.generate-matrix.outputs.connectors_matrix }}
|
||||
@@ -50,7 +38,8 @@ jobs:
|
||||
DD_TAGS: "connector.name:${{ matrix.connector }}"
|
||||
|
||||
strategy:
|
||||
matrix: ${{ fromJson(needs.generate-matrix.outputs.connectors-matrix) }}
|
||||
matrix:
|
||||
connector: ${{ fromJson(needs.generate-matrix.outputs.connectors-matrix) }}
|
||||
max-parallel: 5 # Limit number of parallel jobs
|
||||
fail-fast: false # Don't stop on first failure
|
||||
steps:
|
||||
|
||||
Reference in New Issue
Block a user