106 lines
4.8 KiB
YAML
106 lines
4.8 KiB
YAML
name: Connector Ops CI - Run Regression Tests
|
|
|
|
concurrency:
|
|
# This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow.
|
|
#
|
|
# - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests
|
|
# ensuring that only one run per pull request is active at a time.
|
|
#
|
|
# - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches.
|
|
# This allows us to run multiple workflow dispatches in parallel.
|
|
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
connector_name:
|
|
description: Connector name (e.g. source-faker)
|
|
required: true
|
|
connection_id:
|
|
description: ID of the connection to test; use "auto" to let the connection retriever choose a connection
|
|
required: true
|
|
pr_url:
|
|
description: URL of the PR containing the code change
|
|
required: true
|
|
streams:
|
|
description: Streams to include in regression tests
|
|
use_local_cdk:
|
|
description: Use the local CDK when building the target connector
|
|
default: "false"
|
|
type: boolean
|
|
|
|
jobs:
|
|
regression_tests:
|
|
name: Regression Tests
|
|
runs-on: connector-test-large
|
|
timeout-minutes: 360 # 6 hours
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v4
|
|
- name: Check PAT rate limits
|
|
run: |
|
|
./tools/bin/find_non_rate_limited_PAT \
|
|
${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
|
|
${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
|
|
- name: Extract branch name [WORKFLOW DISPATCH]
|
|
shell: bash
|
|
if: github.event_name == 'workflow_dispatch'
|
|
run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
|
|
id: extract_branch
|
|
|
|
- name: Install Poetry
|
|
id: install_poetry
|
|
uses: snok/install-poetry@v1
|
|
|
|
- name: Make poetry venv in project
|
|
id: poetry_venv
|
|
run: poetry config virtualenvs.in-project true
|
|
|
|
- name: Install Python packages
|
|
id: install_python_packages
|
|
working-directory: airbyte-ci/connectors/pipelines
|
|
run: poetry install
|
|
|
|
- name: Fetch last commit id from remote branch [WORKFLOW DISPATCH]
|
|
if: github.event_name == 'workflow_dispatch'
|
|
id: fetch_last_commit_id_wd
|
|
run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT
|
|
|
|
- name: Setup Stream Parameters
|
|
if: github.event_name == 'workflow_dispatch'
|
|
run: |
|
|
if [ -z "${{ github.event.inputs.streams }}" ]; then
|
|
echo "STREAM_PARAMS=" >> $GITHUB_ENV
|
|
else
|
|
STREAMS=$(echo "${{ github.event.inputs.streams }}" | sed 's/,/ --connector_live_tests.selected-streams=/g')
|
|
echo "STREAM_PARAMS=--connector_live_tests.selected-streams=$STREAMS" >> $GITHUB_ENV
|
|
fi
|
|
|
|
- name: Setup Local CDK Flag
|
|
if: github.event_name == 'workflow_dispatch'
|
|
run: |
|
|
if ${{ github.event.inputs.use_local_cdk }}; then
|
|
echo "USE_LOCAL_CDK_FLAG=--use-local-cdk" >> $GITHUB_ENV
|
|
else
|
|
echo "USE_LOCAL_CDK_FLAG=" >> $GITHUB_ENV
|
|
fi
|
|
|
|
- name: Run Regression Tests [WORKFLOW DISPATCH]
|
|
if: github.event_name == 'workflow_dispatch' # TODO: consider using the matrix strategy (https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs). See https://github.com/airbytehq/airbyte/pull/37659#discussion_r1583380234 for details.
|
|
uses: ./.github/actions/run-airbyte-ci
|
|
with:
|
|
context: "manual"
|
|
#dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
|
|
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
|
|
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
|
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
|
|
gcp_integration_tester_credentials: ${{ secrets.GCLOUD_INTEGRATION_TESTER }}
|
|
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
|
|
git_branch: ${{ steps.extract_branch.outputs.branch }}
|
|
git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }}
|
|
github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
|
|
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
subcommand: connectors ${{ env.USE_LOCAL_CDK_FLAG }} --name ${{ github.event.inputs.connector_name }} test --only-step connector_live_tests --connector_live_tests.test-suite=regression --connector_live_tests.connection-id=${{ github.event.inputs.connection_id }} --connector_live_tests.pr-url=${{ github.event.inputs.pr_url }} ${{ env.STREAM_PARAMS }}
|