226 lines
10 KiB
YAML
226 lines
10 KiB
YAML
name: Run Integration Test
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
connector:
|
|
description: "Airbyte Connector"
|
|
required: true
|
|
repo:
|
|
description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos."
|
|
required: false
|
|
default: "airbytehq/airbyte"
|
|
gitref:
|
|
description: "The git ref to check out from the specified repository."
|
|
required: false
|
|
default: master
|
|
comment-id:
|
|
description: "The comment-id of the slash command. Used to update the comment with the status."
|
|
required: false
|
|
uuid:
|
|
description: "Custom UUID of workflow run. Used because GitHub dispatches endpoint does not return workflow run id."
|
|
required: false
|
|
connector-acceptance-test-version:
|
|
description: "Set a specific connector acceptance test version to use. Enter 'dev' to test, build and use a local version of Connector Acceptance Test."
|
|
required: false
|
|
default: "latest"
|
|
local_cdk:
|
|
description: "Run Connector Acceptance Tests against the CDK version on the current branch."
|
|
required: false
|
|
jobs:
|
|
uuid:
|
|
name: "Custom UUID of workflow run"
|
|
timeout-minutes: 10
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: UUID ${{ github.event.inputs.uuid }}
|
|
run: true
|
|
start-test-runner:
|
|
name: Start Build EC2 Runner
|
|
needs: uuid
|
|
timeout-minutes: 10
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
label: ${{ steps.start-ec2-runner.outputs.label }}
|
|
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
|
|
pipeline-start-timestamp: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
|
|
steps:
|
|
- name: Get start timestamp
|
|
id: get-start-timestamp
|
|
run: echo "::set-output name=start-timestamp::$(date +%s)"
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v3
|
|
with:
|
|
repository: ${{ github.event.inputs.repo }}
|
|
ref: ${{ github.event.inputs.gitref }}
|
|
- name: Check PAT rate limits
|
|
run: |
|
|
./tools/bin/find_non_rate_limited_PAT \
|
|
${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
|
|
${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
|
|
- name: Start AWS Runner
|
|
id: start-ec2-runner
|
|
uses: ./.github/actions/start-aws-runner
|
|
with:
|
|
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
github-token: ${{ env.PAT }}
|
|
integration-test:
|
|
timeout-minutes: 240
|
|
needs: start-test-runner
|
|
runs-on: ${{ needs.start-test-runner.outputs.label }}
|
|
steps:
|
|
- name: Link comment to workflow run
|
|
if: github.event.inputs.comment-id
|
|
uses: peter-evans/create-or-update-comment@v1
|
|
with:
|
|
comment-id: ${{ github.event.inputs.comment-id }}
|
|
body: |
|
|
> :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
|
|
- name: Search for valid connector name format
|
|
id: regex
|
|
uses: AsasInnab/regex-action@v1
|
|
with:
|
|
regex_pattern: "^((connectors|bases)/)?[a-zA-Z0-9-_]+$"
|
|
regex_flags: "i" # required to be set for this plugin
|
|
search_string: ${{ github.event.inputs.connector }}
|
|
- name: Validate input workflow format
|
|
if: steps.regex.outputs.first_match != github.event.inputs.connector
|
|
run: echo "The connector provided has an invalid format!" && exit 1
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v3
|
|
with:
|
|
repository: ${{ github.event.inputs.repo }}
|
|
ref: ${{ github.event.inputs.gitref }}
|
|
- name: Install Java
|
|
uses: actions/setup-java@v3
|
|
with:
|
|
distribution: "zulu"
|
|
java-version: "17"
|
|
- name: Install Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: "3.9"
|
|
- name: Install CI scripts
|
|
# all CI python packages have the prefix "ci_"
|
|
run: |
|
|
pip install --quiet -e ./tools/ci_*
|
|
- name: Write Integration Test Credentials for ${{ github.event.inputs.connector }}
|
|
run: |
|
|
ci_credentials ${{ github.event.inputs.connector }} write-to-storage
|
|
# normalization also runs destination-specific tests, so fetch their creds also
|
|
if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then
|
|
ci_credentials destination-bigquery write-to-storage
|
|
ci_credentials destination-postgres write-to-storage
|
|
ci_credentials destination-snowflake write-to-storage
|
|
fi
|
|
env:
|
|
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
|
|
- name: Test ${{ github.event.inputs.connector }}
|
|
id: test
|
|
env:
|
|
ACTION_RUN_ID: ${{github.run_id}}
|
|
# Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
|
|
TZ: UTC
|
|
ORG_GRADLE_PROJECT_connectorAcceptanceTestVersion: ${{github.event.inputs.connector-acceptance-test-version}}
|
|
S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
uses: Wandalen/wretry.action@v1.0.42
|
|
with:
|
|
command: ./tools/bin/ci_integration_test.sh ${{ github.event.inputs.connector }} ${{ github.event.inputs.local_cdk }}
|
|
attempt_limit: 3
|
|
attempt_delay: 10000 # in ms
|
|
- name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }}
|
|
if: always()
|
|
run: |
|
|
ci_credentials ${{ github.event.inputs.connector }} update-secrets
|
|
# normalization also runs destination-specific tests, so fetch their creds also
|
|
if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then
|
|
ci_credentials destination-bigquery update-secrets
|
|
ci_credentials destination-postgres update-secrets
|
|
ci_credentials destination-snowflake update-secrets
|
|
fi
|
|
env:
|
|
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
|
|
- name: Archive test reports artifacts
|
|
if: github.event.inputs.comment-id && failure()
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: test-reports
|
|
path: |
|
|
**/${{ github.event.inputs.connector }}/build/reports/tests/**/**
|
|
**/${{ github.event.inputs.connector }}/acceptance_tests_logs/**
|
|
**/normalization_test_output/**/dbt_output.log
|
|
**/normalization_test_output/**/destination_output.log
|
|
**/normalization_test_output/**/build/compiled/airbyte_utils/**
|
|
**/normalization_test_output/**/build/run/airbyte_utils/**
|
|
**/normalization_test_output/**/models/generated/**
|
|
- name: Test coverage reports artifacts
|
|
if: github.event.inputs.comment-id && success()
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: test-reports
|
|
path: |
|
|
**/${{ github.event.inputs.connector }}/htmlcov/**
|
|
retention-days: 3
|
|
- name: Run QA checks for ${{ github.event.inputs.connector }}
|
|
id: qa_checks
|
|
if: always()
|
|
run: |
|
|
run-qa-checks ${{ github.event.inputs.connector }}
|
|
- name: Report Observability
|
|
if: always()
|
|
run: ./tools/status/report_observability.sh ${{ github.event.inputs.connector }} ${{github.run_id}} ${{ needs.start-test-runner.outputs.pipeline-start-timestamp }} ${{ github.event.inputs.gitref }} ${{ github.sha }} ${{steps.test.outcome}} ${{steps.qa_checks.outcome}}
|
|
env:
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }}
|
|
AWS_DEFAULT_REGION: "us-east-2"
|
|
- name: Add Success Comment
|
|
if: github.event.inputs.comment-id && success()
|
|
uses: peter-evans/create-or-update-comment@v1
|
|
with:
|
|
comment-id: ${{ github.event.inputs.comment-id }}
|
|
body: |
|
|
> :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
|
|
${{env.PYTHON_UNITTEST_COVERAGE_REPORT}}
|
|
> ${{env.TEST_SUMMARY_INFO}}
|
|
- name: Add Failure Comment
|
|
if: github.event.inputs.comment-id && failure()
|
|
uses: peter-evans/create-or-update-comment@v1
|
|
with:
|
|
comment-id: ${{ github.event.inputs.comment-id }}
|
|
body: |
|
|
> :x: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
|
|
> :bug: ${{env.GRADLE_SCAN_LINK}}
|
|
> ${{env.TEST_SUMMARY_INFO}}
|
|
# In case of self-hosted EC2 errors, remove this block.
|
|
stop-test-runner:
|
|
name: Stop Build EC2 Runner
|
|
timeout-minutes: 10
|
|
needs:
|
|
- start-test-runner # required to get output from the start-runner job
|
|
- integration-test # required to wait when the main job is done
|
|
- uuid
|
|
runs-on: ubuntu-latest
|
|
if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
|
|
steps:
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v1
|
|
with:
|
|
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
aws-region: us-east-2
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v3
|
|
- name: Check PAT rate limits
|
|
run: |
|
|
./tools/bin/find_non_rate_limited_PAT \
|
|
${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
|
|
${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
|
|
- name: Stop EC2 runner
|
|
uses: supertopher/ec2-github-runner@base64v1.0.10
|
|
with:
|
|
mode: stop
|
|
github-token: ${{ env.PAT }}
|
|
label: ${{ needs.start-test-runner.outputs.label }}
|
|
ec2-instance-id: ${{ needs.start-test-runner.outputs.ec2-instance-id }}
|