airbyte/.github/workflows/test-performance-command.yml

# runs ./tools/bin/ci_performance_test.sh
# which is more or less ./gradlew performanceTest limited to connectors with changes
name: Run Connectors Performance Test
on:
  workflow_dispatch:
    inputs:
      connector:
        description: "Airbyte Connector"
        required: true
      repo:
        description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos."
        required: false
        default: "airbytehq/airbyte"
      gitref:
        description: "The git ref to check out from the specified repository."
        required: false
        default: master
      comment-id:
        description: "The comment-id of the slash command. Used to update the comment with the status."
        required: false
      cpulimit:
        description: "Container CPU limit"
        required: false
      memorylimit:
        description: "Memory CPU limit"
        required: false

jobs:
  start-test-runner:
    name: Start Build EC2 Runner
    timeout-minutes: 10
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v3
        with:
          repository: ${{ github.event.inputs.repo }}
          ref: ${{ github.event.inputs.gitref }}
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
      - name: Start AWS Runner
        id: start-ec2-runner
        uses: ./.github/actions/start-aws-runner
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          github-token: ${{ env.PAT }}
          # 80 gb disk
          ec2-image-id: ami-06cf12549e3d9c522
  performance-test:
    timeout-minutes: 240
    needs: start-test-runner
    runs-on: ${{ needs.start-test-runner.outputs.label }}
    steps:
      - name: Search for valid connector name format
        id: regex
        uses: AsasInnab/regex-action@v1
        with:
          regex_pattern: "^((connectors|bases)/)?[a-zA-Z0-9-_]+$"
          regex_flags: "i" # required to be set for this plugin
          search_string: ${{ github.event.inputs.connector }}
      - name: Validate input workflow format
        if: steps.regex.outputs.first_match != github.event.inputs.connector
        run: echo "The connector provided has an invalid format!" && exit 1
      - name: Link comment to workflow run
        if: github.event.inputs.comment-id
        uses: peter-evans/create-or-update-comment@v1
        with:
          comment-id: ${{ github.event.inputs.comment-id }}
          body: |
            > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
      - name: Checkout Airbyte
        uses: actions/checkout@v3
        with:
          repository: ${{ github.event.inputs.repo }}
          ref: ${{ github.event.inputs.gitref }}
      - name: Install Java
        uses: actions/setup-java@v3
        with:
          distribution: "zulu"
          java-version: "17"
      - name: Install Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"
      - name: Install Pyenv
        run: |
          python3 -m pip install --quiet virtualenv==16.7.9 --user
          python3 -m virtualenv venv
          source venv/bin/activate
      - name: Install CI scripts
        # all CI python packages have the prefix "ci_"
        run: |
          source venv/bin/activate
          pip install --quiet -e ./tools/ci_*
      - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }}
        run: |
          source venv/bin/activate
          ci_credentials ${{ github.event.inputs.connector }} write-to-storage
          # normalization also runs destination-specific tests, so fetch their creds also
          if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then
            ci_credentials destination-bigquery write-to-storage
            ci_credentials destination-postgres write-to-storage
            ci_credentials destination-snowflake write-to-storage
          fi
        env:
          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
      - run: |
          ./tools/bin/ci_performance_test.sh ${{ github.event.inputs.connector }} ${{ github.event.inputs.cpulimit }} ${{ github.event.inputs.memorylimit }}
        name: test_performance ${{ github.event.inputs.connector }}
        id: test_performance
        env:
          ACTION_RUN_ID: ${{github.run_id}}
          # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
          TZ: UTC
      - name: Update Integration Test Credentials after test run for ${{ github.event.inputs.connector }}
        if: always()
        run: |
          source venv/bin/activate
          ci_credentials ${{ github.event.inputs.connector }} update-secrets
          # normalization also runs destination-specific tests, so fetch their creds also
          if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ] || [ 'base-normalization' = "${{ github.event.inputs.connector }}" ]; then
            ci_credentials destination-bigquery update-secrets
            ci_credentials destination-postgres update-secrets
            ci_credentials destination-snowflake update-secrets
          fi
        env:
          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
      - name: Archive test reports artifacts
        if: github.event.inputs.comment-id && failure()
        uses: actions/upload-artifact@v3
        with:
          name: test-reports
          path: |
            **/${{ github.event.inputs.connector }}/build/reports/tests/**/**
            **/${{ github.event.inputs.connector }}/acceptance_tests_logs/**
            **/normalization_test_output/**/dbt_output.log
            **/normalization_test_output/**/destination_output.log
            **/normalization_test_output/**/build/compiled/airbyte_utils/**
            **/normalization_test_output/**/build/run/airbyte_utils/**
            **/normalization_test_output/**/models/generated/**

      - name: Test coverage reports artifacts
        if: github.event.inputs.comment-id && success()
        uses: actions/upload-artifact@v3
        with:
          name: test-reports
          path: |
            **/${{ github.event.inputs.connector }}/htmlcov/**
          retention-days: 3

      - name: Report Status
        if: github.ref == 'refs/heads/master' && always()
        run: ./tools/status/report.sh ${{ github.event.inputs.connector }} ${{github.repository}} ${{github.run_id}} ${{steps.test.outcome}}
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: "us-east-2"
      - name: Add Success Comment
        if: github.event.inputs.comment-id && success()
        uses: peter-evans/create-or-update-comment@v1
        with:
          comment-id: ${{ github.event.inputs.comment-id }}
          body: |
            > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
            ${{env.PYTHON_UNITTEST_COVERAGE_REPORT}}
      - name: Add Failure Comment
        if: github.event.inputs.comment-id && failure()
        uses: peter-evans/create-or-update-comment@v1
        with:
          comment-id: ${{ github.event.inputs.comment-id }}
          body: |
            > :x: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
            > :bug: ${{env.GRADLE_SCAN_LINK}}
  # In case of self-hosted EC2 errors, remove this block.
  stop-test-runner:
    name: Stop Build EC2 Runner
    timeout-minutes: 10
    needs:
      - start-test-runner # required to get output from the start-runner job
      - performance-test # required to wait when the main job is done
    runs-on: ubuntu-latest
    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-2
      - name: Checkout Airbyte
        uses: actions/checkout@v3
      - name: Check PAT rate limits
        run: |
          ./tools/bin/find_non_rate_limited_PAT \
            ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \
            ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }}
      - name: Stop EC2 runner
        uses: supertopher/ec2-github-runner@base64v1.0.10
        with:
          mode: stop
          github-token: ${{ env.PAT }}
          label: ${{ needs.start-test-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-test-runner.outputs.ec2-instance-id }}