airbyte/.github/workflows/publish-command.yml

name: Publish Connector Image
on:
  workflow_dispatch:
    inputs:
      connector:
        description: "Airbyte Connector"
        required: true
      run-tests:
        description: "Should run tests when publishing"
        required: true
        default: "true"
      comment-id:
        description: "The comment-id of the slash command. Used to update the comment with the status."
        required: false

jobs:
  ## Gradle Build
  # In case of self-hosted EC2 errors, remove this block.
  start-publish-image-runner:
    name: Start Build EC2 Runner
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-ec2-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2
        with:
          repository: ${{github.event.pull_request.head.repo.full_name}} # always use the branch's repository
      - name: Start AWS Runner
        id: start-ec2-runner
        uses: ./.github/actions/start-aws-runner
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
          ec2-image-id: ami-0d648081937c75a73
  publish-image:
    needs: start-publish-image-runner
    runs-on: ${{ needs.start-publish-image-runner.outputs.label }}
    environment: more-secrets
    steps:
      - name: Set up Cloud SDK
        uses: google-github-actions/setup-gcloud@master
        with:
          service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }}
          export_default_credentials: true
      - name: Search for valid connector name format
        id: regex
        uses: AsasInnab/regex-action@v1
        with:
          regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$"
          regex_flags: "i" # required to be set for this plugin
          search_string: ${{ github.event.inputs.connector }}
      - name: Validate input workflow format
        if: steps.regex.outputs.first_match != github.event.inputs.connector
        run: echo "The connector provided has an invalid format!" && exit 1
      - name: Link comment to workflow run
        if: github.event.inputs.comment-id
        uses: peter-evans/create-or-update-comment@v1
        with:
          comment-id: ${{ github.event.inputs.comment-id }}
          body: |
            > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
      - name: Checkout Airbyte
        uses: actions/checkout@v2
        with:
          repository: ${{github.event.pull_request.head.repo.full_name}} # always use the branch's repository
      # Beside PyEnv, this does not set any runtimes up because it uses an AMI image that has everything pre-installed. See https://github.com/airbytehq/airbyte/issues/4559.
      - name: Install Pyenv
        run: python3 -m pip install virtualenv==16.7.9 --user
      - name: Write Integration Test Credentials # TODO DRY this with test-command.yml
        run: ./tools/bin/ci_credentials.sh ${{ github.event.inputs.connector }}
        env:
          GITHUB_PROVIDED_SECRETS_JSON: ${{ toJson(secrets) }}
          GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
      - run: |
          echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD}
          ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache
        name: publish ${{ github.event.inputs.connector }}
        id: publish
        env:
          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
          # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
          TZ: UTC
      - name: Add Success Comment
        if: github.event.inputs.comment-id && success()
        uses: peter-evans/create-or-update-comment@v1
        with:
          comment-id: ${{ github.event.inputs.comment-id }}
          body: |
            > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
      - name: Add Failure Comment
        if: github.event.inputs.comment-id && !success()
        uses: peter-evans/create-or-update-comment@v1
        with:
          comment-id: ${{ github.event.inputs.comment-id }}
          body: |
            > :x: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
      - name: Slack Notification - Failure
        if: failure()
        uses: rtCamp/action-slack-notify@master
        env:
          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
          SLACK_USERNAME: Buildozer
          SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
          SLACK_COLOR: DC143C
          SLACK_TITLE: "Failed to publish connector ${{ github.event.inputs.connector }} from branch ${{ github.ref	 }}"
          SLACK_FOOTER: ""
  # In case of self-hosted EC2 errors, remove this block.
  stop-publish-image-runner:
    name: Stop Build EC2 Runner
    needs:
      - start-publish-image-runner # required to get output from the start-runner job
      - publish-image # required to wait when the main job is done
    runs-on: ubuntu-latest
    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v1
        with:
          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
          aws-region: us-east-2
      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@v2
        with:
          mode: stop
          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
          label: ${{ needs.start-publish-image-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-publish-image-runner.outputs.ec2-instance-id }}