184 lines
8.4 KiB
YAML
184 lines
8.4 KiB
YAML
name: Publish Connector Image
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
repo:
|
|
description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos."
|
|
required: false
|
|
default: "airbytehq/airbyte"
|
|
gitref:
|
|
description: "The git ref to check out from the specified repository."
|
|
required: false
|
|
default: master
|
|
connector:
|
|
description: "Airbyte Connector"
|
|
required: true
|
|
run-tests:
|
|
description: "Should run tests when publishing"
|
|
required: true
|
|
default: "true"
|
|
comment-id:
|
|
description: "The comment-id of the slash command. Used to update the comment with the status."
|
|
required: false
|
|
|
|
jobs:
|
|
## Gradle Build
|
|
# In case of self-hosted EC2 errors, remove this block.
|
|
start-publish-image-runner:
|
|
name: Start Build EC2 Runner
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
label: ${{ steps.start-ec2-runner.outputs.label }}
|
|
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
with:
|
|
repository: ${{ github.event.inputs.repo }}
|
|
ref: ${{ github.event.inputs.gitref }}
|
|
- name: Start AWS Runner
|
|
id: start-ec2-runner
|
|
uses: ./.github/actions/start-aws-runner
|
|
with:
|
|
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
|
|
# 80 gb disk
|
|
ec2-image-id: ami-0d648081937c75a73
|
|
publish-image:
|
|
needs: start-publish-image-runner
|
|
runs-on: ${{ needs.start-publish-image-runner.outputs.label }}
|
|
environment: more-secrets
|
|
steps:
|
|
- name: Set up Cloud SDK
|
|
uses: google-github-actions/setup-gcloud@master
|
|
with:
|
|
service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }}
|
|
export_default_credentials: true
|
|
- name: Search for valid connector name format
|
|
id: regex
|
|
uses: AsasInnab/regex-action@v1
|
|
with:
|
|
regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$"
|
|
regex_flags: "i" # required to be set for this plugin
|
|
search_string: ${{ github.event.inputs.connector }}
|
|
- name: Validate input workflow format
|
|
if: steps.regex.outputs.first_match != github.event.inputs.connector
|
|
run: echo "The connector provided has an invalid format!" && exit 1
|
|
- name: Link comment to workflow run
|
|
if: github.event.inputs.comment-id
|
|
uses: peter-evans/create-or-update-comment@v1
|
|
with:
|
|
comment-id: ${{ github.event.inputs.comment-id }}
|
|
body: |
|
|
> :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
with:
|
|
repository: ${{ github.event.inputs.repo }}
|
|
ref: ${{ github.event.inputs.gitref }}
|
|
- name: Install Java
|
|
uses: actions/setup-java@v1
|
|
with:
|
|
java-version: '17'
|
|
- name: Install Pyenv and Tox
|
|
# Beside PyEnv, this does not set any runtimes up because it uses an AMI image that has everything pre-installed. See https://github.com/airbytehq/airbyte/issues/4559/
|
|
run: |
|
|
python3 -m pip install --quiet virtualenv==16.7.9 --user
|
|
python3 -m virtualenv venv
|
|
source venv/bin/activate
|
|
pip install --quiet tox==3.24.4
|
|
- name: Test and install CI scripts
|
|
# all CI python packages have the prefix "ci_"
|
|
run: |
|
|
source venv/bin/activate
|
|
tox -r -c ./tools/tox_ci.ini
|
|
pip install --quiet -e ./tools/ci_*
|
|
- name: Write Integration Test Credentials for ${{ github.event.inputs.connector }}
|
|
run: |
|
|
source venv/bin/activate
|
|
ci_credentials ${{ github.event.inputs.connector }}
|
|
env:
|
|
GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
|
|
- name: Prepare Sentry Release
|
|
if: startsWith(github.event.inputs.connector, 'connectors')
|
|
run: |
|
|
curl -sL https://sentry.io/get-cli/ | bash
|
|
source tools/lib/lib.sh
|
|
DOCKERFILE=airbyte-integrations/${{ github.event.inputs.connector }}/Dockerfile
|
|
echo "IMAGE_NAME=$(echo ${{ github.event.inputs.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV
|
|
echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV
|
|
- name: Create Sentry Release
|
|
if: startsWith(github.event.inputs.connector, 'connectors')
|
|
run: |
|
|
# The --ignore-empty flag is not ideal but necessary, because we need to run this command from forked repos,
|
|
# which do not have Sentry integration, and thus Sentry cannot find the proper commit history.
|
|
sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing --ignore-empty
|
|
env:
|
|
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
|
|
SENTRY_ORG: airbyte-5j
|
|
SENTRY_PROJECT: airbyte-connectors
|
|
- name: Publish ${{ github.event.inputs.connector }}
|
|
run: |
|
|
echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u airbytebot -p ${DOCKER_PASSWORD}
|
|
./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache
|
|
id: publish
|
|
env:
|
|
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
|
|
# Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners.
|
|
TZ: UTC
|
|
- name: Finalize Sentry release
|
|
if: startsWith(github.event.inputs.connector, 'connectors')
|
|
run: |
|
|
sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}"
|
|
env:
|
|
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
|
|
SENTRY_ORG: airbyte-5j
|
|
SENTRY_PROJECT: airbyte-connectors
|
|
- name: Add Success Comment
|
|
if: github.event.inputs.comment-id && success()
|
|
uses: peter-evans/create-or-update-comment@v1
|
|
with:
|
|
comment-id: ${{ github.event.inputs.comment-id }}
|
|
body: |
|
|
> :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
|
|
- name: Add Failure Comment
|
|
if: github.event.inputs.comment-id && !success()
|
|
uses: peter-evans/create-or-update-comment@v1
|
|
with:
|
|
comment-id: ${{ github.event.inputs.comment-id }}
|
|
body: |
|
|
> :x: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
|
|
- name: Slack Notification - Failure
|
|
if: failure()
|
|
uses: rtCamp/action-slack-notify@master
|
|
env:
|
|
SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
|
|
SLACK_USERNAME: Buildozer
|
|
SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
|
|
SLACK_COLOR: DC143C
|
|
SLACK_TITLE: "Failed to publish connector ${{ github.event.inputs.connector }} from branch ${{ github.ref }}"
|
|
SLACK_FOOTER: ""
|
|
# In case of self-hosted EC2 errors, remove this block.
|
|
stop-publish-image-runner:
|
|
name: Stop Build EC2 Runner
|
|
needs:
|
|
- start-publish-image-runner # required to get output from the start-runner job
|
|
- publish-image # required to wait when the main job is done
|
|
runs-on: ubuntu-latest
|
|
if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
|
|
steps:
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v1
|
|
with:
|
|
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
aws-region: us-east-2
|
|
- name: Stop EC2 runner
|
|
uses: machulav/ec2-github-runner@v2
|
|
with:
|
|
mode: stop
|
|
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
|
|
label: ${{ needs.start-publish-image-runner.outputs.label }}
|
|
ec2-instance-id: ${{ needs.start-publish-image-runner.outputs.ec2-instance-id }}
|