212 lines
9.0 KiB
YAML
212 lines
9.0 KiB
YAML
name: Publish Connector [EXPERIMENTAL]
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
repo:
|
|
description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos."
|
|
required: false
|
|
default: "airbytehq/airbyte"
|
|
gitref:
|
|
description: "The git ref to check out from the specified repository."
|
|
required: false
|
|
default: master
|
|
connector:
|
|
description: "Airbyte Connector"
|
|
required: true
|
|
bump-version:
|
|
description: "Set to major, minor, or patch to automatically bump connectors version in Dockerfile, definitions.yaml and generate seed spec. You can also do this manually"
|
|
required: false
|
|
default: "false"
|
|
run-tests:
|
|
description: "Should run tests"
|
|
required: false
|
|
default: "true"
|
|
comment-id:
|
|
description: "The comment-id of the slash command. Used to update the comment with the status."
|
|
required: false
|
|
|
|
jobs:
|
|
find_valid_pat:
|
|
name: "Find a PAT with room for actions"
|
|
timeout-minutes: 10
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
pat: ${{ steps.variables.outputs.pat }}
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v3
|
|
- name: Check PAT rate limits
|
|
id: variables
|
|
run: |
|
|
./tools/bin/find_non_rate_limited_PAT \
|
|
${{ secrets.AIRBYTEIO_PAT }} \
|
|
${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
|
|
${{ secrets.SUPERTOPHER_PAT }} \
|
|
${{ secrets.DAVINCHIA_PAT }}
|
|
## Gradle Build
|
|
# In case of self-hosted EC2 errors, remove this block.
|
|
|
|
# start-bump-build-test-connector-runner:
|
|
# name: Start Build EC2 Runner
|
|
# runs-on: ubuntu-latest
|
|
# needs: find_valid_pat
|
|
# outputs:
|
|
# label: ${{ steps.start-ec2-runner.outputs.label }}
|
|
# ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
|
|
# steps:
|
|
# - name: Checkout Airbyte
|
|
# uses: actions/checkout@v3
|
|
# with:
|
|
# repository: ${{ github.event.inputs.repo }}
|
|
# ref: ${{ github.event.inputs.gitref }}
|
|
# - name: Start AWS Runner
|
|
# id: start-ec2-runner
|
|
# uses: ./.github/actions/start-aws-runner
|
|
# with:
|
|
# aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
# aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
# github-token: ${{ needs.find_valid_pat.outputs.pat }}
|
|
# # 80 gb disk
|
|
# ec2-image-id: ami-06cf12549e3d9c522
|
|
# bump-build-test-connector:
|
|
# needs: start-bump-build-test-connector-runner
|
|
# runs-on: ${{ needs.start-bump-build-test-connector-runner.outputs.label }}
|
|
# environment: more-secrets
|
|
# steps:
|
|
# ############################
|
|
# ## SET UP ##
|
|
# ############################
|
|
# - name: Set up Cloud SDK
|
|
# uses: google-github-actions/setup-gcloud@v0
|
|
# with:
|
|
# service_account_key: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY }}
|
|
# export_default_credentials: true
|
|
# - name: Search for valid connector name format
|
|
# id: regex
|
|
# uses: AsasInnab/regex-action@v1
|
|
# with:
|
|
# regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$"
|
|
# regex_flags: "i" # required to be set for this plugin
|
|
# search_string: ${{ github.event.inputs.connector }}
|
|
# - name: Validate input workflow format
|
|
# if: steps.regex.outputs.first_match != github.event.inputs.connector
|
|
# run: echo "The connector provided has an invalid format!" && exit 1
|
|
# - name: Link comment to workflow run
|
|
# if: github.event.inputs.comment-id
|
|
# uses: peter-evans/create-or-update-comment@v1
|
|
# with:
|
|
# comment-id: ${{ github.event.inputs.comment-id }}
|
|
# body: |
|
|
# > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
|
|
# - name: Checkout Airbyte
|
|
# uses: actions/checkout@v3
|
|
# with:
|
|
# repository: ${{ github.event.inputs.repo }}
|
|
# ref: ${{ github.event.inputs.gitref }}
|
|
# token: ${{ secrets.OCTAVIA_PAT }}
|
|
# - name: Install Java
|
|
# uses: actions/setup-java@v3
|
|
# with:
|
|
# distribution: "zulu"
|
|
# java-version: "17"
|
|
# - name: Install Python
|
|
# uses: actions/setup-python@v4
|
|
# with:
|
|
# python-version: "3.9"
|
|
# - name: Install Pyenv and Tox
|
|
# run: |
|
|
# python3 -m pip install --quiet virtualenv==16.7.9 --user
|
|
# python3 -m virtualenv venv
|
|
# source venv/bin/activate
|
|
# pip install --quiet tox==3.24.4
|
|
# - name: Install yq
|
|
# if: github.event.inputs.bump-version != 'false' && success()
|
|
# run: |
|
|
# sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
|
|
# sudo add-apt-repository ppa:rmescandon/yq
|
|
# sudo apt update
|
|
# sudo apt install yq -y
|
|
# - name: Test and install CI scripts
|
|
# # all CI python packages have the prefix "ci_"
|
|
# run: |
|
|
# source venv/bin/activate
|
|
# tox -r -c ./tools/tox_ci.ini
|
|
# pip install --quiet -e ./tools/ci_*
|
|
# - name: Get Credentials for ${{ github.event.inputs.connector }}
|
|
# run: |
|
|
# source venv/bin/activate
|
|
# ci_credentials ${{ github.event.inputs.connector }}
|
|
# env:
|
|
# GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
|
|
# # TODO: seems like this should run in post-merge workflow
|
|
# # - name: Prepare Sentry
|
|
# # if: startsWith(github.event.inputs.connector, 'connectors')
|
|
# # run: |
|
|
# # curl -sL https://sentry.io/get-cli/ | bash
|
|
# # - name: Create Sentry Release
|
|
# # if: startsWith(github.event.inputs.connector, 'connectors')
|
|
# # run: |
|
|
# # sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing
|
|
# # env:
|
|
# # SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }}
|
|
# # SENTRY_ORG: airbyte-5j
|
|
# # SENTRY_PROJECT: airbyte-connectors
|
|
# # - name: Build and Test Success Comment
|
|
# # if: github.event.inputs.comment-id && success()
|
|
# # uses: peter-evans/create-or-update-comment@v1
|
|
# # with:
|
|
# # comment-id: ${{ github.event.inputs.comment-id }}
|
|
# # body: |
|
|
# # > :rocket: Successfully built and tested ${{github.event.inputs.connector}}
|
|
# # - name: Build and Test Failure Comment
|
|
# # if: github.event.inputs.comment-id && !success()
|
|
# # uses: peter-evans/create-or-update-comment@v1
|
|
# # with:
|
|
# # comment-id: ${{ github.event.inputs.comment-id }}
|
|
# # body: |
|
|
# # > :x: Failed to build and test ${{github.event.inputs.connector}}
|
|
# # - name: Slack Notification - Failure
|
|
# # if: failure()
|
|
# # uses: rtCamp/action-slack-notify@master
|
|
# # env:
|
|
# # SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
|
|
# # SLACK_USERNAME: Buildozer
|
|
# # SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
|
|
# # SLACK_COLOR: DC143C
|
|
# # SLACK_TITLE: "Failed to build and test connector ${{ github.event.inputs.connector }} from branch ${{ github.ref }}"
|
|
# # SLACK_FOOTER: ""
|
|
# # - name: Add Final Success Comment
|
|
# # if: github.event.inputs.comment-id && success()
|
|
# # uses: peter-evans/create-or-update-comment@v1
|
|
# # with:
|
|
# # comment-id: ${{ github.event.inputs.comment-id }}
|
|
# # body: |
|
|
# # > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
|
|
# # - name: Set publish label
|
|
# # if: success()
|
|
# # run: |
|
|
# # echo "set some label on PR"
|
|
# # In case of self-hosted EC2 errors, remove this block.
|
|
# stop-bump-build-test-connector-runner:
|
|
# name: Stop Build EC2 Runner
|
|
# needs:
|
|
# - start-bump-build-test-connector-runner # required to get output from the start-runner job
|
|
# - bump-build-test-connector # required to wait when the main job is done
|
|
# - find_valid_pat
|
|
# runs-on: ubuntu-latest
|
|
# if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
|
|
# steps:
|
|
# - name: Configure AWS credentials
|
|
# uses: aws-actions/configure-aws-credentials@v1
|
|
# with:
|
|
# aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
# aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
# aws-region: us-east-2
|
|
# - name: Stop EC2 runner
|
|
# uses: supertopher/ec2-github-runner@base64v1.0.10
|
|
# with:
|
|
# mode: stop
|
|
# github-token: ${{ needs.find_valid_pat.outputs.pat }}
|
|
# label: ${{ needs.start-bump-build-test-connector-runner.outputs.label }}
|
|
# ec2-instance-id: ${{ needs.start-bump-build-test-connector-runner.outputs.ec2-instance-id }}
|