1
0
mirror of synced 2026-01-08 03:06:34 -05:00
Files
airbyte/.github/workflows/gradle.yml
2021-11-02 17:31:40 -07:00

518 lines
19 KiB
YAML

name: Airbyte CI
on:
schedule:
- cron: '0 */1 * * *'
push:
jobs:
## Gradle Build (Connectors Base)
# In case of self-hosted EC2 errors, remove this block.
start-connectors-base-build-runner:
name: "Connectors Base: Start Build EC2 Runner"
timeout-minutes: 10
runs-on: ubuntu-latest
outputs:
label: ${{ steps.start-ec2-runner.outputs.label }}
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
- name: Start AWS Runner
id: start-ec2-runner
uses: ./.github/actions/start-aws-runner
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
ensure-images-exist:
name: "Ensure all required Docker images exist on Dockerhub"
timeout-minutes: 10
runs-on: ubuntu-latest
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
- name: Check images exist
run: ./tools/bin/check_images_exist.sh all
build-connectors-base:
# In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
needs: start-connectors-base-build-runner # required to start the main job when the runner is ready
runs-on: ${{ needs.start-connectors-base-build-runner.outputs.label }} # run the job on the newly created runner
name: "Connectors Base: Build"
timeout-minutes: 90
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
- name: Pip Caching
uses: actions/cache@v2
with:
path: |
~/.cache/pip
key: ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}-
- name: Npm Caching
uses: actions/cache@v2
with:
path: |
~/.npm
key: ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-
# this intentionally does not use restore-keys so we don't mess with gradle caching
- name: Gradle and Python Caching
uses: actions/cache@v2
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
**/.venv
key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }}
- uses: actions/setup-java@v1
with:
java-version: '14'
- uses: actions/setup-node@v1
with:
node-version: '14.7'
- uses: actions/setup-python@v2
with:
python-version: '3.7'
- name: Install Pyenv
run: python3 -m pip install virtualenv==16.7.9 --user
- name: Set up CI Gradle Properties
run: |
mkdir -p ~/.gradle/
cat > ~/.gradle/gradle.properties <<EOF
org.gradle.jvmargs=-Xmx8g -Xss4m
org.gradle.workers.max=8
org.gradle.vfs.watch=false
EOF
- name: Generate Template scaffold
run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan
- name: Format
run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace
- name: Build
run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan
- name: Ensure no file change
run: git --no-pager diff && test -z "$(git --no-pager diff)"
- name: Slack Notification - Failure
if: failure() && github.ref == 'refs/heads/master'
uses: rtCamp/action-slack-notify@master
env:
SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
SLACK_USERNAME: Buildozer
SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
SLACK_COLOR: DC143C
SLACK_TITLE: "Build failure"
SLACK_FOOTER: ""
- name: Slack Notification - Success
if: success() && github.ref == 'refs/heads/master'
uses: rtCamp/action-slack-notify@master
env:
SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
SLACK_USERNAME: Buildbot
SLACK_TITLE: "Build Success"
SLACK_FOOTER: ""
# In case of self-hosted EC2 errors, remove this block.
stop-connectors-base-build-runner:
name: "Connectors Base: Stop Build EC2 Runner"
timeout-minutes: 10
needs:
- start-connectors-base-build-runner # required to get output from the start-runner job
- build-connectors-base # required to wait when the main job is done
runs-on: ubuntu-latest
if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Stop EC2 runner
uses: machulav/ec2-github-runner@v2.3.0
with:
mode: stop
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
label: ${{ needs.start-connectors-base-build-runner.outputs.label }}
ec2-instance-id: ${{ needs.start-connectors-base-build-runner.outputs.ec2-instance-id }}
## Gradle Build (Platform)
# In case of self-hosted EC2 errors, remove this block.
start-platform-build-runner:
name: "Platform: Start Build EC2 Runner"
timeout-minutes: 10
runs-on: ubuntu-latest
outputs:
label: ${{ steps.start-ec2-runner.outputs.label }}
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
- name: Start AWS Runner
id: start-ec2-runner
uses: ./.github/actions/start-aws-runner
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
platform-build:
# In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line.
needs: start-platform-build-runner # required to start the main job when the runner is ready
runs-on: ${{ needs.start-platform-build-runner.outputs.label }} # run the job on the newly created runner
name: "Platform: Build"
timeout-minutes: 90
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
- name: Npm Caching
uses: actions/cache@v2
with:
path: |
~/.npm
key: ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
restore-keys: |
${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-
# this intentionally does not use restore-keys so we don't mess with gradle caching
- name: Gradle Caching
uses: actions/cache@v2
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
**/.venv
key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}
- uses: actions/setup-java@v1
with:
java-version: '14'
- uses: actions/setup-node@v1
with:
node-version: '14.7'
- name: Set up CI Gradle Properties
run: |
mkdir -p ~/.gradle/
cat > ~/.gradle/gradle.properties <<EOF
org.gradle.jvmargs=-Xmx8g -Xss4m
org.gradle.workers.max=8
org.gradle.vfs.watch=false
EOF
- name: Format
run: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace
- name: Ensure no file change
run: git --no-pager diff && test -z "$(git --no-pager diff)"
- name: Build
run: SUB_BUILD=PLATFORM ./gradlew build --scan
- name: Ensure no file change
run: git --no-pager diff && test -z "$(git --no-pager diff)"
# todo (cgardens) - scope by platform.
- name: Check documentation
if: success() && github.ref == 'refs/heads/master'
run: ./tools/site/link_checker.sh check_docs
# This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
# - name: Get Docker Space
# run: docker run --rm busybox df -h
- name: Run End-to-End Acceptance Tests
run: ./tools/bin/acceptance_test.sh
- name: Automatic Migration Acceptance Test
run: MIGRATION_TEST_VERSION=$(grep VERSION .env | tr -d "VERSION=") SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i
- name: Slack Notification - Failure
if: failure() && github.ref == 'refs/heads/master'
uses: rtCamp/action-slack-notify@master
env:
SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
SLACK_USERNAME: Buildozer
SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
SLACK_COLOR: DC143C
SLACK_TITLE: "Build failure"
SLACK_FOOTER: ""
- name: Slack Notification - Success
if: success() && github.ref == 'refs/heads/master'
uses: rtCamp/action-slack-notify@master
env:
SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
SLACK_USERNAME: Buildbot
SLACK_TITLE: "Build Success"
SLACK_FOOTER: ""
# In case of self-hosted EC2 errors, remove this block.
stop-platform-build-runner:
name: "Platform: Stop Build EC2 Runner"
timeout-minutes: 10
needs:
- start-platform-build-runner # required to get output from the start-runner job
- platform-build # required to wait when the main job is done
runs-on: ubuntu-latest
if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Stop EC2 runner
uses: machulav/ec2-github-runner@v2.3.0
with:
mode: stop
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
label: ${{ needs.start-platform-build-runner.outputs.label }}
ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }}
## Frontend Test
## Gradle Build
# In case of self-hosted EC2 errors, remove this block.
start-frontend-test-runner:
name: Start Frontend Test EC2 Runner
timeout-minutes: 10
runs-on: ubuntu-latest
outputs:
label: ${{ steps.start-ec2-runner.outputs.label }}
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
- name: Start AWS Runner
id: start-ec2-runner
uses: ./.github/actions/start-aws-runner
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
frontend-test:
needs: start-frontend-test-runner # required to start the main job when the runner is ready
runs-on: ${{ needs.start-frontend-test-runner.outputs.label }} # run the job on the newly created runner
name: Run Frontend Test
timeout-minutes: 120
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
- uses: actions/setup-java@v1
with:
java-version: '14'
- uses: actions/setup-node@v1
with:
node-version: '14.7'
- name: Install Cypress Test Dependencies
run: sudo apt-get update && sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb
- name: Set up CI Gradle Properties
run: |
mkdir -p ~/.gradle/
cat > ~/.gradle/gradle.properties <<EOF
org.gradle.jvmargs=-Xmx8g -Xss4m
org.gradle.workers.max=8
org.gradle.vfs.watch=false
EOF
- name: Build Platform Docker Images
run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan
- name: Run End-to-End Frontend Tests
run: ./tools/bin/e2e_test.sh
# In case of self-hosted EC2 errors, remove this block.
stop-frontend-test-runner:
name: Stop Frontend Test EC2 Runner
timeout-minutes: 10
needs:
- start-frontend-test-runner # required to get output from the start-runner job
- frontend-test # required to wait when the main job is done
runs-on: ubuntu-latest
if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Stop EC2 runner
uses: machulav/ec2-github-runner@v2.3.0
with:
mode: stop
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
label: ${{ needs.start-frontend-test-runner.outputs.label }}
ec2-instance-id: ${{ needs.start-frontend-test-runner.outputs.ec2-instance-id }}
## Kube Acceptance Tests
# In case of self-hosted EC2 errors, remove this block.
# Docker acceptance tests run as part of the build job.
start-kube-acceptance-test-runner:
name: Start Kube Acceptance Test EC2 Runner
timeout-minutes: 10
runs-on: ubuntu-latest
outputs:
label: ${{ steps.start-ec2-runner.outputs.label }}
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
- name: Start AWS Runner
id: start-ec2-runner
uses: ./.github/actions/start-aws-runner
with:
ec2-image-id: ami-0d4083c04fde515c4
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
kube-acceptance-test:
# In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
needs: start-kube-acceptance-test-runner # required to start the main job when the runner is ready
runs-on: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
environment: more-secrets
name: Acceptance Tests (Kube)
timeout-minutes: 90
steps:
- name: Checkout Airbyte
uses: actions/checkout@v2
- uses: actions/setup-java@v1
with:
java-version: '14'
- uses: actions/setup-node@v1
with:
node-version: '14.7'
- name: Fix EC-2 Runner
run: |
mkdir -p /home/runner
- name: Set up CI Gradle Properties
run: |
mkdir -p ~/.gradle/
cat > ~/.gradle/gradle.properties <<EOF
org.gradle.jvmargs=-Xmx8g -Xss4m
org.gradle.workers.max=8
org.gradle.vfs.watch=false
EOF
- name: Install socat (required for port forwarding)
run: |
sudo apt-get update
sudo apt-get install socat
- name: Create cluster config file
run: |
cat > /tmp/kind-config.yaml <<EOF
kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
nodes:
- role: control-plane
- role: worker
- role: worker
EOF
- name: Check Docker System Info
run: docker system info
- name: KIND Kubernetes Cluster Setup
uses: helm/kind-action@v1.2.0
with:
node_image: kindest/node:v1.21.2
config: /tmp/kind-config.yaml
# In case of self-hosted EC2 errors, remove this env block.
env:
USER: root
HOME: /home/runner
CHANGE_MINIKUBE_NONE_USER: true
- name: Describe kube nodes
run: kubectl describe nodes
env:
USER: root
HOME: /home/runner
- name: Build Platform Docker Images
run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan
- name: Run Kubernetes End-to-End Acceptance Tests
env:
USER: root
HOME: /home/runner
AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
SECRET_STORE_FOR_CONFIGS: ${{ secrets.SECRET_STORE_FOR_CONFIGS }}
run: |
CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
- name: Run Logging Tests
run: ./tools/bin/cloud_storage_logging_test.sh
env:
AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
GOOGLE_CLOUD_STORAGE_TEST_CREDS: ${{ secrets.GOOGLE_CLOUD_STORAGE_TEST_CREDS }}
- name: Show Disk Usage
run: |
df -h
docker system df
- name: Run GCP End-to-End Acceptance Tests
env:
USER: root
HOME: /home/runner
AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
SECRET_STORE_FOR_CONFIGS: ${{ secrets.SECRET_STORE_FOR_CONFIGS }}
run: |
CI=true ./tools/bin/gcp_acceptance_tests.sh
# In case of self-hosted EC2 errors, remove this block.
stop-kube-acceptance-test-runner:
name: Stop Kube Acceptance Test EC2 Runner
timeout-minutes: 10
needs:
- start-kube-acceptance-test-runner # required to get output from the start-runner job
- kube-acceptance-test # required to wait when the main job is done
runs-on: ubuntu-latest
if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- name: Stop EC2 runner
uses: machulav/ec2-github-runner@v2.3.0
with:
mode: stop
github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }}
ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }}