714 lines
28 KiB
YAML
714 lines
28 KiB
YAML
name: Airbyte CI
|
|
|
|
on:
|
|
#ability to start task manually in Web UI
|
|
workflow_dispatch:
|
|
schedule:
|
|
- cron: "0 */1 * * *"
|
|
push:
|
|
branches-ignore:
|
|
- "gitbook/v1"
|
|
|
|
jobs:
|
|
# COMMON TASKS
|
|
ensure-images-exist:
|
|
name: "Ensure all required Docker images exist on Dockerhub"
|
|
timeout-minutes: 10
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
|
|
- name: Check images exist
|
|
run: ./tools/bin/check_images_exist.sh all
|
|
# The output of this job is used to trigger the following builds.
|
|
changes:
|
|
name: "Detect Modified Files"
|
|
# The filtering action does not deal with well scheduled events so skip to avoid errors.
|
|
# See https://github.com/dorny/paths-filter/issues/100 for more info.
|
|
# This is okay this workflow is only scheduled on master, where we want to build everything
|
|
# so filtering is not required. Use always() in each start block to force the start task.
|
|
if: github.event_name != 'schedule'
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
backend: ${{ steps.filter.outputs.backend }}
|
|
build: ${{ steps.filter.outputs.build }}
|
|
cli: ${{ steps.filter.outputs.cli }}
|
|
connectors: ${{ steps.filter.outputs.connectors }}
|
|
db: ${{ steps.filter.outputs.db }}
|
|
frontend: ${{ steps.filter.outputs.frontend }}
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
- uses: dorny/paths-filter@v2
|
|
id: filter
|
|
with:
|
|
# Note, the following glob expression within a filters are ORs.
|
|
filters: |
|
|
backend:
|
|
- 'airbyte-!(cdk|integrations|webapp|webapp-e2e-tests)/**'
|
|
build:
|
|
- '.github/**'
|
|
- 'buildSrc/**'
|
|
- 'tools/**'
|
|
- '*.gradle'
|
|
cli:
|
|
- 'airbyte-api/**'
|
|
- 'octavia-cli/**'
|
|
connectors:
|
|
- 'airbyte-cdk/**'
|
|
- 'airbyte-protocol/**'
|
|
- 'airbyte-integrations/**'
|
|
db:
|
|
- 'airbyte-db/**'
|
|
frontend:
|
|
- 'airbyte-webapp/**'
|
|
- 'airbyte-webapp-e2e-tests/**'
|
|
find_valid_pat:
|
|
name: "Find a PAT with room for actions"
|
|
timeout-minutes: 10
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
pat: ${{ steps.variables.outputs.pat }}
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
- name: Check PAT rate limits
|
|
id: variables
|
|
run: |
|
|
./tools/bin/find_non_rate_limited_PAT \
|
|
${{ secrets.AIRBYTEIO_PAT }} \
|
|
${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
|
|
${{ secrets.SUPERTOPHER_PAT }} \
|
|
${{ secrets.DAVINCHIA_PAT }}
|
|
|
|
# Uncomment to debug.
|
|
# changes-output:
|
|
# name: "Debug Change Detection Logic"
|
|
# needs: changes
|
|
# runs-on: ubuntu-latest
|
|
# steps:
|
|
# - uses: actions/checkout@v2
|
|
# - run: |
|
|
# echo '${{ toJSON(needs) }}'
|
|
|
|
## BUILDS
|
|
octavia-cli-build:
|
|
needs: changes
|
|
runs-on: ubuntu-latest
|
|
# Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
|
|
if: needs.changes.outputs.cli == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
|
|
name: "Octavia CLI: Build"
|
|
timeout-minutes: 90
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
|
|
- name: Cache Build Artifacts
|
|
uses: ./.github/actions/cache-build-artifacts
|
|
with:
|
|
cache-key: ${{ secrets.CACHE_VERSION }}
|
|
cache-python: "false"
|
|
|
|
- uses: actions/setup-java@v1
|
|
with:
|
|
java-version: "17"
|
|
|
|
- uses: actions/setup-python@v2
|
|
with:
|
|
python-version: "3.9"
|
|
|
|
- name: Set up CI Gradle Properties
|
|
run: |
|
|
mkdir -p ~/.gradle/
|
|
cat > ~/.gradle/gradle.properties <<EOF
|
|
org.gradle.jvmargs=-Xmx8g -Xss4m \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
|
|
org.gradle.workers.max=8
|
|
org.gradle.vfs.watch=false
|
|
EOF
|
|
|
|
- name: Format
|
|
run: SUB_BUILD=OCTAVIA_CLI ./gradlew format --scan --info --stacktrace
|
|
|
|
- name: Ensure no file change
|
|
run: ./tools/bin/check_for_file_changes
|
|
|
|
- name: Build
|
|
run: |
|
|
SUB_BUILD=OCTAVIA_CLI ./gradlew :octavia-cli:build javadoc --scan
|
|
|
|
- name: Build Platform Docker Images
|
|
run: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan
|
|
|
|
- name: Run integration tests
|
|
run: ./tools/bin/integration_tests_octavia.sh
|
|
|
|
# Connectors Base
|
|
# In case of self-hosted EC2 errors, remove this block.
|
|
start-connectors-base-build-runner:
|
|
name: "Connectors Base: Start Build EC2 Runner"
|
|
needs:
|
|
- changes
|
|
- find_valid_pat
|
|
# Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
|
|
if: |
|
|
needs.changes.outputs.build == 'true' || needs.changes.outputs.connectors == 'true' || needs.changes.outputs.db == 'true' || (always() && github.ref == 'refs/heads/master')
|
|
timeout-minutes: 10
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
label: ${{ steps.start-ec2-runner.outputs.label }}
|
|
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
- name: Start AWS Runner
|
|
id: start-ec2-runner
|
|
uses: ./.github/actions/start-aws-runner
|
|
with:
|
|
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
github-token: ${{ needs.find_valid_pat.outputs.pat }}
|
|
|
|
build-connectors-base:
|
|
# In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
|
|
needs: start-connectors-base-build-runner # required to start the main job when the runner is ready
|
|
runs-on: ${{ needs.start-connectors-base-build-runner.outputs.label }} # run the job on the newly created runner
|
|
name: "Connectors Base: Build"
|
|
timeout-minutes: 90
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
|
|
- name: Cache Build Artifacts
|
|
uses: ./.github/actions/cache-build-artifacts
|
|
with:
|
|
cache-key: ${{ secrets.CACHE_VERSION }}
|
|
|
|
- uses: actions/setup-java@v1
|
|
with:
|
|
java-version: "17"
|
|
|
|
- uses: actions/setup-node@v2
|
|
with:
|
|
node-version: "lts/gallium"
|
|
|
|
- uses: actions/setup-python@v2
|
|
with:
|
|
python-version: "3.9"
|
|
|
|
- name: Install Pyenv
|
|
run: python3 -m pip install virtualenv==16.7.9 --user
|
|
|
|
- name: Install automake
|
|
run: apt-get install -y automake build-essential libtool libtool-bin autoconf
|
|
|
|
- name: Set up CI Gradle Properties
|
|
run: |
|
|
mkdir -p ~/.gradle/
|
|
cat > ~/.gradle/gradle.properties <<EOF
|
|
org.gradle.jvmargs=-Xmx8g -Xss4m \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
|
|
org.gradle.workers.max=8
|
|
org.gradle.vfs.watch=false
|
|
EOF
|
|
|
|
- name: Generate Template scaffold
|
|
run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan
|
|
|
|
- name: Format
|
|
run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace
|
|
|
|
- name: Build
|
|
run: SUB_BUILD=CONNECTORS_BASE ./gradlew build -x :airbyte-workers:test --scan
|
|
|
|
- name: Ensure no file change
|
|
run: git --no-pager diff && test -z "$(git --no-pager diff)"
|
|
|
|
# In case of self-hosted EC2 errors, remove this block.
|
|
stop-connectors-base-build-runner:
|
|
name: "Connectors Base: Stop Build EC2 Runner"
|
|
timeout-minutes: 10
|
|
needs:
|
|
- start-connectors-base-build-runner # required to get output from the start-runner job
|
|
- build-connectors-base # required to wait when the main job is done
|
|
- find_valid_pat
|
|
runs-on: ubuntu-latest
|
|
# Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
|
|
# Thus, we check for skipped here.
|
|
if: ${{ always() && needs.start-connectors-base-build-runner.result != 'skipped'}}
|
|
steps:
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v1
|
|
with:
|
|
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
aws-region: us-east-2
|
|
- name: Stop EC2 runner
|
|
uses: supertopher/ec2-github-runner@base64v1.0.10
|
|
with:
|
|
mode: stop
|
|
github-token: ${{ needs.find_valid_pat.outputs.pat }}
|
|
label: ${{ needs.start-connectors-base-build-runner.outputs.label }}
|
|
ec2-instance-id: ${{ needs.start-connectors-base-build-runner.outputs.ec2-instance-id }}
|
|
|
|
## Frontend Test
|
|
# In case of self-hosted EC2 errors, remove this block.
|
|
start-frontend-runner:
|
|
name: "Frontend: Start EC2 Runner"
|
|
needs:
|
|
- changes
|
|
- find_valid_pat
|
|
# Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
|
|
if: |
|
|
needs.changes.outputs.frontend == 'true' || needs.changes.outputs.build == 'true' || github.ref == 'refs/heads/master'
|
|
|| (always() && needs.changes.outputs.backend == 'true')
|
|
timeout-minutes: 10
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
label: ${{ steps.start-ec2-runner.outputs.label }}
|
|
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
- name: Start AWS Runner
|
|
id: start-ec2-runner
|
|
uses: ./.github/actions/start-aws-runner
|
|
with:
|
|
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
github-token: ${{ needs.find_valid_pat.outputs.pat }}
|
|
frontend-build:
|
|
name: "Frontend: Build"
|
|
needs: start-frontend-runner
|
|
runs-on: ${{ needs.start-frontend-runner.outputs.label }}
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
|
|
- name: Cache Build Artifacts
|
|
uses: ./.github/actions/cache-build-artifacts
|
|
with:
|
|
cache-key: ${{ secrets.CACHE_VERSION }}
|
|
cache-python: "false"
|
|
|
|
- uses: actions/setup-java@v1
|
|
with:
|
|
java-version: "17"
|
|
|
|
- uses: actions/setup-node@v2
|
|
with:
|
|
node-version: "lts/gallium"
|
|
|
|
- name: Set up CI Gradle Properties
|
|
run: |
|
|
mkdir -p ~/.gradle/
|
|
cat > ~/.gradle/gradle.properties <<EOF
|
|
org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
|
|
org.gradle.workers.max=8
|
|
org.gradle.vfs.watch=false
|
|
EOF
|
|
|
|
- name: Build :airbyte-webapp
|
|
run: SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-webapp:build --scan
|
|
frontend-test:
|
|
name: "Frontend: Run End-to-End Tests"
|
|
needs:
|
|
- start-frontend-runner # required to have runner started
|
|
runs-on: ${{ needs.start-frontend-runner.outputs.label }} # run the job on the newly created runner
|
|
timeout-minutes: 120
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
|
|
- name: Cache Build Artifacts
|
|
uses: ./.github/actions/cache-build-artifacts
|
|
with:
|
|
cache-key: ${{ secrets.CACHE_VERSION }}
|
|
cache-python: "false"
|
|
|
|
- uses: actions/setup-java@v1
|
|
with:
|
|
java-version: "17"
|
|
|
|
- uses: actions/setup-node@v2
|
|
with:
|
|
node-version: "lts/gallium"
|
|
|
|
- name: Set up CI Gradle Properties
|
|
run: |
|
|
mkdir -p ~/.gradle/
|
|
cat > ~/.gradle/gradle.properties <<EOF
|
|
org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
|
|
org.gradle.workers.max=8
|
|
org.gradle.vfs.watch=false
|
|
EOF
|
|
|
|
- name: Build Platform Docker Images
|
|
run: SUB_BUILD=PLATFORM ./gradlew --no-daemon assemble --scan
|
|
|
|
- name: Run End-to-End Frontend Tests
|
|
env:
|
|
CYPRESS_WEBAPP_KEY: ${{ secrets.CYPRESS_WEBAPP_KEY }}
|
|
run: ./tools/bin/e2e_test.sh
|
|
# In case of self-hosted EC2 errors, remove this block.
|
|
stop-frontend-runner:
|
|
name: "Frontend: Stop Runner"
|
|
timeout-minutes: 10
|
|
needs:
|
|
- start-frontend-runner # required to get output from the start-runner job
|
|
- frontend-test # required to wait when the e2e-test job is done
|
|
- frontend-build # required to wait when then build job is done
|
|
- find_valid_pat
|
|
runs-on: ubuntu-latest
|
|
# Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
|
|
# Thus, we check for skipped here.
|
|
if: ${{ always() && needs.start-frontend-runner.result != 'skipped'}}
|
|
steps:
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v1
|
|
with:
|
|
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
aws-region: us-east-2
|
|
- name: Stop EC2 runner
|
|
uses: supertopher/ec2-github-runner@base64v1.0.10
|
|
with:
|
|
mode: stop
|
|
github-token: ${{ needs.find_valid_pat.outputs.pat }}
|
|
label: ${{ needs.start-frontend-runner.outputs.label }}
|
|
ec2-instance-id: ${{ needs.start-frontend-runner.outputs.ec2-instance-id }}
|
|
|
|
## FOLLOWING BUILDS ARE ALL PLATFORM BUILDS.
|
|
|
|
# Main Platform
|
|
# In case of self-hosted EC2 errors, remove this block.
|
|
start-platform-build-runner:
|
|
name: "Platform: Start Build EC2 Runner"
|
|
needs:
|
|
- changes
|
|
- find_valid_pat
|
|
# Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
|
|
if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
|
|
timeout-minutes: 10
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
label: ${{ steps.start-ec2-runner.outputs.label }}
|
|
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
- name: Start AWS Runner
|
|
id: start-ec2-runner
|
|
uses: ./.github/actions/start-aws-runner
|
|
with:
|
|
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
github-token: ${{ needs.find_valid_pat.outputs.pat }}
|
|
platform-build:
|
|
name: "Platform: Build"
|
|
# In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line.
|
|
needs: start-platform-build-runner # required to start the main job when the runner is ready
|
|
runs-on: ${{ needs.start-platform-build-runner.outputs.label }} # run the job on the newly created runner
|
|
timeout-minutes: 90
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
|
|
- name: Cache Build Artifacts
|
|
uses: ./.github/actions/cache-build-artifacts
|
|
with:
|
|
cache-key: ${{ secrets.CACHE_VERSION }}
|
|
cache-python: "false"
|
|
|
|
- uses: actions/setup-java@v1
|
|
with:
|
|
java-version: "17"
|
|
|
|
- uses: actions/setup-node@v2
|
|
with:
|
|
node-version: "lts/gallium"
|
|
|
|
- name: Set up CI Gradle Properties
|
|
run: |
|
|
mkdir -p ~/.gradle/
|
|
cat > ~/.gradle/gradle.properties <<EOF
|
|
org.gradle.jvmargs=-Xmx8g -Xss4m \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
|
|
org.gradle.workers.max=8
|
|
org.gradle.vfs.watch=false
|
|
EOF
|
|
|
|
- name: Format
|
|
run: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace
|
|
|
|
- name: Ensure no file change
|
|
run: git --no-pager diff && test -z "$(git --no-pager diff)"
|
|
|
|
- name: Build
|
|
run: |
|
|
SUB_BUILD=PLATFORM ./gradlew build javadoc --scan
|
|
|
|
- name: Integration test
|
|
run: SUB_BUILD=PLATFORM ./gradlew newIntegrationTest
|
|
|
|
- name: Slow integration test
|
|
if: contains(github.ref, 'bump-version') || contains(github.ref, 'master')
|
|
run: SUB_BUILD=PLATFORM ./gradlew slowIntegrationTest
|
|
|
|
- name: Test if Seed spec is updated
|
|
run: SUB_BUILD=PLATFORM ./gradlew format && git --no-pager diff && test -z "$(git --no-pager diff)"
|
|
|
|
# todo (cgardens) - scope by platform.
|
|
- name: Check documentation
|
|
if: success() && github.ref == 'refs/heads/master'
|
|
run: ./tools/site/link_checker.sh check_docs
|
|
|
|
# This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
|
|
# - name: Get Docker Space
|
|
# run: docker run --rm busybox df -h
|
|
|
|
- name: Run End-to-End Acceptance Tests
|
|
run: ./tools/bin/acceptance_test.sh
|
|
|
|
- name: Automatic Migration Acceptance Test
|
|
run: SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i
|
|
|
|
# In case of self-hosted EC2 errors, remove this block.
|
|
stop-platform-build-runner:
|
|
name: "Platform: Stop Build EC2 Runner"
|
|
timeout-minutes: 10
|
|
needs:
|
|
- start-platform-build-runner # required to get output from the start-runner job
|
|
- platform-build # required to wait when the main job is done
|
|
- find_valid_pat
|
|
runs-on: ubuntu-latest
|
|
# Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
|
|
# Thus, we check for skipped here.
|
|
if: ${{ always() && needs.start-platform-build-runner.result != 'skipped'}}
|
|
steps:
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v1
|
|
with:
|
|
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
aws-region: us-east-2
|
|
- name: Stop EC2 runner
|
|
uses: supertopher/ec2-github-runner@base64v1.0.10
|
|
with:
|
|
mode: stop
|
|
github-token: ${{ needs.find_valid_pat.outputs.pat }}
|
|
label: ${{ needs.start-platform-build-runner.outputs.label }}
|
|
ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }}
|
|
|
|
## Kube Acceptance Tests
|
|
# Docker acceptance tests run as part of the build job.
|
|
# In case of self-hosted EC2 errors, remove this block.
|
|
start-kube-acceptance-test-runner:
|
|
name: "Platform: Start Kube Acceptance Test Runner"
|
|
needs:
|
|
- changes
|
|
- find_valid_pat
|
|
# Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
|
|
if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
|
|
timeout-minutes: 10
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
label: ${{ steps.start-ec2-runner.outputs.label }}
|
|
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
- name: Start AWS Runner
|
|
id: start-ec2-runner
|
|
uses: ./.github/actions/start-aws-runner
|
|
with:
|
|
# github-self-hosted-runner-ubuntu-20-with-150gdisk-docker-20.10.7-and-socat
|
|
ec2-image-id: ami-0c1a9bc22624339d8
|
|
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
github-token: ${{ needs.find_valid_pat.outputs.pat }}
|
|
kube-acceptance-test:
|
|
name: "Platform: Acceptance Tests (Kube)"
|
|
# In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
|
|
needs: start-kube-acceptance-test-runner # required to start the main job when the runner is ready
|
|
runs-on: ${{ needs.start-kube-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
|
|
environment: more-secrets
|
|
timeout-minutes: 90
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v2
|
|
|
|
- name: Cache Build Artifacts
|
|
uses: ./.github/actions/cache-build-artifacts
|
|
with:
|
|
cache-key: ${{ secrets.CACHE_VERSION }}
|
|
cache-python: "false"
|
|
|
|
- uses: actions/setup-java@v1
|
|
with:
|
|
java-version: "17"
|
|
|
|
- uses: actions/setup-node@v2
|
|
with:
|
|
node-version: "lts/gallium"
|
|
|
|
- name: Fix EC-2 Runner
|
|
run: |
|
|
mkdir -p /home/runner
|
|
|
|
- name: Set up CI Gradle Properties
|
|
run: |
|
|
mkdir -p ~/.gradle/
|
|
cat > ~/.gradle/gradle.properties <<EOF
|
|
org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
|
|
--add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
|
|
org.gradle.workers.max=8
|
|
org.gradle.vfs.watch=false
|
|
EOF
|
|
|
|
- name: Create cluster config file
|
|
run: |
|
|
cat > /tmp/kind-config.yaml <<EOF
|
|
kind: Cluster
|
|
apiVersion: kind.x-k8s.io/v1alpha4
|
|
nodes:
|
|
- role: control-plane
|
|
- role: worker
|
|
EOF
|
|
|
|
- name: KIND Kubernetes Cluster Setup
|
|
uses: helm/kind-action@v1.2.0
|
|
with:
|
|
node_image: kindest/node:v1.21.2
|
|
config: /tmp/kind-config.yaml
|
|
# In case of self-hosted EC2 errors, remove this env block.
|
|
env:
|
|
USER: root
|
|
HOME: /home/runner
|
|
CHANGE_MINIKUBE_NONE_USER: true
|
|
|
|
- name: Build Platform Docker Images
|
|
run: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
|
|
|
|
- name: Run Kubernetes End-to-End Acceptance Tests
|
|
env:
|
|
USER: root
|
|
HOME: /home/runner
|
|
# AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS
|
|
AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
|
|
SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
|
|
SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
|
|
run: |
|
|
CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube.sh
|
|
|
|
- uses: actions/upload-artifact@v2
|
|
if: failure()
|
|
with:
|
|
name: Kubernetes Logs
|
|
path: /tmp/kubernetes_logs/*
|
|
# In case of self-hosted EC2 errors, remove this block.
|
|
stop-kube-acceptance-test-runner:
|
|
name: "Platform: Stop Kube Acceptance Test EC2 Runner"
|
|
timeout-minutes: 10
|
|
needs:
|
|
- start-kube-acceptance-test-runner # required to get output from the start-runner job
|
|
- kube-acceptance-test # required to wait when the main job is done
|
|
- find_valid_pat
|
|
runs-on: ubuntu-latest
|
|
# Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
|
|
# Thus, we check for skipped here.
|
|
if: ${{ always() && needs.start-kube-acceptance-test-runner.result != 'skipped'}}
|
|
steps:
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v1
|
|
with:
|
|
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
aws-region: us-east-2
|
|
- name: Stop EC2 runner
|
|
uses: supertopher/ec2-github-runner@base64v1.0.10
|
|
with:
|
|
mode: stop
|
|
github-token: ${{ needs.find_valid_pat.outputs.pat }}
|
|
label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }}
|
|
ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }}
|
|
|
|
notify-failure-slack-channel:
|
|
name: "Notify Slack Channel on Build Failures"
|
|
runs-on: ubuntu-latest
|
|
needs:
|
|
- build-connectors-base
|
|
- frontend-build
|
|
- octavia-cli-build
|
|
- platform-build
|
|
- kube-acceptance-test
|
|
if: ${{ failure() && github.ref == 'refs/heads/master' }}
|
|
steps:
|
|
- name: Publish to OSS Build Failure Slack Channel
|
|
uses: abinoda/slack-action@master
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
|
|
with:
|
|
args: >-
|
|
{\"channel\":\"C03BEADRPNY\", \"blocks\":[
|
|
{\"type\":\"divider\"},
|
|
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Merge to OSS Master failed! :bangbang: \n\n\"}},
|
|
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
|
|
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-shocked: \n\"}},
|
|
{\"type\":\"divider\"}]}
|
|
|
|
notify-failure-slack-channel-fixed-broken-build:
|
|
name: "Notify Slack Channel on Build Fixes"
|
|
runs-on: ubuntu-latest
|
|
needs:
|
|
- build-connectors-base
|
|
- frontend-build
|
|
- octavia-cli-build
|
|
- platform-build
|
|
- kube-acceptance-test
|
|
if: success()
|
|
steps:
|
|
- name: Get Previous Workflow Status
|
|
uses: Mercymeilya/last-workflow-status@v0.3
|
|
id: last_status
|
|
with:
|
|
github_token: ${{ secrets.GITHUB_TOKEN }}
|
|
# To avoid clogging up the channel, only publish build success if the previous build was a failure since this means the build was fixed.
|
|
- name: Publish Build Fixed Message to OSS Build Failure Slack Channel
|
|
if: ${{ steps.last_status.outputs.last_status == 'failure' }}
|
|
uses: abinoda/slack-action@master
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
|
|
with:
|
|
args: >-
|
|
{\"channel\":\"C03BEADRPNY\", \"blocks\":[
|
|
{\"type\":\"divider\"},
|
|
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" OSS Master Fixed! :white_check_mark: \n\n\"}},
|
|
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}},
|
|
{\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-rocket: <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|View Action Run> :octavia-rocket: \n\"}},
|
|
{\"type\":\"divider\"}]}
|