diff --git a/.github/workflows/connector_nightly_builds_dagger.yml b/.github/workflows/connectors_manual_tests.yml similarity index 84% rename from .github/workflows/connector_nightly_builds_dagger.yml rename to .github/workflows/connectors_manual_tests.yml index 753cb6ed501..791398bd565 100644 --- a/.github/workflows/connector_nightly_builds_dagger.yml +++ b/.github/workflows/connectors_manual_tests.yml @@ -1,9 +1,6 @@ -name: POC Connectors CI - nightly builds +name: Connectors CI - Manual tests on: - schedule: - # 11AM UTC is 12AM CET, 4AM PST. - - cron: "0 11 * * *" workflow_dispatch: inputs: runs-on: @@ -14,11 +11,11 @@ on: default: --concurrency=10 --release-stage=generally_available --release-stage=beta required: true -run-name: Test connectors ${{ inputs.test-connectors-options || 'Nightly builds GA and Beta connectors' }} +run-name: "Test connectors ${{ inputs.test-connectors-options}} on ${{ inputs.runs-on }}" jobs: test_connectors: - name: Test connectors ${{ inputs.test-connectors-options || 'Nightly builds GA and Beta connectors' }} + name: "Test connectors ${{ inputs.test-connectors-options}} on ${{ inputs.runs-on }}" timeout-minutes: 600 # 10 hours runs-on: ${{ inputs.runs-on || 'dev-large-runner' }} steps: @@ -47,7 +44,7 @@ jobs: token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - name: Install ci-connector-ops package run: pip install ./tools/ci_connector_ops\[pipelines]\ - - name: Test connectors + - name: "Test connectors ${{ inputs.test-connectors-options}}" run: | export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0" @@ -57,7 +54,7 @@ jobs: mkdir -p "$DAGGER_TMP_BINDIR" curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" fi - airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors ${{ inputs.test-connectors-options || '--concurrency=10 --release-stage=generally_available --release-stage=beta' }} test + airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors ${{ inputs.test-connectors-options }} test env: _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} @@ -67,5 +64,5 @@ jobs: TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status" CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} - CI_CONTEXT: "nightly_builds" + CI_CONTEXT: "manual" CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} diff --git a/.github/workflows/connectors_nightly_builds.yml b/.github/workflows/connectors_nightly_builds.yml new file mode 100644 index 00000000000..a0fb2f4778e --- /dev/null +++ b/.github/workflows/connectors_nightly_builds.yml @@ -0,0 +1,90 @@ +name: Connectors CI - nightly builds for GA and Beta connectors + +on: + schedule: + # 11AM UTC is 12AM CET, 4AM PST. + - cron: "0 11 * * *" + workflow_dispatch: + inputs: + runs-on: + type: string + default: dev-large-runner + required: true + +run-name: "Connectors CI - nightly builds for GA and Beta connectors - ${{ inputs.runs-on }}" + +jobs: + test_connectors: + name: Test GA and Beta connectors + timeout-minutes: 600 # 10 hours + runs-on: ${{ inputs.runs-on || 'dev-large-runner' }} + steps: + - name: Get start timestamp + id: get-start-timestamp + run: echo "::set-output name=start-timestamp::$(date +%s)" + - name: Login to DockerHub + run: "docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}" + env: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Extract branch name + shell: bash + if: github.event_name == 'workflow_dispatch' + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + - name: Install Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10" + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + - name: Install ci-connector-ops package + run: pip install ./tools/ci_connector_ops\[pipelines]\ + - name: Test Python connectors + run: | + export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" + DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0" + DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}" + export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger" + if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then + mkdir -p "$DAGGER_TMP_BINDIR" + curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" + fi + airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors --concurrency=8 --release-stage=generally_available --release-stage=beta --language=python --languague=low-code test + env: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "us-east-2" + TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status" + CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} + CI_CONTEXT: "nightly_builds" + CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} + - name: Test Java connectors + run: | + export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" + DAGGER_CLI_COMMIT="6ed6264f1c4efbf84d310a104b57ef1bc57d57b0" + DAGGER_TMP_BINDIR="/tmp/dagger_${DAGGER_CLI_COMMIT}" + export _EXPERIMENTAL_DAGGER_CLI_BIN="$DAGGER_TMP_BINDIR/dagger" + if [ ! -f "$_EXPERIMENTAL_DAGGER_CLI_BIN" ]; then + mkdir -p "$DAGGER_TMP_BINDIR" + curl "https://dl.dagger.io/dagger/main/${DAGGER_CLI_COMMIT}/dagger_${DAGGER_CLI_COMMIT}_$(uname -s | tr A-Z a-z)_$(uname -m | sed s/x86_64/amd64/).tar.gz" | tar xvz -C "$DAGGER_TMP_BINDIR" + fi + airbyte-ci --is-ci --gha-workflow-run-id=${{ github.run_id }} connectors --concurrency=1 --release-stage=generally_available --release-stage=beta --language=java test + env: + _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + AWS_ACCESS_KEY_ID: ${{ secrets.STATUS_API_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.STATUS_API_AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "us-east-2" + TEST_REPORTS_BUCKET_NAME: "airbyte-connector-build-status" + CI_GITHUB_ACCESS_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + CI_GIT_BRANCH: ${{ steps.extract_branch.outputs.branch }} + CI_CONTEXT: "nightly_builds" + CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}