391 lines
17 KiB
YAML
391 lines
17 KiB
YAML
name: Publish Python CDK Manually
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
repo:
|
|
description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos."
|
|
required: false
|
|
default: "airbytehq/airbyte"
|
|
gitref:
|
|
description: "The git ref to check out from the specified repository."
|
|
required: false
|
|
default: master
|
|
release-type:
|
|
type: choice
|
|
description: "Choose the type of version upgrade : major|minor|patch"
|
|
options:
|
|
- none
|
|
- major
|
|
- minor
|
|
- patch
|
|
required: true
|
|
changelog-message:
|
|
description: "Changelog message to be added to CHANGELOG.md"
|
|
required: false
|
|
|
|
concurrency:
|
|
group: publish-airbyte-cdk
|
|
cancel-in-progress: false
|
|
|
|
jobs:
|
|
build-cdk:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Install Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: "3.10"
|
|
- name: Install Poetry
|
|
id: install_poetry
|
|
uses: snok/install-poetry@v1
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v3
|
|
with:
|
|
repository: ${{ github.event.inputs.repo }}
|
|
ref: ${{ github.event.inputs.gitref }}
|
|
- name: Install Dependencies
|
|
id: install_dependencies
|
|
working-directory: airbyte-cdk/python
|
|
run: poetry install
|
|
- name: Build CDK Package
|
|
working-directory: airbyte-cdk/python
|
|
run: poetry run poe build
|
|
- name: Post failure to Slack channel dev-connectors-extensibility
|
|
if: ${{ failure() }}
|
|
uses: slackapi/slack-github-action@v1.23.0
|
|
continue-on-error: true
|
|
with:
|
|
channel-id: C04J1M66D8B
|
|
payload: |
|
|
{
|
|
"text": "Error during `build-cdk` while publishing Python CDK!",
|
|
"blocks": [
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "Error while publishing Python CDK!"
|
|
}
|
|
},
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
|
|
|
|
bump-version:
|
|
needs: build-cdk
|
|
if: github.event.inputs.release-type != 'none'
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
new_cdk_version: ${{ steps.bumpversion.outputs.NEW_VERSION }}
|
|
steps:
|
|
- uses: actions/setup-python@v4
|
|
with:
|
|
python-version: "3.10"
|
|
- name: Install Poetry
|
|
id: install_poetry
|
|
uses: snok/install-poetry@v1
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v3
|
|
with:
|
|
repository: ${{ github.event.inputs.repo }}
|
|
ref: ${{ github.event.inputs.gitref }}
|
|
token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }} # This token is what allows us to commit directly to master
|
|
- name: "Publish Python CDK: bump Poetry package version"
|
|
id: bumpversion
|
|
run: |
|
|
cd airbyte-cdk/python
|
|
# Bump package version
|
|
poetry version ${{ github.event.inputs.release-type }}
|
|
new_version="$(poetry version -s)"
|
|
awk -v NEW_VERSION="$new_version" -v CHANGELOG_MESSAGE="${{ github.event.inputs.changelog-message }}" 'NR==3{print "## " NEW_VERSION "\n" CHANGELOG_MESSAGE "\n"}1' CHANGELOG.md > tmp && mv tmp CHANGELOG.md
|
|
echo NEW_VERSION=$new_version >> $GITHUB_OUTPUT
|
|
- name: Commit and Push Changes
|
|
uses: stefanzweifel/git-auto-commit-action@v4
|
|
with:
|
|
file_pattern: airbyte-cdk/python/pyproject.toml airbyte-cdk/python/CHANGELOG.md
|
|
commit_message: 🤖 ${{ github.event.inputs.release-type }} bump Python CDK to version ${{ steps.bumpversion.outputs.NEW_VERSION }}
|
|
commit_user_name: Octavia Squidington III
|
|
commit_user_email: octavia-squidington-iii@users.noreply.github.com
|
|
- name: Post failure to Slack channel dev-connectors-extensibility
|
|
if: ${{ failure() }}
|
|
uses: slackapi/slack-github-action@v1.23.0
|
|
continue-on-error: true
|
|
with:
|
|
channel-id: C04J1M66D8B
|
|
payload: |
|
|
{
|
|
"text": "Error during `bump-version` while publishing Python CDK!",
|
|
"blocks": [
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "Error while publishing Python CDK!"
|
|
}
|
|
},
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
|
|
|
|
publish-cdk:
|
|
name: Publish Python CDK to PyPi
|
|
needs: bump-version
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v3
|
|
with:
|
|
repository: ${{ github.event.inputs.repo }}
|
|
ref: ${{ github.event.inputs.gitref }}
|
|
- name: Build and publish to pypi
|
|
uses: JRubics/poetry-publish@v2.0
|
|
with:
|
|
pypi_token: ${{ secrets.PYPI_TOKEN }}
|
|
python_version: "3.10"
|
|
package_directory: "airbyte-cdk/python"
|
|
- name: Post failure to Slack channel dev-connectors-extensibility
|
|
if: ${{ failure() }}
|
|
uses: slackapi/slack-github-action@v1.23.0
|
|
continue-on-error: true
|
|
with:
|
|
channel-id: C04J1M66D8B
|
|
payload: |
|
|
{
|
|
"text": "Error during `publish-cdk` while publishing Python CDK!",
|
|
"blocks": [
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "Error while publishing Python CDK!"
|
|
}
|
|
},
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
|
|
|
|
bump-manifest-source:
|
|
name: Bump CDK dependency of source-declarative-manifest
|
|
needs:
|
|
- bump-version
|
|
- publish-cdk
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/setup-python@v4
|
|
with:
|
|
python-version: "3.10"
|
|
- name: Install Poetry
|
|
id: install_poetry
|
|
uses: snok/install-poetry@v1
|
|
- name: Checkout Airbyte
|
|
uses: actions/checkout@v3
|
|
with:
|
|
repository: ${{ github.event.inputs.repo }}
|
|
ref: ${{ github.event.inputs.gitref }}
|
|
token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }} # This token is what allows us to commit directly to master
|
|
- name: Bump CDK dependency of source-declarative-manifest
|
|
timeout-minutes: 10
|
|
run: |
|
|
cd airbyte-integrations/connectors/source-declarative-manifest
|
|
echo "Attempting to pull the newly published version of airbyte-cdk."
|
|
while true; do
|
|
# --no-cache to force looking for the new version
|
|
poetry add airbyte-cdk==${{needs.bump-version.outputs.new_cdk_version}} --no-cache && break
|
|
# Loop to wait for the new version to be available to poetry
|
|
echo "Couldn't add new dependency. This is normal if the dependency could not (yet) be found. Retrying in 10 seconds..."
|
|
sleep 10
|
|
done
|
|
echo "Successfully updated the CDK dependency of source-declarative-manifest to ${{needs.bump-version.outputs.new_cdk_version}}"
|
|
- name: Bump version of source-declarative-manifest
|
|
uses: ./.github/actions/run-airbyte-ci
|
|
with:
|
|
context: "master" # TODO: figure out why changing this yells with `The ci_gcs_credentials was not set on this PipelineContext.`
|
|
# Disable the dagger_cloud_token to disable remote cache access.
|
|
# See https://github.com/airbytehq/airbyte-internal-issues/issues/6439#issuecomment-2109503985 for context
|
|
#dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
|
|
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
|
|
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
|
|
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
|
|
gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
|
|
github_token: ${{ secrets.GITHUB_TOKEN }}
|
|
metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
|
|
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
|
|
slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
|
|
spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
|
|
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
|
|
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
|
|
# There is no pull request number as we do this manually, so will just reference when we started doing it manually for now
|
|
subcommand: "connectors --concurrency=1 --execute-timeout=3600 --name=source-declarative-manifest bump-version ${{ github.event.inputs.release-type }} 'Bump CDK version to ${{needs.bump-version.outputs.new_cdk_version}}' --pr-number=36501"
|
|
python_registry_token: ${{ secrets.PYPI_TOKEN }}
|
|
- name: Commit and Push Changes
|
|
uses: stefanzweifel/git-auto-commit-action@v4
|
|
with:
|
|
file_pattern: docs/integrations/sources/low-code.md airbyte-integrations/connectors/source-declarative-manifest/*
|
|
commit_message: 🤖 Cut version ${{needs.bump-version.outputs.new_cdk_version}} of source-declarative-manifest
|
|
commit_user_name: Octavia Squidington III
|
|
commit_user_email: octavia-squidington-iii@users.noreply.github.com
|
|
- name: Post failure to Slack channel dev-connectors-extensibility
|
|
if: ${{ failure() }}
|
|
uses: slackapi/slack-github-action@v1.23.0
|
|
continue-on-error: true
|
|
with:
|
|
channel-id: C04J1M66D8B
|
|
payload: |
|
|
{
|
|
"text": ":warning: A new version of Python CDK has been released but `source-declarative-manifest` and Connector Builder haven't been automatically updated",
|
|
"blocks": [
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "A new version of Python CDK has been released with <https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/CHANGELOG.md|changelog>: ${{ github.event.inputs.changelog-message }}\n\n"
|
|
}
|
|
},
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": ":warning: Could not bump version of `source-declarative-manifest`.>\n"
|
|
}
|
|
},
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
|
|
|
|
update-connector-builder:
|
|
needs:
|
|
- bump-version
|
|
- bump-manifest-source
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/setup-python@v4
|
|
with:
|
|
python-version: "3.9"
|
|
- name: Checkout Airbyte Platform Internal
|
|
uses: actions/checkout@v3
|
|
with:
|
|
repository: airbytehq/airbyte-platform-internal
|
|
token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
|
|
- name: Update CDK version
|
|
run: |
|
|
PREVIOUS_VERSION=$(cat oss/airbyte-connector-builder-resources/CDK_VERSION)
|
|
sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/Dockerfile
|
|
sed -i "s/${PREVIOUS_VERSION}/${{needs.bump-version.outputs.new_cdk_version}}/g" cloud/airbyte-connector-builder-server-wrapped/Dockerfile
|
|
sed -i "s/airbyte-cdk==${PREVIOUS_VERSION}/airbyte-cdk==${{needs.bump-version.outputs.new_cdk_version}}/g" oss/airbyte-connector-builder-server/requirements.in
|
|
echo ${{needs.bump-version.outputs.new_cdk_version}} > oss/airbyte-connector-builder-resources/CDK_VERSION
|
|
cd oss/airbyte-connector-builder-server
|
|
pip install pip-tools
|
|
pip-compile --upgrade
|
|
- name: Create Pull Request
|
|
id: create-pull-request
|
|
uses: peter-evans/create-pull-request@v6
|
|
with:
|
|
token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }}
|
|
commit-message: "chore: update CDK version following release"
|
|
title: "chore: update CDK version following release"
|
|
body: This is an automatically generated PR triggered by a CDK release
|
|
branch: automatic-cdk-release
|
|
base: master
|
|
delete-branch: true
|
|
- name: Post success to Slack channel dev-connectors-extensibility
|
|
uses: slackapi/slack-github-action@v1.23.0
|
|
continue-on-error: true
|
|
with:
|
|
channel-id: C04J1M66D8B
|
|
payload: |
|
|
{
|
|
"text": "A new version of Python CDK has been released!",
|
|
"blocks": [
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "A new version of Python CDK has been released with <https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/CHANGELOG.md|changelog>: ${{ github.event.inputs.changelog-message }}\n\n"
|
|
}
|
|
},
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "A PR has also been created for the <${{ steps.create-pull-request.outputs.pull-request-url }}|Connector Builder>\n"
|
|
}
|
|
},
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
|
|
- name: Post failure to Slack channel dev-connectors-extensibility
|
|
if: ${{ failure() }}
|
|
uses: slackapi/slack-github-action@v1.23.0
|
|
continue-on-error: true
|
|
with:
|
|
channel-id: C04J1M66D8B
|
|
payload: |
|
|
{
|
|
"text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated",
|
|
"blocks": [
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "A new version of Python CDK has been released with <https://github.com/airbytehq/airbyte/blob/master/airbyte-cdk/python/CHANGELOG.md|changelog>: ${{ github.event.inputs.changelog-message }}\n\n"
|
|
}
|
|
},
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": ":warning: Could not automatically create a PR for Connector Builder>\n"
|
|
}
|
|
},
|
|
{
|
|
"type": "section",
|
|
"text": {
|
|
"type": "mrkdwn",
|
|
"text": "See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
|