Files
steampipe/.github/workflows/release_cli_and_assets.yml

718 lines
21 KiB
YAML

name: Steampipe Release (CLI and Assets)
on:
workflow_dispatch:
inputs:
version:
description: "The version to release (must be prefixed with 'v')"
required: true
env:
PROJECT_ID: steampipe
CORE_REPO: us-docker.pkg.dev/steampipe/steampipe
ORG: turbot
ASSET_IMAGE_NAME: assets
CONFIG_SCHEMA_VERSION: "2020-11-18"
VERSION: ${{ github.event.inputs.version }}
STEAMPIPE_UPDATE_CHECK: false
GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
SPIPETOOLS_PG_CONN_STRING: ${{ secrets.SPIPETOOLS_PG_CONN_STRING }}
SPIPETOOLS_TOKEN: ${{ secrets.SPIPETOOLS_TOKEN }}
jobs:
verify_input:
name: Verify Inputs
runs-on: ubuntu-latest
steps:
- name: Trim asset version prefix and Validate
run: |-
echo $VERSION
trim=${VERSION#"v"}
echo $trim
if [[ $trim =~ ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
echo "Version OK: $trim"
else
echo "Invalid version: $trim"
exit 1
fi
echo "VERSION=${trim}" >> $GITHUB_ENV
- name: Validate Branch
uses: actions/checkout@v3
with:
ref: ${{ github.event.ref }}
verify_asset_unreleased:
name: Verify Assets Unreleased
needs: [verify_input]
runs-on: ubuntu-latest
outputs:
status: ${{ steps.verify.conclusion }}
steps:
- name: Trim asset version prefix and Validate
run: |-
echo $VERSION
trim=${VERSION#"v"}
echo $trim
if [[ $trim =~ ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
echo "Version OK: $trim"
else
echo "Invalid version: $trim"
exit 1
fi
echo "VERSION=${trim}" >> $GITHUB_ENV
- name: 'Authenticate to Google Cloud'
uses: 'google-github-actions/auth@v1'
with:
service_account: ${{ secrets.GCP_GITHUB_ACTION_PUSH_ARTIFACTS }}
credentials_json: ${{ secrets.GCP_GITHUB_ACTION_PUSH_ARTIFACTS_CREDENTIAL_JSON }}
- name: Get GCloud CLI
uses: google-github-actions/setup-gcloud@v1.1.0
with:
project_id: ${{ env.PROJECT_ID }}
- name: Setup GCloud CLI
run: |-
gcloud config list
gcloud components install beta
gcloud components install alpha --quiet
gcloud beta auth configure-docker us-docker.pkg.dev
- name: Delete the tag if it already exists
id: verify
continue-on-error: true
run: |-
[ $(gcloud beta artifacts docker tags list ${{ env.CORE_REPO }}/${{ env.ASSET_IMAGE_NAME }} --project ${{ env.PROJECT_ID }} --format=json | jq 'map(select(.tag | endswith("${{ env.VERSION }}"))) | length') -eq 1 ]
gcloud beta artifacts docker tags delete ${{ env.CORE_REPO }}/${{ env.ASSET_IMAGE_NAME }}:"${{ env.VERSION }}"
build_and_release_assets:
name: Build and Release Assets
needs: [verify_input,verify_asset_unreleased]
if: needs.verify_asset_unreleased.outputs.status == 'success'
runs-on: ubuntu-latest
steps:
- name: Trim asset version prefix and Validate
run: |-
echo $VERSION
trim=${VERSION#"v"}
echo $trim
if [[ $trim =~ ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
echo "Version OK: $trim"
else
echo "Invalid version: $trim"
exit 1
fi
echo "VERSION=${trim}" >> $GITHUB_ENV
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.event.ref }}
- name: 'Authenticate to Google Cloud'
uses: 'google-github-actions/auth@v1'
with:
service_account: ${{ secrets.GCP_GITHUB_ACTION_PUSH_ARTIFACTS }}
credentials_json: ${{ secrets.GCP_GITHUB_ACTION_PUSH_ARTIFACTS_CREDENTIAL_JSON }}
- name: Get GCloud CLI
uses: google-github-actions/setup-gcloud@v1.1.0
with:
project_id: ${{ env.PROJECT_ID }}
- name: Setup GCloud CLI
run: |-
gcloud config list
gcloud components install beta
gcloud components install alpha --quiet
gcloud beta auth configure-docker us-docker.pkg.dev
- name: Verify ORAS
run: |-
gcloud beta auth configure-docker us-docker.pkg.dev
oras version
- name: Setup Node
uses: actions/setup-node@v3
with:
node-version: '16'
- name: Setup Yarn
working-directory: ./ui/dashboard
run: |-
rm -rf ./build
- name: YARN Install
working-directory: ./ui/dashboard
run: |-
unset CI
yarn install
- name: Generate dashboard UI icons
working-directory: ./ui/dashboard
run: |-
make setup
- name: Run Unit Tests
working-directory: ./ui/dashboard
run: |
yarn install
CI=true yarn test
- name: YARN Build
working-directory: ./ui/dashboard
run: |-
unset CI
yarn build
env:
REACT_APP_HEAP_ID: ${{ secrets.HEAP_ANALYTICS_PRODUCTION_ID }}
REACT_APP_VERSION: ${{ env.VERSION }}
- name: Move Build Assets
run: |-
mkdir -p dashboard_ui_build
mv ./ui/dashboard/build/* dashboard_ui_build/
ls -la dashboard_ui_build
- name: Create a version file
run: |-
JSON_STRING=$( jq -n \
--arg version "$VERSION" \
'{
"version":$version,
}' )
echo $JSON_STRING > ./dashboard_ui_build/versions.json
- name: Create Image Config File
run: |-
JSON_STRING=$( jq -n \
--arg name "${{ env.ASSET_IMAGE_NAME }}" \
--arg organization "$ORG" \
--arg version "$VERSION" \
--arg schemaVersion "$CONFIG_SCHEMA_VERSION" \
'{schemaVersion: $schemaVersion, assets: { name: $name, organization: $organization, version: $version} }' )
echo $JSON_STRING > config.json
- name: Create Image Annotations File
run: |-
JSON_STRING=$( jq -n \
--arg title "${{ env.ASSET_IMAGE_NAME }}" \
--arg desc "$ORG" \
--arg version "$VERSION" \
--arg timestamp "$(date +%FT%TZ)" \
--arg vendor "Turbot HQ, Inc." \
'{
"$manifest": {
"org.opencontainers.image.title": $title,
"org.opencontainers.image.description": $desc,
"org.opencontainers.image.version": $version,
"org.opencontainers.image.created": $timestamp,
"org.opencontainers.image.vendor": $vendor
}
}' )
echo $JSON_STRING > annotations.json
- name: Generate Image REF
id: image_ref
run: |
cat config.json
cat annotations.json
cat dashboard_ui_build/versions.json
REF="${{ env.CORE_REPO }}/${{ env.ASSET_IMAGE_NAME }}:$GITHUB_RUN_ID"
echo "REF=$REF" >> $GITHUB_OUTPUT
- name: Push to registry
run: |-
oras push ${{ steps.image_ref.outputs.REF }} \
--config config.json:application/vnd.turbot.steampipe.config.v1+json \
--annotation-file annotations.json \
dashboard_ui_build:application/vnd.turbot.steampipe.assets.report.layer.v1+tar \
- name: Set version tags on Pushed Image
uses: ./.github/actions/semver-tags
id: semver
with:
image-to-tag: "${{ steps.image_ref.outputs.REF }}"
image-version: ${{ env.VERSION }}
create_test_build:
name: Create Test Build
runs-on: ubuntu-latest
needs: [verify_input]
steps:
- name: Trim asset version prefix and Validate
run: |-
echo $VERSION
trim=${VERSION#"v"}
echo $trim
if [[ $trim =~ ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
echo "Version OK: $trim"
else
echo "Invalid version: $trim"
exit 1
fi
echo "VERSION=${trim}" >> $GITHUB_ENV
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.event.ref }}
- name: Unshallow
run: git fetch
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 1.19
- name: Run CLI Unit Tests
run: |
go clean -testcache
go test -timeout 30s ./... -test.v
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@v4
with:
version: latest
args: release --snapshot --clean --skip-publish
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Move build artifacts
run: |
mkdir ~/artifacts
mv $GITHUB_WORKSPACE/dist/steampipe_linux_amd64.tar.gz ~/artifacts/linux.tar.gz
mv $GITHUB_WORKSPACE/dist/steampipe_darwin_amd64.zip ~/artifacts/darwin.zip
- name: List Build Artifacts
run: ls -l ~/artifacts
- name: Save Linux Build Artifact
uses: actions/upload-artifact@v3
with:
name: build-artifact-linux
path: ~/artifacts/linux.tar.gz
if-no-files-found: error
- name: Save MacOS Build Artifact
uses: actions/upload-artifact@v3
with:
name: build-artifact-darwin
path: ~/artifacts/darwin.zip
if-no-files-found: error
run_acceptance_tests:
name: Acceptance tests
needs: [create_test_build,build_and_release_assets]
strategy:
fail-fast: false
matrix:
platform: [ macos-latest ]
test_block:
- "migration"
- "service_and_plugin"
- "search_path"
- "chaos_and_query"
- "dynamic_schema"
- "dynamic_aggregators"
- "cache"
- "mod_install"
- "mod"
- "check"
- "performance"
- "workspace"
- "introspection"
- "cloud"
- "snapshot"
- "dashboard"
- "dashboard_parsing_validation"
- "schema_cloning"
- "exit_codes"
- "force_stop"
exclude:
- platform: macos-latest
test_block: migration
- platform: macos-latest
test_block: force_stop
runs-on: ${{ matrix.platform }}
steps:
- name: Trim asset version prefix and Validate
run: |-
echo $VERSION
trim=${VERSION#"v"}
echo $trim
if [[ $trim =~ ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
echo "Version OK: $trim"
else
echo "Invalid version: $trim"
exit 1
fi
echo "VERSION=${trim}" >> $GITHUB_ENV
- name: Checkout
uses: actions/checkout@v3
with:
submodules: true
ref: ${{ github.event.ref }}
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 1.19
- name: Setup BATS
uses: mig4/setup-bats@v1
with:
bats-version: 1.2.1
- name: Prepare for downloads
id: prepare-for-downloads
run: |
mkdir ~/artifacts
- name: Download Darwin Build Artifacts
uses: actions/download-artifact@v3
if: ${{ matrix.platform == 'macos-latest' }}
with:
name: build-artifact-darwin
path: ~/artifacts
- name: Extract Darwin Artifacts and Install Binary
if: ${{ matrix.platform == 'macos-latest' }}
run: |
mkdir ~/build
unzip ~/artifacts/darwin.zip -d ~/build
- name: Set PATH
run: |
echo "PATH=$PATH:$HOME/build:$GTIHUB_WORKSPACE/tests/acceptance/lib/bats-core/libexec" >> $GITHUB_ENV
- name: Go install jd
run: |
go install github.com/josephburnett/jd@latest
- name: Install DB
id: install-db
continue-on-error: false
run: |
steampipe service start
steampipe plugin install chaos chaosdynamic
steampipe service stop
- name: Run Test Suite
id: run-test-suite
timeout-minutes: 15
run: |
chmod +x $GITHUB_WORKSPACE/tests/acceptance/run.sh
$GITHUB_WORKSPACE/tests/acceptance/run.sh ${{ matrix.test_block }}.bats
steampipe service stop --force
#
# docker_acceptance_test_debian:
# name: Acceptance tests(Docker - Debian)
# needs: [create_test_build,build_and_release_assets]
# runs-on: ubuntu-latest
# steps:
# - name: Checkout
# uses: actions/checkout@v3
# with:
# ref: ${{ github.event.ref }}
#
# - name: Prepare for downloads
# id: prepare-for-downloads
# run: |
# mkdir ~/artifacts
#
# - name: Download Linux Build Artifacts
# uses: actions/download-artifact@v3
# with:
# name: build-artifact-linux
# path: ~/artifacts
#
# - name: Extract Ubuntu Artifacts and Install Binary
# run: |
# mkdir ~/build
# tar -xf ~/artifacts/linux.tar.gz -C ~/build
#
# - name: Copy binary
# run: |
# cp ~/build/steampipe tests/dockertesting/debian
#
# - name: Build steampipe docker images
# run: |
# docker build -t spdebian tests/dockertesting/debian --build-arg TARGETBRANCH="${{ github.event.ref }}"
# docker images
#
# - name: Create a container and run the test suite
# timeout-minutes: 15
# run: |
# docker run --name steampipe-deb `docker images -q spdebian`
# echo "docker run complete"
#
# docker_acceptance_test_oraclelinux:
# name: Acceptance tests(Docker - OracleLinux)
# needs: [create_test_build,build_and_release_assets]
# runs-on: ubuntu-latest
# steps:
# - name: Checkout
# uses: actions/checkout@v3
# with:
# ref: ${{ github.event.ref }}
#
# - name: Prepare for downloads
# id: prepare-for-downloads
# run: |
# mkdir ~/artifacts
#
# - name: Download Linux Build Artifacts
# uses: actions/download-artifact@v3
# with:
# name: build-artifact-linux
# path: ~/artifacts
#
# - name: Extract Ubuntu Artifacts and Install Binary
# run: |
# mkdir ~/build
# tar -xf ~/artifacts/linux.tar.gz -C ~/build
#
# - name: Copy binary
# run: |
# cp ~/build/steampipe tests/dockertesting/oraclelinux
#
# - name: Build steampipe docker images
# run: |
# docker build -t sporacle tests/dockertesting/oraclelinux --build-arg TARGETBRANCH="${{ github.event.ref }}"
# docker images
#
# - name: Create a container and run the test suite
# timeout-minutes: 15
# run: |
# docker run --name steampipe-oracle `docker images -q sporacle`
# echo "docker run complete"
create_release_tag:
name: Tag Release
needs: [run_acceptance_tests]
runs-on: ubuntu-latest
steps:
- name: Trim asset version prefix and Validate
run: |-
echo $VERSION
trim=${VERSION#"v"}
echo $trim
if [[ $trim =~ ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
echo "Version OK: $trim"
else
echo "Invalid version: $trim"
exit 1
fi
echo "VERSION=${trim}" >> $GITHUB_ENV
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.event.ref }}
- name: Unshallow
run: git fetch --prune --unshallow
- name: Tag Release
run: |
git config user.name "Steampipe GitHub Actions Bot"
git config user.email noreply@github.com
git tag ${{ github.event.inputs.version }}
git push origin ${{ github.event.inputs.version }}
ensure_branch_in_homebrew:
name: Ensure branch exists in homebrew-tap
needs: [create_release_tag]
runs-on: ubuntu-latest
steps:
- name: Parse semver string
id: semver_parser
uses: booxmedialtd/ws-action-parse-semver@v1
with:
input_string: ${{ github.event.inputs.version }}
- name: Checkout
if: steps.semver_parser.outputs.prerelease == ''
uses: actions/checkout@v3
with:
repository: turbot/homebrew-tap
token: ${{ secrets.GH_ACCESS_TOKEN }}
ref: main
- name: Delete base branch if exists
if: steps.semver_parser.outputs.prerelease == ''
run: |
git fetch --all
git push origin --delete bump-brew
git push origin --delete $VERSION
continue-on-error: true
- name: Create base branch
if: steps.semver_parser.outputs.prerelease == ''
run: |
git checkout -b bump-brew
git push --set-upstream origin bump-brew
build_and_release_cli:
name: Release CLI
needs: [create_release_tag, ensure_branch_in_homebrew]
runs-on: ubuntu-latest
steps:
- name: Trim asset version prefix and Validate
run: |-
echo $VERSION
trim=${VERSION#"v"}
echo $trim
if [[ $trim =~ ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
echo "Version OK: $trim"
else
echo "Invalid version: $trim"
exit 1
fi
echo "VERSION=${trim}" >> $GITHUB_ENV
- name: Checkout
uses: actions/checkout@v3
with:
ref: ${{ github.event.inputs.version }}
- name: Unshallow
run: git fetch --prune --unshallow
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: 1.19
- name: Run GoReleaser
uses: goreleaser/goreleaser-action@v4
with:
version: latest
args: release --clean
env:
GITHUB_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
# - name: 'Authenticate to Google Cloud'
# uses: 'google-github-actions/auth@v1'
# with:
# service_account: ${{ secrets.GCP_GITHUB_ACTION_PUSH_ARTIFACTS }}
# credentials_json: ${{ secrets.GCP_GITHUB_ACTION_PUSH_ARTIFACTS_CREDENTIAL_JSON }}
# - name: Get GCloud CLI
# uses: google-github-actions/setup-gcloud@v1.1.0
# with:
# project_id: ${{ env.PROJECT_ID }}
# - name: Setup GCloud CLI
# run: |-
# gcloud config list
# gcloud components install beta
# gcloud components install alpha --quiet
# gcloud beta auth configure-docker us-docker.pkg.dev
# - name: Upload rpm and deb packages to artifact registry
# run: |
# gcloud beta artifacts yum upload steampipe-yum-repo --location=us --source=$GITHUB_WORKSPACE/dist/steampipe_linux_amd64.rpm --project steampipe
# gcloud beta artifacts apt upload steampipe-apt-repo --location=us --source=$GITHUB_WORKSPACE/dist/steampipe_linux_amd64.deb --project steampipe
create_pr_in_homebrew:
name: Create PR in homebrew-tap
needs: [ensure_branch_in_homebrew, build_and_release_cli]
runs-on: ubuntu-latest
env:
Version: ${{ github.event.inputs.version }}
steps:
- name: Parse semver string
id: semver_parser
uses: booxmedialtd/ws-action-parse-semver@v1
with:
input_string: ${{ github.event.inputs.version }}
- name: Checkout
if: steps.semver_parser.outputs.prerelease == ''
uses: actions/checkout@v3
with:
repository: turbot/homebrew-tap
token: ${{ secrets.GH_ACCESS_TOKEN }}
ref: main
- name: Create a new branch off the base branch
if: steps.semver_parser.outputs.prerelease == ''
run: |
git fetch --all
git checkout bump-brew
git checkout -b $VERSION
git push --set-upstream origin $VERSION
- name: Close pull request if already exists
if: steps.semver_parser.outputs.prerelease == ''
run: |
gh pr close $VERSION
continue-on-error: true
- name: Create pull request
if: steps.semver_parser.outputs.prerelease == ''
run: |
gh pr create --base main --head $VERSION --title "$Version" --body "Update formula"
update_pr_for_versioning:
name: Update PR
needs: [create_pr_in_homebrew]
runs-on: ubuntu-latest
env:
Version: ${{ github.event.inputs.version }}
steps:
- name: Parse semver string
id: semver_parser
uses: booxmedialtd/ws-action-parse-semver@v1
with:
input_string: ${{ github.event.inputs.version }}
- name: Checkout
if: steps.semver_parser.outputs.prerelease == ''
uses: actions/checkout@v3
with:
repository: turbot/homebrew-tap
token: ${{ secrets.GH_ACCESS_TOKEN }}
ref: ${{ github.event.inputs.version }}
- name: Update live version
if: steps.semver_parser.outputs.prerelease == ''
run: |
scripts/formula_versioning.sh
git config --global user.email "puskar@turbot.com"
git config --global user.name "Puskar Basu"
git add .
git commit -m "Versioning brew formulas"
git push origin $VERSION
clean_up:
# let's clean up the artifacts.
# in case this step isn't reached,
# artifacts automatically expire after 90 days anyway
# refer:
# https://docs.github.com/en/actions/configuring-and-managing-workflows/persisting-workflow-data-using-artifacts#downloading-and-deleting-artifacts-after-a-workflow-run-is-complete
name: Clean Up Artifacts
needs: update_pr_for_versioning
runs-on: ubuntu-latest
steps:
- name: Clean up Linux Build
uses: geekyeggo/delete-artifact@v2
with:
name: build-artifact-linux
- name: Clean up Darwin Build
uses: geekyeggo/delete-artifact@v2
with:
name: build-artifact-darwin