diff --git a/.github/actions/equivalence-test/action.yml b/.github/actions/equivalence-test/action.yml new file mode 100644 index 0000000000..42af6aba6f --- /dev/null +++ b/.github/actions/equivalence-test/action.yml @@ -0,0 +1,58 @@ +name: equivalence-test +description: "Execute the suite of Terraform equivalence tests in testing/equivalence-tests" +inputs: + target-terraform-version: + description: "The version of Terraform to use in execution." + required: true + target-terraform-branch: + description: "The branch within this repository to update and compare." + required: true + target-equivalence-test-version: + description: "The version of the Terraform equivalence tests to use." + default: "0.3.0" + target-os: + description: "Current operating system" + default: "linux" + target-arch: + description: "Current architecture" + default: "amd64" +runs: + using: "composite" + steps: + - name: "download equivalence test binary" + shell: bash + run: | + ./.github/scripts/equivalence-test.sh download_equivalence_test_binary \ + ${{ inputs.target-equivalence-test-version }} \ + ./bin/equivalence-tests \ + ${{ inputs.target-os }} \ + ${{ inputs.target-arch }} + - name: "download terraform binary" + shell: bash + run: | + ./.github/scripts/equivalence-test.sh download_terraform_binary \ + ${{ inputs.target-terraform-version }} \ + ./bin/terraform \ + ${{ inputs.target-os }} \ + ${{ inputs.target-arch }} + - name: "run and update equivalence tests" + shell: bash + run: | + ./bin/equivalence-tests update \ + --tests=testing/equivalence-tests/tests \ + --goldens=testing/equivalence-tests/outputs \ + --binary=$(pwd)/bin/terraform + + changed=$(git diff --quiet -- testing/equivalence-tests/outputs || echo true) + if [[ $changed == "true" ]]; then + echo "found changes, and pushing new golden files into branch ${{ inputs.target-terraform-branch }}." + + git config user.email "52939924+teamterraform@users.noreply.github.com" + git config user.name "The Terraform Team" + + git add ./testing/equivalence-tests/outputs + git commit -m "Automated equivalence test golden file update for release ${{ inputs.target-terraform-version }}." + git push + else + echo "found no changes, so not pushing any updates." + fi diff --git a/.github/scripts/equivalence-test.sh b/.github/scripts/equivalence-test.sh new file mode 100755 index 0000000000..78b547695d --- /dev/null +++ b/.github/scripts/equivalence-test.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +set -uo pipefail + +function usage { + cat <<-'EOF' +Usage: ./equivalence-test.sh [] [] + +Description: + This script will handle various commands related to the execution of the + Terraform equivalence tests. + +Commands: + get_target_branch + get_target_branch returns the default target branch for a given Terraform + version. + + target_branch=$(./equivalence-test.sh get_target_branch v1.4.3); target_branch=v1.4 + target_branch=$(./equivalence-test.sh get_target_branch 1.4.3); target_branch=v1.4 + + download_equivalence_test_binary + download_equivalence_test_binary downloads the equivalence testing binary + for a given version and places it at the target path. + + ./equivalence-test.sh download_equivalence_test_binary 0.3.0 ./bin/terraform-equivalence-testing linux amd64 + + download_terraform_binary + download_terraform_binary downloads the terraform release binary for a given + version and places it at the target path. + + ./equivalence-test.sh download_terraform_binary 1.4.3 ./bin/terraform linux amd64 +EOF +} + +function download_equivalence_test_binary { + VERSION="${1:-}" + TARGET="${2:-}" + OS="${3:-}" + ARCH="${4:-}" + + if [[ -z "$VERSION" || -z "$TARGET" || -z "$OS" || -z "$ARCH" ]]; then + echo "missing at least one of [, , , ] arguments" + usage + exit 1 + fi + + curl \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/hashicorp/terraform-equivalence-testing/releases" > releases.json + + ASSET="terraform-equivalence-testing_v${VERSION}_${OS}_${ARCH}.zip" + ASSET_ID=$(jq -r --arg VERSION "v$VERSION" --arg ASSET "$ASSET" '.[] | select(.name == $VERSION) | .assets[] | select(.name == $ASSET) | .id' releases.json) + + mkdir -p zip + curl -L \ + -H "Accept: application/octet-stream" \ + "https://api.github.com/repos/hashicorp/terraform-equivalence-testing/releases/assets/$ASSET_ID" > "zip/$ASSET" + + mkdir -p bin + unzip -p "zip/$ASSET" terraform-equivalence-testing > "$TARGET" + chmod u+x "$TARGET" + rm -r zip + rm releases.json +} + +function download_terraform_binary { + VERSION="${1:-}" + TARGET="${2:-}" + OS="${3:-}" + ARCH="${4:-}" + + if [[ -z "$VERSION" || -z "$TARGET" || -z "$OS" || -z "$ARCH" ]]; then + echo "missing at least one of [, , , ] arguments" + usage + exit 1 + fi + + mkdir -p zip + curl "https://releases.hashicorp.com/terraform/${VERSION}/terraform_${VERSION}_${OS}_${ARCH}.zip" > "zip/terraform.zip" + + mkdir -p bin + unzip -p "zip/terraform.zip" terraform > "$TARGET" + chmod u+x "$TARGET" + rm -r zip +} + +function get_target_branch { + VERSION="${1:-}" + + if [ -z "$VERSION" ]; then + echo "missing argument" + usage + exit 1 + fi + + + # Split off the build metadata part, if any + # (we won't actually include it in our final version, and handle it only for + # completeness against semver syntax.) + IFS='+' read -ra VERSION BUILD_META <<< "$VERSION" + + # Separate out the prerelease part, if any + IFS='-' read -r BASE_VERSION PRERELEASE <<< "$VERSION" + + # Separate out major, minor and patch versions. + IFS='.' read -r MAJOR_VERSION MINOR_VERSION PATCH_VERSION <<< "$BASE_VERSION" + + if [[ "$PRERELEASE" == *"alpha"* ]]; then + TARGET_BRANCH=main + else + if [[ $MAJOR_VERSION = v* ]]; then + TARGET_BRANCH=${MAJOR_VERSION}.${MINOR_VERSION} + else + TARGET_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} + fi + fi + + echo "$TARGET_BRANCH" +} + +function main { + case "$1" in + get_target_branch) + if [ "${#@}" != 2 ]; then + echo "invalid number of arguments" + usage + exit 1 + fi + + get_target_branch "$2" + + ;; + download_equivalence_test_binary) + if [ "${#@}" != 5 ]; then + echo "invalid number of arguments" + usage + exit 1 + fi + + download_equivalence_test_binary "$2" "$3" "$4" "$5" + + ;; + download_terraform_binary) + if [ "${#@}" != 5 ]; then + echo "invalid number of arguments" + usage + exit 1 + fi + + download_terraform_binary "$2" "$3" "$4" "$5" + + ;; + *) + echo "unrecognized command $*" + usage + exit 1 + + ;; + esac +} + +main "$@" +exit $? diff --git a/.github/workflows/crt-hook-equivalence-tests.yml b/.github/workflows/crt-hook-equivalence-tests.yml new file mode 100644 index 0000000000..a4607cc937 --- /dev/null +++ b/.github/workflows/crt-hook-equivalence-tests.yml @@ -0,0 +1,45 @@ +name: crt-hook-equivalence-tests + +on: + repository_dispatch: + types: + - crt-hook-equivalence-tests::terraform::* + +permissions: + contents: write + +jobs: + parse-metadata: + name: "Parse metadata.json" + runs-on: ubuntu-latest + outputs: + version: ${{ steps.parse.outputs.version }} + target-branch: ${{ steps.parse.outputs.target-branch }} + steps: + - name: parse + id: parse + env: + METADATA_PAYLOAD: ${{ toJSON(github.event.client_payload.payload) }} + run: | + VERSION=$(echo ${METADATA_PAYLOAD} | jq -r '.version') + TARGET_BRANCH=$(./.github/scripts/equivalence-test.sh get-target-branch "$VERSION") + + echo "target-branch=$TARGET_BRANCH" >> "GITHUB_OUTPUT" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + run-equivalence-tests: + runs-on: ubuntu-latest + name: "Run equivalence tests" + needs: + - parse-metadata + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + ref: ${{ needs.parse-metadata.outputs.target-branch }} + - uses: ./.github/actions/equivalence-test + with: + target-terraform-version: ${{ needs.parse-metadata.outputs.version }} + target-terraform-branch: ${{ needs.parse-metadata.outputs.target-branch }} + target-equivalence-test-version: 0.3.0 + target-os: linux + target-arch: amd64 diff --git a/.github/workflows/equivalence-test.yml b/.github/workflows/equivalence-test.yml deleted file mode 100644 index b2ca0a73a1..0000000000 --- a/.github/workflows/equivalence-test.yml +++ /dev/null @@ -1,255 +0,0 @@ -name: Terraform Equivalence Tests - -on: - workflow_dispatch: - inputs: - build-run-id: - type: string - required: true - description: "build-run-id: The `Build Terraform CLI Packages` run to retrieve built Terraform binaries from." - workflow_run: - workflows: [Build Terraform CLI Packages] - types: - - completed - -permissions: - actions: read - contents: write - -env: - terraform-equivalence-testing-version: v0.3.0 - target-os: linux - target-arch: amd64 - -jobs: - get-metadata: - name: "Determine Terraform version and other metadata" - runs-on: ubuntu-latest - - outputs: - check-run-equivalence-tests: ${{ steps.check-run-equivalence-tests.outputs.check-run-equivalence-tests }} - terraform-version: ${{ steps.terraform-metadata.outputs.terraform-version }} - terraform-artifact-id: ${{ steps.terraform-metadata.outputs.terraform-artifact-id }} - target-branch: ${{ steps.target-branch.outputs.target-branch }} - - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.workflow_run.head_branch }} - - - id: build-run-id - name: "Get build job ID" - run: | - if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - # Then we map all our outputs from the user supplied inputs. - BUILD_RUN_ID=${{ inputs.build-run-id }} - else - # Quick sanity check, if the workflow_run that triggered this action - # failed then we shouldn't carry on. - if [[ "${{ github.event.workflow_run.conclusion }}" != "success" ]]; then - echo "check-run-equivalence-tests=false" >> $GITHUB_OUTPUT - exit 0 - fi - - BUILD_RUN_ID=${{ github.event.workflow_run.id }} - fi - - echo "build-run-id=${BUILD_RUN_ID}" >> $GITHUB_OUTPUT - - - id: terraform-metadata - name: "Get Terraform version and artifact metadata" - run: | - curl \ - -H "Accept: application/vnd.github+json" \ - "https://api.github.com/repos/hashicorp/terraform/actions/runs/${{ steps.build-run-id.outputs.build-run-id }}/artifacts" > artifacts.json - - METADATA_ARTIFACT_ID=$(jq -r '.artifacts | .[] | select(.name == "metadata.json") | .id' artifacts.json) - - curl -L \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ - "https://api.github.com/repos/hashicorp/terraform/actions/artifacts/$METADATA_ARTIFACT_ID/zip" > "metadata.json.zip" - - unzip metadata.json.zip - - BRANCH=$(jq -r '.branch' metadata.json) - VERSION=$(jq -r '.version' metadata.json) - - TERRAFORM_ARTIFACT_NAME="terraform_${VERSION}_${{ env.target-os }}_${{ env.target-arch }}.zip" - TERRAFORM_ARTIFACT_ID=$(jq -r --arg ARTIFACT "$TERRAFORM_ARTIFACT_NAME" '.artifacts | .[] | select(.name == $ARTIFACT) | .id' artifacts.json) - - TERRAFORM_VERSION="v$VERSION" - - echo "branch=${BRANCH}" >> $GITHUB_OUTPUT - echo "version=${VERSION}" >> $GITHUB_OUTPUT - echo "terraform-artifact-id=${TERRAFORM_ARTIFACT_ID}" >> $GITHUB_OUTPUT - echo "terraform-version=${TERRAFORM_VERSION}" >> $GITHUB_OUTPUT - - - id: check-run-equivalence-tests - name: "Check if we even should run the equivalence tests" - run: | - VERSION=${{ steps.terraform-metadata.outputs.version }} - BRANCH=${{ steps.terraform-metadata.outputs.branch }} - if [[ "$BRANCH" == "refs/tags/v$VERSION" ]]; then - # We only execute the equivalence tests if the build run that - # triggered this was triggered by a tag. - RUN_EQUIVALENCE_TESTS=true - else - RUN_EQUIVALENCE_TESTS=false - fi - echo "check-run-equivalence-tests=${RUN_EQUIVALENCE_TESTS}" >> $GITHUB_OUTPUT - - - id: target-branch - name: "Get the target branch we should write the equivalence tests to" - run: | - # One last thing to do is to work out which branch we want to operate - # against. This could be `main` for an alpha build, or a release - # branch (eg. v1.1, v1.2, v1.3) for any other kind of build. - VERSION="${{ steps.terraform-metadata.outputs.version }}" - - # Split off the build metadata part, if any - # (we won't actually include it in our final version, and handle it only for - # completeness against semver syntax.) - IFS='+' read -ra VERSION BUILD_META <<< "$VERSION" - - # Separate out the prerelease part, if any - IFS='-' read -r BASE_VERSION PRERELEASE <<< "$VERSION" - - # Separate out major, minor and patch versions. - IFS='.' read -r MAJOR_VERSION MINOR_VERSION PATCH_VERSION <<< "$BASE_VERSION" - - if [[ "$PRERELEASE" == *"alpha"* ]]; then - TARGET_BRANCH=main - else - TARGET_BRANCH=v${MAJOR_VERSION}.${MINOR_VERSION} - fi - - echo "target-branch=${TARGET_BRANCH}" >> $GITHUB_OUTPUT - - prepare-equivalence-tests: - name: "Prepare equivalence testing binary" - if: ${{ needs.get-metadata.outputs.check-run-equivalence-tests == 'true' }} - runs-on: ubuntu-latest - needs: - - get-metadata - - steps: - - name: "Download terraform-equivalence-testing binary" - run: | - curl \ - -H "Accept: application/vnd.github+json" \ - "https://api.github.com/repos/hashicorp/terraform-equivalence-testing/releases" > releases.json - - VERSION="${{ env.terraform-equivalence-testing-version }}" - ASSET="terraform-equivalence-testing_${VERSION}_${{ env.target-os }}_${{ env.target-arch }}.zip" - ASSET_ID=$(jq -r --arg VERSION "$VERSION" --arg ASSET "$ASSET" '.[] | select(.name == $VERSION) | .assets[] | select(.name == $ASSET) | .id' releases.json) - - curl -L \ - -H "Accept: application/octet-stream" \ - "https://api.github.com/repos/hashicorp/terraform-equivalence-testing/releases/assets/$ASSET_ID" > "$ASSET" - - - name: "Unzip terraform-equivalence-testing binary" - run: | - ASSET="terraform-equivalence-testing_${{ env.terraform-equivalence-testing-version }}_${{ env.target-os }}_${{ env.target-arch }}.zip" - unzip -p "$ASSET" terraform-equivalence-testing > terraform-equivalence-testing - - - name: "Upload terraform-equivalence-testing binary" - uses: actions/upload-artifact@v2 - with: - name: terraform-equivalence-testing - path: terraform-equivalence-testing - - prepare-terraform: - name: "Prepare Terraform binary" - if: ${{ needs.get-metadata.outputs.check-run-equivalence-tests == 'true' }} - runs-on: ubuntu-latest - needs: - - get-metadata - - env: - terraform-version: ${{ needs.get-metadata.outputs.terraform-version }} - terraform-artifact-id: ${{ needs.get-metadata.outputs.terraform-artifact-id }} - - steps: - - name: "Download terraform binary" - run: | - VERSION="${{ env.terraform-version }}" # The Terraform artifacts don't have the `v` prefix. - ARTIFACT="terraform_${VERSION#v}_${{ env.target-os }}_${{ env.target-arch }}.zip" - - curl -L \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ - "https://api.github.com/repos/hashicorp/terraform/actions/artifacts/${{ env.terraform-artifact-id }}/zip" > "$ARTIFACT.zip" - - - name: "Unzip terraform binary" - run: | - VERSION="${{ env.terraform-version }}" # The Terraform artifacts don't have the `v` prefix. - ARTIFACT="terraform_${VERSION#v}_${{ env.target-os }}_${{ env.target-arch }}.zip" - - # We actually have nested zip files, as the Github API downloads the - # artifacts in a zip file and the Terraform build action embeds the - # terraform binary in a zip file also. - - unzip $ARTIFACT.zip - unzip $ARTIFACT - - - name: "Upload terraform binary" - uses: actions/upload-artifact@v2 - with: - name: terraform - path: terraform - - run-equivalence-tests: - name: "Run equivalence tests" - if: ${{ needs.get-metadata.outputs.check-run-equivalence-tests == 'true' }} - runs-on: ubuntu-latest - needs: - - get-metadata - - prepare-terraform - - prepare-equivalence-tests - - env: - target-branch: ${{ needs.get-metadata.outputs.target-branch }} - terraform-version: ${{ needs.get-metadata.outputs.terraform-version }} - - steps: - - name: "Checkout repository at target branch ${{ env.target-branch }}" - uses: actions/checkout@v3 - with: - ref: ${{ env.target-branch }} - - - name: "Download Terraform binary" - uses: actions/download-artifact@v2 - with: - name: terraform - path: . - - - name: "Download terraform-equivalence-testing binary" - uses: actions/download-artifact@v2 - with: - name: terraform-equivalence-testing - path: . - - - name: "Run and update equivalence tests" - run: | - chmod u+x ./terraform-equivalence-testing - chmod u+x ./terraform - - ./terraform-equivalence-testing update \ - --tests=testing/equivalence-tests/tests \ - --goldens=testing/equivalence-tests/outputs \ - --binary=$(pwd)/terraform - - changed=$(git diff --quiet -- testing/equivalence-tests/outputs || echo true) - if [[ $changed == "true" ]]; then - echo "found changes, and pushing new golden files into branch ${{ env.target-branch }}." - - git config user.email "52939924+teamterraform@users.noreply.github.com" - git config user.name "The Terraform Team" - - git add ./testing/equivalence-tests/outputs - git commit -m"Automated equivalence test golden file update for release ${{ env.terraform-version }}." - git push - else - echo "found no changes, so not pushing any updates." - fi diff --git a/.github/workflows/manual-equivalence-tests.yml b/.github/workflows/manual-equivalence-tests.yml new file mode 100644 index 0000000000..fb6b0d5b3b --- /dev/null +++ b/.github/workflows/manual-equivalence-tests.yml @@ -0,0 +1,37 @@ +name: manual-equivalence-tests + +on: + workflow_dispatch: + inputs: + target-branch: + type: string + description: "Which branch should be updated?" + required: true + terraform-version: + type: string + description: "Terraform version to run against (no v prefix, eg. 1.4.4)." + required: true + equivalence-test-version: + type: string + description: 'Equivalence testing framework version to use (no v prefix, eg. 0.3.0).' + default: "0.3.0" + required: true + +permissions: + contents: write # We push updates to the equivalence tests back into the repository. + +jobs: + run-equivalence-tests: + name: "Run equivalence tests" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + ref: ${{ inputs.target-branch }} + - uses: ./.github/actions/equivalence-test + with: + target-terraform-version: ${{ inputs.terraform-version }} + target-terraform-branch: ${{ inputs.target-branch }} + target-equivalence-test-version: ${{ inputs.equivalence-test-version }} + target-os: linux + target-arch: amd64