Refactor Staging deployment workflow to support open source PRs (#20459)

* Add a Staging build workflow

* Remove all commented out code from build workflow

It will be handled in https://github.com/github/docs-engineering/issues/726

* Use pinned version of upload-artifact action

* Tweaks to build

* Minor deployment script refactoring

* Update the Staging deployment workflow

* Missed refactoring tweak

* Add relevant comments

* Update Heroku app naming convention for Actions deploy to include 'gha-' prefix

* Update Heroku app ConfigVars and SourceBlob for optional prebuilt app

* Remove obsolete 'dist/' dir from PR build artifact

See https://github.com/github/docs-internal/pull/20405

* Ensure a new enough version of npm is used

* Switch to creating a tarball for upload

* Remove obsolete 'layouts' dir from file list

* Ditch the verbosity for 'tar'... too many files

* Add tarball support to deploy

* Add esm workaround to deploy script

See https://github.com/actions/github-script/issues/168

* Temporarily ignore staging deploy workflow from workflow linter

* Update deployment to use a Heroku Build Source instead of a GitHub Actions Artifact

* Update undeploy workflow to use ESM workaround

See https://github.com/actions/github-script/issues/168

* Add 'esm' package to optionalDependencies to better support workaround

See https://github.com/actions/github-script/issues/168

* Add Slack notifications for workflow failures

* Wrap AppSetup polling in try-catch

* Improve dyno monitoring

* Rename 'script/deploy' to have a .js extension #esm

* Update script references to include the extension

* Use non-deprecated Sources API for Heroku

* Use normal quotes

* Stub in a step to mark deployment inactive after timing out

* Apply suggestions from code review

Co-authored-by: Rachael Sewell <rachmari@github.com>

Co-authored-by: Rachael Sewell <rachmari@github.com>

.github/workflows/staging-build-pr.yml

@@ -0,0 +1,88 @@
+name: Staging - Build PR
+
+# **What it does**: Builds PRs before deploying them.
+# **Why we have it**: Because it's not safe to share our deploy secrets with forked repos: https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+# **Who does it impact**: All contributors.
+
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - unlocked
+
+jobs:
+  build:
+    if: ${{ github.repository == 'github/docs-internal' || github.repository == 'github/docs' }}
+    name: Build
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    concurrency:
+      group: staging_${{ github.head_ref }}
+      cancel-in-progress: true
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+
+      - name: Setup node
+        uses: actions/setup-node@38d90ce44d5275ad62cc48384b3d8a58c500bb5f
+        with:
+          node-version: 16.x
+          cache: npm
+
+      # Required for `npm pkg ...` command support
+      - name: Update to npm@^7.20.0
+        run: npm install --global npm@^7.20.0
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build
+        run: npm run build
+
+      - name: Remove development-only dependencies
+        run: npm prune --production
+
+      - name: Remove all npm scripts
+        run: npm pkg delete scripts
+
+      - name: Set npm script for Heroku build to noop
+        run: npm set-script heroku-postbuild "echo 'Application was pre-built!'"
+
+      - name: Create an archive
+        run: |
+          tar -cf app.tar \
+            node_modules/ \
+            .next/ \
+            assets/ \
+            content/ \
+            data/ \
+            includes/ \
+            lib/ \
+            middleware/ \
+            translations/ \
+            server.mjs \
+            package*.json \
+            feature-flags.json \
+            next.config.js \
+            app.json \
+            Procfile
+
+      # Upload only the files needed to run this application.
+      # We are not willing to trust the rest (e.g. script/) for the remainder
+      # of the deployment process.
+      - name: Upload build artifact
+        uses: actions/upload-artifact@27121b0bdffd731efa15d66772be8dc71245d074
+        with:
+          name: pr_build
+          path: app.tar
+
+      - name: Send Slack notification if workflow fails
+        uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd
+        if: ${{ failure() }}
+        with:
+          channel: ${{ secrets.DOCS_STAGING_DEPLOYMENT_FAILURES_SLACK_CHANNEL_ID }}
+          bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
+          color: failure
+          text: Staging build failed for PR ${{ github.event.pull_request.html_url }} at commit ${{ github.sha }}. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/staging-deploy-pr.yml

@@ -5,96 +5,148 @@ name: Staging - Deploy PR
 # **Who does it impact**: All contributors.
 
 on:
-  pull_request:
+  workflow_run:
+    workflows:
+      - 'Staging - Build PR'
     types:
-      - opened
-      - reopened
-      - synchronize
-      - unlocked
-  workflow_dispatch:
-    inputs:
-      pullRequestUrl:
-        description: 'Pull Request URL'
-        required: true
-        default: 'https://github.com/github/docs/pull/1234'
-      forceRebuild:
-        description: 'Force the Heroku App to be rebuilt from scratch? (true/false)'
-        required: false
-        default: 'false'
+      - completed
+
+env:
+  EARLY_ACCESS_SCRIPT_PATH: script/early-access/clone-for-build.js
+  # In this specific workflow relationship, the `github.event.workflow_run.pull_requests`
+  # array will always contain only 1 item! Specifically, it will contain the PR associated
+  # with the `github.event.workflow_run.head_branch` that triggered the preceding
+  # `pull_request` event that triggered the "Staging - Build PR" workflow.
+  PR_URL: ${{ github.event.workflow_run.repository.html_url }}/pull/${{ github.event.workflow_run.pull_requests[0].number }}
 
 jobs:
-  validate-inputs:
-    if: ${{ github.repository == 'github/docs-internal' || github.repository == 'github/docs' }}
-    name: Validate inputs
+  prepare:
+    if: |
+      ${{
+        github.event.workflow_run.conclusion == 'success' &&
+        (github.repository == 'github/docs-internal' || github.repository == 'github/docs')
+      }}
     runs-on: ubuntu-latest
-    timeout-minutes: 2
+    timeout-minutes: 5
+    concurrency:
+      group: staging_${{ github.event.workflow_run.head_branch }}
+      cancel-in-progress: true
     outputs:
-      headRef: ${{ steps.validate.outputs.headRef }}
+      source_blob_url: ${{ steps.build-source.outputs.download_url }}
     steps:
-      - if: ${{ github.event_name == 'workflow_dispatch' }}
-        name: Check out repo
-        uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+      - name: Download build artifact
+        uses: dawidd6/action-download-artifact@b9571484721e8187f1fd08147b497129f8972c74
         with:
-          # Enables cloning the Early Access repo later with the relevant PAT
-          persist-credentials: 'false'
+          workflow: ${{ github.event.workflow_run.workflow_id }}
+          run_id: ${{ github.event.workflow_run.id }}
+          name: pr_build
+          path: ./
 
-      - if: ${{ github.event_name == 'workflow_dispatch' }}
-        name: Setup node
+      - name: Show contents
+        run: ls -l
+
+      - name: Extract the archive
+        run: |
+          tar -xf app.tar -C ./
+          rm app.tar
+
+      - name: Show contents again
+        run: ls -l
+
+      - if: ${{ github.repository == 'github/docs-internal' }}
+        name: Setup node to clone early access
         uses: actions/setup-node@38d90ce44d5275ad62cc48384b3d8a58c500bb5f
         with:
           node-version: 16.x
           cache: npm
 
-      - if: ${{ github.event_name == 'workflow_dispatch' }}
-        name: Install dependencies
-        run: npm ci
+      - if: ${{ github.repository == 'github/docs-internal' }}
+        name: Download the script to clone early access
+        uses: Bhacaz/checkout-files@c8f01756bfd894ba746d5bf48205e19000b0742b
+        with:
+          files: ${{ env.EARLY_ACCESS_SCRIPT_PATH }}
 
-      - if: ${{ github.event_name == 'workflow_dispatch' }}
-        name: Validate and get head.ref
-        id: validate
+      # Add any dependencies that are needed for this workflow below
+      - if: ${{ github.repository == 'github/docs-internal' }}
+        name: Install temporary development-only dependencies
+        run: npm install --no-save rimraf
+
+      - if: ${{ github.repository == 'github/docs-internal' }}
+        name: Clone early access
+        run: node ${{ env.EARLY_ACCESS_SCRIPT_PATH }}
+        env:
+          DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }}
+          GIT_BRANCH: ${{ github.event.workflow_run.head_branch }}
+
+      # Remove any dependencies installed for this workflow below
+      - if: ${{ github.repository == 'github/docs-internal' }}
+        name: Remove development-only dependencies
+        run: npm prune --production
+
+      - if: ${{ github.repository == 'github/docs-internal' }}
+        name: Delete the script to clone early access
+        run: rm ${{ env.EARLY_ACCESS_SCRIPT_PATH }}
+
+      - name: Create a gzipped archive
+        run: tar -cfz app.tar.gz ./
+
+      - name: Install Heroku client development-only dependency
+        run: npm install --no-save heroku-client
+
+      - name: Create a Heroku build source
+        id: build-source
         uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d
         env:
-          PR_URL: ${{ github.event.inputs.pullRequestUrl }}
-          FORCE_REBUILD: ${{ github.event.inputs.forceRebuild }}
+          HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }}
         with:
           script: |
-            const parsePrUrl = require('./script/deployment/parse-pr-url')
+            const { owner, repo } = context.repo
 
-            // Manually resolve workflow_dispatch inputs
-            const { PR_URL, FORCE_REBUILD } = process.env
-
-            if (!['true', 'false'].includes(FORCE_REBUILD)) {
-              throw new Error(`'forceRebuild' input must be either 'true' or 'false' but was '${FORCE_REBUILD}'`)
+            if (owner !== 'github') {
+              throw new Error(`Repository owner must be 'github' but was: ${owner}`)
+            }
+            if (repo !== 'docs-internal' && repo !== 'docs') {
+              throw new Error(`Repository name must be either 'docs-internal' or 'docs' but was: ${repo}`)
             }
 
-            const { owner, repo, pullNumber } = parsePrUrl(PR_URL)
-            if (!owner || !repo || !pullNumber) {
-              throw new Error(`'pullRequestUrl' input must match URL format 'https://github.com/github/(docs|docs-internal)/pull/123' but was '${PR_URL}'`)
-            }
+            const Heroku = require('heroku-client')
+            const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN })
 
-            const { data: pullRequest } = await github.pulls.get({
-              owner,
-              repo,
-              pull_number: pullNumber
-            })
+            const { source_blob: sourceBlob } = await heroku.post('/sources')
+            const { put_url: uploadUrl, get_url: downloadUrl } = sourceBlob
 
-            core.setOutput('headRef', pullRequest.head.ref)
+            core.setOutput('upload_url', uploadUrl)
+            core.setOutput('download_url', downloadUrl)
+
+      # See: https://devcenter.heroku.com/articles/build-and-release-using-the-api#sources-endpoint
+      - name: Upload to the Heroku build source
+        run: |
+          curl '${{ steps.build-source.outputs.upload_url }}' \
+            --fail \
+            -X PUT \
+            -H 'Content-Type:' \
+            -H 'Authorization: Bearer ${{ secrets.HEROKU_API_TOKEN }}' \
+            --data-binary @app.tar.gz
+
+      - name: Send Slack notification if workflow fails
+        uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd
+        if: ${{ failure() }}
+        with:
+          channel: ${{ secrets.DOCS_STAGING_DEPLOYMENT_FAILURES_SLACK_CHANNEL_ID }}
+          bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
+          color: failure
+          text: Staging preparation failed for PR ${{ env.PR_URL }} at commit ${{ github.sha }}. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
 
   deploy:
-    if: ${{ github.repository == 'github/docs-internal' || github.repository == 'github/docs' }}
-    needs: validate-inputs
-    name: Deploy
+    needs: prepare
     runs-on: ubuntu-latest
     timeout-minutes: 10
     concurrency:
-      group: staging_${{ needs.validate-inputs.outputs.headRef || github.head_ref }}
+      group: staging_${{ github.event.workflow_run.head_branch }}
       cancel-in-progress: true
     steps:
-      - name: Check out repo
+      - name: Check out repo's default branch
         uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
-        with:
-          # Enables cloning the Early Access repo later with the relevant PAT
-          persist-credentials: 'false'
 
       - name: Setup node
         uses: actions/setup-node@38d90ce44d5275ad62cc48384b3d8a58c500bb5f
@@ -105,16 +157,19 @@ jobs:
       - name: Install dependencies
         run: npm ci
 
+      - name: Install one-off development-only dependencies
+        run: npm install --no-save esm
+
       - name: Deploy
+        id: deploy
         uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }}
-          DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }}
           HYDRO_ENDPOINT: ${{ secrets.HYDRO_ENDPOINT }}
           HYDRO_SECRET: ${{ secrets.HYDRO_SECRET }}
-          PR_URL: ${{ github.event.inputs.pullRequestUrl }}
-          FORCE_REBUILD: ${{ github.event.inputs.forceRebuild }}
+          PR_URL: ${{ env.PR_URL }}
+          SOURCE_BLOB_URL: ${{ needs.prepare.outputs.source_blob_url }}
         with:
           script: |
             const { GITHUB_TOKEN, HEROKU_API_TOKEN } = process.env
@@ -129,9 +184,13 @@ jobs:
               throw new Error('You must supply a HEROKU_API_TOKEN environment variable!')
             }
 
-            const parsePrUrl = require('./script/deployment/parse-pr-url')
-            const getOctokit = require('./script/helpers/github')
-            const deployToStaging = require('./script/deployment/deploy-to-staging')
+            // Workaround to allow us to load ESM files with `require(...)`
+            const esm = require('esm')
+            require = esm({})
+
+            const { default: parsePrUrl } = require('./script/deployment/parse-pr-url')
+            const { default: getOctokit } = require('./script/helpers/github')
+            const { default: deployToStaging } = require('./script/deployment/deploy-to-staging')
 
             // This helper uses the `GITHUB_TOKEN` implicitly!
             // We're using our usual version of Octokit vs. the provided `github`
@@ -139,33 +198,24 @@ jobs:
             const octokit = getOctokit()
 
             try {
-              let pullRequest = null
-              let forceRebuild = false
-
-              // Manually resolve workflow_dispatch inputs
-              if (context.eventName === 'workflow_dispatch') {
-                const { PR_URL, FORCE_REBUILD } = process.env
-
-                forceRebuild = FORCE_REBUILD === 'true'
-
-                const { owner, repo, pullNumber } = parsePrUrl(PR_URL)
-                if (!owner || !repo || !pullNumber) {
-                  throw new Error(`'pullRequestUrl' input must match URL format 'https://github.com/github/(docs|docs-internal)/pull/123' but was '${PR_URL}'`)
-                }
-
-                const { data: pr } = await octokit.pulls.get({
-                  owner,
-                  repo,
-                  pull_number: pullNumber
-                })
-                pullRequest = pr
+              const { PR_URL, SOURCE_BLOB_URL } = process.env
+              const { owner, repo, pullNumber } = parsePrUrl(PR_URL)
+              if (!owner || !repo || !pullNumber) {
+                throw new Error(`'pullRequestUrl' input must match URL format 'https://github.com/github/(docs|docs-internal)/pull/123' but was '${PR_URL}'`)
               }
 
+              const { data: pullRequest } = await octokit.pulls.get({
+                owner,
+                repo,
+                pull_number: pullNumber
+              })
+
               await deployToStaging({
-                herokuToken: HEROKU_API_TOKEN,
                 octokit,
-                pullRequest: pullRequest || context.payload.pull_request,
-                forceRebuild,
+                pullRequest,
+                forceRebuild: false,
+                // These parameters will ONLY be set by Actions
+                sourceBlobUrl: SOURCE_BLOB_URL,
                 runId: context.runId
               })
             } catch (error) {
@@ -173,3 +223,21 @@ jobs:
               console.error(error)
               throw error
             }
+
+      - name: Mark the deployment as inactive if timed out
+        if: ${{ steps.deploy.outcome == 'cancelled' }}
+        uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d
+        with:
+          script: |
+            // TODO: Find the relevant deployment
+            // TODO: Create a new deployment status for it as "inactive"
+            return 'TODO'
+
+      - name: Send Slack notification if workflow fails
+        uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd
+        if: ${{ failure() }}
+        with:
+          channel: ${{ secrets.DOCS_STAGING_DEPLOYMENT_FAILURES_SLACK_CHANNEL_ID }}
+          bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
+          color: failure
+          text: Staging deployment failed for PR ${{ env.PR_URL }} at commit ${{ github.sha }}. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}

.github/workflows/staging-undeploy-pr.yml

@@ -5,7 +5,7 @@ name: Staging - Undeploy PR
 # **Who does it impact**: All contributors.
 
 on:
-  pull_request:
+  pull_request_target:
     types:
       - closed
       - locked
@@ -20,10 +20,10 @@ jobs:
       group: staging_${{ github.head_ref }}
       cancel-in-progress: true
     steps:
-      - name: Check out repo
+      - name: Check out repo's default branch
         uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
         with:
-          # Enables cloning the Early Access repo later with the relevant PAT
+          # For enhanced security: https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
           persist-credentials: 'false'
 
       - name: Setup node
@@ -35,6 +35,9 @@ jobs:
       - name: Install dependencies
         run: npm ci
 
+      - name: Install one-off development-only dependencies
+        run: npm install --no-save esm
+
       - name: Undeploy
         uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d
         env:
@@ -54,8 +57,12 @@ jobs:
               throw new Error('You must supply a HEROKU_API_TOKEN environment variable!')
             }
 
-            const getOctokit = require('./script/helpers/github')
-            const undeployFromStaging = require('./script/deployment/undeploy-from-staging')
+            // Workaround to allow us to load ESM files with `require(...)`
+            const esm = require('esm')
+            require = esm({})
+
+            const { default: getOctokit } = require('./script/helpers/github')
+            const { default: undeployFromStaging } = require('./script/deployment/undeploy-from-staging')
 
             // This helper uses the `GITHUB_TOKEN` implicitly!
             // We're using our usual version of Octokit vs. the provided `github`
@@ -64,7 +71,6 @@ jobs:
 
             try {
               await undeployFromStaging({
-                herokuToken: HEROKU_API_TOKEN,
                 octokit,
                 pullRequest: context.payload.pull_request,
                 runId: context.runId

.github/workflows/workflow-lint.yml

@@ -28,4 +28,4 @@ jobs:
       - name: Run linter
         uses: cschleiden/actions-linter@caffd707beda4fc6083926a3dff48444bc7c24aa
         with:
-          workflows: '[".github/workflows/*.yml", ".github/workflows/*.yaml", "!.github/workflows/remove-from-fr-board.yaml"]'
+          workflows: '[".github/workflows/*.yml", ".github/workflows/*.yaml", "!.github/workflows/remove-from-fr-board.yaml", "!.github/workflows/staging-deploy-pr.yml"]'