diff --git a/.github/actions-scripts/staging-commit-status-success.js b/.github/actions-scripts/staging-commit-status-success.js new file mode 100755 index 0000000000..a0e3cdd9a9 --- /dev/null +++ b/.github/actions-scripts/staging-commit-status-success.js @@ -0,0 +1,36 @@ +#!/usr/bin/env node + +import getOctokit from '../../script/helpers/github.js' + +const { GITHUB_TOKEN } = process.env + +// Exit if GitHub Actions PAT is not found +if (!GITHUB_TOKEN) { + throw new Error('You must supply a GITHUB_TOKEN environment variable!') +} + +// This helper uses the `GITHUB_TOKEN` implicitly! +// We're using our usual version of Octokit vs. the provided `github` +// instance to avoid versioning discrepancies. +const octokit = getOctokit() + +const { CONTEXT_NAME, ACTIONS_RUN_LOG, HEAD_SHA } = process.env +if (!CONTEXT_NAME) { + throw new Error('$CONTEXT_NAME not set') +} +if (!ACTIONS_RUN_LOG) { + throw new Error('$ACTIONS_RUN_LOG not set') +} +if (!HEAD_SHA) { + throw new Error('$HEAD_SHA not set') +} + +await octokit.repos.createCommitStatus({ + owner, + repo, + sha: HEAD_SHA, + context: CONTEXT_NAME, + state: 'success', + description: 'Successfully deployed! See logs.', + target_url: ACTIONS_RUN_LOG, +}) diff --git a/.github/actions-scripts/staging-deploy.js b/.github/actions-scripts/staging-deploy.js index 4ba5e4a8e7..26dadc4157 100755 --- a/.github/actions-scripts/staging-deploy.js +++ b/.github/actions-scripts/staging-deploy.js @@ -21,7 +21,7 @@ if (!HEROKU_API_TOKEN) { // instance to avoid versioning discrepancies. const octokit = getOctokit() -const { RUN_ID, PR_URL, SOURCE_BLOB_URL, CONTEXT_NAME, ACTIONS_RUN_LOG, HEAD_SHA } = process.env +const { RUN_ID, PR_URL, SOURCE_BLOB_URL } = process.env if (!RUN_ID) { throw new Error('$RUN_ID not set') } @@ -31,15 +31,6 @@ if (!PR_URL) { if (!SOURCE_BLOB_URL) { throw new Error('$SOURCE_BLOB_URL not set') } -if (!CONTEXT_NAME) { - throw new Error('$CONTEXT_NAME not set') -} -if (!ACTIONS_RUN_LOG) { - throw new Error('$ACTIONS_RUN_LOG not set') -} -if (!HEAD_SHA) { - throw new Error('$HEAD_SHA not set') -} const { owner, repo, pullNumber } = parsePrUrl(PR_URL) if (!owner || !repo || !pullNumber) { @@ -62,13 +53,3 @@ await deployToStaging({ sourceBlobUrl: SOURCE_BLOB_URL, runId: RUN_ID, }) - -await octokit.repos.createCommitStatus({ - owner, - repo, - sha: HEAD_SHA, - context: CONTEXT_NAME, - state: 'success', - description: 'Successfully deployed! See logs.', - target_url: ACTIONS_RUN_LOG, -}) diff --git a/.github/workflows/prod-build-deploy.yml b/.github/workflows/prod-build-deploy.yml index 349f32ae05..98101a18a8 100644 --- a/.github/workflows/prod-build-deploy.yml +++ b/.github/workflows/prod-build-deploy.yml @@ -52,6 +52,12 @@ jobs: DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }} GIT_BRANCH: main + - name: Cache nextjs build + uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353 + with: + path: .next/cache + key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }} + - name: Build run: npm run build diff --git a/.github/workflows/staging-build-and-deploy-pr.yml b/.github/workflows/staging-build-and-deploy-pr.yml new file mode 100644 index 0000000000..fee6a9ffaa --- /dev/null +++ b/.github/workflows/staging-build-and-deploy-pr.yml @@ -0,0 +1,227 @@ +name: Staging - Build and Deploy PR (fast and private-only) + +# **What it does**: Builds and deploys PRs to staging but ONLY for docs-internal +# **Why we have it**: Most PRs are made on the private repo. Let's make those extra fast if we can worry less about security. +# **Who does it impact**: All staff. + +# This whole workflow is only guaranteed to be secure in the *private +# repo* and because we repo-sync these files over the to the public one, +# IT'S CRUCIALLY IMPORTANT THAT THIS WORKFLOW IS ONLY ENABLED IN docs-internal! + +on: + # Ideally, we'd like to use 'pull_request' because we can more easily + # test changes to this workflow without relying on merges to 'main'. + # But this is guaranteed to be safer and won't have the problem of + # necessary secrets not being available. + # Perhaps some day when we're confident this workflow will always + # work in a regular PR, we can switch to that. + pull_request_target: + +permissions: + actions: read + contents: read + deployments: write + pull-requests: read + statuses: write + +# This allows one Build workflow run to interrupt another +# These are different from the concurrency in that here it checks if the +# whole workflow runs again. The "inner concurrency" is used for +# undeployments to cleaning up resources. +concurrency: + group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label }}' + cancel-in-progress: true + +jobs: + build-and-deploy: + # Important. This whole file is only supposed to run in the PRIVATE repo. + if: ${{ github.repository == 'github/docs-internal' }} + + # The assumption here is that self-hosted is faster (e.g CPU power) + # that the regular ones. And it matters in this workflow because + # we do heavy CPU stuff with `npm run build` and `tar` + # runs-on: ubuntu-latest + runs-on: self-hosted + + timeout-minutes: 5 + # This interrupts Build, Deploy, and pre-write Undeploy workflow runs in + # progress for this PR branch. + concurrency: + group: 'PR Staging @ ${{ github.event.pull_request.head.label }}' + cancel-in-progress: true + steps: + - name: Check out repo + uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 + with: + ref: ${{ github.event.pull_request.head.sha }} + lfs: 'true' + # To prevent issues with cloning early access content later + persist-credentials: 'false' + + - name: Check out LFS objects + run: git lfs checkout + + - name: Setup node + uses: actions/setup-node@04c56d2f954f1e4c69436aa54cfef261a018f458 + with: + node-version: 16.13.x + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Cache nextjs build + uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353 + with: + path: .next/cache + key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }} + + - name: Build + run: npm run build + + - name: Clone early access + run: node script/early-access/clone-for-build.js + env: + DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }} + GIT_BRANCH: ${{ github.event.pull_request.head.sha }} + + - name: Check that the PR isn't blocking deploys + # We can't use ${{...}} on this if statement because of this bug + # https://github.com/cschleiden/actions-linter/issues/114 + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'automated-block-deploy') + run: | + echo "The PR appears to have the label 'automated-block-deploy'" + echo "Will not proceed to deploy the PR." + exit 2 + + - name: Create a Heroku build source + id: build-source + uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d + env: + HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }} + with: + script: | + const { owner, repo } = context.repo + + if (owner !== 'github') { + throw new Error(`Repository owner must be 'github' but was: ${owner}`) + } + if (repo !== 'docs-internal') { + throw new Error(`Repository name must be 'docs-internal' but was: ${repo}`) + } + + const Heroku = require('heroku-client') + const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN }) + + try { + const { source_blob: sourceBlob } = await heroku.post('/sources') + const { put_url: uploadUrl, get_url: downloadUrl } = sourceBlob + + core.setOutput('upload_url', uploadUrl) + core.setOutput('download_url', downloadUrl) + } catch (error) { + if (error.statusCode === 503) { + console.error('💀 Heroku may be down! Please check its Status page: https://status.heroku.com/') + } + throw error + } + + - name: Remove development-only dependencies + run: npm prune --production + + - name: Remove all npm scripts + run: npm pkg delete scripts + + - name: Set npm script for Heroku build to noop + run: npm set-script heroku-postbuild "echo 'Application was pre-built!'" + + - name: Delete heavy things we won't need deployed + run: | + + # The dereferenced file is not used in runtime once the + # decorated file has been created from it. + rm -rf lib/rest/static/dereferenced + + # Translations are never tested in Staging builds + # but let's keep the empty directory. + rm -rf translations + mkdir translations + + # Delete all the big search indexes that are NOT English (`*-en-*`) + pushd lib/search/indexes + ls | grep -v '\-en\-' | xargs rm + popd + + # Note! Some day it would be nice to be able to delete + # all the heavy assets because they bloat the tarball. + # But it's not obvious how to test it then. For now, we'll have + # to accept that every staging build has a copy of the images. + + # The assumption here is that a staging build will not + # need these legacy redirects. Only the redirects from + # front-matter will be at play. + # These static redirects json files are notoriously large + # and they make the tarball unnecessarily large. + echo '[]' > lib/redirects/static/archived-frontmatter-fallbacks.json + echo '{}' > lib/redirects/static/developer.json + echo '{}' > lib/redirects/static/archived-redirects-from-213-to-217.json + + # This will turn every `lib/**/static/*.json` into + # an equivalent `lib/**/static/*.json.br` file. + # Once the server starts, it'll know to fall back to reading + # the `.br` equivalent if the `.json` file isn't present. + node .github/actions-scripts/compress-large-files.js + + - name: Make the tarball for Heroku + run: | + # We can't delete the .next/cache directory from the workflow + # because it's needed for caching, but we can at least exclude it + # from the tarball. Then it can be cached but not weigh down the + # tarball we intend to deploy. + tar -zc --exclude=.next/cache --file=app.tar.gz \ + node_modules/ \ + .next/ \ + assets/ \ + content/ \ + data/ \ + includes/ \ + lib/ \ + middleware/ \ + translations/ \ + server.mjs \ + package*.json \ + .npmrc \ + feature-flags.json \ + next.config.js \ + app.json \ + Procfile + + du -sh app.tar.gz + + # See: https://devcenter.heroku.com/articles/build-and-release-using-the-api#sources-endpoint + - name: Upload to the Heroku build source + env: + UPLOAD_URL: ${{ steps.build-source.outputs.upload_url }} + run: | + curl "$UPLOAD_URL" \ + -X PUT \ + -H 'Content-Type:' \ + --data-binary @app.tar.gz + + # 'npm install' is faster than 'npm ci' because it only needs to + # *append* what's missing from ./node_modules/ + - name: Re-install dependencies so we get devDependencies back + run: npm install --no-audit --no-fund --only=dev + + - name: Deploy + id: deploy + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }} + HYDRO_ENDPOINT: ${{ secrets.HYDRO_ENDPOINT }} + HYDRO_SECRET: ${{ secrets.HYDRO_SECRET }} + PR_URL: ${{ github.event.pull_request.html_url }} + SOURCE_BLOB_URL: ${{ steps.build-source.outputs.download_url }} + ALLOWED_POLLING_FAILURES_PER_PHASE: '15' + RUN_ID: ${{ github.run_id }} + run: .github/actions-scripts/staging-deploy.js diff --git a/.github/workflows/staging-build-pr.yml b/.github/workflows/staging-build-pr.yml index a39ed5a0f7..807beba04a 100644 --- a/.github/workflows/staging-build-pr.yml +++ b/.github/workflows/staging-build-pr.yml @@ -4,6 +4,8 @@ name: Staging - Build PR # **Why we have it**: Because it's not safe to share our deploy secrets with forked repos: https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ # **Who does it impact**: All contributors. +# IT'S CRUCIALLY IMPORTANT THAT THIS WORKFLOW IS ONLY ENABLED IN docs! + on: pull_request: types: @@ -31,7 +33,9 @@ concurrency: jobs: build-pr: - if: ${{ github.repository == 'github/docs-internal' || github.repository == 'github/docs' }} + # Important. This whole file is only supposed to run in the PUBLIC repo. + if: ${{ github.repository == 'github/docs' }} + runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'github/docs-internal'] }} timeout-minutes: 5 # This interrupts Build, Deploy, and pre-write Undeploy workflow runs in @@ -45,7 +49,7 @@ jobs: # Make sure only approved files are changed if it's in github/docs - name: Check changed files - if: ${{ github.repository == 'github/docs' && github.event.pull_request.user.login != 'Octomerger' }} + if: ${{ github.event.pull_request.user.login != 'Octomerger' }} uses: dorny/paths-filter@eb75a1edc117d3756a18ef89958ee59f9500ba58 id: filter with: @@ -107,44 +111,6 @@ jobs: - name: Set npm script for Heroku build to noop run: npm set-script heroku-postbuild "echo 'Application was pre-built!'" - - name: Delete heavy things we won't need deployed - if: ${{ github.repository == 'github/docs-internal' }} - run: | - - # The dereferenced file is not used in runtime once the - # decorated file has been created from it. - rm -fr lib/rest/static/dereferenced - - # Translations are never tested in Staging builds - # but let's keep the empty directory. - rm -fr translations - mkdir translations - - # Delete all the big search indexes that are NOT English (`*-en-*`) - pushd lib/search/indexes - ls | grep -v '\-en\-' | xargs rm - popd - - # Note! Some day it would be nice to be able to delete - # all the heavy assets because they bloat the tarball. - # But it's not obvious how to test it then. For now, we'll have - # to accept that every staging build has a copy of the images. - - # The assumption here is that a staging build will not - # need these legacy redirects. Only the redirects from - # front-matter will be at play. - # These static redirects json files are notoriously large - # and they make the tarball unnecessarily large. - echo '[]' > lib/redirects/static/archived-frontmatter-fallbacks.json - echo '{}' > lib/redirects/static/developer.json - echo '{}' > lib/redirects/static/archived-redirects-from-213-to-217.json - - # This will turn every `lib/**/static/*.json` into - # an equivalent `lib/**/static/*.json.br` file. - # Once the server starts, it'll know to fall back to reading - # the `.br` equivalent if the `.json` file isn't present. - node .github/actions-scripts/compress-large-files.js - - name: Create an archive # Only bother if this is actually a pull request if: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/staging-deploy-pr.yml b/.github/workflows/staging-deploy-pr.yml index 6e8ba10812..ed5de66b12 100644 --- a/.github/workflows/staging-deploy-pr.yml +++ b/.github/workflows/staging-deploy-pr.yml @@ -4,6 +4,8 @@ name: Staging - Deploy PR # **Why we have it**: To deploy with high visibility in case of failures. # **Who does it impact**: All contributors. +# IT'S CRUCIALLY IMPORTANT THAT THIS WORKFLOW IS ONLY ENABLED IN docs! + on: workflow_run: workflows: @@ -46,10 +48,12 @@ jobs: # This is needed because the workflow we depend on # (see on.workflow_run.workflows) might be running from pushes on # main. That's because it needs to do that to popular the cache. - if: > - ${{ github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.conclusion == 'success' }} - + if: >- + ${{ + github.repository == 'github/docs' && + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + }} runs-on: ubuntu-latest outputs: number: ${{ steps.pr.outputs.number }} @@ -164,8 +168,7 @@ jobs: if: >- ${{ needs.pr-metadata.outputs.number != '0' && - github.event.workflow_run.conclusion == 'failure' && - (github.repository == 'github/docs-internal' || github.repository == 'github/docs') + github.event.workflow_run.conclusion == 'failure' }} runs-on: ubuntu-latest timeout-minutes: 1 @@ -208,8 +211,7 @@ jobs: if: >- ${{ needs.pr-metadata.outputs.number != '0' && - github.event.workflow_run.conclusion == 'success' && - (github.repository == 'github/docs-internal' || github.repository == 'github/docs') + github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest # This timeout should match or exceed the value of the timeout for Undeploy @@ -262,7 +264,7 @@ jobs: prepare-for-deploy: needs: [pr-metadata, check-pr-before-prepare] if: ${{ needs.check-pr-before-prepare.outputs.pull_request_state == 'open' }} - runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'github/docs-internal'] }} + runs-on: ubuntu-latest timeout-minutes: 5 # This interrupts Build, Deploy, and pre-write Undeploy workflow runs in # progress for this PR branch. @@ -308,22 +310,10 @@ jobs: node-version: 16.13.x cache: npm - # Install any dependencies that are needed for the early access script - - if: ${{ github.repository == 'github/docs-internal' }} - name: Install temporary dependencies - run: npm install --no-save dotenv rimraf - # Install any additional dependencies *before* downloading the build artifact - name: Install Heroku client development-only dependency run: npm install --no-save heroku-client - - if: ${{ github.repository == 'github/docs-internal' }} - name: Clone early access - run: node script/early-access/clone-for-build.js - env: - DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }} - GIT_BRANCH: ${{ needs.pr-metadata.outputs.head_ref }} - # Download the previously built "app.tar" - name: Download build artifact uses: dawidd6/action-download-artifact@af92a8455a59214b7b932932f2662fdefbd78126 @@ -333,35 +323,8 @@ jobs: name: pr_build path: ${{ runner.temp }} - # For security reasons, only extract the tar from docs-internal - # This allows us to add search indexes and early access content to the build - - if: ${{ github.repository == 'github/docs-internal' }} - name: Extract user-changes to temp directory - run: | - mkdir $RUNNER_TEMP/app - tar -x --file=$RUNNER_TEMP/app.tar -C "$RUNNER_TEMP/app/" - - # Move the LFS content into the temp directory in chunks (destructively) - - if: ${{ github.repository == 'github/docs-internal' }} - name: Move the LFS objects - run: | - git lfs ls-files --name-only | xargs -n 1 -I {} sh -c 'mkdir -p "$RUNNER_TEMP/app/$(dirname {})"; mv {} "$RUNNER_TEMP/app/$(dirname {})/"' - - # Move the early access content into the temp directory (destructively) - - if: ${{ github.repository == 'github/docs-internal' }} - name: Move the early access content - run: | - mv assets/images/early-access "$RUNNER_TEMP/app/assets/images/" - mv content/early-access "$RUNNER_TEMP/app/content/" - mv data/early-access "$RUNNER_TEMP/app/data/" - - - if: ${{ github.repository == 'github/docs-internal' }} - name: Create a gzipped archive (docs-internal) - run: tar -cz --file app.tar.gz "$RUNNER_TEMP/app/" - - # gzip the app.tar from github/docs so we're working with the same format - - if: ${{ github.repository == 'github/docs' }} - name: Create a gzipped archive (docs) + # gzip the app.tar to meet Heroku's expected format + - name: Create a gzipped archive (docs) run: gzip -9 < "$RUNNER_TEMP/app.tar" > app.tar.gz - name: Create a Heroku build source @@ -376,8 +339,8 @@ jobs: if (owner !== 'github') { throw new Error(`Repository owner must be 'github' but was: ${owner}`) } - if (repo !== 'docs-internal' && repo !== 'docs') { - throw new Error(`Repository name must be either 'docs-internal' or 'docs' but was: ${repo}`) + if (repo !== 'docs') { + throw new Error(`Repository name must be 'docs' but was: ${repo}`) } const Heroku = require('heroku-client') @@ -466,7 +429,7 @@ jobs: deploy: needs: [pr-metadata, prepare-for-deploy, check-pr-before-deploy] if: ${{ needs.check-pr-before-deploy.outputs.pull_request_state == 'open' }} - runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'github/docs-internal'] }} + runs-on: ubuntu-latest timeout-minutes: 10 # This interrupts Build, Deploy, and pre-write Undeploy workflow runs in # progress for this PR branch. @@ -495,13 +458,18 @@ jobs: HYDRO_SECRET: ${{ secrets.HYDRO_SECRET }} PR_URL: ${{ needs.pr-metadata.outputs.url }} SOURCE_BLOB_URL: ${{ needs.prepare-for-deploy.outputs.source_blob_url }} - CONTEXT_NAME: ${{ env.CONTEXT_NAME }} - ACTIONS_RUN_LOG: ${{ env.ACTIONS_RUN_LOG }} - HEAD_SHA: ${{ needs.pr-metadata.outputs.head_sha }} ALLOWED_POLLING_FAILURES_PER_PHASE: '15' RUN_ID: ${{ github.run_id }} run: .github/actions-scripts/staging-deploy.js + - name: Create successful commit status + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CONTEXT_NAME: ${{ env.CONTEXT_NAME }} + ACTIONS_RUN_LOG: ${{ env.ACTIONS_RUN_LOG }} + HEAD_SHA: ${{ needs.pr-metadata.outputs.head_sha }} + run: .github/actions-scripts/staging-commit-status-success.js + - name: Mark the deployment as inactive if timed out uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d if: ${{ steps.deploy.outcome == 'cancelled' }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 804b349ba9..a407ba2b28 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -86,7 +86,7 @@ jobs: uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353 with: path: .next/cache - key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }}-${{ hashFiles('.github/workflows/test.yml') }} + key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }} - name: Run build script run: npm run build diff --git a/script/deployment/deploy-to-staging.js b/script/deployment/deploy-to-staging.js index 9cc7b14bf3..5e3df9a09b 100644 --- a/script/deployment/deploy-to-staging.js +++ b/script/deployment/deploy-to-staging.js @@ -1,10 +1,12 @@ #!/usr/bin/env node -import sleep from 'await-sleep' import got from 'got' import Heroku from 'heroku-client' import { setOutput } from '@actions/core' import createStagingAppName from './create-staging-app-name.js' +// Equivalent of the 'await-sleep' module without the install +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) + const SLEEP_INTERVAL = 5000 const HEROKU_LOG_LINES_TO_SHOW = 25