From 624581e52716d11f3dd06f70f1c1610b370aeee9 Mon Sep 17 00:00:00 2001 From: Mike Surowiec Date: Thu, 10 Feb 2022 17:36:07 -0600 Subject: [PATCH] Remove most of Heroku references (#25208) * remove most of heroku references * remove more stuff * update codeowners --- .github/CODEOWNERS | 1 - .github/actions-scripts/prod-deploy.js | 51 -- .../staging-commit-status-success.js | 42 -- .github/actions-scripts/staging-deploy.js | 55 -- .github/workflows/ping-staging-apps.yml | 31 - .github/workflows/prod-build-deploy.yml | 199 ----- .../remove-stale-staging-resources.yml | 69 -- .../workflows/staging-build-and-deploy-pr.yml | 210 ------ .github/workflows/staging-build-pr.yml | 135 ---- .github/workflows/staging-deploy-pr.yml | 466 ------------ .github/workflows/test-windows.yml | 76 -- .github/workflows/test.yml | 2 - .../triage-unallowed-contributions.yml | 8 +- .../triage-unallowed-internal-changes.yml | 15 +- Dockerfile | 4 +- Procfile | 1 - app.json | 17 - package-lock.json | 60 -- package.json | 2 - script/deployment/create-staging-app-name.js | 18 - script/deployment/deploy-to-production.js | 465 ------------ script/deployment/deploy-to-staging.js | 680 ------------------ script/deployment/parse-pr-url.js | 21 - script/early-access/clone-for-build.js | 140 ---- script/ping-staging-apps.js | 41 -- script/remove-stale-staging-apps.js | 145 ---- script/remove-stale-staging-envs.js | 266 ------- tests/rendering/robots-txt.js | 4 +- 28 files changed, 6 insertions(+), 3218 deletions(-) delete mode 100755 .github/actions-scripts/prod-deploy.js delete mode 100755 .github/actions-scripts/staging-commit-status-success.js delete mode 100755 .github/actions-scripts/staging-deploy.js delete mode 100644 .github/workflows/ping-staging-apps.yml delete mode 100644 .github/workflows/prod-build-deploy.yml delete mode 100644 .github/workflows/remove-stale-staging-resources.yml delete mode 100644 .github/workflows/staging-build-and-deploy-pr.yml delete mode 100644 .github/workflows/staging-build-pr.yml delete mode 100644 .github/workflows/staging-deploy-pr.yml delete mode 100644 .github/workflows/test-windows.yml delete mode 100644 Procfile delete mode 100644 app.json delete mode 100644 script/deployment/create-staging-app-name.js delete mode 100755 script/deployment/deploy-to-production.js delete mode 100644 script/deployment/deploy-to-staging.js delete mode 100644 script/deployment/parse-pr-url.js delete mode 100755 script/early-access/clone-for-build.js delete mode 100755 script/ping-staging-apps.js delete mode 100755 script/remove-stale-staging-apps.js delete mode 100755 script/remove-stale-staging-envs.js diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4c6ff3c4fd..f54c51ee4e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,7 +11,6 @@ /script/ @github/docs-engineering /includes/ @github/docs-engineering /lib/search/popular-pages.json @github/docs-engineering -app.json @github/docs-engineering Dockerfile @github/docs-engineering package-lock.json @github/docs-engineering package.json @github/docs-engineering diff --git a/.github/actions-scripts/prod-deploy.js b/.github/actions-scripts/prod-deploy.js deleted file mode 100755 index 387e1397be..0000000000 --- a/.github/actions-scripts/prod-deploy.js +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env node - -import getOctokit from '../../script/helpers/github.js' -import deployToProduction from '../../script/deployment/deploy-to-production.js' - -const { - GITHUB_TOKEN, - HEROKU_API_TOKEN, - HEROKU_PRODUCTION_APP_NAME, - SOURCE_BLOB_URL, - DELAY_FOR_PREBOOT, - RUN_ID, -} = process.env - -// Exit if GitHub Actions PAT is not found -if (!GITHUB_TOKEN) { - throw new Error('You must supply a GITHUB_TOKEN environment variable!') -} - -// Exit if Heroku API token is not found -if (!HEROKU_API_TOKEN) { - throw new Error('You must supply a HEROKU_API_TOKEN environment variable!') -} - -// Exit if Heroku App name is not found -if (!HEROKU_PRODUCTION_APP_NAME) { - throw new Error('You must supply a HEROKU_PRODUCTION_APP_NAME environment variable!') -} - -if (!RUN_ID) { - throw new Error('$RUN_ID not set') -} - -// This helper uses the `GITHUB_TOKEN` implicitly! -// We're using our usual version of Octokit vs. the provided `github` -// instance to avoid versioning discrepancies. -const octokit = getOctokit() - -try { - await deployToProduction({ - octokit, - includeDelayForPreboot: DELAY_FOR_PREBOOT !== 'false', - // These parameters will ONLY be set by Actions - sourceBlobUrl: SOURCE_BLOB_URL, - runId: RUN_ID, - }) -} catch (error) { - console.error(`Failed to deploy to production: ${error.message}`) - console.error(error) - throw error -} diff --git a/.github/actions-scripts/staging-commit-status-success.js b/.github/actions-scripts/staging-commit-status-success.js deleted file mode 100755 index e7e1a6ba97..0000000000 --- a/.github/actions-scripts/staging-commit-status-success.js +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env node - -import * as github from '@actions/github' - -import getOctokit from '../../script/helpers/github.js' - -const { GITHUB_TOKEN } = process.env - -// Exit if GitHub Actions PAT is not found -if (!GITHUB_TOKEN) { - throw new Error('You must supply a GITHUB_TOKEN environment variable!') -} - -// This helper uses the `GITHUB_TOKEN` implicitly! -// We're using our usual version of Octokit vs. the provided `github` -// instance to avoid versioning discrepancies. -const octokit = getOctokit() - -const { CONTEXT_NAME, ACTIONS_RUN_LOG, HEAD_SHA } = process.env -if (!CONTEXT_NAME) { - throw new Error('$CONTEXT_NAME not set') -} -if (!ACTIONS_RUN_LOG) { - throw new Error('$ACTIONS_RUN_LOG not set') -} -if (!HEAD_SHA) { - throw new Error('$HEAD_SHA not set') -} - -const { context } = github -const owner = context.repo.owner -const repo = context.payload.repository.name - -await octokit.repos.createCommitStatus({ - owner, - repo, - sha: HEAD_SHA, - context: CONTEXT_NAME, - state: 'success', - description: 'Successfully deployed! See logs.', - target_url: ACTIONS_RUN_LOG, -}) diff --git a/.github/actions-scripts/staging-deploy.js b/.github/actions-scripts/staging-deploy.js deleted file mode 100755 index 26dadc4157..0000000000 --- a/.github/actions-scripts/staging-deploy.js +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env node - -import parsePrUrl from '../../script/deployment/parse-pr-url.js' -import getOctokit from '../../script/helpers/github.js' -import deployToStaging from '../../script/deployment/deploy-to-staging.js' - -const { GITHUB_TOKEN, HEROKU_API_TOKEN } = process.env - -// Exit if GitHub Actions PAT is not found -if (!GITHUB_TOKEN) { - throw new Error('You must supply a GITHUB_TOKEN environment variable!') -} - -// Exit if Heroku API token is not found -if (!HEROKU_API_TOKEN) { - throw new Error('You must supply a HEROKU_API_TOKEN environment variable!') -} - -// This helper uses the `GITHUB_TOKEN` implicitly! -// We're using our usual version of Octokit vs. the provided `github` -// instance to avoid versioning discrepancies. -const octokit = getOctokit() - -const { RUN_ID, PR_URL, SOURCE_BLOB_URL } = process.env -if (!RUN_ID) { - throw new Error('$RUN_ID not set') -} -if (!PR_URL) { - throw new Error('$PR_URL not set') -} -if (!SOURCE_BLOB_URL) { - throw new Error('$SOURCE_BLOB_URL not set') -} - -const { owner, repo, pullNumber } = parsePrUrl(PR_URL) -if (!owner || !repo || !pullNumber) { - throw new Error( - `'pullRequestUrl' input must match URL format 'https://github.com/github/(docs|docs-internal)/pull/123' but was '${PR_URL}'` - ) -} - -const { data: pullRequest } = await octokit.pulls.get({ - owner, - repo, - pull_number: pullNumber, -}) - -await deployToStaging({ - octokit, - pullRequest, - forceRebuild: false, - // These parameters will ONLY be set by Actions - sourceBlobUrl: SOURCE_BLOB_URL, - runId: RUN_ID, -}) diff --git a/.github/workflows/ping-staging-apps.yml b/.github/workflows/ping-staging-apps.yml deleted file mode 100644 index b5654ff1e2..0000000000 --- a/.github/workflows/ping-staging-apps.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Ping staging apps - -# **What it does**: This keeps our staging applications from automatically spinning down. -# **Why we have it**: Staging applications can hiberate without use. -# **Who does it impact**: Anyone with a pull request in docs-internal. - -on: - schedule: - - cron: '10,30,50 * * * *' # every twenty minutes - -permissions: - contents: read - -jobs: - ping_staging_apps: - name: Ping - if: github.repository == 'github/docs-internal' - runs-on: ubuntu-latest - env: - HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }} - steps: - - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - - name: Setup node - uses: actions/setup-node@1f8c6b94b26d0feae1e387ca63ccbdc44d27b561 - with: - node-version: 16.13.x - cache: npm - - name: npm ci - run: npm ci - - name: Run script - run: script/ping-staging-apps.js diff --git a/.github/workflows/prod-build-deploy.yml b/.github/workflows/prod-build-deploy.yml deleted file mode 100644 index 31094bf7db..0000000000 --- a/.github/workflows/prod-build-deploy.yml +++ /dev/null @@ -1,199 +0,0 @@ -name: Production - Build and Deploy - -# **What it does**: Builds and deploys the default branch to production -# **Why we have it**: To enable us to deploy the latest to production whenever necessary rather than relying on PR merges. -# **Who does it impact**: All contributors. - -on: - push: - branches: - - main - workflow_dispatch: - -permissions: - contents: read - deployments: write - -# This allows a subsequently queued workflow run to take priority over -# previously queued runs but NOT interrupt currently executing runs -concurrency: - group: '${{ github.workflow }}' - cancel-in-progress: false - -jobs: - build-and-deploy: - if: ${{ github.repository == 'github/docs-internal'}} - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Check out repo - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - with: - persist-credentials: 'false' - lfs: 'true' - - - name: Check out LFS objects - run: git lfs checkout - - - name: Setup node - uses: actions/setup-node@1f8c6b94b26d0feae1e387ca63ccbdc44d27b561 - with: - node-version: 16.13.x - cache: npm - - # Required for `npm pkg ...` command support - - name: Update to npm@^7.20.0 - run: npm install --global npm@^7.20.0 - - - name: Install dependencies - run: npm ci - - - name: Clone early access - run: node script/early-access/clone-for-build.js - env: - DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }} - GIT_BRANCH: main - - - name: Cache nextjs build - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed - with: - path: .next/cache - key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }} - - - name: Build - run: npm run build - - - name: Remove development-only dependencies - run: npm prune --production - - - name: Remove all npm scripts - run: npm pkg delete scripts - - - name: Set npm script for Heroku build to noop - run: npm set-script heroku-postbuild "echo 'Application was pre-built!'" - - - name: Create a gzipped archive - run: | - tar -cz --file=app.tar.gz \ - node_modules/ \ - .next/ \ - assets/ \ - content/ \ - data/ \ - includes/ \ - lib/ \ - middleware/ \ - translations/ \ - server.mjs \ - package*.json \ - .npmrc \ - feature-flags.json \ - next.config.js \ - app.json \ - Procfile - - - name: Install the development dependencies again - run: npm install - - - name: Create a Heroku build source - id: build-source - uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d - env: - HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }} - with: - script: | - const { owner, repo } = context.repo - - if (owner !== 'github') { - throw new Error(`Repository owner must be 'github' but was: ${owner}`) - } - if (repo !== 'docs-internal') { - throw new Error(`Repository name must be 'docs-internal' but was: ${repo}`) - } - - const Heroku = require('heroku-client') - const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN }) - - try { - const { source_blob: sourceBlob } = await heroku.post('/sources') - const { put_url: uploadUrl, get_url: downloadUrl } = sourceBlob - - core.setOutput('upload_url', uploadUrl) - core.setOutput('download_url', downloadUrl) - } catch (error) { - if (error.statusCode === 503) { - console.error('💀 Heroku may be down! Please check its Status page: https://status.heroku.com/') - } - throw error - } - - # See: https://devcenter.heroku.com/articles/build-and-release-using-the-api#sources-endpoint - - name: Upload to the Heroku build source - env: - UPLOAD_URL: ${{ steps.build-source.outputs.upload_url }} - run: | - curl "$UPLOAD_URL" \ - -X PUT \ - -H 'Content-Type:' \ - --data-binary @app.tar.gz - - - name: Deploy - id: deploy - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }} - HEROKU_PRODUCTION_APP_NAME: ${{ secrets.HEROKU_PRODUCTION_APP_NAME }} - HYDRO_ENDPOINT: ${{ secrets.HYDRO_ENDPOINT }} - HYDRO_SECRET: ${{ secrets.HYDRO_SECRET }} - SOURCE_BLOB_URL: ${{ steps.build-source.outputs.download_url }} - DELAY_FOR_PREBOOT: 'true' - ALLOWED_POLLING_FAILURES_PER_PHASE: '15' - RUN_ID: ${{ github.run_id }} - run: .github/actions-scripts/prod-deploy.js - - - name: Mark the deployment as inactive if timed out - uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d - if: ${{ steps.deploy.outcome == 'cancelled' }} - env: - DEPLOYMENT_ID: ${{ steps.deploy.outputs.deploymentId }} - LOG_URL: ${{ steps.deploy.outputs.logUrl }} - with: - script: | - const { DEPLOYMENT_ID, LOG_URL } = process.env - const { owner, repo } = context.repo - - if (!DEPLOYMENT_ID) { - throw new Error('A deployment wasn't created before a timeout occurred!') - } - - await github.repos.createDeploymentStatus({ - owner, - repo, - deployment_id: DEPLOYMENT_ID, - state: 'error', - description: 'The deployment step timed out. See workflow logs.', - log_url: LOG_URL, - // The 'ant-man' preview is required for `state` values of 'inactive', as well as - // the use of the `log_url`, `environment_url`, and `auto_inactive` parameters. - // The 'flash' preview is required for `state` values of 'in_progress' and 'queued'. - mediaType: { - previews: ['ant-man', 'flash'], - }, - }) - console.log('⏲️ Deployment status: error - The deployment timed out...') - - # - name: Purge Fastly edge cache - # env: - # FASTLY_TOKEN: ${{ secrets.FASTLY_TOKEN }} - # FASTLY_SERVICE_ID: ${{ secrets.FASTLY_SERVICE_ID }} - # FASTLY_SURROGATE_KEY: 'every-deployment' - # run: .github/actions-scripts/purge-fastly-edge-cache.js - - - name: Send Slack notification if workflow failed - uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340 - if: ${{ failure() }} - with: - channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }} - bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }} - color: failure - text: Production deployment failed at commit ${{ github.sha }}. See https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/.github/workflows/remove-stale-staging-resources.yml b/.github/workflows/remove-stale-staging-resources.yml deleted file mode 100644 index 5375869132..0000000000 --- a/.github/workflows/remove-stale-staging-resources.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Remove stale staging resources - -# **What it does**: -# This cleans up any rogue staging applications and environments that outlasted -# the closure of their corresponding pull requests. -# **Why we have it**: -# Staging applications and environments should be destroyed after their -# corresponding pull request is closed or merged, especially to save money spent -# on Heroku App staging deployments for closed PRs. -# **Who does it impact**: -# Anyone with a closed, spammy, or deleted pull request in docs or docs-internal. - -on: - schedule: - - cron: '15,45 * * * *' # every thirty minutes at :15 and :45 - -permissions: - actions: read - contents: read - deployments: write - pull-requests: write - -jobs: - remove_stale_staging_apps: - name: Remove stale staging apps - if: ${{ github.repository == 'github/docs-internal' }} - runs-on: ubuntu-latest - steps: - - name: Check out repo's default branch - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - - - name: Setup Node - uses: actions/setup-node@1f8c6b94b26d0feae1e387ca63ccbdc44d27b561 - with: - node-version: 16.13.x - cache: npm - - - name: Install dependencies - run: npm ci - - - name: Run script - run: script/remove-stale-staging-apps.js - env: - HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - remove_stale_staging_envs: - name: Remove stale staging environments - runs-on: ubuntu-latest - steps: - - name: Check out repo's default branch - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - - - name: Setup Node - uses: actions/setup-node@1f8c6b94b26d0feae1e387ca63ccbdc44d27b561 - with: - node-version: 16.13.x - cache: npm - - - name: Install dependencies - run: npm ci - - - name: Run script - run: script/remove-stale-staging-envs.js - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ELEVATED_TOKEN: ${{ secrets.DOCS_BOT_FR }} - REPO: ${{ github.repository }} - RUN_ID: ${{ github.run_id }} diff --git a/.github/workflows/staging-build-and-deploy-pr.yml b/.github/workflows/staging-build-and-deploy-pr.yml deleted file mode 100644 index 45db1bf149..0000000000 --- a/.github/workflows/staging-build-and-deploy-pr.yml +++ /dev/null @@ -1,210 +0,0 @@ -name: Staging - Build and Deploy PR (fast and private-only) - -# **What it does**: Builds and deploys PRs to staging but ONLY for docs-internal -# **Why we have it**: Most PRs are made on the private repo. Let's make those extra fast if we can worry less about security. -# **Who does it impact**: All staff. - -# This whole workflow is only guaranteed to be secure in the *private -# repo* and because we repo-sync these files over the to the public one, -# IT'S IMPORTANT THAT THIS WORKFLOW IS ONLY ENABLED IN docs-internal! - -on: - # The advantage of 'pull_request' over 'pull_request_target' is that we - # can make changes to this file and test them in a pull request, instead - # of relying on landing it in 'main' first. - # From a security point of view, its arguably safer this way because - # unlike 'pull_request_target', these only have secrets if the pull - # request creator has permission to access secrets. - pull_request: - -permissions: - actions: read - contents: read - deployments: write - pull-requests: read - statuses: write - -# This allows a subsequently queued workflow run to interrupt previous runs -concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' - cancel-in-progress: true - -jobs: - build-and-deploy-pr: - # Important. This whole file is only supposed to run in the PRIVATE repo. - if: ${{ github.repository == 'github/docs-internal' }} - - # The assumption here is that self-hosted is faster (e.g CPU power) - # that the regular ones. And it matters in this workflow because - # we do heavy CPU stuff with `npm run build` and `tar` - # runs-on: ubuntu-latest - runs-on: self-hosted - - timeout-minutes: 5 - - steps: - - name: Check out repo - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - with: - lfs: 'true' - # To prevent issues with cloning early access content later - persist-credentials: 'false' - - - name: Check out LFS objects - run: git lfs checkout - - - name: Setup node - uses: actions/setup-node@1f8c6b94b26d0feae1e387ca63ccbdc44d27b561 - with: - node-version: 16.13.x - cache: npm - - - name: Install dependencies - run: npm ci - - - name: Cache nextjs build - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed - with: - path: .next/cache - key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }} - - - name: Build - run: npm run build - - - name: Clone early access - run: node script/early-access/clone-for-build.js - env: - DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }} - GIT_BRANCH: ${{ github.head_ref || github.ref }} - - - name: Create a Heroku build source - id: build-source - uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d - env: - HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }} - with: - script: | - const { owner, repo } = context.repo - - if (owner !== 'github') { - throw new Error(`Repository owner must be 'github' but was: ${owner}`) - } - if (repo !== 'docs-internal') { - throw new Error(`Repository name must be 'docs-internal' but was: ${repo}`) - } - - const Heroku = require('heroku-client') - const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN }) - - try { - const { source_blob: sourceBlob } = await heroku.post('/sources') - const { put_url: uploadUrl, get_url: downloadUrl } = sourceBlob - - core.setOutput('upload_url', uploadUrl) - core.setOutput('download_url', downloadUrl) - } catch (error) { - if (error.statusCode === 503) { - console.error('💀 Heroku may be down! Please check its Status page: https://status.heroku.com/') - } - throw error - } - - - name: Remove development-only dependencies - run: npm prune --production - - - name: Remove all npm scripts - run: npm pkg delete scripts - - - name: Set npm script for Heroku build to noop - run: npm set-script heroku-postbuild "echo 'Application was pre-built!'" - - - name: Delete heavy things we won't need deployed - run: | - - # The dereferenced file is not used in runtime once the - # decorated file has been created from it. - rm -rf lib/rest/static/dereferenced - - # Translations are never tested in Staging builds - # but let's keep the empty directory. - rm -rf translations - mkdir translations - - # Delete all the big search indexes that are NOT English (`*-en-*`) - pushd lib/search/indexes - ls | grep -ve '\-en\b' | xargs rm - popd - - # Note! Some day it would be nice to be able to delete - # all the heavy assets because they bloat the tarball. - # But it's not obvious how to test it then. For now, we'll have - # to accept that every staging build has a copy of the images. - - # The assumption here is that a staging build will not - # need these legacy redirects. Only the redirects from - # front-matter will be at play. - # These static redirects json files are notoriously large - # and they make the tarball unnecessarily large. - echo '[]' > lib/redirects/static/archived-frontmatter-fallbacks.json - echo '{}' > lib/redirects/static/developer.json - echo '{}' > lib/redirects/static/archived-redirects-from-213-to-217.json - - # This will turn every `lib/**/static/*.json` into - # an equivalent `lib/**/static/*.json.br` file. - # Once the server starts, it'll know to fall back to reading - # the `.br` equivalent if the `.json` file isn't present. - node .github/actions-scripts/compress-large-files.js - - - name: Make the tarball for Heroku - run: | - # We can't delete the .next/cache directory from the workflow - # because it's needed for caching, but we can at least exclude it - # from the tarball. Then it can be cached but not weigh down the - # tarball we intend to deploy. - tar -zc --exclude=.next/cache --file=app.tar.gz \ - node_modules/ \ - .next/ \ - assets/ \ - content/ \ - data/ \ - includes/ \ - lib/ \ - middleware/ \ - translations/ \ - server.mjs \ - package*.json \ - .npmrc \ - feature-flags.json \ - next.config.js \ - app.json \ - Procfile - - du -sh app.tar.gz - - # See: https://devcenter.heroku.com/articles/build-and-release-using-the-api#sources-endpoint - - name: Upload to the Heroku build source - env: - UPLOAD_URL: ${{ steps.build-source.outputs.upload_url }} - run: | - curl "$UPLOAD_URL" \ - -X PUT \ - -H 'Content-Type:' \ - --data-binary @app.tar.gz - - # 'npm install' is faster than 'npm ci' because it only needs to - # *append* what's missing from ./node_modules/ - - name: Re-install dependencies so we get devDependencies back - run: npm install --no-audit --no-fund --only=dev - - - name: Deploy - id: deploy - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }} - HYDRO_ENDPOINT: ${{ secrets.HYDRO_ENDPOINT }} - HYDRO_SECRET: ${{ secrets.HYDRO_SECRET }} - PR_URL: ${{ github.event.pull_request.html_url }} - SOURCE_BLOB_URL: ${{ steps.build-source.outputs.download_url }} - ALLOWED_POLLING_FAILURES_PER_PHASE: '15' - RUN_ID: ${{ github.run_id }} - run: .github/actions-scripts/staging-deploy.js diff --git a/.github/workflows/staging-build-pr.yml b/.github/workflows/staging-build-pr.yml deleted file mode 100644 index 91c54cc037..0000000000 --- a/.github/workflows/staging-build-pr.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: Staging - Build PR - -# **What it does**: Builds PRs before deploying them. -# **Why we have it**: Because it's not safe to share our deploy secrets with forked repos: https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ -# **Who does it impact**: All contributors. - -# IT'S CRUCIALLY IMPORTANT THAT THIS WORKFLOW IS ONLY ENABLED IN docs! - -on: - pull_request: - -permissions: - contents: read - -# This allows a subsequently queued workflow run to interrupt previous runs -# These are different from the concurrency in that here it checks if the -# whole workflow runs again. The "inner concurrency" is used for -# undeployments to cleaning up resources. -concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' - cancel-in-progress: true - -jobs: - build-pr: - # Important. This whole file is only supposed to run in the PUBLIC repo. - if: ${{ github.repository == 'github/docs' }} - - runs-on: ${{ fromJSON('["ubuntu-latest", "self-hosted"]')[github.repository == 'github/docs-internal'] }} - timeout-minutes: 5 - # This interrupts Build and Deploy workflow runs in progress for this PR branch. - concurrency: - group: 'PR Staging @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' - cancel-in-progress: true - steps: - - name: Check out repo - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - - # Make sure only approved files are changed if it's in github/docs - - name: Check changed files - if: ${{ github.event.pull_request.user.login != 'Octomerger' }} - uses: dorny/paths-filter@eb75a1edc117d3756a18ef89958ee59f9500ba58 - id: filter - with: - # Base branch used to get changed files - base: 'main' - - # Enables setting an output in the format in `${FILTER_NAME}_files - # with the names of the matching files formatted as JSON array - list-files: json - - # Returns list of changed files matching each filter - filters: | - notAllowed: - - '*.js' - - '*.mjs' - - '*.cjs' - - '*.ts' - - '*.tsx' - - '*.json' - - '.npmrc' - - '.babelrc*' - - '.env*' - - 'script/**' - - 'Procfile' - - # When there are changes to files we can't accept - - name: Fail when disallowed files are changed - if: ${{ steps.filter.outputs.notAllowed == 'true' }} - run: exit 1 - - - name: Setup node - uses: actions/setup-node@1f8c6b94b26d0feae1e387ca63ccbdc44d27b561 - with: - node-version: 16.13.x - cache: npm - - # Required for `npm pkg ...` command support - - name: Update to npm@^7.20.0 - run: npm install --global npm@^7.20.0 - - - name: Install dependencies - run: npm ci - - - name: Cache nextjs build - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed - with: - path: .next/cache - key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }} - - - name: Build - run: npm run build - - - name: Remove development-only dependencies - run: npm prune --production - - - name: Remove all npm scripts - run: npm pkg delete scripts - - - name: Set npm script for Heroku build to noop - run: npm set-script heroku-postbuild "echo 'Application was pre-built!'" - - - name: Create an archive - run: | - tar -c --file=app.tar \ - node_modules/ \ - .next/ \ - assets/ \ - content/ \ - data/ \ - includes/ \ - lib/ \ - middleware/ \ - translations/ \ - server.mjs \ - package*.json \ - .npmrc \ - feature-flags.json \ - next.config.js \ - app.json \ - Procfile - - # We can't delete the .next/cache directory from the workflow - # because it's needed for caching, but we can at least delete it - # from within the tarball. Then it can be cached but not - # weigh down the tarball we intend to deploy. - tar --delete --file=app.tar .next/cache - - # Upload only the files needed to run this application. - # We are not willing to trust the rest (e.g. script/) for the remainder - # of the deployment process. - - name: Upload build artifact - uses: actions/upload-artifact@82c141cc518b40d92cc801eee768e7aafc9c2fa2 - with: - name: pr_build - path: app.tar diff --git a/.github/workflows/staging-deploy-pr.yml b/.github/workflows/staging-deploy-pr.yml deleted file mode 100644 index 52c382a675..0000000000 --- a/.github/workflows/staging-deploy-pr.yml +++ /dev/null @@ -1,466 +0,0 @@ -name: Staging - Deploy PR - -# **What it does**: To deploy PRs to a Heroku staging environment. -# **Why we have it**: To deploy with high visibility in case of failures. -# **Who does it impact**: All contributors. - -# IT'S CRUCIALLY IMPORTANT THAT THIS WORKFLOW IS ONLY ENABLED IN docs! - -on: - workflow_run: - workflows: - - 'Staging - Build PR' - types: - - completed - -permissions: - actions: read - contents: read - deployments: write - pull-requests: read - statuses: write - -# IMPORTANT: Intentionally OMIT a `concurrency` configuration from this workflow's -# top-level as we do not have any guarantee of identifying values being available -# within the `github.event` context for PRs from forked repos! -# -# The implication of this shortcoming is that we may have multiple workflow runs -# of this running at the same time for different commits within the same PR. -# However, once they reach the `concurrency` configurations deeper down within -# this workflow's jobs, then we can expect concurrent short-circuiting to begin. - -env: - CONTEXT_NAME: '${{ github.workflow }} / deploy (${{ github.event.workflow_run.event }})' - ACTIONS_RUN_LOG: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - BUILD_ACTIONS_RUN_ID: ${{ github.event.workflow_run.id }} - BUILD_ACTIONS_RUN_LOG: https://github.com/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }} - -jobs: - pr-metadata: - # This is needed because the workflow we depend on - # (see on.workflow_run.workflows) might be running from pushes on - # main. That's because it needs to do that to popular the cache. - if: >- - ${{ - github.repository == 'github/docs' && - github.event.workflow_run.event == 'pull_request' && - github.event.workflow_run.conclusion == 'success' - }} - runs-on: ubuntu-latest - outputs: - number: ${{ steps.pr.outputs.number }} - url: ${{ steps.pr.outputs.url }} - state: ${{ steps.pr.outputs.state }} - head_sha: ${{ steps.pr.outputs.head_sha }} - head_branch: ${{ steps.pr.outputs.head_branch }} - head_label: ${{ steps.pr.outputs.head_label }} - head_ref: ${{ steps.pr.outputs.head_ref }} - steps: - - name: Find the originating pull request - id: pr - uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d - env: - BUILD_ACTIONS_RUN_ID: ${{ env.BUILD_ACTIONS_RUN_ID }} - with: - script: | - - // Curious about what version of node you get - console.log('Node version:', process.version) - - // In order to find out the PR info for a forked repo, we must query - // the API for more info based on the originating workflow run - const { BUILD_ACTIONS_RUN_ID } = process.env - const { owner, repo } = context.repo - const { data: run } = await github.actions.getWorkflowRun({ - owner, - repo, - run_id: BUILD_ACTIONS_RUN_ID, - }) - - // Gather PR-identifying information from the workflow run - const { - head_branch: headBranch, - head_sha: headSha, - head_repository: { - owner: { login: prRepoOwner }, - name: prRepoName - } - } = run - - const prIsInternal = owner === prRepoOwner && repo === prRepoName - let headLabel = `${prRepoOwner}:${headBranch}` - - // If the PR is external, prefix its head branch name with the - // forked repo owner's login and their fork repo name e.g. - // "octocat/my-fork:docs". We need to include the fork repo - // name as well to account for an API issue (this will work fine - // if they don't have a different fork repo name). - if (!prIsInternal) { - headLabel = `${prRepoOwner}/${prRepoName}:${headBranch}` - } - - // If the PR is external, prefix its head branch name with the - // forked repo owner's login, e.g. "octocat:docs" - const headRef = prIsInternal ? headBranch : headLabel - - // Retrieve matching PRs (up to 30) - const { data: pulls } = await github.pulls.list({ - owner, - repo, - head: headLabel, - sort: 'updated', - direction: 'desc', - per_page: 30 - }) - - // Find the open PR, if any, otherwise choose the most recently updated - const targetPull = pulls.find(pr => pr.state === 'open') || pulls[0] || {} - - const pullNumber = targetPull.number || 0 - const pullUrl = targetPull.html_url || 'about:blank' - const pullState = targetPull.state || 'closed' - - core.setOutput('number', pullNumber.toString()) - core.setOutput('url', pullUrl) - core.setOutput('state', pullState) - core.setOutput('head_sha', headSha) - core.setOutput('head_branch', headBranch) - core.setOutput('head_label', headLabel) - core.setOutput('head_ref', headRef) - - debug-originating-trigger: - needs: pr-metadata - runs-on: ubuntu-latest - steps: - - name: Dump info about the originating workflow run - env: - PR_NUMBER: ${{ needs.pr-metadata.outputs.number }} - PR_URL: ${{ needs.pr-metadata.outputs.url }} - PR_STATE: ${{ needs.pr-metadata.outputs.state }} - HEAD_SHA: ${{ needs.pr-metadata.outputs.head_sha }} - HEAD_BRANCH: ${{ needs.pr-metadata.outputs.head_branch }} - HEAD_LABEL: ${{ needs.pr-metadata.outputs.head_label }} - HEAD_REF: ${{ needs.pr-metadata.outputs.head_ref }} - BUILD_ACTIONS_RUN_ID: ${{ env.BUILD_ACTIONS_RUN_ID }} - BUILD_ACTIONS_RUN_LOG: ${{ env.BUILD_ACTIONS_RUN_LOG }} - run: | - echo "Originating workflow info:" - echo " - PR_NUMBER = $PR_NUMBER" - echo " - PR_URL = $PR_URL" - echo " - PR_STATE = $PR_STATE" - echo " - HEAD_SHA = $HEAD_SHA" - echo " - HEAD_BRANCH = $HEAD_BRANCH" - echo " - HEAD_LABEL = $HEAD_LABEL" - echo " - HEAD_REF = $HEAD_REF" - echo " - BUILD_ACTIONS_RUN_ID = $BUILD_ACTIONS_RUN_ID" - echo " - BUILD_ACTIONS_RUN_LOG = $BUILD_ACTIONS_RUN_LOG" - - notify-of-failed-builds: - needs: pr-metadata - if: >- - ${{ - needs.pr-metadata.outputs.number != '0' && - github.event.workflow_run.conclusion == 'failure' - }} - runs-on: ubuntu-latest - timeout-minutes: 1 - # Specifically omitting a concurrency group here in case the build was not - # successful BECAUSE a subsequent build already canceled it - steps: - - name: Verify build workflow run was not cancelled - id: check-workflow-run - uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d - env: - BUILD_ACTIONS_RUN_ID: ${{ env.BUILD_ACTIONS_RUN_ID }} - with: - script: | - const { owner, repo } = context.repo - const { data: { jobs: buildJobs } } = await github.actions.listJobsForWorkflowRun({ - owner, - repo, - run_id: process.env.BUILD_ACTIONS_RUN_ID, - filter: 'latest' - }) - const wasCancelled = ( - buildJobs.length > 0 && - buildJobs.every(({ status, conclusion }) => { - return status === 'completed' && conclusion === 'cancelled' - }) - ) - core.setOutput('cancelled', wasCancelled.toString()) - - - if: ${{ steps.check-workflow-run.outputs.cancelled == 'false' }} - name: Send Slack notification if build workflow failed - uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340 - with: - channel: ${{ secrets.DOCS_STAGING_DEPLOYMENT_FAILURES_SLACK_CHANNEL_ID }} - bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }} - color: failure - text: Staging build failed for PR ${{ needs.pr-metadata.outputs.url }} at commit ${{ needs.pr-metadata.outputs.head_sha }}. See ${{ env.BUILD_ACTIONS_RUN_LOG }}. This run was ${{ env.ACTIONS_RUN_LOG }}. - - prepare-for-deploy: - needs: pr-metadata - if: ${{ needs.pr-metadata.outputs.state == 'open' }} - runs-on: ubuntu-latest - timeout-minutes: 5 - # This interrupts Build and Deploy workflow runs in progress for this PR branch. - concurrency: - group: 'PR Staging @ ${{ needs.pr-metadata.outputs.head_label }}' - cancel-in-progress: true - outputs: - source_blob_url: ${{ steps.build-source.outputs.download_url }} - steps: - - name: Create initial status - uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d - env: - CONTEXT_NAME: ${{ env.CONTEXT_NAME }} - ACTIONS_RUN_LOG: ${{ env.ACTIONS_RUN_LOG }} - HEAD_SHA: ${{ needs.pr-metadata.outputs.head_sha }} - with: - script: | - const { CONTEXT_NAME, ACTIONS_RUN_LOG, HEAD_SHA } = process.env - const { owner, repo } = context.repo - await github.repos.createCommitStatus({ - owner, - repo, - sha: HEAD_SHA, - context: CONTEXT_NAME, - state: 'pending', - description: 'The app is being deployed. See logs.', - target_url: ACTIONS_RUN_LOG - }) - - - name: Check out repo's default branch - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - with: - # To prevent issues with cloning early access content later - persist-credentials: 'false' - lfs: 'true' - - - name: Check out LFS objects - run: git lfs checkout - - - name: Setup node - uses: actions/setup-node@1f8c6b94b26d0feae1e387ca63ccbdc44d27b561 - with: - node-version: 16.13.x - cache: npm - - # Install any additional dependencies *before* downloading the build artifact - - name: Install Heroku client development-only dependency - run: npm install --no-save heroku-client - - # Download the previously built "app.tar" - - name: Download build artifact - uses: dawidd6/action-download-artifact@af92a8455a59214b7b932932f2662fdefbd78126 - with: - workflow: ${{ github.event.workflow_run.workflow_id }} - run_id: ${{ env.BUILD_ACTIONS_RUN_ID }} - name: pr_build - path: ${{ runner.temp }} - - # gzip the app.tar to meet Heroku's expected format - - name: Create a gzipped archive (docs) - run: gzip -9 < "$RUNNER_TEMP/app.tar" > app.tar.gz - - - name: Create a Heroku build source - id: build-source - uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d - env: - HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }} - with: - script: | - const { owner, repo } = context.repo - - if (owner !== 'github') { - throw new Error(`Repository owner must be 'github' but was: ${owner}`) - } - if (repo !== 'docs') { - throw new Error(`Repository name must be 'docs' but was: ${repo}`) - } - - const Heroku = require('heroku-client') - const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN }) - - try { - const { source_blob: sourceBlob } = await heroku.post('/sources') - const { put_url: uploadUrl, get_url: downloadUrl } = sourceBlob - - core.setOutput('upload_url', uploadUrl) - core.setOutput('download_url', downloadUrl) - } catch (error) { - if (error.statusCode === 503) { - console.error('💀 Heroku may be down! Please check its Status page: https://status.heroku.com/') - } - throw error - } - - # See: https://devcenter.heroku.com/articles/build-and-release-using-the-api#sources-endpoint - - name: Upload to the Heroku build source - env: - UPLOAD_URL: ${{ steps.build-source.outputs.upload_url }} - run: | - curl "$UPLOAD_URL" \ - -X PUT \ - -H 'Content-Type:' \ - --data-binary @app.tar.gz - - - name: Create failure status - uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d - if: ${{ failure() }} - env: - CONTEXT_NAME: ${{ env.CONTEXT_NAME }} - ACTIONS_RUN_LOG: ${{ env.ACTIONS_RUN_LOG }} - HEAD_SHA: ${{ needs.pr-metadata.outputs.head_sha }} - with: - script: | - const { CONTEXT_NAME, ACTIONS_RUN_LOG, HEAD_SHA } = process.env - const { owner, repo } = context.repo - await github.repos.createCommitStatus({ - owner, - repo, - sha: HEAD_SHA, - context: CONTEXT_NAME, - state: 'error', - description: 'Failed to deploy. See logs.', - target_url: ACTIONS_RUN_LOG - }) - - - name: Send Slack notification if deployment preparation job failed - uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340 - if: ${{ failure() }} - with: - channel: ${{ secrets.DOCS_STAGING_DEPLOYMENT_FAILURES_SLACK_CHANNEL_ID }} - bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }} - color: failure - text: Staging preparation failed for PR ${{ needs.pr-metadata.outputs.url }} at commit ${{ needs.pr-metadata.outputs.head_sha }}. See ${{ env.ACTIONS_RUN_LOG }}. - - check-pr-before-deploy: - needs: [pr-metadata, prepare-for-deploy] - runs-on: ubuntu-latest - timeout-minutes: 1 - # This interrupts Build and Deploy workflow runs in progress for this PR branch. - concurrency: - group: 'PR Staging @ ${{ needs.pr-metadata.outputs.head_label }}' - cancel-in-progress: true - outputs: - pull_request_state: ${{ steps.check-pr.outputs.state }} - steps: - - name: Check pull request state - id: check-pr - uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d - env: - PR_NUMBER: ${{ needs.pr-metadata.outputs.number }} - with: - script: | - const { owner, repo } = context.repo - const { data: pullRequest } = await github.pulls.get({ - owner, - repo, - pull_number: process.env.PR_NUMBER - }) - core.setOutput('state', pullRequest.state) - - deploy: - needs: [pr-metadata, prepare-for-deploy, check-pr-before-deploy] - if: ${{ needs.check-pr-before-deploy.outputs.pull_request_state == 'open' }} - runs-on: ubuntu-latest - timeout-minutes: 10 - # This interrupts Build and Deploy workflow runs in progress for this PR branch. - concurrency: - group: 'PR Staging @ ${{ needs.pr-metadata.outputs.head_label }}' - cancel-in-progress: true - steps: - - name: Check out repo's default branch - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - - - name: Setup node - uses: actions/setup-node@1f8c6b94b26d0feae1e387ca63ccbdc44d27b561 - with: - node-version: 16.13.x - cache: npm - - - name: Install dependencies - run: npm ci - - - name: Deploy - id: deploy - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HEROKU_API_TOKEN: ${{ secrets.HEROKU_API_TOKEN }} - HYDRO_ENDPOINT: ${{ secrets.HYDRO_ENDPOINT }} - HYDRO_SECRET: ${{ secrets.HYDRO_SECRET }} - PR_URL: ${{ needs.pr-metadata.outputs.url }} - SOURCE_BLOB_URL: ${{ needs.prepare-for-deploy.outputs.source_blob_url }} - ALLOWED_POLLING_FAILURES_PER_PHASE: '15' - RUN_ID: ${{ github.run_id }} - run: .github/actions-scripts/staging-deploy.js - - - name: Create successful commit status - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CONTEXT_NAME: ${{ env.CONTEXT_NAME }} - ACTIONS_RUN_LOG: ${{ env.ACTIONS_RUN_LOG }} - HEAD_SHA: ${{ needs.pr-metadata.outputs.head_sha }} - run: .github/actions-scripts/staging-commit-status-success.js - - - name: Mark the deployment as inactive if timed out - uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d - if: ${{ steps.deploy.outcome == 'cancelled' }} - env: - DEPLOYMENT_ID: ${{ steps.deploy.outputs.deploymentId }} - LOG_URL: ${{ steps.deploy.outputs.logUrl }} - with: - script: | - const { DEPLOYMENT_ID, LOG_URL } = process.env - const { owner, repo } = context.repo - - if (!DEPLOYMENT_ID) { - throw new Error('A deployment wasn't created before a timeout occurred!') - } - - await github.repos.createDeploymentStatus({ - owner, - repo, - deployment_id: DEPLOYMENT_ID, - state: 'error', - description: 'The deployment step timed out. See workflow logs.', - log_url: LOG_URL, - // The 'ant-man' preview is required for `state` values of 'inactive', as well as - // the use of the `log_url`, `environment_url`, and `auto_inactive` parameters. - // The 'flash' preview is required for `state` values of 'in_progress' and 'queued'. - mediaType: { - previews: ['ant-man', 'flash'], - }, - }) - console.log('⏲️ Deployment status: error - The deployment timed out...') - - - name: Create failure status - uses: actions/github-script@2b34a689ec86a68d8ab9478298f91d5401337b7d - if: ${{ failure() }} - env: - CONTEXT_NAME: ${{ env.CONTEXT_NAME }} - ACTIONS_RUN_LOG: ${{ env.ACTIONS_RUN_LOG }} - HEAD_SHA: ${{ needs.pr-metadata.outputs.head_sha }} - with: - script: | - const { CONTEXT_NAME, ACTIONS_RUN_LOG, HEAD_SHA } = process.env - const { owner, repo } = context.repo - await github.repos.createCommitStatus({ - owner, - repo, - sha: HEAD_SHA, - context: CONTEXT_NAME, - state: 'error', - description: 'Failed to deploy. See logs.', - target_url: ACTIONS_RUN_LOG - }) - - - name: Send Slack notification if deployment job failed - uses: someimportantcompany/github-actions-slack-message@f8d28715e7b8a4717047d23f48c39827cacad340 - if: ${{ failure() }} - with: - channel: ${{ secrets.DOCS_STAGING_DEPLOYMENT_FAILURES_SLACK_CHANNEL_ID }} - bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }} - color: failure - text: Staging deployment failed for PR ${{ needs.pr-metadata.outputs.url }} at commit ${{ needs.pr-metadata.outputs.head_sha }}. See ${{ env.ACTIONS_RUN_LOG }}. diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml deleted file mode 100644 index 52d77dd42c..0000000000 --- a/.github/workflows/test-windows.yml +++ /dev/null @@ -1,76 +0,0 @@ -# NOTE: Changes to this file should also be applied to './test.yml' - -name: Node.js Tests - Windows - -# **What it does**: This runs our tests on Windows. -# **Why we have it**: We want to support Windows contributors to docs. -# **Who does it impact**: Anyone working on docs on a Windows device. - -on: - workflow_dispatch: - pull_request: - schedule: - - cron: '50 19 * * *' # once a day at 19:50 UTC / 11:50 PST - -permissions: - contents: read - -# This allows a subsequently queued workflow run to interrupt previous runs -concurrency: - group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' - cancel-in-progress: true - -jobs: - test: - runs-on: windows-latest - if: (github.event_name != 'pull_request') || (github.event_name == 'pull_request' && (contains(github.event.pull_request.labels.*.name, 'Windows') || contains(github.event.pull_request.labels.*.name, 'windows'))) - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - test-group: - [ - content, - graphql, - meta, - rendering, - routing, - unit, - linting, - translations, - ] - steps: - - name: Check out repo - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 - with: - # Enables cloning the Early Access repo later with the relevant PAT - persist-credentials: 'false' - - - name: Setup node - uses: actions/setup-node@1f8c6b94b26d0feae1e387ca63ccbdc44d27b561 - with: - node-version: 16.13.x - cache: npm - - - name: Install dependencies - run: npm ci - - - name: Cache nextjs build - uses: actions/cache@937d24475381cd9c75ae6db12cb4e79714b926ed - with: - path: .next/cache - key: ${{ runner.os }}-nextjs-${{ hashFiles('package*.json') }} - - - if: ${{ github.repository == 'github/docs-internal' }} - name: Clone early access - run: npm run heroku-postbuild - env: - DOCUBOT_REPO_PAT: ${{ secrets.DOCUBOT_REPO_PAT }} - GIT_BRANCH: ${{ github.head_ref || github.ref }} - - - if: ${{ github.repository != 'github/docs-internal' }} - name: Run build script - run: npm run build - - - name: Run tests - run: npm test -- tests/${{ matrix.test-group }}/ diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b080e73df7..1db6685fae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,3 @@ -# NOTE: Changes to this file should also be applied to './test-windows.yml' - name: Node.js Tests # **What it does**: Runs our tests. diff --git a/.github/workflows/triage-unallowed-contributions.yml b/.github/workflows/triage-unallowed-contributions.yml index 0cb30cdd6f..caae155b4a 100644 --- a/.github/workflows/triage-unallowed-contributions.yml +++ b/.github/workflows/triage-unallowed-contributions.yml @@ -10,7 +10,6 @@ on: - '.github/actions-scripts/**' - '.github/workflows/**' - '.github/CODEOWNERS' - - 'app.json' - 'assets/fonts/**' - 'data/graphql/**' - 'Dockerfile*' @@ -20,7 +19,6 @@ on: - 'lib/webhooks/**' - 'lib/search/indexes/**' - 'package*.json' - - 'Procfile' - 'script/**' - 'translations/**' @@ -58,7 +56,6 @@ jobs: - '.github/actions-scripts/**' - '.github/workflows/**' - '.github/CODEOWNERS' - - 'app.json' - 'assets/fonts/**' - 'data/graphql/**' - 'Dockerfile*' @@ -68,7 +65,6 @@ jobs: - 'lib/webhooks/**' - 'lib/search/indexes/**' - 'package*.json' - - 'Procfile' - 'scripts/**' - 'translations/**' @@ -83,7 +79,6 @@ jobs: '.github/actions-scripts/**', '.github/workflows/**', '.github/CODEOWNERS', - 'app.json', 'assets/fonts/**', 'data/graphql/**', 'Dockerfile*', @@ -93,7 +88,6 @@ jobs: 'lib/webhooks/**', 'lib/search/indexes/**', 'package*.json', - 'Procfile', 'scripts/**', 'translations/**', ] @@ -111,7 +105,7 @@ jobs: body: reviewMessage, }) - workflowFailMessage = `${workflowFailMessage} Please see ${createdComment.data.html_url} for details.` + workflowFailMessage = `${workflowFailMessage} Please see ${createdComment.data.html_url} for details.` } catch(err) { console.log("Error creating comment.", err) } diff --git a/.github/workflows/triage-unallowed-internal-changes.yml b/.github/workflows/triage-unallowed-internal-changes.yml index f839c016e9..4a505b2afc 100644 --- a/.github/workflows/triage-unallowed-internal-changes.yml +++ b/.github/workflows/triage-unallowed-internal-changes.yml @@ -1,7 +1,7 @@ name: Check for unallowed internal changes -# **What it does**: If someone changes app.json or search indexes, we fail the check. -# **Why we have it**: app.json should rarely be edited, so we'll require an admin merge if the file really needs to be changed. The search indexes are synced every 4 hours, so changes should not need to be made. +# **What it does**: If someone changes search indexes, we fail the check. +# **Why we have it**: The search indexes are synced every 4 hours, so changes should not need to be made. # **Who does it impact**: Docs engineering and content writers. on: @@ -44,19 +44,8 @@ jobs: # Returns list of changed files matching each filter filters: | - notAllowed: - - 'app.json' notAllowedSearchSyncLabel: - 'lib/search/indexes/**' - notAllowed: - needs: check-internal-changes - if: ${{ needs.check-internal-changes.outputs.notAllowed == 'true' }} - runs-on: ubuntu-latest - steps: - - name: Fail if unallowed changes were made - run: | - echo "Please admin merge if you really need to update app.json!" - exit 1 notAllowedSearchSyncLabel: needs: check-internal-changes if: ${{ needs.check-internal-changes.outputs.notAllowedSearchSyncLabel == 'true' }} diff --git a/Dockerfile b/Dockerfile index 985bde3af7..cad07f06c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,4 @@ -# This Dockerfile can be used for docker-based deployments to platforms -# like Now or Moda, but it is currently _not_ used by our Heroku deployments -# It uses two multi-stage builds: `install` and the main build to keep the image size down. +# This Dockerfile is used for docker-based deployments to Azure for both preview environments and production # -------------------------------------------------------------------------------- # BASE IMAGE diff --git a/Procfile b/Procfile deleted file mode 100644 index 4bec004287..0000000000 --- a/Procfile +++ /dev/null @@ -1 +0,0 @@ -web: NODE_ENV=production node server.mjs diff --git a/app.json b/app.json deleted file mode 100644 index 14f8ba9f54..0000000000 --- a/app.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "docs.github.com", - "env": { - "NODE_ENV": "production", - "ENABLED_LANGUAGES": "en", - "WEB_CONCURRENCY": "1" - }, - "buildpacks": [ - { "url": "heroku/nodejs" } - ], - "formation": { - "web": { - "quantity": 1, - "size": "standard-2x" - } - } -} diff --git a/package-lock.json b/package-lock.json index ffb2aff60c..785d7798dc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -135,7 +135,6 @@ "git-diff": "^2.0.6", "glob": "^7.2.0", "graphql": "^16.3.0", - "heroku-client": "^3.1.0", "http-status-code": "^2.1.0", "husky": "^7.0.4", "japanese-characters": "^1.1.0", @@ -10576,19 +10575,6 @@ "node": ">=10.0.0" } }, - "node_modules/heroku-client": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/heroku-client/-/heroku-client-3.1.0.tgz", - "integrity": "sha512-UfGKwUm5duzzSVI8uUXlNAE1mus6uPxmZPji4vuG1ArV5DYL1rXsZShp0OoxraWdEwYoxCUrM6KGztC68x5EZQ==", - "dev": true, - "dependencies": { - "is-retry-allowed": "^1.0.0", - "tunnel-agent": "^0.6.0" - }, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/hexoid": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz", @@ -11433,15 +11419,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-retry-allowed": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", - "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/is-shared-array-buffer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.1.tgz", @@ -21037,18 +21014,6 @@ "node": ">=0.6.11 <=0.7.0 || >=0.7.3" } }, - "node_modules/tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "dev": true, - "dependencies": { - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": "*" - } - }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -30703,16 +30668,6 @@ "resolved": "https://registry.npmjs.org/helmet/-/helmet-4.6.0.tgz", "integrity": "sha512-HVqALKZlR95ROkrnesdhbbZJFi/rIVSoNq6f3jA/9u6MIbTsPh3xZwihjeI5+DO/2sOV6HMHooXcEOuwskHpTg==" }, - "heroku-client": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/heroku-client/-/heroku-client-3.1.0.tgz", - "integrity": "sha512-UfGKwUm5duzzSVI8uUXlNAE1mus6uPxmZPji4vuG1ArV5DYL1rXsZShp0OoxraWdEwYoxCUrM6KGztC68x5EZQ==", - "dev": true, - "requires": { - "is-retry-allowed": "^1.0.0", - "tunnel-agent": "^0.6.0" - } - }, "hexoid": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz", @@ -31292,12 +31247,6 @@ "has-tostringtag": "^1.0.0" } }, - "is-retry-allowed": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", - "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==", - "dev": true - }, "is-shared-array-buffer": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.1.tgz", @@ -38575,15 +38524,6 @@ "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", "dev": true }, - "tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "dev": true, - "requires": { - "safe-buffer": "^5.0.1" - } - }, "type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index 59dba3dcf7..d4414fcf92 100644 --- a/package.json +++ b/package.json @@ -137,7 +137,6 @@ "git-diff": "^2.0.6", "glob": "^7.2.0", "graphql": "^16.3.0", - "heroku-client": "^3.1.0", "http-status-code": "^2.1.0", "husky": "^7.0.4", "japanese-characters": "^1.1.0", @@ -193,7 +192,6 @@ "build": "next build", "debug": "cross-env NODE_ENV=development ENABLED_LANGUAGES='en,ja' nodemon --inspect server.mjs", "dev": "npm start", - "heroku-postbuild": "node script/early-access/clone-for-build.js && npm run build", "lint": "eslint '**/*.{js,mjs,ts,tsx}'", "lint-translation": "cross-env NODE_OPTIONS=--experimental-vm-modules TEST_TRANSLATION=true jest tests/linting/lint-files.js", "pa11y-ci": "pa11y-ci", diff --git a/script/deployment/create-staging-app-name.js b/script/deployment/create-staging-app-name.js deleted file mode 100644 index cee33514bc..0000000000 --- a/script/deployment/create-staging-app-name.js +++ /dev/null @@ -1,18 +0,0 @@ -import GithubSlugger from 'github-slugger' -const slugify = GithubSlugger.slug - -const APP_NAME_MAX_LENGTH = 30 - -export default function ({ prefix = '', repo, pullNumber, branch }) { - return ( - `${prefix}${repo}-${pullNumber}--${slugify(branch)}` - // Shorten the string to the max allowed length - .slice(0, APP_NAME_MAX_LENGTH) - // Convert underscores to dashes - .replace(/_/g, '-') - // Remove trailing dashes - .replace(/-+$/, '') - // Make it all lowercase - .toLowerCase() - ) -} diff --git a/script/deployment/deploy-to-production.js b/script/deployment/deploy-to-production.js deleted file mode 100755 index b23482bc98..0000000000 --- a/script/deployment/deploy-to-production.js +++ /dev/null @@ -1,465 +0,0 @@ -#!/usr/bin/env node -import got from 'got' -import Heroku from 'heroku-client' -import { setOutput } from '@actions/core' - -const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) - -const SLEEP_INTERVAL = 5000 -const HEROKU_LOG_LINES_TO_SHOW = 25 -const DELAY_FOR_PREBOOT_SWAP = 135000 // 2:15 - -// Allow for a few 404 (Not Found), 429 (Too Many Requests), etc. responses from -// the semi-unreliable Heroku API when we're polling for status updates -const ALLOWED_MISSING_RESPONSE_COUNT = - parseInt(process.env.ALLOWED_POLLING_FAILURES_PER_PHASE, 10) || 10 -const ALLOWABLE_ERROR_CODES = [404, 429, 500, 503] - -export default async function deployToProduction({ - octokit, - includeDelayForPreboot = true, - // These parameters will only be set by Actions - sourceBlobUrl = null, - runId = null, -}) { - // Start a timer so we can report how long the deployment takes - const startTime = Date.now() - const [owner, repo, branch] = ['github', 'docs-internal', 'main'] - - let sha - try { - const { - data: { sha: latestSha }, - } = await octokit.repos.getCommit({ - owner, - repo, - ref: branch, - }) - sha = latestSha - - if (!sha) { - throw new Error('Latest commit SHA could not be found') - } - } catch (error) { - console.error(`Error: ${error}`) - console.log(`🛑 There was an error getting latest commit.`) - process.exit(1) - } - - // Put together application configuration variables - const isPrebuilt = !!sourceBlobUrl - const { DOCUBOT_REPO_PAT } = process.env - const appConfigVars = { - // Track the git branch - GIT_BRANCH: branch, - // If prebuilt: prevent the Heroku Node.js buildpack from installing devDependencies - NPM_CONFIG_PRODUCTION: isPrebuilt.toString(), - // If prebuilt: prevent the Heroku Node.js buildpack from using `npm ci` as it would - // delete all of the vendored "node_modules/" directory. - USE_NPM_INSTALL: isPrebuilt.toString(), - // If not prebuilt, include the PAT required for cloning the `docs-early-access` repo. - // Otherwise, set it to `null` to unset it from the environment for security. - DOCUBOT_REPO_PAT: (!isPrebuilt && DOCUBOT_REPO_PAT) || null, - } - - const workflowRunLog = runId ? `https://github.com/${owner}/${repo}/actions/runs/${runId}` : null - let deploymentId = null - let logUrl = workflowRunLog - - const appName = process.env.HEROKU_PRODUCTION_APP_NAME - const environment = 'production' - const homepageUrl = 'https://docs.github.com/' - - try { - const title = `branch '${branch}' at commit '${sha}' in the '${environment}' environment` - - console.log(`About to deploy ${title}...`) - - // Kick off a pending GitHub Deployment right away, so the PR author - // will have instant feedback that their work is being deployed. - const { data: deployment } = await octokit.repos.createDeployment({ - owner, - repo, - description: `Deploying ${title}`, - ref: sha, - - // In the GitHub API, there can only be one active deployment per environment. - environment, - - // The status contexts to verify against commit status checks. If you omit - // this parameter, GitHub verifies all unique contexts before creating a - // deployment. To bypass checking entirely, pass an empty array. Defaults - // to all unique contexts. - required_contexts: [], - - // Do not try to merge the base branch into the feature branch - auto_merge: false, - }) - console.log('GitHub Deployment created', deployment) - - // Store this ID for later updating - deploymentId = deployment.id - - // Set some output variables for workflow steps that run after this script - if (process.env.GITHUB_ACTIONS) { - setOutput('deploymentId', deploymentId) - setOutput('logUrl', logUrl) - } - - await octokit.repos.createDeploymentStatus({ - owner, - repo, - deployment_id: deploymentId, - state: 'in_progress', - description: 'Deploying the app...', - // The 'ant-man' preview is required for `state` values of 'inactive', as well as - // the use of the `log_url`, `environment_url`, and `auto_inactive` parameters. - // The 'flash' preview is required for `state` values of 'in_progress' and 'queued'. - mediaType: { - previews: ['ant-man', 'flash'], - }, - }) - console.log('🚀 Deployment status: in_progress - Preparing to deploy the app...') - - // Time to talk to Heroku... - const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN }) - let build = null - - if (!sourceBlobUrl) { - try { - sourceBlobUrl = await getTarballUrl({ - octokit, - owner, - repo, - sha, - }) - } catch (error) { - throw new Error(`Failed to generate source blob URL. Error: ${error}`) - } - } - - console.log('Updating Heroku app configuration variables...') - - // Reconfigure environment variables - // https://devcenter.heroku.com/articles/platform-api-reference#config-vars-update - try { - await heroku.patch(`/apps/${appName}/config-vars`, { - body: appConfigVars, - }) - } catch (error) { - announceIfHerokuIsDown(error) - throw new Error(`Failed to update Heroku app configuration variables. Error: ${error}`) - } - - console.log('Reconfigured') - console.log('Building Heroku app...') - - try { - build = await heroku.post(`/apps/${appName}/builds`, { - body: { - source_blob: { - url: sourceBlobUrl, - }, - }, - }) - } catch (error) { - announceIfHerokuIsDown(error) - throw new Error(`Failed to create Heroku build. Error: ${error}`) - } - - console.log('Heroku build created', build) - - const buildStartTime = Date.now() // Close enough... - const buildId = build.id - logUrl = build.output_stream_url - - console.log('🚀 Deployment status: in_progress - Building a new Heroku slug...') - - // Poll until the Build's status changes from "pending" to "succeeded" or "failed". - let buildAcceptableErrorCount = 0 - while (!build || !build.release || !build.release.id) { - await sleep(SLEEP_INTERVAL) - try { - build = await heroku.get(`/apps/${appName}/builds/${buildId}`) - } catch (error) { - // Allow for a few bad responses from the Heroku API - if (isAllowableHerokuError(error)) { - buildAcceptableErrorCount += 1 - if (buildAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { - console.warn( - `Ignoring allowable Heroku error #${buildAcceptableErrorCount}: ${error.statusCode}` - ) - continue - } - } - announceIfHerokuIsDown(error) - throw new Error(`Failed to get build status. Error: ${error}`) - } - - if (build && build.status === 'failed') { - throw new Error( - `Failed to build after ${Math.round( - (Date.now() - buildStartTime) / 1000 - )} seconds. See Heroku logs for more information:\n${logUrl}` - ) - } - - console.log( - `Heroku build status: ${(build || {}).status} (after ${Math.round( - (Date.now() - buildStartTime) / 1000 - )} seconds)` - ) - } - - console.log( - `Finished Heroku build after ${Math.round((Date.now() - buildStartTime) / 1000)} seconds.`, - build - ) - console.log('Heroku release detected', build.release) - - const releaseStartTime = Date.now() // Close enough... - const releaseId = build.release.id - let release = null - - // Poll until the associated Release's status changes from "pending" to "succeeded" or "failed". - let releaseAcceptableErrorCount = 0 - while (!release || release.status === 'pending') { - await sleep(SLEEP_INTERVAL) - try { - const result = await heroku.get(`/apps/${appName}/releases/${releaseId}`) - - // Update the deployment status but only on the first retrieval - if (!release) { - logUrl = result.output_stream_url - - console.log('Heroku Release created', result) - - console.log('🚀 Deployment status: in_progress - Releasing the built Heroku slug...') - } - - release = result - } catch (error) { - // Allow for a few bad responses from the Heroku API - if (isAllowableHerokuError(error)) { - releaseAcceptableErrorCount += 1 - if (releaseAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { - console.warn( - `Ignoring allowable Heroku error #${releaseAcceptableErrorCount}: ${error.statusCode}` - ) - continue - } - } - announceIfHerokuIsDown(error) - throw new Error(`Failed to get release status. Error: ${error}`) - } - - if (release && release.status === 'failed') { - throw new Error( - `Failed to release after ${Math.round( - (Date.now() - releaseStartTime) / 1000 - )} seconds. See Heroku logs for more information:\n${logUrl}` - ) - } - - console.log( - `Release status: ${(release || {}).status} (after ${Math.round( - (Date.now() - releaseStartTime) / 1000 - )} seconds)` - ) - } - - console.log( - `Finished Heroku release after ${Math.round( - (Date.now() - releaseStartTime) / 1000 - )} seconds.`, - release - ) - - // Monitor dyno state for this release to ensure it reaches "up" rather than crashing. - // This will help us catch issues with faulty startup code and/or the package manifest. - const dynoBootStartTime = Date.now() - console.log('Checking Heroku dynos...') - logUrl = workflowRunLog - - console.log('🚀 Deployment status: in_progress - Monitoring the Heroku dyno start-up...') - - // Keep checking while there are still dynos in non-terminal states - let newDynos = [] - let dynoAcceptableErrorCount = 0 - while (newDynos.length === 0 || newDynos.some((dyno) => dyno.state === 'starting')) { - await sleep(SLEEP_INTERVAL) - try { - const dynoList = await heroku.get(`/apps/${appName}/dynos`) - const dynosForThisRelease = dynoList.filter((dyno) => dyno.release.id === releaseId) - - // To track them afterward - newDynos = dynosForThisRelease - - console.log( - `Dyno states: ${JSON.stringify(newDynos.map((dyno) => dyno.state))} (after ${Math.round( - (Date.now() - dynoBootStartTime) / 1000 - )} seconds)` - ) - } catch (error) { - // Allow for a few bad responses from the Heroku API - if (isAllowableHerokuError(error)) { - dynoAcceptableErrorCount += 1 - if (dynoAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { - console.warn( - `Ignoring allowable Heroku error #${dynoAcceptableErrorCount}: ${error.statusCode}` - ) - continue - } - } - announceIfHerokuIsDown(error) - throw new Error(`Failed to find dynos for this release. Error: ${error}`) - } - } - - const crashedDynos = newDynos.filter((dyno) => ['crashed', 'restarting'].includes(dyno.state)) - const runningDynos = newDynos.filter((dyno) => dyno.state === 'up') - - // If any dynos crashed on start-up, fail the deployment - if (crashedDynos.length > 0) { - const errorMessage = `At least ${crashedDynos.length} Heroku dyno(s) crashed on start-up!` - - console.error(errorMessage) - - // Attempt to dump some of the Heroku log here for debugging - try { - const logSession = await heroku.post(`/apps/${appName}/log-sessions`, { - body: { - dyno: crashedDynos[0].name, - lines: HEROKU_LOG_LINES_TO_SHOW, - tail: false, - }, - }) - - logUrl = logSession.logplex_url - - const logText = await got(logUrl).text() - console.error( - `Here are the last ${HEROKU_LOG_LINES_TO_SHOW} lines of the Heroku log:\n\n${logText}` - ) - } catch (error) { - announceIfHerokuIsDown(error) - // Don't fail because of this error - console.error(`Failed to retrieve the Heroku logs for the crashed dynos. Error: ${error}`) - } - - throw new Error(errorMessage) - } - - console.log( - `At least ${runningDynos.length} Heroku dyno(s) are ready after ${Math.round( - (Date.now() - dynoBootStartTime) / 1000 - )} seconds.` - ) - - // IMPORTANT: - // If Heroku Preboot is enabled, then there is an additional delay of at - // least 2 minutes before the new dynos are swapped into active serving. - // If we move off Heroku in the future, this should be revisited and - // updated/removed as relevant to align with the new hosting platform. - if (includeDelayForPreboot) { - console.log(`Waiting for Heroku Preboot to swap dynos (${DELAY_FOR_PREBOOT_SWAP} ms)...`) - await sleep(DELAY_FOR_PREBOOT_SWAP) - - // TODO: - // Is there a faster alternative than this arbitrary delay? For example, - // is there some Heroku API we can query to see when this release is - // considered to be the live one, or when the old dynos are shut down? - } else { - console.warn( - '⚠️ Bypassing the wait for Heroku Preboot....\nPlease understand that your changes will not be visible for at least another 2 minutes!' - ) - } - - // Report success! - const successMessage = `Deployment succeeded after ${Math.round( - (Date.now() - startTime) / 1000 - )} seconds.` - console.log(successMessage) - - await octokit.repos.createDeploymentStatus({ - owner, - repo, - deployment_id: deploymentId, - state: 'success', - description: successMessage, - ...(logUrl && { log_url: logUrl }), - environment_url: homepageUrl, - // The 'ant-man' preview is required for `state` values of 'inactive', as well as - // the use of the `log_url`, `environment_url`, and `auto_inactive` parameters. - // The 'flash' preview is required for `state` values of 'in_progress' and 'queued'. - mediaType: { - previews: ['ant-man', 'flash'], - }, - }) - - console.log(`🚀 Deployment status: success - ${successMessage}`) - console.log(`Visit the newly deployed app at: ${homepageUrl}`) - } catch (error) { - // Report failure! - const failureMessage = `Deployment failed after ${Math.round( - (Date.now() - startTime) / 1000 - )} seconds. See logs for more information.` - console.error(failureMessage) - - try { - if (deploymentId) { - await octokit.repos.createDeploymentStatus({ - owner, - repo, - deployment_id: deploymentId, - state: 'error', - description: failureMessage, - ...(logUrl && { log_url: logUrl }), - environment_url: homepageUrl, - // The 'ant-man' preview is required for `state` values of 'inactive', as well as - // the use of the `log_url`, `environment_url`, and `auto_inactive` parameters. - // The 'flash' preview is required for `state` values of 'in_progress' and 'queued'. - mediaType: { - previews: ['ant-man', 'flash'], - }, - }) - - console.log( - `🚀 Deployment status: error - ${failureMessage}` + (logUrl ? ` Logs: ${logUrl}` : '') - ) - } - } catch (error) { - console.error(`Failed to finalize GitHub Deployment Status as a failure. Error: ${error}`) - } - - // Re-throw the error to bubble up - throw error - } -} - -async function getTarballUrl({ octokit, owner, repo, sha }) { - // Get a URL for the tarballed source code bundle - const { - headers: { location: tarballUrl }, - } = await octokit.repos.downloadTarballArchive({ - owner, - repo, - ref: sha, - // Override the underlying `node-fetch` module's `redirect` option - // configuration to prevent automatically following redirects. - request: { - redirect: 'manual', - }, - }) - return tarballUrl -} - -function isAllowableHerokuError(error) { - return error && ALLOWABLE_ERROR_CODES.includes(error.statusCode) -} - -function announceIfHerokuIsDown(error) { - if (error && error.statusCode === 503) { - console.error('💀 Heroku may be down! Please check its Status page: https://status.heroku.com/') - } -} diff --git a/script/deployment/deploy-to-staging.js b/script/deployment/deploy-to-staging.js deleted file mode 100644 index 5e3df9a09b..0000000000 --- a/script/deployment/deploy-to-staging.js +++ /dev/null @@ -1,680 +0,0 @@ -#!/usr/bin/env node -import got from 'got' -import Heroku from 'heroku-client' -import { setOutput } from '@actions/core' -import createStagingAppName from './create-staging-app-name.js' - -// Equivalent of the 'await-sleep' module without the install -const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) - -const SLEEP_INTERVAL = 5000 -const HEROKU_LOG_LINES_TO_SHOW = 25 - -// Allow for a few 404 (Not Found), 429 (Too Many Requests), etc. responses from -// the semi-unreliable Heroku API when we're polling for status updates -const ALLOWED_MISSING_RESPONSE_COUNT = - parseInt(process.env.ALLOWED_POLLING_FAILURES_PER_PHASE, 10) || 10 -const ALLOWABLE_ERROR_CODES = [404, 429, 500, 503] - -export default async function deployToStaging({ - octokit, - pullRequest, - forceRebuild = false, - // These parameters will only be set by Actions - sourceBlobUrl = null, - runId = null, -}) { - // Start a timer so we can report how long the deployment takes - const startTime = Date.now() - - // Extract some important properties from the PR - const { - number: pullNumber, - base: { - repo: { - name: repo, - owner: { login: owner }, - }, - }, - state, - head: { ref: branch, sha }, - user: author, - } = pullRequest - - // Verify the PR is still open - if (state !== 'open') { - throw new Error(`This pull request is not open. State is: '${state}'`) - } - - // Put together application configuration variables - const isPrivateRepo = owner === 'github' && repo === 'docs-internal' - const isPrebuilt = !!sourceBlobUrl - const { DOCUBOT_REPO_PAT, HYDRO_ENDPOINT, HYDRO_SECRET } = process.env - const appConfigVars = { - // Track the git branch - GIT_BRANCH: branch, - // If prebuilt: prevent the Heroku Node.js buildpack from installing devDependencies - NPM_CONFIG_PRODUCTION: isPrebuilt.toString(), - // If prebuilt: prevent the Heroku Node.js buildpack from using `npm ci` as it would - // delete all of the vendored "node_modules/" directory. - USE_NPM_INSTALL: isPrebuilt.toString(), - // IMPORTANT: This secret should only be set in the private repo! - // If not prebuilt, include the PAT required for cloning the `docs-early-access` repo. - // Otherwise, set it to `null` to unset it from the environment for security. - DOCUBOT_REPO_PAT: (isPrivateRepo && !isPrebuilt && DOCUBOT_REPO_PAT) || null, - // IMPORTANT: These secrets should only be set in the private repo! - // These are required for Hydro event tracking - ...(isPrivateRepo && HYDRO_ENDPOINT && HYDRO_SECRET && { HYDRO_ENDPOINT, HYDRO_SECRET }), - } - - const workflowRunLog = runId ? `https://github.com/${owner}/${repo}/actions/runs/${runId}` : null - let deploymentId = null - let logUrl = workflowRunLog - let appIsNewlyCreated = false - - const appName = createStagingAppName({ repo, pullNumber, branch }) - const environment = appName - const homepageUrl = `https://${appName}.herokuapp.com/` - - try { - const title = `branch '${branch}' at commit '${sha}' in the '${environment}' staging environment` - - console.log(`About to deploy ${title}...`) - - // Kick off a pending GitHub Deployment right away, so the PR author - // will have instant feedback that their work is being deployed. - const { data: deployment } = await octokit.repos.createDeployment({ - owner, - repo, - - description: `Deploying ${title}`, - - // Use a commit SHA instead of a branch name as the ref for more precise - // feedback, and also because the branch may have already been deleted. - ref: sha, - - // In the GitHub API, there can only be one active deployment per environment. - // For our many staging apps, we must use the unique appName as the environment. - environment, - - // The status contexts to verify against commit status checks. If you omit - // this parameter, GitHub verifies all unique contexts before creating a - // deployment. To bypass checking entirely, pass an empty array. Defaults - // to all unique contexts. - required_contexts: [], - - // Do not try to merge the base branch into the feature branch - auto_merge: false, - }) - console.log('GitHub Deployment created', deployment) - - // Store this ID for later updating - deploymentId = deployment.id - - // Set some output variables for workflow steps that run after this script - if (process.env.GITHUB_ACTIONS) { - setOutput('deploymentId', deploymentId) - setOutput('logUrl', logUrl) - } - - await octokit.repos.createDeploymentStatus({ - owner, - repo, - deployment_id: deploymentId, - state: 'in_progress', - description: 'Deploying the app...', - // The 'ant-man' preview is required for `state` values of 'inactive', as well as - // the use of the `log_url`, `environment_url`, and `auto_inactive` parameters. - // The 'flash' preview is required for `state` values of 'in_progress' and 'queued'. - mediaType: { - previews: ['ant-man', 'flash'], - }, - }) - console.log('🚀 Deployment status: in_progress - Preparing to deploy the app...') - - // Time to talk to Heroku... - const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN }) - let appSetup = null - let build = null - - // Is there already a Heroku App for this PR? - let appExists = true - try { - await heroku.get(`/apps/${appName}`) - } catch (error) { - announceIfHerokuIsDown(error) - appExists = false - } - - // If there is an existing app but we want to forcibly rebuild, delete the app first - if (appExists && forceRebuild) { - console.log('🚀 Deployment status: in_progress - Destroying existing Heroku app...') - - try { - await heroku.delete(`/apps/${appName}`) - appExists = false - - console.log(`Heroku app '${appName}' deleted for forced rebuild`) - } catch (error) { - announceIfHerokuIsDown(error) - throw new Error( - `Failed to delete Heroku app '${appName}' for forced rebuild. Error: ${error}` - ) - } - } - - if (!sourceBlobUrl) { - try { - sourceBlobUrl = await getTarballUrl({ - octokit, - owner, - repo, - sha, - }) - } catch (error) { - throw new Error(`Failed to generate source blob URL. Error: ${error}`) - } - } - - // If an app does not exist, create one! - // This action will also trigger a build as a by-product. - if (!appExists) { - appIsNewlyCreated = true - - console.log(`Heroku app '${appName}' does not exist. Creating a new AppSetup...`) - - console.log('🚀 Deployment status: in_progress - Creating a new Heroku app...') - - const appSetupStartTime = Date.now() - try { - appSetup = await heroku.post('/app-setups', { - body: { - app: { - name: appName, - }, - source_blob: { - url: sourceBlobUrl, - }, - - // Pass some environment variables to staging apps via Heroku - // config variables. - overrides: { - // AppSetup API cannot handle `null` values for config vars - env: removeEmptyProperties(appConfigVars), - }, - }, - }) - console.log('Heroku AppSetup created', appSetup) - - // This probably will not be available yet - build = appSetup.build - } catch (error) { - announceIfHerokuIsDown(error) - throw new Error(`Failed to create Heroku app '${appName}'. Error: ${error}`) - } - - // Add PR author (if staff) as a collaborator on the new staging app - try { - if (author.site_admin === true) { - await heroku.post(`/apps/${appName}/collaborators`, { - body: { - user: `${author.login}@github.com`, - // We don't want an email invitation for every new staging app - silent: true, - }, - }) - console.log(`Added PR author @${author.login} as a Heroku app collaborator`) - } - } catch (error) { - announceIfHerokuIsDown(error) - // It's fine if this fails, it shouldn't block the app from deploying! - console.warn( - `Warning: failed to add PR author as a Heroku app collaborator. Error: ${error}` - ) - } - - // A new Build is created as a by-product of creating an AppSetup. - // Poll until there is a Build object attached to the AppSetup. - let setupAcceptableErrorCount = 0 - while (!appSetup || !build || !build.id) { - await sleep(SLEEP_INTERVAL) - try { - appSetup = await heroku.get(`/app-setups/${appSetup.id}`) - build = appSetup.build - } catch (error) { - // Allow for a few bad responses from the Heroku API - if (isAllowableHerokuError(error)) { - setupAcceptableErrorCount += 1 - if (setupAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { - console.warn( - `Ignoring allowable Heroku error #${setupAcceptableErrorCount}: ${error.statusCode}` - ) - continue - } - } - announceIfHerokuIsDown(error) - throw new Error(`Failed to get AppSetup status. Error: ${error}`) - } - - if (appSetup && appSetup.status === 'failed') { - const manifestErrors = appSetup.manifest_errors || [] - const hasManifestErrors = Array.isArray(manifestErrors) && manifestErrors.length > 0 - const manifestErrorMessage = hasManifestErrors - ? `\nManifest errors:\n - ${manifestErrors.join('\n - ')}` - : '' - throw new Error( - `Failed to setup app after ${Math.round( - (Date.now() - appSetupStartTime) / 1000 - )} seconds. -Reason: ${appSetup.failure_message}${manifestErrorMessage} -See Heroku logs for more information:\n${logUrl}` - ) - } - - console.log( - `AppSetup status: ${appSetup.status} (after ${Math.round( - (Date.now() - appSetupStartTime) / 1000 - )} seconds)` - ) - } - - console.log('Heroku AppSetup finished', appSetup) - console.log('Heroku build detected', build) - } else { - // If the app does exist, just manually trigger a new build - console.log(`Heroku app '${appName}' already exists.`) - - console.log('Updating Heroku app configuration variables...') - - // Reconfigure environment variables - // https://devcenter.heroku.com/articles/platform-api-reference#config-vars-update - try { - await heroku.patch(`/apps/${appName}/config-vars`, { - body: appConfigVars, - }) - } catch (error) { - announceIfHerokuIsDown(error) - throw new Error(`Failed to update Heroku app configuration variables. Error: ${error}`) - } - - console.log('Reconfigured') - console.log('Building Heroku app...') - - try { - build = await heroku.post(`/apps/${appName}/builds`, { - body: { - source_blob: { - url: sourceBlobUrl, - }, - }, - }) - } catch (error) { - announceIfHerokuIsDown(error) - throw new Error(`Failed to create Heroku build. Error: ${error}`) - } - - console.log('Heroku build created', build) - } - - const buildStartTime = Date.now() // Close enough... - const buildId = build.id - logUrl = build.output_stream_url - - console.log('🚀 Deployment status: in_progress - Building a new Heroku slug...') - - // Poll until the Build's status changes from "pending" to "succeeded" or "failed". - let buildAcceptableErrorCount = 0 - while (!build || !build.release || !build.release.id) { - await sleep(SLEEP_INTERVAL) - try { - build = await heroku.get(`/apps/${appName}/builds/${buildId}`) - } catch (error) { - // Allow for a few bad responses from the Heroku API - if (isAllowableHerokuError(error)) { - buildAcceptableErrorCount += 1 - if (buildAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { - console.warn( - `Ignoring allowable Heroku error #${buildAcceptableErrorCount}: ${error.statusCode}` - ) - continue - } - } - announceIfHerokuIsDown(error) - throw new Error(`Failed to get build status. Error: ${error}`) - } - - if (build && build.status === 'failed') { - throw new Error( - `Failed to build after ${Math.round( - (Date.now() - buildStartTime) / 1000 - )} seconds. See Heroku logs for more information:\n${logUrl}` - ) - } - - console.log( - `Heroku build status: ${(build || {}).status} (after ${Math.round( - (Date.now() - buildStartTime) / 1000 - )} seconds)` - ) - } - - console.log( - `Finished Heroku build after ${Math.round((Date.now() - buildStartTime) / 1000)} seconds.`, - build - ) - console.log('Heroku release detected', build.release) - - const releaseStartTime = Date.now() // Close enough... - let releaseId = build.release.id - let release = null - - // Poll until the associated Release's status changes from "pending" to "succeeded" or "failed". - let releaseAcceptableErrorCount = 0 - while (!release || release.status === 'pending') { - await sleep(SLEEP_INTERVAL) - try { - const result = await heroku.get(`/apps/${appName}/releases/${releaseId}`) - - // Update the deployment status but only on the first retrieval - if (!release) { - logUrl = result.output_stream_url - - console.log('Heroku Release created', result) - - console.log('🚀 Deployment status: in_progress - Releasing the built Heroku slug...') - } - - release = result - } catch (error) { - // Allow for a few bad responses from the Heroku API - if (isAllowableHerokuError(error)) { - releaseAcceptableErrorCount += 1 - if (releaseAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { - console.warn( - `Ignoring allowable Heroku error #${releaseAcceptableErrorCount}: ${error.statusCode}` - ) - continue - } - } - announceIfHerokuIsDown(error) - throw new Error(`Failed to get release status. Error: ${error}`) - } - - if (release && release.status === 'failed') { - throw new Error( - `Failed to release after ${Math.round( - (Date.now() - releaseStartTime) / 1000 - )} seconds. See Heroku logs for more information:\n${logUrl}` - ) - } - - console.log( - `Release status: ${(release || {}).status} (after ${Math.round( - (Date.now() - releaseStartTime) / 1000 - )} seconds)` - ) - } - - console.log( - `Finished Heroku release after ${Math.round( - (Date.now() - releaseStartTime) / 1000 - )} seconds.`, - release - ) - - // Monitor dyno state for this release to ensure it reaches "up" rather than crashing. - // This will help us catch issues with faulty startup code and/or the package manifest. - const dynoBootStartTime = Date.now() - console.log('Checking Heroku dynos...') - logUrl = workflowRunLog - - console.log('🚀 Deployment status: in_progress - Monitoring the Heroku dyno start-up...') - - // Keep checking while there are still dynos in non-terminal states - let newDynos = [] - let dynoAcceptableErrorCount = 0 - while (newDynos.length === 0 || newDynos.some((dyno) => dyno.state === 'starting')) { - await sleep(SLEEP_INTERVAL) - try { - const dynoList = await heroku.get(`/apps/${appName}/dynos`) - const dynosForThisRelease = dynoList.filter((dyno) => dyno.release.id === releaseId) - - // To track them afterward - newDynos = dynosForThisRelease - - // Dynos for this release OR a newer release - const relevantDynos = dynoList.filter((dyno) => dyno.release.version >= release.version) - - // If this Heroku app was just newly created, often a secondary release - // is requested to enable automatically managed SSL certificates. The - // release description will read: - // "Enable allow-multiple-sni-endpoints feature" - // - // If that is the case, we need to update to monitor that secondary - // release instead. - if (relevantDynos.length > 0 && dynosForThisRelease.length === 0) { - // If the app is NOT newly created, fail fast! - if (!appIsNewlyCreated) { - throw new Error('The dynos for this release disappeared unexpectedly') - } - - // Check for the secondary release - let nextRelease = null - try { - nextRelease = await heroku.get(`/apps/${appName}/releases/${release.version + 1}`) - } catch (error) { - announceIfHerokuIsDown(error) - throw new Error( - `Could not find a secondary release to explain the disappearing dynos. Error: ${error}` - ) - } - - if (nextRelease) { - if (nextRelease.description === 'Enable allow-multiple-sni-endpoints feature') { - // Track dynos for the next release instead - release = nextRelease - releaseId = nextRelease.id - - console.warn('Switching to monitor secondary release...') - - // Allow the loop to repeat to fetch the dynos for the secondary release - } else { - // Otherwise, assume another release replaced this one but it - // PROBABLY would've succeeded...? - newDynos.forEach((dyno) => { - dyno.state = 'up' - }) - } - } - // else just keep monitoring and hope for the best - } - - console.log( - `Dyno states: ${JSON.stringify(newDynos.map((dyno) => dyno.state))} (after ${Math.round( - (Date.now() - dynoBootStartTime) / 1000 - )} seconds)` - ) - } catch (error) { - // Allow for a few bad responses from the Heroku API - if (isAllowableHerokuError(error)) { - dynoAcceptableErrorCount += 1 - if (dynoAcceptableErrorCount <= ALLOWED_MISSING_RESPONSE_COUNT) { - console.warn( - `Ignoring allowable Heroku error #${dynoAcceptableErrorCount}: ${error.statusCode}` - ) - continue - } - } - announceIfHerokuIsDown(error) - throw new Error(`Failed to find dynos for this release. Error: ${error}`) - } - } - - const crashedDynos = newDynos.filter((dyno) => ['crashed', 'restarting'].includes(dyno.state)) - const runningDynos = newDynos.filter((dyno) => dyno.state === 'up') - - // If any dynos crashed on start-up, fail the deployment - if (crashedDynos.length > 0) { - const errorMessage = `At least ${crashedDynos.length} Heroku dyno(s) crashed on start-up!` - - console.error(errorMessage) - - // Attempt to dump some of the Heroku log here for debugging - try { - const logSession = await heroku.post(`/apps/${appName}/log-sessions`, { - body: { - dyno: crashedDynos[0].name, - lines: HEROKU_LOG_LINES_TO_SHOW, - tail: false, - }, - }) - - logUrl = logSession.logplex_url - - const logText = await got(logUrl).text() - console.error( - `Here are the last ${HEROKU_LOG_LINES_TO_SHOW} lines of the Heroku log:\n\n${logText}` - ) - } catch (error) { - announceIfHerokuIsDown(error) - // Don't fail because of this error - console.error(`Failed to retrieve the Heroku logs for the crashed dynos. Error: ${error}`) - } - - throw new Error(errorMessage) - } - - console.log( - `At least ${runningDynos.length} Heroku dyno(s) are ready after ${Math.round( - (Date.now() - dynoBootStartTime) / 1000 - )} seconds.` - ) - - // Send a series of requests to trigger the server warmup routines - console.log('🚀 Deployment status: in_progress - Triggering server warmup routines...') - - const warmupStartTime = Date.now() - console.log(`Making warmup requests to: ${homepageUrl}`) - try { - await got(homepageUrl, { - timeout: 10000, // Maximum 10 second timeout per request - retry: { - limit: 7, // About 2 minutes 7 seconds of delay, plus active request time for 8 requests - statusCodes: [404, 421].concat(got.defaults.options.retry.statusCodes), // prepend extras - }, - hooks: { - beforeRetry: [ - (options, error = {}, retryCount = '?') => { - const statusCode = error.statusCode || (error.response || {}).statusCode || -1 - console.log( - `Retrying after warmup request attempt #${retryCount} (${statusCode}) after ${Math.round( - (Date.now() - warmupStartTime) / 1000 - )} seconds...` - ) - }, - ], - }, - }) - console.log( - `Warmup requests passed after ${Math.round((Date.now() - warmupStartTime) / 1000)} seconds` - ) - } catch (error) { - throw new Error( - `Warmup requests failed after ${Math.round( - (Date.now() - warmupStartTime) / 1000 - )} seconds. Error: ${error}` - ) - } - - // Report success! - const successMessage = `Deployment succeeded after ${Math.round( - (Date.now() - startTime) / 1000 - )} seconds.` - console.log(successMessage) - - await octokit.repos.createDeploymentStatus({ - owner, - repo, - deployment_id: deploymentId, - state: 'success', - description: successMessage, - ...(logUrl && { log_url: logUrl }), - environment_url: homepageUrl, - // The 'ant-man' preview is required for `state` values of 'inactive', as well as - // the use of the `log_url`, `environment_url`, and `auto_inactive` parameters. - // The 'flash' preview is required for `state` values of 'in_progress' and 'queued'. - mediaType: { - previews: ['ant-man', 'flash'], - }, - }) - - console.log(`🚀 Deployment status: success - ${successMessage}`) - console.log(`Visit the newly deployed app at: ${homepageUrl}`) - } catch (error) { - // Report failure! - const failureMessage = `Deployment failed after ${Math.round( - (Date.now() - startTime) / 1000 - )} seconds. See logs for more information.` - console.error(failureMessage) - - try { - if (deploymentId) { - await octokit.repos.createDeploymentStatus({ - owner, - repo, - deployment_id: deploymentId, - state: 'error', - description: failureMessage, - ...(logUrl && { log_url: logUrl }), - environment_url: homepageUrl, - // The 'ant-man' preview is required for `state` values of 'inactive', as well as - // the use of the `log_url`, `environment_url`, and `auto_inactive` parameters. - // The 'flash' preview is required for `state` values of 'in_progress' and 'queued'. - mediaType: { - previews: ['ant-man', 'flash'], - }, - }) - - console.log( - `🚀 Deployment status: error - ${failureMessage}` + (logUrl ? ` Logs: ${logUrl}` : '') - ) - } - } catch (error) { - console.error(`Failed to finalize GitHub DeploymentStatus as a failure. Error: ${error}`) - } - - // Re-throw the error to bubble up - throw error - } -} - -async function getTarballUrl({ octokit, owner, repo, sha }) { - // Get a URL for the tarballed source code bundle - const { - headers: { location: tarballUrl }, - } = await octokit.repos.downloadTarballArchive({ - owner, - repo, - ref: sha, - // Override the underlying `node-fetch` module's `redirect` option - // configuration to prevent automatically following redirects. - request: { - redirect: 'manual', - }, - }) - return tarballUrl -} - -function isAllowableHerokuError(error) { - return error && ALLOWABLE_ERROR_CODES.includes(error.statusCode) -} - -function announceIfHerokuIsDown(error) { - if (error && error.statusCode === 503) { - console.error('💀 Heroku may be down! Please check its Status page: https://status.heroku.com/') - } -} - -function removeEmptyProperties(obj) { - return Object.fromEntries(Object.entries(obj).filter(([key, val]) => val != null)) -} diff --git a/script/deployment/parse-pr-url.js b/script/deployment/parse-pr-url.js deleted file mode 100644 index af7f876dff..0000000000 --- a/script/deployment/parse-pr-url.js +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env node -const USERNAME_FORMAT = '([A-Za-z0-9-]+)' -const REPO_NAME_FORMAT = '([A-Za-z0-9._-]+)' -const PR_NUMBER_FORMAT = '(\\d+)' - -const ALLOWED_PR_URL_FORMAT = new RegExp( - '^' + - '[\'"]?' + - `https://github\\.com/${USERNAME_FORMAT}/${REPO_NAME_FORMAT}/pull/${PR_NUMBER_FORMAT}` + - '[\'"]?' + - '$' -) - -export default function parsePullRequestUrl(prUrl) { - const [, /* fullMatch */ owner, repo, pr] = (prUrl || '').match(ALLOWED_PR_URL_FORMAT) || [] - return { - owner, - repo, - pullNumber: parseInt(pr, 10) || undefined, - } -} diff --git a/script/early-access/clone-for-build.js b/script/early-access/clone-for-build.js deleted file mode 100755 index 5a2be19625..0000000000 --- a/script/early-access/clone-for-build.js +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env node - -// [start-readme] -// -// This script is run as a postbuild script during staging and deployments on Heroku. It clones a branch -// in the early-access repo that matches the current branch in the docs repo; if one can't be found, it -// clones the `main` branch. -// -// [end-readme] - -import dotenv from 'dotenv' -import { execSync } from 'child_process' -import rimraf from 'rimraf' -import fs from 'fs' -import path from 'path' -import os from 'os' - -dotenv.config() -const { - DOCUBOT_REPO_PAT, - HEROKU_PRODUCTION_APP, - GIT_BRANCH, // Set by Actions and/or the deployer with the name of the docs-internal branch -} = process.env - -// Exit if PAT is not found -if (!DOCUBOT_REPO_PAT) { - console.log('Skipping early access, not authorized') - process.exit(0) -} - -const EA_PRODUCTION_BRANCH = 'main' - -// If a branch name is not provided in the environment, attempt to get -// the local branch name; or default to 'main' -let currentBranch = (GIT_BRANCH || '').replace(/^refs\/heads\//, '') -if (!currentBranch) { - try { - currentBranch = execSync('git branch --show-current').toString() - } catch (err) { - // Ignore but log - console.warn('Error checking for local branch:', err.message) - } -} -if (!currentBranch) { - currentBranch = EA_PRODUCTION_BRANCH -} - -// Early Access details -const earlyAccessOwner = 'github' -const earlyAccessRepoName = 'docs-early-access' -const earlyAccessDirName = 'early-access' -const earlyAccessFullRepo = `https://${DOCUBOT_REPO_PAT}@github.com/${earlyAccessOwner}/${earlyAccessRepoName}` - -// On our Azure self-hosted runners, os.tmpdir() doesn't work reliably. On Heroku, os.homedir doesn't work reliably. -const earlyAccessCloningParentDir = process.env.CI ? os.homedir() : os.tmpdir() -const earlyAccessCloningDir = path.join(earlyAccessCloningParentDir, earlyAccessRepoName) - -const destinationDirNames = ['content', 'data', 'assets/images'] -const destinationDirsMap = destinationDirNames.reduce((map, dirName) => { - map[dirName] = path.join(process.cwd(), dirName, earlyAccessDirName) - return map -}, {}) - -// Production vs. staging environment -// TODO test that this works as expected -const environment = HEROKU_PRODUCTION_APP ? 'production' : 'staging' - -// Early access branch to clone -let earlyAccessBranch = HEROKU_PRODUCTION_APP ? EA_PRODUCTION_BRANCH : currentBranch - -// Confirm that the branch exists in the remote -let branchExists = execSync( - `git ls-remote --heads ${earlyAccessFullRepo} ${earlyAccessBranch}` -).toString() - -// If the branch did NOT exist, try checking for the default branch instead -if (!branchExists && earlyAccessBranch !== EA_PRODUCTION_BRANCH) { - console.warn( - `The branch '${earlyAccessBranch}' was not found in ${earlyAccessOwner}/${earlyAccessRepoName}!` - ) - console.warn(`Attempting the default branch ${EA_PRODUCTION_BRANCH} instead...`) - - earlyAccessBranch = EA_PRODUCTION_BRANCH - branchExists = execSync( - `git ls-remote --heads ${earlyAccessFullRepo} ${earlyAccessBranch}` - ).toString() -} - -// If no suitable branch was found, bail out now -if (!branchExists) { - console.error( - `The branch '${earlyAccessBranch}' was not found in ${earlyAccessOwner}/${earlyAccessRepoName}!` - ) - console.error('Exiting!') - process.exit(1) -} - -// Remove any previously cloned copies of the early access repo -rimraf.sync(earlyAccessCloningDir) - -// Clone the repo -console.log(`Setting up: ${earlyAccessCloningDir}`) -execSync( - `git clone --single-branch --branch ${earlyAccessBranch} ${earlyAccessFullRepo} ${earlyAccessRepoName}`, - { - cwd: earlyAccessCloningParentDir, - } -) -console.log(`Using early-access ${environment} branch: '${earlyAccessBranch}'`) - -// Remove all existing early access directories from this repo -destinationDirNames.forEach((key) => rimraf.sync(destinationDirsMap[key])) - -// Move the latest early access source directories into this repo -destinationDirNames.forEach((dirName) => { - const sourceDir = path.join(earlyAccessCloningDir, dirName) - const destDir = destinationDirsMap[dirName] - - // If the source directory doesn't exist, skip it - if (!fs.existsSync(sourceDir)) { - console.warn(`Early access directory '${dirName}' does not exist. Skipping...`) - return - } - - // Ensure the base directory exists - fs.mkdirSync(path.join(process.cwd(), dirName), { recursive: true }) - - // Move the directory from the cloned source to the destination - fs.renameSync(sourceDir, destDir) - - // Confirm the newly moved directory exist - if (fs.existsSync(destDir)) { - console.log(`Successfully moved early access directory '${dirName}' into this repo`) - } else { - throw new Error(`Failed to move early access directory '${dirName}'!`) - } -}) - -// Remove the source content again for good hygiene -rimraf.sync(earlyAccessCloningDir) diff --git a/script/ping-staging-apps.js b/script/ping-staging-apps.js deleted file mode 100755 index d919013597..0000000000 --- a/script/ping-staging-apps.js +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env node - -// [start-readme] -// -// This script finds all Heroku staging apps and pings them to make sure they're always "warmed" and responsive to requests. -// -// [end-readme] - -import dotenv from 'dotenv' -import assert from 'assert' -import got from 'got' -import { chain } from 'lodash-es' -import chalk from 'chalk' -import Heroku from 'heroku-client' - -dotenv.config() - -assert(process.env.HEROKU_API_TOKEN) - -const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN }) - -main() - -async function main() { - const apps = chain(await heroku.get('/apps')) - .orderBy('name') - .value() - - async function ping(app) { - // ?warmup param has no effect but makes it easier to find these requests in the logs - const url = `https://${app.name}.herokuapp.com/en?warmup` - try { - const response = await got(url) - console.log(chalk.green(url, response.statusCode)) - } catch (error) { - console.log(chalk.red(url, error.response.statusCode)) - } - } - - Promise.all(apps.map(ping)) -} diff --git a/script/remove-stale-staging-apps.js b/script/remove-stale-staging-apps.js deleted file mode 100755 index 37be1c475f..0000000000 --- a/script/remove-stale-staging-apps.js +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env node - -// [start-readme] -// -// This script removes all stale Heroku staging apps that outlasted the closure -// of their corresponding pull requests, or correspond to spammy pull requests. -// -// [end-readme] - -import dotenv from 'dotenv' -import { chain } from 'lodash-es' -import chalk from 'chalk' -import Heroku from 'heroku-client' -import getOctokit from './helpers/github.js' - -dotenv.config() - -// Check for required Heroku API token -if (!process.env.HEROKU_API_TOKEN) { - console.error( - 'Error! You must have a HEROKU_API_TOKEN environment variable for deployer-level access.' - ) - process.exit(1) -} -// Check for required GitHub PAT -if (!process.env.GITHUB_TOKEN) { - console.error('Error! You must have a GITHUB_TOKEN environment variable for repo access.') - process.exit(1) -} - -const heroku = new Heroku({ token: process.env.HEROKU_API_TOKEN }) -// This helper uses the `GITHUB_TOKEN` implicitly -const octokit = getOctokit() - -const protectedAppNames = ['help-docs'] - -main() - -async function main() { - const apps = chain(await heroku.get('/apps')) - .orderBy('name') - .value() - - const prInfoMatch = /^(?:gha-|ghd-)?(?docs(?:-internal)?)-(?\d+)--.*$/ - - const appsPlusPullIds = apps.map((app) => { - const match = prInfoMatch.exec(app.name) - const { repo, pullNumber } = (match || {}).groups || {} - - return { - app, - repo, - pullNumber: parseInt(pullNumber, 10) || null, - } - }) - - const appsWithPullIds = appsPlusPullIds.filter((appi) => appi.repo && appi.pullNumber > 0) - - const nonMatchingAppNames = appsPlusPullIds - .filter((appi) => !(appi.repo && appi.pullNumber > 0)) - .map((appi) => appi.app.name) - .filter((name) => !protectedAppNames.includes(name)) - - let staleCount = 0 - let spammyCount = 0 - for (const awpi of appsWithPullIds) { - const { isStale, isSpammy } = await assessPullRequest(awpi.repo, awpi.pullNumber) - - if (isSpammy) spammyCount++ - if (isStale) staleCount++ - - if (isSpammy || isStale) { - await deleteHerokuApp(awpi.app.name) - } - } - - const matchingCount = appsWithPullIds.length - const counts = { - total: matchingCount, - alive: matchingCount - staleCount, - stale: { - total: staleCount, - spammy: spammyCount, - closed: staleCount - spammyCount, - }, - } - console.log(`🧮 COUNTS!\n${JSON.stringify(counts, null, 2)}`) - - const nonMatchingCount = nonMatchingAppNames.length - if (nonMatchingCount > 0) { - console.log( - '⚠️ 👀', - chalk.yellow( - `Non-matching app names (${nonMatchingCount}):\n - ${nonMatchingAppNames.join('\n - ')}` - ) - ) - } -} - -function displayParams(params) { - const { owner, repo, pull_number: pullNumber } = params - return `${owner}/${repo}#${pullNumber}` -} - -async function assessPullRequest(repo, pullNumber) { - const params = { - owner: 'github', - repo: repo, - pull_number: pullNumber, - } - - let isStale = false - let isSpammy = false - try { - const { data: pullRequest } = await octokit.pulls.get(params) - - if (pullRequest && pullRequest.state === 'closed') { - isStale = true - console.debug(chalk.green(`STALE: ${displayParams(params)} is closed`)) - } - } catch (error) { - // Using a standard GitHub PAT, PRs from spammy users will respond as 404 - if (error.status === 404) { - isStale = true - isSpammy = true - console.debug(chalk.yellow(`STALE: ${displayParams(params)} is spammy or deleted`)) - } else { - console.debug(chalk.red(`ERROR: ${displayParams(params)} - ${error.message}`)) - } - } - - return { isStale, isSpammy } -} - -async function deleteHerokuApp(appName) { - try { - await heroku.delete(`/apps/${appName}`) - console.log('✅', chalk.green(`Removed stale app "${appName}"`)) - } catch (error) { - console.log( - '❌', - chalk.red(`ERROR: Failed to remove stale app "${appName}" - ${error.message}`) - ) - } -} diff --git a/script/remove-stale-staging-envs.js b/script/remove-stale-staging-envs.js deleted file mode 100755 index d10f4742fa..0000000000 --- a/script/remove-stale-staging-envs.js +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env node - -// [start-readme] -// -// This script removes all stale GitHub deployment environments that outlasted -// the closure of their corresponding pull requests, or correspond to spammy -// pull requests. -// -// [end-readme] - -import dotenv from 'dotenv' -import chalk from 'chalk' -import getOctokit from './helpers/github.js' - -dotenv.config() - -// Check for required GitHub PAT -if (!process.env.GITHUB_TOKEN) { - console.error('Error! You must have a GITHUB_TOKEN environment variable for repo access.') - process.exit(1) -} - -if (!process.env.ELEVATED_TOKEN) { - console.error( - 'Error! You must have a ELEVATED_TOKEN environment variable for removing deployment environments.' - ) - process.exit(1) -} - -if (!process.env.REPO) { - console.error('Error! You must have a REPO environment variable.') - process.exit(1) -} - -if (!process.env.RUN_ID) { - console.error('Error! You must have a RUN_ID environment variable.') - process.exit(1) -} - -// This helper uses the `GITHUB_TOKEN` implicitly -const octokit = getOctokit() - -const protectedEnvNames = ['production'] -const maxEnvironmentsToProcess = 50 - -// How long must a PR be closed without being merged to be considered stale? -const ONE_HOUR = 60 * 60 * 1000 -const prClosureStaleTime = 2 * ONE_HOUR - -main() - -async function main() { - const owner = 'github' - const [repoOwner, repo] = (process.env.REPO || '').split('/') - - if (repoOwner !== owner) { - console.error(`Error! The repository owner must be "${owner}" but was "${repoOwner}".`) - process.exit(1) - } - - const logUrl = `https://github.com/${owner}/${repo}/actions/runs/${process.env.RUN_ID}` - - const prInfoMatch = /^(?:gha-|ghd-)?(?docs(?:-internal)?)-(?\d+)--.*$/ - - let exceededLimit = false - let matchingCount = 0 - let staleCount = 0 - let spammyCount = 0 - const nonMatchingEnvNames = [] - - for await (const response of octokit.paginate.iterator(octokit.repos.getAllEnvironments, { - owner, - repo, - })) { - const { data: environments } = response - - const envsPlusPullIds = environments.map((env) => { - const match = prInfoMatch.exec(env.name) - const { repo: repoName, pullNumber } = (match || {}).groups || {} - - return { - env, - repo: repoName, - pullNumber: parseInt(pullNumber, 10) || null, - } - }) - - const envsWithPullIds = envsPlusPullIds.filter( - (eppi) => eppi.repo === repo && eppi.pullNumber > 0 - ) - matchingCount += envsWithPullIds.length - - nonMatchingEnvNames.push( - ...envsPlusPullIds - .filter((eppi) => !(eppi.repo && eppi.pullNumber > 0)) - .map((eppi) => eppi.env.name) - .filter((name) => !protectedEnvNames.includes(name)) - ) - - for (const ewpi of envsWithPullIds) { - const { isStale, isSpammy } = await assessPullRequest(ewpi.pullNumber) - - if (isSpammy) spammyCount++ - if (isStale) staleCount++ - - if (isSpammy || isStale) { - await deleteEnvironment(ewpi.env.name) - } - - if (spammyCount + staleCount >= maxEnvironmentsToProcess) { - exceededLimit = true - break - } - } - - if (exceededLimit) { - console.log( - '🛑', - chalk.bgRed(`STOP! Exceeded limit, halting after ${maxEnvironmentsToProcess}.`) - ) - break - } - } - - const counts = { - total: matchingCount, - alive: matchingCount - staleCount, - stale: { - total: staleCount, - spammy: spammyCount, - closed: staleCount - spammyCount, - }, - } - console.log(`🧮 COUNTS!\n${JSON.stringify(counts, null, 2)}`) - - const nonMatchingCount = nonMatchingEnvNames.length - if (nonMatchingCount > 0) { - console.log( - '⚠️ 👀', - chalk.yellow( - `Non-matching env names (${nonMatchingCount}):\n - ${nonMatchingEnvNames.join('\n - ')}` - ) - ) - } - - function displayParams(params) { - const { owner, repo, pull_number: pullNumber } = params - return `${owner}/${repo}#${pullNumber}` - } - - async function assessPullRequest(pullNumber) { - const params = { - owner, - repo, - pull_number: pullNumber, - } - - let isStale = false - let isSpammy = false - try { - const { data: pullRequest } = await octokit.pulls.get(params) - - if (pullRequest && pullRequest.state === 'closed') { - const isMerged = pullRequest.merged === true - const closureAge = Date.now() - Date.parse(pullRequest.closed_at) - isStale = isMerged || closureAge >= prClosureStaleTime - - if (isStale) { - console.debug(chalk.green(`STALE: ${displayParams(params)} is closed`)) - } else { - console.debug( - chalk.blue(`NOT STALE: ${displayParams(params)} is closed but not yet stale`) - ) - } - } - } catch (error) { - // Using a standard GitHub PAT, PRs from spammy users will respond as 404 - if (error.status === 404) { - isStale = true - isSpammy = true - console.debug(chalk.yellow(`STALE: ${displayParams(params)} is spammy or deleted`)) - } else { - console.debug(chalk.red(`ERROR: ${displayParams(params)} - ${error.message}`)) - } - } - - return { isStale, isSpammy } - } - - async function deleteEnvironment(envName) { - try { - let deploymentCount = 0 - - // Get all of the Deployments to signal this environment's complete deactivation - for await (const response of octokit.paginate.iterator(octokit.repos.listDeployments, { - owner, - repo, - - // In the GitHub API, there can only be one active deployment per environment. - // For our many staging apps, we must use the unique appName as the environment. - environment: envName, - })) { - const { data: deployments } = response - - // Deactivate ALL of the deployments - for (const deployment of deployments) { - // Deactivate this Deployment with an 'inactive' DeploymentStatus - await octokit.repos.createDeploymentStatus({ - owner, - repo, - deployment_id: deployment.id, - state: 'inactive', - description: 'The app was undeployed', - log_url: logUrl, - // The 'ant-man' preview is required for `state` values of 'inactive', as well as - // the use of the `log_url`, `environment_url`, and `auto_inactive` parameters. - // The 'flash' preview is required for `state` values of 'in_progress' and 'queued'. - mediaType: { - previews: ['ant-man', 'flash'], - }, - }) - - // Delete this Deployment - await octokit.repos.deleteDeployment({ - owner, - repo, - deployment_id: deployment.id, - }) - - deploymentCount++ - } - } - - // Delete this Environment - try { - await octokit.repos.deleteAnEnvironment({ - // Must use a PAT with more elevated permissions than GITHUB_TOKEN can achieve! - headers: { - authorization: `token ${process.env.ELEVATED_TOKEN}`, - }, - owner, - repo, - environment_name: envName, - }) - } catch (error) { - if (error.status !== 404) { - throw error - } - } - - console.log( - '✅', - chalk.green( - `Removed stale deployment environment "${envName}" (${deploymentCount} deployments)` - ) - ) - } catch (error) { - console.log( - '❌', - chalk.red( - `ERROR: Failed to remove stale deployment environment "${envName}" - ${error.message}` - ) - ) - } - } -} diff --git a/tests/rendering/robots-txt.js b/tests/rendering/robots-txt.js index d7319d663f..c7e3871e93 100644 --- a/tests/rendering/robots-txt.js +++ b/tests/rendering/robots-txt.js @@ -35,9 +35,9 @@ describe('robots.txt', () => { }) }) - it('disallows indexing of herokuapp.com domains', async () => { + it('disallows indexing of azurecontainer.io domains', async () => { const req = { - hostname: 'docs-internal-12345--my-branch.herokuapp.com', + hostname: 'docs-internal-preview-12345-asdfz.azurecontainer.io', path: '/robots.txt', } const res = new MockExpressResponse()