diff --git a/.github/workflows/prod-build-deploy.yml b/.github/workflows/prod-build-deploy.yml index 894dede0b8..5c4c9d001b 100644 --- a/.github/workflows/prod-build-deploy.yml +++ b/.github/workflows/prod-build-deploy.yml @@ -129,9 +129,10 @@ jobs: HYDRO_ENDPOINT: ${{ secrets.HYDRO_ENDPOINT }} HYDRO_SECRET: ${{ secrets.HYDRO_SECRET }} SOURCE_BLOB_URL: ${{ steps.build-source.outputs.download_url }} + DELAY_FOR_PREBOOT: 'true' with: script: | - const { GITHUB_TOKEN, HEROKU_API_TOKEN, SOURCE_BLOB_URL } = process.env + const { GITHUB_TOKEN, HEROKU_API_TOKEN, SOURCE_BLOB_URL, DELAY_FOR_PREBOOT } = process.env // Exit if GitHub Actions PAT is not found if (!GITHUB_TOKEN) { @@ -158,6 +159,7 @@ jobs: try { await deployToProduction({ octokit, + includeDelayForPreboot: DELAY_FOR_PREBOOT !== 'false', // These parameters will ONLY be set by Actions sourceBlobUrl: SOURCE_BLOB_URL, runId: context.runId @@ -205,7 +207,6 @@ jobs: FASTLY_TOKEN: ${{ secrets.FASTLY_TOKEN }} FASTLY_SERVICE_ID: ${{ secrets.FASTLY_SERVICE_ID }} FASTLY_SURROGATE_KEY: 'all-the-things' - DELAY_FOR_PREBOOT: 'true' with: script: | // Workaround to allow us to load ESM files with `require(...)` @@ -215,9 +216,7 @@ jobs: const { default: purgeEdgeCache } = require('./script/deployment/purge-edge-cache') try { - await purgeEdgeCache({ - includeDelayForPreboot: process.env.DELAY_FOR_PREBOOT !== 'false' - }) + await purgeEdgeCache() } catch (error) { console.error(`Failed to purge the edge cache: ${error.message}`) console.error(error) diff --git a/script/deployment/deploy-to-production.js b/script/deployment/deploy-to-production.js index 0f45ebeca7..bb7f5d194c 100755 --- a/script/deployment/deploy-to-production.js +++ b/script/deployment/deploy-to-production.js @@ -6,6 +6,7 @@ import { setOutput } from '@actions/core' const SLEEP_INTERVAL = 5000 const HEROKU_LOG_LINES_TO_SHOW = 25 +const DELAY_FOR_PREBOOT_SWAP = 135000 // 2:15 // Allow for a few 404 (Not Found) or 429 (Too Many Requests) responses from the // semi-unreliable Heroku API when we're polling for status updates @@ -13,6 +14,7 @@ const ALLOWED_MISSING_RESPONSE_COUNT = 5 export default async function deployToProduction({ octokit, + includeDelayForPreboot = true, // These parameters will only be set by Actions sourceBlobUrl = null, runId = null, @@ -332,11 +334,20 @@ export default async function deployToProduction({ )} seconds.` ) - // - // TODO: - // Should we consider adding an explicit 2-minute pause here to allow for - // Heroku Preboot to actually swap in the new dynos? - // + // IMPORTANT: + // If Heroku Preboot is enabled, then there is an additional delay of at + // least 2 minutes before the new dynos are swapped into active serving. + // If we move off Heroku in the future, this should be revisited and + // updated/removed as relevant to align with the new hosting platform. + if (includeDelayForPreboot) { + console.log(`Waiting for Heroku Preboot to swap dynos (${DELAY_FOR_PREBOOT_SWAP} ms)...`) + await sleep(DELAY_FOR_PREBOOT_SWAP) + + // TODO: + // Is there a faster alternative than this arbitrary delay? For example, + // is there some Heroku API we can query to see when this release is + // considered to be the live one, or when the old dynos are shut down? + } // Report success! const successMessage = `Deployment succeeded after ${Math.round( diff --git a/script/deployment/purge-edge-cache.js b/script/deployment/purge-edge-cache.js index 7024417882..e46972452b 100644 --- a/script/deployment/purge-edge-cache.js +++ b/script/deployment/purge-edge-cache.js @@ -2,7 +2,6 @@ import sleep from 'await-sleep' import got from 'got' const ONE_SECOND = 1000 -const ONE_MINUTE = 60 * ONE_SECOND async function purgeFastlyBySurrogateKey({ apiToken, serviceId, surrogateKey }) { const key = surrogateKey @@ -17,17 +16,10 @@ async function purgeFastlyBySurrogateKey({ apiToken, serviceId, surrogateKey }) return got.post(requestPath, { headers, json: true }) } -// This delay (includeDelayForPreboot) can potentially be removed in the -// future if the deployment workflow is updated to include a delay to offset -// Heroku Preboot before this script runs. -export default async function purgeEdgeCache({ includeDelayForPreboot = true } = {}) { - // If Heroku Preboot is enabled, then there is an additional delay of at - // least 2 minutes before the new dynos are swapped into active serving. - const delayForPrebootSwap = 2 * ONE_MINUTE + 30 * ONE_SECOND - +export default async function purgeEdgeCache() { // Give the app some extra time to wake up before the thundering herd of // Fastly requests. - const delayBeforeFirstPurge = ONE_MINUTE + const delayBeforeFirstPurge = 30 * ONE_SECOND // Evidence has shown that it's necessary to purge twice to ensure all // customers see fresh content. @@ -47,11 +39,6 @@ export default async function purgeEdgeCache({ includeDelayForPreboot = true } = surrogateKey: FASTLY_SURROGATE_KEY, } - if (includeDelayForPreboot) { - console.log('Waiting for Heroku Preboot to swap dynos...') - await sleep(delayForPrebootSwap) - } - console.log('Waiting extra time to prevent a Thundering Herd problem...') await sleep(delayBeforeFirstPurge)