diff --git a/.github/actions-scripts/purge-fastly-edge-cache-per-language.js b/.github/actions-scripts/purge-fastly-edge-cache-per-language.js new file mode 100755 index 0000000000..eb10b3fc65 --- /dev/null +++ b/.github/actions-scripts/purge-fastly-edge-cache-per-language.js @@ -0,0 +1,33 @@ +#!/usr/bin/env node + +import { languageKeys } from '../../lib/languages.js' + +import { makeLanguageSurrogateKey } from '../../middleware/set-fastly-surrogate-key.js' +import purgeEdgeCache from '../../script/deployment/purge-edge-cache.js' + +// This covers things like `/api/webhooks` which isn't language specific. +await purgeEdgeCache(makeLanguageSurrogateKey()) + +const languages = process.env.LANGUAGES + ? languagesFromString(process.env.LANGUAGES) + : // Make sure `en` is first because contributors write mostly in English + // and they're most likely wanting to see their landed changes appear + // in production as soon as possible. + languageKeys.sort((a) => (a === 'en' ? -1 : 1)) + +for (const language of languages) { + await purgeEdgeCache(makeLanguageSurrogateKey(language)) +} + +function languagesFromString(str) { + const languages = str + .split(/,/) + .map((x) => x.trim()) + .filter(Boolean) + if (!languages.every((lang) => languageKeys.includes(lang))) { + throw new Error( + `Unrecognized language code (${languages.find((lang) => !languageKeys.includes(lang))})` + ) + } + return languages +} diff --git a/.github/actions-scripts/purge-fastly-edge-cache.js b/.github/actions-scripts/purge-fastly-edge-cache.js index e3de5c2e5d..30869ee94f 100755 --- a/.github/actions-scripts/purge-fastly-edge-cache.js +++ b/.github/actions-scripts/purge-fastly-edge-cache.js @@ -1,5 +1,20 @@ #!/usr/bin/env node - +import { SURROGATE_ENUMS } from '../../middleware/set-fastly-surrogate-key.js' import purgeEdgeCache from '../../script/deployment/purge-edge-cache.js' -await purgeEdgeCache() +// This will purge every response that *contains* `SURROGATE_ENUMS.DEFAULT`. +// We normally send Surrogate-Key values like: +// +// every-deployment language:en +// every-deployment language:fr +// every-deployment language:ja +// or +// every-deployment no-language +// +// But if you send a purge request for just: +// +// every-deployment +// +// It will cover all surrogate keys that contain that. +// So this the nuclear option for all keys with this prefix. +await purgeEdgeCache(SURROGATE_ENUMS.DEFAULT) diff --git a/.github/workflows/azure-prod-build-deploy.yml b/.github/workflows/azure-prod-build-deploy.yml index 7c7aef2e88..2802f0777e 100644 --- a/.github/workflows/azure-prod-build-deploy.yml +++ b/.github/workflows/azure-prod-build-deploy.yml @@ -143,13 +143,6 @@ jobs: run: | az webapp deployment slot swap --slot canary --target-slot production -n ghdocs-prod -g docs-prod - - name: Purge Fastly edge cache - env: - FASTLY_TOKEN: ${{ secrets.FASTLY_TOKEN }} - FASTLY_SERVICE_ID: ${{ secrets.FASTLY_SERVICE_ID }} - FASTLY_SURROGATE_KEY: 'every-deployment' - run: npm install got && .github/actions-scripts/purge-fastly-edge-cache.js - send-slack-notification-on-failure: needs: [azure-prod-build-and-deploy] runs-on: ubuntu-latest diff --git a/.github/workflows/purge-fastly.yml b/.github/workflows/purge-fastly.yml new file mode 100644 index 0000000000..440907dd20 --- /dev/null +++ b/.github/workflows/purge-fastly.yml @@ -0,0 +1,55 @@ +name: Purge Fastly + +# **What it does**: Sends a soft-purge to Fastly. +# **Why we have it**: So that, right after a production deploy, we start afresh +# **Who does it impact**: Writers and engineers. + +on: + workflow_dispatch: + inputs: + nuke_all: + description: "Nuke all 'every-deployment' keys independent of language" + required: false + type: boolean + default: false + languages: + description: "Comma separated languages. E.g. 'en,ja, es' (defaults to all)" + required: false + default: '' + workflow_run: + workflows: ['Azure Production - Build and Deploy'] + types: + - completed + +permissions: + contents: read + +env: + FASTLY_TOKEN: ${{ secrets.FASTLY_TOKEN }} + FASTLY_SERVICE_ID: ${{ secrets.FASTLY_SERVICE_ID }} + +jobs: + send-purges: + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748 + + - name: Setup node + uses: actions/setup-node@17f8bd926464a1afa4c6a11669539e9c1ba77048 + with: + node-version: '16.17.0' + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Purge Fastly edge cache independent of language + if: ${{ github.event.inputs.nuke_all }} + run: .github/actions-scripts/purge-fastly-edge-cache.js + + - name: Purge Fastly edge cache per language + if: ${{ !github.event.inputs.nuke_all }} + env: + LANGUAGES: ${{ github.event.inputs.languages }} + run: .github/actions-scripts/purge-fastly-edge-cache-per-language.js diff --git a/middleware/index.js b/middleware/index.js index 982b4c6125..90b84298ea 100644 --- a/middleware/index.js +++ b/middleware/index.js @@ -11,7 +11,10 @@ import morgan from 'morgan' import datadog from './connect-datadog.js' import helmet from './helmet.js' import cookieParser from './cookie-parser.js' -import { setDefaultFastlySurrogateKey } from './set-fastly-surrogate-key.js' +import { + setDefaultFastlySurrogateKey, + setLanguageFastlySurrogateKey, +} from './set-fastly-surrogate-key.js' import reqUtils from './req-utils.js' import recordRedirect from './record-redirect.js' import handleErrors from './handle-errors.js' @@ -234,6 +237,10 @@ export default function (app) { app.get('/_ip', instrument(remoteIP, './remoteIP')) app.get('/_build', instrument(buildInfo, './buildInfo')) + // Things like `/api` sets their own Fastly surrogate keys. + // Now that the `req.language` is known, set it for the remaining endpoints + app.use(setLanguageFastlySurrogateKey) + // Check for a dropped connection before proceeding (again) app.use(haltOnDroppedConnection) diff --git a/middleware/set-fastly-surrogate-key.js b/middleware/set-fastly-surrogate-key.js index 63f1f8d5b9..82b0accd80 100644 --- a/middleware/set-fastly-surrogate-key.js +++ b/middleware/set-fastly-surrogate-key.js @@ -26,6 +26,18 @@ export function setFastlySurrogateKey(res, enumKey, isCustomKey = false) { } export function setDefaultFastlySurrogateKey(req, res, next) { - res.set(KEY, SURROGATE_ENUMS.DEFAULT) + res.set(KEY, `${SURROGATE_ENUMS.DEFAULT} ${makeLanguageSurrogateKey()}`) return next() } + +export function setLanguageFastlySurrogateKey(req, res, next) { + res.set(KEY, `${SURROGATE_ENUMS.DEFAULT} ${makeLanguageSurrogateKey(req.language)}`) + return next() +} + +export function makeLanguageSurrogateKey(langCode = null) { + if (!langCode) { + return 'no-language' + } + return `language:${langCode}` +} diff --git a/script/deployment/purge-edge-cache.js b/script/deployment/purge-edge-cache.js index 8b6c5feb97..8cdc8852b4 100644 --- a/script/deployment/purge-edge-cache.js +++ b/script/deployment/purge-edge-cache.js @@ -1,8 +1,19 @@ import got from 'got' -const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) +// Because we use Origin Shielding, it's recommended that you purge twice +// so it purges the edge nodes *and* the origin. +// The documentation says: +// +// One solution to this race condition problem is simply to purge +// twice. For purge-all operations, the two purges should be +// around 30 seconds apart and, for single object and surrogate +// key purges, around 2 seconds apart. +// +// See https://developer.fastly.com/learning/concepts/purging/#shielding +const DELAY_BEFORE_FIRST_PURGE = 5 * 1000 +const DELAY_BEFORE_SECOND_PURGE = 30 * 1000 -const ONE_SECOND = 1000 +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) async function purgeFastlyBySurrogateKey({ apiToken, serviceId, surrogateKey }) { const key = surrogateKey @@ -17,40 +28,50 @@ async function purgeFastlyBySurrogateKey({ apiToken, serviceId, surrogateKey }) return got.post(requestPath, { headers, json: true }) } -export default async function purgeEdgeCache() { - // Give the app some extra time to wake up before the thundering herd of - // Fastly requests. - const delayBeforeFirstPurge = 30 * ONE_SECOND +export default async function purgeEdgeCache( + key, + { + purgeTwice = true, + delayBeforeFirstPurge = DELAY_BEFORE_FIRST_PURGE, + delayBeforeSecondPurge = DELAY_BEFORE_SECOND_PURGE, + } = {} +) { + const surrogateKey = key || process.env.FASTLY_SURROGATE_KEY + if (!surrogateKey) { + throw new Error('No key set and/or no FASTLY_SURROGATE_KEY env var set') + } + console.log(`Fastly purgeEdgeCache initialized for ${surrogateKey}...`) - // Evidence has shown that it's necessary to purge twice to ensure all - // customers see fresh content. - const delayBeforeSecondPurge = 5 * ONE_SECOND - - console.log('Fastly purgeEdgeCache initialized...') - - const { FASTLY_TOKEN, FASTLY_SERVICE_ID, FASTLY_SURROGATE_KEY } = process.env - if (!FASTLY_TOKEN || !FASTLY_SERVICE_ID || !FASTLY_SURROGATE_KEY) { - console.log('Fastly env vars not detected; skipping purgeEdgeCache step') - return + const { FASTLY_TOKEN, FASTLY_SERVICE_ID } = process.env + if (!FASTLY_TOKEN || !FASTLY_SERVICE_ID) { + throw new Error('Fastly env vars not detected; skipping purgeEdgeCache step') } const purgingParams = { apiToken: FASTLY_TOKEN, serviceId: FASTLY_SERVICE_ID, - surrogateKey: FASTLY_SURROGATE_KEY, + surrogateKey, } - console.log('Waiting extra time to prevent a Thundering Herd problem...') - await sleep(delayBeforeFirstPurge) + // Give the app some extra time to wake up before the thundering herd of + // Fastly requests. + if (delayBeforeFirstPurge) { + console.log('Waiting extra time to prevent a Thundering Herd problem...') + await sleep(delayBeforeFirstPurge) + } console.log('Attempting first Fastly purge...') const firstPurge = await purgeFastlyBySurrogateKey(purgingParams) console.log('First Fastly purge result:', firstPurge.body || firstPurge) - console.log('Waiting to purge a second time...') - await sleep(delayBeforeSecondPurge) + // Evidence has shown that it's necessary to purge twice to ensure all + // customers see fresh content. + if (purgeTwice) { + console.log('Waiting to purge a second time...') + await sleep(delayBeforeSecondPurge) - console.log('Attempting second Fastly purge...') - const secondPurge = await purgeFastlyBySurrogateKey(purgingParams) - console.log('Second Fastly purge result:', secondPurge.body || secondPurge) + console.log('Attempting second Fastly purge...') + const secondPurge = await purgeFastlyBySurrogateKey(purgingParams) + console.log('Second Fastly purge result:', secondPurge.body || secondPurge) + } } diff --git a/tests/content/webhooks.js b/tests/content/webhooks.js index 975ce42ad4..d3e4215186 100644 --- a/tests/content/webhooks.js +++ b/tests/content/webhooks.js @@ -1,5 +1,8 @@ import { get } from '../helpers/e2etest.js' -import { SURROGATE_ENUMS } from '../../middleware/set-fastly-surrogate-key.js' +import { + SURROGATE_ENUMS, + makeLanguageSurrogateKey, +} from '../../middleware/set-fastly-surrogate-key.js' import { describe, expect } from '@jest/globals' describe('webhooks v1 middleware', () => { @@ -23,7 +26,9 @@ describe('webhooks v1 middleware', () => { expect(res.headers['cache-control']).toMatch(/max-age=[1-9]/) expect(res.headers['surrogate-control']).toContain('public') expect(res.headers['surrogate-control']).toMatch(/max-age=[1-9]/) - expect(res.headers['surrogate-key']).toBe(SURROGATE_ENUMS.DEFAULT) + const surrogateKeySplit = res.headers['surrogate-key'].split(/\s/g) + expect(surrogateKeySplit.includes(SURROGATE_ENUMS.DEFAULT)).toBeTruthy() + expect(surrogateKeySplit.includes(makeLanguageSurrogateKey())).toBeTruthy() }) test('get non-fpt version webhook', async () => { diff --git a/tests/helpers/caching-headers.js b/tests/helpers/caching-headers.js index 6386227b93..02d6060d8f 100644 --- a/tests/helpers/caching-headers.js +++ b/tests/helpers/caching-headers.js @@ -8,7 +8,7 @@ export function checkCachingHeaders(res, defaultSurrogateKey = false, minMaxAge // that it's a reasonably large number of seconds. expect(maxAgeSeconds).toBeGreaterThanOrEqual(minMaxAge) // Because it doesn't have have a unique URL - expect(res.headers['surrogate-key']).toBe( + expect(res.headers['surrogate-key'].split(/\s/g)[0]).toBe( defaultSurrogateKey ? SURROGATE_ENUMS.DEFAULT : SURROGATE_ENUMS.MANUAL ) } diff --git a/tests/rendering/server.js b/tests/rendering/server.js index 37058f64db..748433a305 100644 --- a/tests/rendering/server.js +++ b/tests/rendering/server.js @@ -5,7 +5,10 @@ import { describeViaActionsOnly } from '../helpers/conditional-runs.js' import { loadPages } from '../../lib/page-data.js' import CspParse from 'csp-parse' import { productMap } from '../../lib/all-products.js' -import { SURROGATE_ENUMS } from '../../middleware/set-fastly-surrogate-key.js' +import { + SURROGATE_ENUMS, + makeLanguageSurrogateKey, +} from '../../middleware/set-fastly-surrogate-key.js' import { getPathWithoutVersion } from '../../lib/path-utils.js' import { describe, jest } from '@jest/globals' @@ -138,7 +141,10 @@ describe('server', () => { const res = await get('/en') expect(res.statusCode).toBe(200) expect(res.headers['cache-control']).toMatch(/public, max-age=/) - expect(res.headers['surrogate-key']).toBe(SURROGATE_ENUMS.DEFAULT) + + const surrogateKeySplit = res.headers['surrogate-key'].split(/\s/g) + expect(surrogateKeySplit.includes(SURROGATE_ENUMS.DEFAULT)).toBeTruthy() + expect(surrogateKeySplit.includes(makeLanguageSurrogateKey('en'))).toBeTruthy() }) test('does not render duplicate or tags', async () => { @@ -950,7 +956,10 @@ describe('static routes', () => { expect(res.headers['set-cookie']).toBeUndefined() expect(res.headers['cache-control']).toContain('public') expect(res.headers['cache-control']).toMatch(/max-age=\d+/) - expect(res.headers['surrogate-key']).toBe(SURROGATE_ENUMS.DEFAULT) + + const surrogateKeySplit = res.headers['surrogate-key'].split(/\s/g) + expect(surrogateKeySplit.includes(SURROGATE_ENUMS.DEFAULT)).toBeTruthy() + expect(surrogateKeySplit.includes(makeLanguageSurrogateKey())).toBeTruthy() }) it('serves schema files from the /data/graphql directory at /public', async () => { diff --git a/tests/unit/anchor-redirect.js b/tests/unit/anchor-redirect.js index cc77ed0e65..2b9a2b1a31 100644 --- a/tests/unit/anchor-redirect.js +++ b/tests/unit/anchor-redirect.js @@ -1,7 +1,10 @@ import { describe, expect } from '@jest/globals' import { get } from '../helpers/e2etest.js' -import { SURROGATE_ENUMS } from '../../middleware/set-fastly-surrogate-key.js' +import { + SURROGATE_ENUMS, + makeLanguageSurrogateKey, +} from '../../middleware/set-fastly-surrogate-key.js' import clientSideRedirects from '../../lib/redirects/static/client-side-rest-api-redirects.json' describe('anchor-redirect middleware', () => { @@ -52,6 +55,8 @@ describe('anchor-redirect middleware', () => { expect(res.headers['cache-control']).toMatch(/max-age=[1-9]/) expect(res.headers['surrogate-control']).toContain('public') expect(res.headers['surrogate-control']).toMatch(/max-age=[1-9]/) - expect(res.headers['surrogate-key']).toBe(SURROGATE_ENUMS.DEFAULT) + const surrogateKeySplit = res.headers['surrogate-key'].split(/\s/g) + expect(surrogateKeySplit.includes(SURROGATE_ENUMS.DEFAULT)).toBeTruthy() + expect(surrogateKeySplit.includes(makeLanguageSurrogateKey())).toBeTruthy() }) })