diff --git a/.github/actions/rendered-content-link-checker.js b/.github/actions/rendered-content-link-checker.js index 8bf8cbddd8..d897ab0ba8 100755 --- a/.github/actions/rendered-content-link-checker.js +++ b/.github/actions/rendered-content-link-checker.js @@ -6,6 +6,8 @@ import cheerio from 'cheerio' import coreLib from '@actions/core' import got, { RequestError } from 'got' import chalk from 'chalk' +import { Low } from 'lowdb' +import { JSONFile } from 'lowdb/node' import shortVersions from '../../middleware/contextualizers/short-versions.js' import contextualize from '../../middleware/context.js' @@ -31,6 +33,29 @@ Object.entries(STATIC_PREFIXES).forEach(([key, value]) => { } }) +// By default, we don't cache external link checks to disk. +// By setting this env var to something >0, it enables the disk-based +// caching of external links. +const EXTERNAL_LINK_CHECKER_MAX_AGE_MS = + parseInt(process.env.EXTERNAL_LINK_CHECKER_MAX_AGE_DAYS || 0) * 24 * 60 * 60 * 1000 +const EXTERNAL_LINK_CHECKER_DB = + process.env.EXTERNAL_LINK_CHECKER_DB || 'external-link-checker-db.json' + +const adapter = new JSONFile(EXTERNAL_LINK_CHECKER_DB) +const externalLinkCheckerDB = new Low(adapter) + +// Given a number and a percentage, return the same number with a *percentage* +// max change of making a bit larger or smaller. +// E.g. `jitter(55, 10)` will return a value between `[55 - 55/10: 55 + 55/10]` +// This is useful to avoid the caching timestamps all getting the same +// numbers from the day it started which means that they don't ALL expire +// on the same day but start to expire in a bit of a "random pattern" so +// you don't get all or nothing. +function jitter(base, percentage) { + const r = percentage / 100 + const negative = Math.random() > 0.5 ? -1 : 1 + return base + base * Math.random() * r * negative +} // Return a function that can as quickly as possible check if a certain // href input should be skipped. // Do this so we can use a `Set` and a `iterable.some()` for a speedier @@ -186,12 +211,17 @@ async function main(core, octokit, uploadArtifact, opts = {}) { ) } + await externalLinkCheckerDB.read() + externalLinkCheckerDB.data ||= { urls: {} } + debugTimeStart(core, 'processPages') const flawsGroups = await Promise.all( - pages.map((page) => processPage(core, page, pageMap, redirects, opts)) + pages.map((page) => processPage(core, page, pageMap, redirects, opts, externalLinkCheckerDB)) ) debugTimeEnd(core, 'processPages') + await externalLinkCheckerDB.write() + const flaws = flawsGroups.flat() printGlobalCacheHitRatio(core) @@ -518,12 +548,12 @@ function getPages(pageList, languages, filters, files, max) { .slice(0, max ? Math.min(max, pageList.length) : pageList.length) } -async function processPage(core, page, pageMap, redirects, opts) { +async function processPage(core, page, pageMap, redirects, opts, db) { const { verbose, verboseUrl, bail } = opts const allFlawsEach = await Promise.all( page.permalinks.map((permalink) => { - return processPermalink(core, permalink, page, pageMap, redirects, opts) + return processPermalink(core, permalink, page, pageMap, redirects, opts, db) }) ) @@ -545,7 +575,7 @@ async function processPage(core, page, pageMap, redirects, opts) { return allFlaws } -async function processPermalink(core, permalink, page, pageMap, redirects, opts) { +async function processPermalink(core, permalink, page, pageMap, redirects, opts, db) { const { level = 'critical', checkAnchors, @@ -583,7 +613,8 @@ async function processPermalink(core, permalink, page, pageMap, redirects, opts) pageMap, checkAnchors, checkExternalLinks, - { verbose, patient } + { verbose, patient }, + db ) if (flaw) { @@ -727,7 +758,8 @@ async function checkHrefLink( pageMap, checkAnchors = false, checkExternalLinks = false, - { verbose = false, patient = false } = {} + { verbose = false, patient = false } = {}, + db = null ) { if (href === '#') { if (checkAnchors) { @@ -781,13 +813,53 @@ async function checkHrefLink( if (linksToSkip(href)) { return } - const { ok, ...info } = await checkExternalURL(core, href, { verbose, patient }) + const { ok, ...info } = await checkExternalURLCached(core, href, { verbose, patient }, db) if (!ok) { return { CRITICAL: `Broken external link (${JSON.stringify(info)})`, isExternal: true } } } } +// Can't do this memoization within the checkExternalURL because it can +// return a Promise since it already collates multiple URLs under the +// same cache key. +async function checkExternalURLCached(core, href, { verbose, patient }, db) { + const cacheMaxAge = EXTERNAL_LINK_CHECKER_MAX_AGE_MS + const timestamp = new Date().getTime() + const url = href.split('#')[0] + + if (cacheMaxAge) { + const tooOld = timestamp - Math.floor(jitter(cacheMaxAge, 10)) + if (db && db.data.urls[url]) { + if (db.data.urls[url].timestamp > tooOld) { + if (verbose) { + core.debug(`External URL ${url} in cache`) + } + return db.data.urls[url].result + } else if (verbose) { + core.info(`External URL ${url} in cache but too old`) + // Delete it so the cache file don't bloat infinitely + delete db.data.urls[url] + } + } + } + + const result = await checkExternalURL(core, href, { verbose, patient }) + + if (cacheMaxAge) { + // By only cache storing successful results, we give the system a chance + // to try 40xx and 50x errors another go. + if (db && result.ok) { + db.data.urls[url] = { + timestamp, + result, + } + } + } + + return result +} + const _fetchCache = new Map() async function checkExternalURL(core, url, { verbose = false, patient = false } = {}) { if (!url.startsWith('https://')) throw new Error('Invalid URL') diff --git a/.github/workflows/link-check-daily.yml b/.github/workflows/link-check-daily.yml index 18f3360665..623bef0fd2 100644 --- a/.github/workflows/link-check-daily.yml +++ b/.github/workflows/link-check-daily.yml @@ -54,6 +54,12 @@ jobs: if: ${{ github.repository == 'github/docs-internal' }} run: .github/actions-scripts/merge-early-access.sh + - name: Restore disk-cache file for external link checking + uses: actions/cache@48af2dc4a9e8278b89d7fa154b955c30c6aaab09 + with: + path: external-link-checker-db.json + key: external-link-checker-${{ hashFiles('.github/actions/rendered-content-link-checker.js') }} + - name: Run link checker env: LEVEL: 'critical' @@ -67,6 +73,10 @@ jobs: CREATE_REPORT: true CHECK_EXTERNAL_LINKS: true PATIENT: true + # This means that we'll *re-check* external URLs once a week. + # But mind you that the number has a 10% chance of "jitter" + # to avoid a stampeding herd when they all expire some day. + EXTERNAL_LINK_CHECKER_MAX_AGE_DAYS: 7 timeout-minutes: 30 run: node .github/actions/rendered-content-link-checker.js diff --git a/.gitignore b/.gitignore index d26ed900b4..a11e3bcdda 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ user-code/ # Logs from scripts script/logs/ +external-link-checker-db.json diff --git a/components/context/MainContext.tsx b/components/context/MainContext.tsx index 51a62d5560..44e803927f 100644 --- a/components/context/MainContext.tsx +++ b/components/context/MainContext.tsx @@ -96,7 +96,6 @@ export type MainContextT = { languageCode: string relativePath: string title: string - pageVersionTitle: string pageVersion: string href: string }> @@ -151,14 +150,7 @@ export const getMainContext = async (req: any, res: any): Promise topics: req.context.page.topics || [], introPlainText: req.context.page?.introPlainText, permalinks: req.context.page?.permalinks.map((obj: any) => - pick(obj, [ - 'title', - 'pageVersionTitle', - 'pageVersion', - 'href', - 'relativePath', - 'languageCode', - ]) + pick(obj, ['title', 'pageVersion', 'href', 'relativePath', 'languageCode']) ), hidden: req.context.page.hidden || false, noEarlyAccessBanner: req.context.page.noEarlyAccessBanner || false, diff --git a/components/page-header/VersionPicker.tsx b/components/page-header/VersionPicker.tsx index 767826efb6..276c782635 100644 --- a/components/page-header/VersionPicker.tsx +++ b/components/page-header/VersionPicker.tsx @@ -20,8 +20,8 @@ export const VersionPicker = ({ variant }: Props) => { } const allLinks = (page.permalinks || []).map((permalink) => ({ - text: permalink.pageVersionTitle, - selected: allVersions[currentVersion].versionTitle === permalink.pageVersionTitle, + text: allVersions[permalink.pageVersion].versionTitle, + selected: currentVersion === permalink.pageVersion, href: permalink.href, arrow: false, info: false, diff --git a/content/rest/overview/libraries.md b/content/rest/overview/libraries.md index e5817e491c..33c51a7614 100644 --- a/content/rest/overview/libraries.md +++ b/content/rest/overview/libraries.md @@ -13,16 +13,16 @@ topics: - API --- -
- The Gundamcat -

Octokit comes in many flavors

-

Use the official Octokit library, or choose between any of the available third party libraries.

- -
+![The Gundamcat](/assets/images/gundamcat.png) + +## Octokit comes in many flavors + +Use the official Octokit library, or choose between any of the available third party libraries. + +- **Python** → [octokit.py](https://github.com/khornberg/octokit.py) +- **Ruby** → [octokit.rb](https://github.com/octokit/octokit.rb) +- **.NET** → [octokit.net](https://github.com/octokit/octokit.net) +- **JavaScript** → [octokit/octokit.js](https://github.com/octokit/octokit.js) ## Third-party libraries diff --git a/lib/permalink.js b/lib/permalink.js index 4f85978d1c..7c76136374 100644 --- a/lib/permalink.js +++ b/lib/permalink.js @@ -1,7 +1,6 @@ import assert from 'assert' import path from 'path' import patterns from './patterns.js' -import { allVersions } from './all-versions.js' import removeFPTFromPath from './remove-fpt-from-path.js' /* @@ -15,8 +14,7 @@ page.permalinks is an array of objects that looks like this: "relativePath": "billing/managing-billing-for-your-github-account/index.md", "title": "Managing billing for your GitHub account", "hrefWithoutLanguage": "/billing/managing-billing-for-your-github-account", - "href": "/en/billing/managing-billing-for-your-github-account", - "pageVersionTitle": "Free, Pro, & Team" + "href": "/en/billing/managing-billing-for-your-github-account" }, { "languageCode": "en", @@ -24,8 +22,7 @@ page.permalinks is an array of objects that looks like this: "relativePath": "billing/managing-billing-for-your-github-account/index.md", "title": "Managing billing for your GitHub account", "hrefWithoutLanguage": "/enterprise-cloud@latest/billing/managing-billing-for-your-github-account", - "href": "/en/enterprise-cloud@latest/billing/managing-billing-for-your-github-account", - "pageVersionTitle": "Enterprise Cloud" + "href": "/en/enterprise-cloud@latest/billing/managing-billing-for-your-github-account" } ... and so on for each of the content file's supported versions. ] @@ -46,8 +43,6 @@ class Permalink { this.hrefWithoutLanguage === '/' ? '' : this.hrefWithoutLanguage }` - this.pageVersionTitle = allVersions[pageVersion].versionTitle - return this } diff --git a/package-lock.json b/package-lock.json index 0f94cbf25d..9d231e7329 100644 --- a/package-lock.json +++ b/package-lock.json @@ -50,6 +50,7 @@ "liquidjs": "9.22.1", "lodash": "^4.17.21", "lodash-es": "^4.17.21", + "lowdb": "5.0.5", "lunr": "^2.3.9", "lunr-languages": "^1.9.0", "mdast-util-from-markdown": "^1.2.0", @@ -3480,201 +3481,6 @@ "resolved": "https://registry.npmjs.org/@next/env/-/env-12.2.4.tgz", "integrity": "sha512-/gApFXWk5CCLFQJL5IYJXxPQuG5tz5nPX4l27A9Zm/+wJxiwFrRSP54AopDxIv4JRp/rGwcgk/lZS/0Clw8jYA==" }, - "node_modules/@next/swc-android-arm-eabi": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-android-arm-eabi/-/swc-android-arm-eabi-12.2.4.tgz", - "integrity": "sha512-P4YSFNpmXXSnn3P1qsOAqz+MX3On9fHrlc8ovb/CFJJoU+YLCR53iCEwfw39e0IZEgDA7ttgr108plF8mxaX0g==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-android-arm64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-android-arm64/-/swc-android-arm64-12.2.4.tgz", - "integrity": "sha512-4o2n14E18O+8xHlf6dgJsWPXN9gmSmfIe2Z0EqKDIPBBkFt/2CyrH0+vwHnL2l7xkDHhOGfZYcYIWVUR5aNu0A==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-darwin-arm64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-12.2.4.tgz", - "integrity": "sha512-DcUO6MGBL9E3jj5o86MUnTOy4WawIJJhyCcFYO4f51sbl7+uPIYIx40eo98A6NwJEXazCqq1hLeqOaNTAIvDiQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-darwin-x64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-12.2.4.tgz", - "integrity": "sha512-IUlFMqeLjdIzDorrGC2Dt+2Ae3DbKQbRzCzmDq4/CP1+jJGeDXo/2AHnlE+WYnwQAC4KtAz6pbVnd3KstZWsVA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-freebsd-x64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-freebsd-x64/-/swc-freebsd-x64-12.2.4.tgz", - "integrity": "sha512-475vwyWcjnyDVDWLgAATP0HI8W1rwByc+uXk1B6KkAVFhkoDgH387LW0uNqxavK+VxCzj3avQXX/58XDvxtSlg==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm-gnueabihf": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm-gnueabihf/-/swc-linux-arm-gnueabihf-12.2.4.tgz", - "integrity": "sha512-qZW+L3iG3XSGtlOPmD5RRWXyk6ZNdscLV0BQjuDvP+exTg+uixqHXOHz0/GVATIJEBQOF0Kew7jAXVXEP+iRTQ==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm64-gnu": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-12.2.4.tgz", - "integrity": "sha512-fEPRjItWYaKyyG9N+2HIA59OBHIhk7WC+Rh+LwXsh0pQe870Ykpek3KQs0umjsrEGe57NyMomq3f80/N8taDvA==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm64-musl": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-12.2.4.tgz", - "integrity": "sha512-rnCTzXII0EBCcFn9P5s/Dho2kPUMSX/bP0iOAj8wEI/IxUEfEElbin89zJoNW30cycHu19xY8YP4K2+hzciPzQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-x64-gnu": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-12.2.4.tgz", - "integrity": "sha512-PhXX6NSuIuhHInxPY2VkG2Bl7VllsD3Cjx+pQcS1wTym7Zt7UoLvn05PkRrkiyIkvR+UXnqPUM3TYiSbnemXEw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-x64-musl": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-12.2.4.tgz", - "integrity": "sha512-GmC/QROiUZpFirHRfPQqMyCXZ+5+ndbBZrMvL74HtQB/CKXB8K1VM+rvy9Gp/5OaU8Rxp48IcX79NOfI2LiXlA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-arm64-msvc": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-12.2.4.tgz", - "integrity": "sha512-9XKoCXbNZuaMRPtcKQz3+hgVpkMosaLlcxHFXT8/j4w61k7/qvEbrkMDS9WHNrD/xVcLycwhPRgXcns2K1BdBQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-ia32-msvc": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-12.2.4.tgz", - "integrity": "sha512-hEyRieZKH9iw4AzvXaQ+Fyb98k0G/o9QcRGxA1/O/O/elf1+Qvuwb15phT8GbVtIeNziy66XTPOhKKfdr8KyUg==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-x64-msvc": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-12.2.4.tgz", - "integrity": "sha512-5Pl1tdMJWLy4rvzU1ecx0nHWgDPqoYuvYoXE/5X0Clu9si/yOuBIj573F2kOTY7mu0LX2wgCJVSnyK0abHBxIw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -5650,9 +5456,9 @@ } }, "node_modules/babel-loader": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-9.0.1.tgz", - "integrity": "sha512-szYjslOXFlj/po5KfrVmiuBAcI6GVHFuAgC96Qd6mMPHdwl4lmAJkYtvjQ1RxxPjgdkKjd3LQgXDE4jxEutNuw==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-9.1.0.tgz", + "integrity": "sha512-Antt61KJPinUMwHwIIz9T5zfMgevnfZkEVWYDWlG888fgdvRRGD0JTuf/fFozQnfT+uq64sk1bmdHDy/mOEWnA==", "dev": true, "dependencies": { "find-cache-dir": "^3.3.2", @@ -9564,20 +9370,6 @@ "devOptional": true, "license": "ISC" }, - "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/function-bind": { "version": "1.1.1", "license": "MIT" @@ -14245,6 +14037,20 @@ "loose-envify": "cli.js" } }, + "node_modules/lowdb": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/lowdb/-/lowdb-5.0.5.tgz", + "integrity": "sha512-7EWKmIMhNKA8TXFhL8t0p6N2LC53l3ZqsWQGSksGhhjrcms9rbKlyrAh2PzSGK5v0KPJ2W5VItBnC3NDRzOnzQ==", + "dependencies": { + "steno": "^3.0.0" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/typicode" + } + }, "node_modules/lower-case": { "version": "2.0.2", "dev": true, @@ -18737,6 +18543,17 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/steno": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/steno/-/steno-3.0.0.tgz", + "integrity": "sha512-uZtn7Ht9yXLiYgOsmo8btj4+f7VxyYheMt8g6F1ANjyqByQXEE2Gygjgenp3otHH1TlHsS4JAaRGv5wJ1wvMNw==", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/typicode" + } + }, "node_modules/stream-combiner": { "version": "0.0.4", "dev": true, @@ -23139,84 +22956,6 @@ "resolved": "https://registry.npmjs.org/@next/env/-/env-12.2.4.tgz", "integrity": "sha512-/gApFXWk5CCLFQJL5IYJXxPQuG5tz5nPX4l27A9Zm/+wJxiwFrRSP54AopDxIv4JRp/rGwcgk/lZS/0Clw8jYA==" }, - "@next/swc-android-arm-eabi": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-android-arm-eabi/-/swc-android-arm-eabi-12.2.4.tgz", - "integrity": "sha512-P4YSFNpmXXSnn3P1qsOAqz+MX3On9fHrlc8ovb/CFJJoU+YLCR53iCEwfw39e0IZEgDA7ttgr108plF8mxaX0g==", - "optional": true - }, - "@next/swc-android-arm64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-android-arm64/-/swc-android-arm64-12.2.4.tgz", - "integrity": "sha512-4o2n14E18O+8xHlf6dgJsWPXN9gmSmfIe2Z0EqKDIPBBkFt/2CyrH0+vwHnL2l7xkDHhOGfZYcYIWVUR5aNu0A==", - "optional": true - }, - "@next/swc-darwin-arm64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-12.2.4.tgz", - "integrity": "sha512-DcUO6MGBL9E3jj5o86MUnTOy4WawIJJhyCcFYO4f51sbl7+uPIYIx40eo98A6NwJEXazCqq1hLeqOaNTAIvDiQ==", - "optional": true - }, - "@next/swc-darwin-x64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-12.2.4.tgz", - "integrity": "sha512-IUlFMqeLjdIzDorrGC2Dt+2Ae3DbKQbRzCzmDq4/CP1+jJGeDXo/2AHnlE+WYnwQAC4KtAz6pbVnd3KstZWsVA==", - "optional": true - }, - "@next/swc-freebsd-x64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-freebsd-x64/-/swc-freebsd-x64-12.2.4.tgz", - "integrity": "sha512-475vwyWcjnyDVDWLgAATP0HI8W1rwByc+uXk1B6KkAVFhkoDgH387LW0uNqxavK+VxCzj3avQXX/58XDvxtSlg==", - "optional": true - }, - "@next/swc-linux-arm-gnueabihf": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm-gnueabihf/-/swc-linux-arm-gnueabihf-12.2.4.tgz", - "integrity": "sha512-qZW+L3iG3XSGtlOPmD5RRWXyk6ZNdscLV0BQjuDvP+exTg+uixqHXOHz0/GVATIJEBQOF0Kew7jAXVXEP+iRTQ==", - "optional": true - }, - "@next/swc-linux-arm64-gnu": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-12.2.4.tgz", - "integrity": "sha512-fEPRjItWYaKyyG9N+2HIA59OBHIhk7WC+Rh+LwXsh0pQe870Ykpek3KQs0umjsrEGe57NyMomq3f80/N8taDvA==", - "optional": true - }, - "@next/swc-linux-arm64-musl": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-12.2.4.tgz", - "integrity": "sha512-rnCTzXII0EBCcFn9P5s/Dho2kPUMSX/bP0iOAj8wEI/IxUEfEElbin89zJoNW30cycHu19xY8YP4K2+hzciPzQ==", - "optional": true - }, - "@next/swc-linux-x64-gnu": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-12.2.4.tgz", - "integrity": "sha512-PhXX6NSuIuhHInxPY2VkG2Bl7VllsD3Cjx+pQcS1wTym7Zt7UoLvn05PkRrkiyIkvR+UXnqPUM3TYiSbnemXEw==", - "optional": true - }, - "@next/swc-linux-x64-musl": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-12.2.4.tgz", - "integrity": "sha512-GmC/QROiUZpFirHRfPQqMyCXZ+5+ndbBZrMvL74HtQB/CKXB8K1VM+rvy9Gp/5OaU8Rxp48IcX79NOfI2LiXlA==", - "optional": true - }, - "@next/swc-win32-arm64-msvc": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-12.2.4.tgz", - "integrity": "sha512-9XKoCXbNZuaMRPtcKQz3+hgVpkMosaLlcxHFXT8/j4w61k7/qvEbrkMDS9WHNrD/xVcLycwhPRgXcns2K1BdBQ==", - "optional": true - }, - "@next/swc-win32-ia32-msvc": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-12.2.4.tgz", - "integrity": "sha512-hEyRieZKH9iw4AzvXaQ+Fyb98k0G/o9QcRGxA1/O/O/elf1+Qvuwb15phT8GbVtIeNziy66XTPOhKKfdr8KyUg==", - "optional": true - }, - "@next/swc-win32-x64-msvc": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-12.2.4.tgz", - "integrity": "sha512-5Pl1tdMJWLy4rvzU1ecx0nHWgDPqoYuvYoXE/5X0Clu9si/yOuBIj573F2kOTY7mu0LX2wgCJVSnyK0abHBxIw==", - "optional": true - }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -24768,9 +24507,9 @@ } }, "babel-loader": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-9.0.1.tgz", - "integrity": "sha512-szYjslOXFlj/po5KfrVmiuBAcI6GVHFuAgC96Qd6mMPHdwl4lmAJkYtvjQ1RxxPjgdkKjd3LQgXDE4jxEutNuw==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-9.1.0.tgz", + "integrity": "sha512-Antt61KJPinUMwHwIIz9T5zfMgevnfZkEVWYDWlG888fgdvRRGD0JTuf/fFozQnfT+uq64sk1bmdHDy/mOEWnA==", "dev": true, "requires": { "find-cache-dir": "^3.3.2", @@ -27509,13 +27248,6 @@ "version": "1.0.0", "devOptional": true }, - "fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "optional": true - }, "function-bind": { "version": "1.1.1" }, @@ -30861,6 +30593,14 @@ "js-tokens": "^3.0.0 || ^4.0.0" } }, + "lowdb": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/lowdb/-/lowdb-5.0.5.tgz", + "integrity": "sha512-7EWKmIMhNKA8TXFhL8t0p6N2LC53l3ZqsWQGSksGhhjrcms9rbKlyrAh2PzSGK5v0KPJ2W5VItBnC3NDRzOnzQ==", + "requires": { + "steno": "^3.0.0" + } + }, "lower-case": { "version": "2.0.2", "dev": true, @@ -33757,6 +33497,11 @@ "state-toggle": { "version": "1.0.3" }, + "steno": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/steno/-/steno-3.0.0.tgz", + "integrity": "sha512-uZtn7Ht9yXLiYgOsmo8btj4+f7VxyYheMt8g6F1ANjyqByQXEE2Gygjgenp3otHH1TlHsS4JAaRGv5wJ1wvMNw==" + }, "stream-combiner": { "version": "0.0.4", "dev": true, diff --git a/package.json b/package.json index c70eeda35e..ef1d9737c4 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "liquidjs": "9.22.1", "lodash": "^4.17.21", "lodash-es": "^4.17.21", + "lowdb": "5.0.5", "lunr": "^2.3.9", "lunr-languages": "^1.9.0", "mdast-util-from-markdown": "^1.2.0", diff --git a/script/dev-toc/generate.js b/script/dev-toc/generate.js index 3f27157680..272328c90f 100755 --- a/script/dev-toc/generate.js +++ b/script/dev-toc/generate.js @@ -36,6 +36,16 @@ async function main() { const res = {} const req = { language: 'en', cookies: {} } + async function recurse(tree) { + const { page } = tree + tree.renderedFullTitle = page.rawTitle.includes('{') + ? await await liquid.parseAndRender(page.rawTitle, req.context) + : page.rawTitle + for (const node of tree.childPages || []) { + await recurse(node) + } + } + for (const version of allVersionKeys) { req.pagePath = version === fpt ? '/' : `/${version}` @@ -52,6 +62,8 @@ async function main() { // Add the tree to the req.context. req.context.currentEnglishTree = req.context.siteTree.en[req.context.currentVersion] + await recurse(req.context.currentEnglishTree) + // Add any defaultOpenSections to the context. req.context.defaultOpenSections = defaultOpenSections diff --git a/tests/unit/permalink.js b/tests/unit/permalink.js index eb541de7ae..0fb44e8c0d 100644 --- a/tests/unit/permalink.js +++ b/tests/unit/permalink.js @@ -27,7 +27,6 @@ describe('Permalink class', () => { test('derives info for non-enterprise versioned homepage', () => { const permalink = new Permalink('en', nonEnterpriseDefaultVersion, 'index.md', 'Hello World') - expect(permalink.pageVersionTitle).toBe('Free, Pro, & Team') expect(permalink.href).toBe('/en') }) @@ -38,7 +37,6 @@ describe('Permalink class', () => { 'index.md', 'Hello World' ) - expect(permalink.pageVersionTitle).toBe(`Enterprise Server ${enterpriseServerReleases.latest}`) expect(permalink.href).toBe(`/en/enterprise-server@${enterpriseServerReleases.latest}`) }) @@ -49,7 +47,6 @@ describe('Permalink class', () => { 'github/index.md', 'Hello World' ) - expect(permalink.pageVersionTitle).toBe('Free, Pro, & Team') expect(permalink.href).toBe('/en/github') }) @@ -60,7 +57,6 @@ describe('Permalink class', () => { 'github/index.md', 'Hello World' ) - expect(permalink.pageVersionTitle).toBe(`Enterprise Server ${enterpriseServerReleases.latest}`) expect(permalink.href).toBe(`/en/enterprise-server@${enterpriseServerReleases.latest}/github`) })