diff --git a/package.json b/package.json index 1d7b151971..679e8f9bf4 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "playwright-test": "playwright test --config src/fixtures/playwright.config.ts --project=\"Google Chrome\"", "post-lints": "node src/content-linter/scripts/post-lints.js", "postinstall": "cp package-lock.json .installed.package-lock.json && echo \"Updated .installed.package-lock.json\" # see husky/post-checkout and husky/post-merge", - "precompute-pageinfo": "node src/pageinfo/scripts/precompute-pageinfo.js", + "precompute-pageinfo": "tsx src/pageinfo/scripts/precompute-pageinfo.ts", "prepare": "husky install src/workflows/husky", "prettier": "prettier -w \"**/*.{ts,tsx,js,mjs,scss,yml,yaml}\"", "prettier-check": "prettier -c \"**/*.{ts,tsx,js,mjs,scss,yml,yaml}\"", diff --git a/src/frame/lib/page-data.js b/src/frame/lib/page-data.js index 8735504188..6a3b4751f2 100644 --- a/src/frame/lib/page-data.js +++ b/src/frame/lib/page-data.js @@ -37,7 +37,7 @@ const translatableFrontmatterKeys = Object.entries(frontmatterSchema.schema.prop * first since it's the most expensive work. This gets us a nested object with pages attached that we can use * as the basis for the siteTree after we do some versioning. We can also use it to derive the pageList. */ -export async function loadUnversionedTree(languagesOnly = null) { +export async function loadUnversionedTree(languagesOnly = []) { if (languagesOnly && !Array.isArray(languagesOnly)) { throw new Error("'languagesOnly' has to be an array") } @@ -47,7 +47,7 @@ export async function loadUnversionedTree(languagesOnly = null) { const languagesValues = Object.entries(languages) .filter(([language]) => { - return !languagesOnly || languagesOnly.includes(language) + return !languagesOnly.length || languagesOnly.includes(language) }) .map(([, data]) => { return data @@ -295,7 +295,7 @@ export async function versionPages(obj, version, langCode) { } // Derive a flat array of Page objects in all languages. -export async function loadPageList(unversionedTree, languagesOnly = null) { +export async function loadPageList(unversionedTree, languagesOnly = []) { if (languagesOnly && !Array.isArray(languagesOnly)) { throw new Error("'languagesOnly' has to be an array") } @@ -303,7 +303,7 @@ export async function loadPageList(unversionedTree, languagesOnly = null) { const pageList = [] await Promise.all( - (languagesOnly || Object.keys(languages)).map(async (langCode) => { + ((languagesOnly.length && languagesOnly) || Object.keys(languages)).map(async (langCode) => { await addToCollection(rawTree[langCode], pageList) }), ) @@ -335,7 +335,7 @@ export function createMapFromArray(pageList) { return pageMap } -export async function loadPageMap(pageList, languagesOnly = null) { +export async function loadPageMap(pageList, languagesOnly = []) { const pages = pageList || (await loadPageList(languagesOnly)) const pageMap = createMapFromArray(pages) return pageMap diff --git a/src/pageinfo/middleware.js b/src/pageinfo/middleware.js index 6ce31cafee..d852997b19 100644 --- a/src/pageinfo/middleware.js +++ b/src/pageinfo/middleware.js @@ -22,7 +22,7 @@ const router = express.Router() // Note that if the file does not exist, it will be ignored and // every pageinfo is computed every time. // Note! The only reason this variable is exported is so that -// it can be imported by the script scripts/precompute-pageinfo.js +// it can be imported by the script scripts/precompute-pageinfo.ts export const CACHE_FILE_PATH = '.pageinfo-cache.json.br' const validationMiddleware = (req, res, next) => { diff --git a/src/pageinfo/scripts/precompute-pageinfo.js b/src/pageinfo/scripts/precompute-pageinfo.ts old mode 100755 new mode 100644 similarity index 53% rename from src/pageinfo/scripts/precompute-pageinfo.js rename to src/pageinfo/scripts/precompute-pageinfo.ts index 24a31f097b..e95c027ad6 --- a/src/pageinfo/scripts/precompute-pageinfo.js +++ b/src/pageinfo/scripts/precompute-pageinfo.ts @@ -13,7 +13,7 @@ * Why cache?: Despite being a fast computation (3 Liquid + Markdown renders), * it still adds up. And it's safe and cheap to precompute in advance. * - * Why only the English?: To make the file not too large. + * Why only the English by default?: To make the file not too large. * Given how good these things compress, we might consider, in the * future, to do all languages. * @@ -29,20 +29,55 @@ import fs from 'fs' import { brotliCompressSync } from 'zlib' -import { loadPages, loadUnversionedTree } from '#src/frame/lib/page-data.js' +import chalk from 'chalk' +import { program, Option } from 'commander' + +import { languageKeys } from 'src/languages/lib/languages.js' +import { loadPages, loadUnversionedTree } from 'src/frame/lib/page-data.js' import { CACHE_FILE_PATH, getPageInfo } from '../middleware.js' -main() +program + .description('Generates a JSON file with precompute pageinfo data by pathname') + .addOption( + new Option('-l, --language ', 'Which languages to focus on') + .choices(languageKeys) + .default(['en']), + ) + .option('-o, --output-file ', 'path to output file', CACHE_FILE_PATH) + .parse(process.argv) + +type Options = { + outputFile: string + languages: string[] +} +const opts = program.opts() + +main({ + outputFile: opts.outputFile, + languages: opts.language, +}) const CI = Boolean(JSON.parse(process.env.CI || 'false')) -async function main() { - const unversionedTree = await loadUnversionedTree(['en']) - const pageList = await loadPages(unversionedTree, ['en']) +type PageInfo = { + title: string + intro: string + product: string +} + +async function main(options: Options) { + const { outputFile, languages } = options + if (outputFile !== CACHE_FILE_PATH) { + console.warn(chalk.yellow(`Writing to ${outputFile} instead of ${CACHE_FILE_PATH}`)) + } + const unversionedTree = await loadUnversionedTree(languages) + const pageList = await loadPages(unversionedTree, languages) let label = `Compute pageinfos for ${pageList.length.toLocaleString()} pages` console.time(label) - const pageinfos = {} + const pageinfos: { + [pathname: string]: PageInfo + } = {} for (const page of pageList) { const pathname = page.permalinks[0].href try { @@ -57,14 +92,20 @@ async function main() { } console.timeEnd(label) - label = `Serialize, compress, and write to ${CACHE_FILE_PATH}` + label = `Serialize, compress, and write to ${outputFile}` console.time(label) const payload = CI ? JSON.stringify(pageinfos) : JSON.stringify(pageinfos, null, 2) - const payloadBuffer = Buffer.from(payload, 'utf-8') - const payloadCompressed = brotliCompressSync(payloadBuffer) - fs.writeFileSync(CACHE_FILE_PATH, payloadCompressed) + if (outputFile.endsWith('.json')) { + fs.writeFileSync(outputFile, payload) + } else { + const payloadBuffer = Buffer.from(payload, 'utf-8') + const payloadCompressed = brotliCompressSync(payloadBuffer) + fs.writeFileSync(outputFile, payloadCompressed) + } console.timeEnd(label) console.log( - `Wrote ${Object.keys(pageinfos).length.toLocaleString()} pageinfos to ${CACHE_FILE_PATH}`, + chalk.green( + `Wrote ${Object.keys(pageinfos).length.toLocaleString()} pageinfos to ${outputFile}`, + ), ) }