diff --git a/package-lock.json b/package-lock.json index 1a618bd7af..82870b6fbe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "ajv-formats": "^2.1.1", "bottleneck": "2.19.5", "cheerio": "^1.0.0-rc.11", + "cheerio-to-text": "0.1.0", "classnames": "^2.3.1", "connect-datadog": "0.0.9", "cookie-parser": "^1.4.6", @@ -3480,201 +3481,6 @@ "resolved": "https://registry.npmjs.org/@next/env/-/env-12.2.4.tgz", "integrity": "sha512-/gApFXWk5CCLFQJL5IYJXxPQuG5tz5nPX4l27A9Zm/+wJxiwFrRSP54AopDxIv4JRp/rGwcgk/lZS/0Clw8jYA==" }, - "node_modules/@next/swc-android-arm-eabi": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-android-arm-eabi/-/swc-android-arm-eabi-12.2.4.tgz", - "integrity": "sha512-P4YSFNpmXXSnn3P1qsOAqz+MX3On9fHrlc8ovb/CFJJoU+YLCR53iCEwfw39e0IZEgDA7ttgr108plF8mxaX0g==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-android-arm64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-android-arm64/-/swc-android-arm64-12.2.4.tgz", - "integrity": "sha512-4o2n14E18O+8xHlf6dgJsWPXN9gmSmfIe2Z0EqKDIPBBkFt/2CyrH0+vwHnL2l7xkDHhOGfZYcYIWVUR5aNu0A==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-darwin-arm64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-12.2.4.tgz", - "integrity": "sha512-DcUO6MGBL9E3jj5o86MUnTOy4WawIJJhyCcFYO4f51sbl7+uPIYIx40eo98A6NwJEXazCqq1hLeqOaNTAIvDiQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-darwin-x64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-12.2.4.tgz", - "integrity": "sha512-IUlFMqeLjdIzDorrGC2Dt+2Ae3DbKQbRzCzmDq4/CP1+jJGeDXo/2AHnlE+WYnwQAC4KtAz6pbVnd3KstZWsVA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-freebsd-x64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-freebsd-x64/-/swc-freebsd-x64-12.2.4.tgz", - "integrity": "sha512-475vwyWcjnyDVDWLgAATP0HI8W1rwByc+uXk1B6KkAVFhkoDgH387LW0uNqxavK+VxCzj3avQXX/58XDvxtSlg==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm-gnueabihf": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm-gnueabihf/-/swc-linux-arm-gnueabihf-12.2.4.tgz", - "integrity": "sha512-qZW+L3iG3XSGtlOPmD5RRWXyk6ZNdscLV0BQjuDvP+exTg+uixqHXOHz0/GVATIJEBQOF0Kew7jAXVXEP+iRTQ==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm64-gnu": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-12.2.4.tgz", - "integrity": "sha512-fEPRjItWYaKyyG9N+2HIA59OBHIhk7WC+Rh+LwXsh0pQe870Ykpek3KQs0umjsrEGe57NyMomq3f80/N8taDvA==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm64-musl": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-12.2.4.tgz", - "integrity": "sha512-rnCTzXII0EBCcFn9P5s/Dho2kPUMSX/bP0iOAj8wEI/IxUEfEElbin89zJoNW30cycHu19xY8YP4K2+hzciPzQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-x64-gnu": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-12.2.4.tgz", - "integrity": "sha512-PhXX6NSuIuhHInxPY2VkG2Bl7VllsD3Cjx+pQcS1wTym7Zt7UoLvn05PkRrkiyIkvR+UXnqPUM3TYiSbnemXEw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-x64-musl": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-12.2.4.tgz", - "integrity": "sha512-GmC/QROiUZpFirHRfPQqMyCXZ+5+ndbBZrMvL74HtQB/CKXB8K1VM+rvy9Gp/5OaU8Rxp48IcX79NOfI2LiXlA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-arm64-msvc": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-12.2.4.tgz", - "integrity": "sha512-9XKoCXbNZuaMRPtcKQz3+hgVpkMosaLlcxHFXT8/j4w61k7/qvEbrkMDS9WHNrD/xVcLycwhPRgXcns2K1BdBQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-ia32-msvc": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-12.2.4.tgz", - "integrity": "sha512-hEyRieZKH9iw4AzvXaQ+Fyb98k0G/o9QcRGxA1/O/O/elf1+Qvuwb15phT8GbVtIeNziy66XTPOhKKfdr8KyUg==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-x64-msvc": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-12.2.4.tgz", - "integrity": "sha512-5Pl1tdMJWLy4rvzU1ecx0nHWgDPqoYuvYoXE/5X0Clu9si/yOuBIj573F2kOTY7mu0LX2wgCJVSnyK0abHBxIw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -6835,6 +6641,14 @@ "url": "https://github.com/sponsors/fb55" } }, + "node_modules/cheerio-to-text": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/cheerio-to-text/-/cheerio-to-text-0.1.0.tgz", + "integrity": "sha512-qKdlJcKn5kvhYea4DpQHnhnLtF9bCZSQ+PK4/78FCDSINnf5UWrKb25/2hkg2eqZi0XxoCuM2hmi7Rg9xb9nyg==", + "peerDependencies": { + "cheerio": "1.0.0-rc.12" + } + }, "node_modules/chokidar": { "version": "3.5.1", "devOptional": true, @@ -9487,20 +9301,6 @@ "devOptional": true, "license": "ISC" }, - "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/function-bind": { "version": "1.1.1", "license": "MIT" @@ -20436,9 +20236,9 @@ } }, "node_modules/website-scraper/node_modules/normalize-url": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-7.0.3.tgz", - "integrity": "sha512-RiCOdwdPnzvwcBFJE4iI1ss3dMVRIrEzFpn8ftje6iBfzBInqlnRrNhxcLwBEKjPPXQKzm1Ptlxtaiv9wdcj5w==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-7.2.0.tgz", + "integrity": "sha512-uhXOdZry0L6M2UIo9BTt7FdpBDiAGN/7oItedQwPKh8jh31ZlvC8U9Xl/EJ3aijDHaywXTW3QbZ6LuCocur1YA==", "optional": true, "engines": { "node": ">=12.20" @@ -20581,9 +20381,9 @@ } }, "node_modules/ws": { - "version": "7.5.8", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.8.tgz", - "integrity": "sha512-ri1Id1WinAX5Jqn9HejiGb8crfRio0Qgu8+MtL36rlTA6RLsMdWt1Az/19A2Qij6uSHUMphEFaTKa4WG+UNHNw==", + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", "optional": true, "engines": { "node": ">=8.3.0" @@ -23128,84 +22928,6 @@ "resolved": "https://registry.npmjs.org/@next/env/-/env-12.2.4.tgz", "integrity": "sha512-/gApFXWk5CCLFQJL5IYJXxPQuG5tz5nPX4l27A9Zm/+wJxiwFrRSP54AopDxIv4JRp/rGwcgk/lZS/0Clw8jYA==" }, - "@next/swc-android-arm-eabi": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-android-arm-eabi/-/swc-android-arm-eabi-12.2.4.tgz", - "integrity": "sha512-P4YSFNpmXXSnn3P1qsOAqz+MX3On9fHrlc8ovb/CFJJoU+YLCR53iCEwfw39e0IZEgDA7ttgr108plF8mxaX0g==", - "optional": true - }, - "@next/swc-android-arm64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-android-arm64/-/swc-android-arm64-12.2.4.tgz", - "integrity": "sha512-4o2n14E18O+8xHlf6dgJsWPXN9gmSmfIe2Z0EqKDIPBBkFt/2CyrH0+vwHnL2l7xkDHhOGfZYcYIWVUR5aNu0A==", - "optional": true - }, - "@next/swc-darwin-arm64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-12.2.4.tgz", - "integrity": "sha512-DcUO6MGBL9E3jj5o86MUnTOy4WawIJJhyCcFYO4f51sbl7+uPIYIx40eo98A6NwJEXazCqq1hLeqOaNTAIvDiQ==", - "optional": true - }, - "@next/swc-darwin-x64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-12.2.4.tgz", - "integrity": "sha512-IUlFMqeLjdIzDorrGC2Dt+2Ae3DbKQbRzCzmDq4/CP1+jJGeDXo/2AHnlE+WYnwQAC4KtAz6pbVnd3KstZWsVA==", - "optional": true - }, - "@next/swc-freebsd-x64": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-freebsd-x64/-/swc-freebsd-x64-12.2.4.tgz", - "integrity": "sha512-475vwyWcjnyDVDWLgAATP0HI8W1rwByc+uXk1B6KkAVFhkoDgH387LW0uNqxavK+VxCzj3avQXX/58XDvxtSlg==", - "optional": true - }, - "@next/swc-linux-arm-gnueabihf": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm-gnueabihf/-/swc-linux-arm-gnueabihf-12.2.4.tgz", - "integrity": "sha512-qZW+L3iG3XSGtlOPmD5RRWXyk6ZNdscLV0BQjuDvP+exTg+uixqHXOHz0/GVATIJEBQOF0Kew7jAXVXEP+iRTQ==", - "optional": true - }, - "@next/swc-linux-arm64-gnu": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-12.2.4.tgz", - "integrity": "sha512-fEPRjItWYaKyyG9N+2HIA59OBHIhk7WC+Rh+LwXsh0pQe870Ykpek3KQs0umjsrEGe57NyMomq3f80/N8taDvA==", - "optional": true - }, - "@next/swc-linux-arm64-musl": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-12.2.4.tgz", - "integrity": "sha512-rnCTzXII0EBCcFn9P5s/Dho2kPUMSX/bP0iOAj8wEI/IxUEfEElbin89zJoNW30cycHu19xY8YP4K2+hzciPzQ==", - "optional": true - }, - "@next/swc-linux-x64-gnu": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-12.2.4.tgz", - "integrity": "sha512-PhXX6NSuIuhHInxPY2VkG2Bl7VllsD3Cjx+pQcS1wTym7Zt7UoLvn05PkRrkiyIkvR+UXnqPUM3TYiSbnemXEw==", - "optional": true - }, - "@next/swc-linux-x64-musl": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-12.2.4.tgz", - "integrity": "sha512-GmC/QROiUZpFirHRfPQqMyCXZ+5+ndbBZrMvL74HtQB/CKXB8K1VM+rvy9Gp/5OaU8Rxp48IcX79NOfI2LiXlA==", - "optional": true - }, - "@next/swc-win32-arm64-msvc": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-12.2.4.tgz", - "integrity": "sha512-9XKoCXbNZuaMRPtcKQz3+hgVpkMosaLlcxHFXT8/j4w61k7/qvEbrkMDS9WHNrD/xVcLycwhPRgXcns2K1BdBQ==", - "optional": true - }, - "@next/swc-win32-ia32-msvc": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-12.2.4.tgz", - "integrity": "sha512-hEyRieZKH9iw4AzvXaQ+Fyb98k0G/o9QcRGxA1/O/O/elf1+Qvuwb15phT8GbVtIeNziy66XTPOhKKfdr8KyUg==", - "optional": true - }, - "@next/swc-win32-x64-msvc": { - "version": "12.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-12.2.4.tgz", - "integrity": "sha512-5Pl1tdMJWLy4rvzU1ecx0nHWgDPqoYuvYoXE/5X0Clu9si/yOuBIj573F2kOTY7mu0LX2wgCJVSnyK0abHBxIw==", - "optional": true - }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -25624,6 +25346,12 @@ "domutils": "^3.0.1" } }, + "cheerio-to-text": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/cheerio-to-text/-/cheerio-to-text-0.1.0.tgz", + "integrity": "sha512-qKdlJcKn5kvhYea4DpQHnhnLtF9bCZSQ+PK4/78FCDSINnf5UWrKb25/2hkg2eqZi0XxoCuM2hmi7Rg9xb9nyg==", + "requires": {} + }, "chokidar": { "version": "3.5.1", "devOptional": true, @@ -27426,13 +27154,6 @@ "version": "1.0.0", "devOptional": true }, - "fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "optional": true - }, "function-bind": { "version": "1.1.1" }, @@ -34874,9 +34595,9 @@ } }, "normalize-url": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-7.0.3.tgz", - "integrity": "sha512-RiCOdwdPnzvwcBFJE4iI1ss3dMVRIrEzFpn8ftje6iBfzBInqlnRrNhxcLwBEKjPPXQKzm1Ptlxtaiv9wdcj5w==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-7.2.0.tgz", + "integrity": "sha512-uhXOdZry0L6M2UIo9BTt7FdpBDiAGN/7oItedQwPKh8jh31ZlvC8U9Xl/EJ3aijDHaywXTW3QbZ6LuCocur1YA==", "optional": true }, "parse5": { @@ -34980,9 +34701,9 @@ } }, "ws": { - "version": "7.5.8", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.8.tgz", - "integrity": "sha512-ri1Id1WinAX5Jqn9HejiGb8crfRio0Qgu8+MtL36rlTA6RLsMdWt1Az/19A2Qij6uSHUMphEFaTKa4WG+UNHNw==", + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", "optional": true, "requires": {} }, diff --git a/package.json b/package.json index e01771c015..7203fb6b71 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "ajv-formats": "^2.1.1", "bottleneck": "2.19.5", "cheerio": "^1.0.0-rc.11", + "cheerio-to-text": "0.1.0", "classnames": "^2.3.1", "connect-datadog": "0.0.9", "cookie-parser": "^1.4.6", diff --git a/script/search/find-indexable-pages.js b/script/search/find-indexable-pages.js index 72ab2dc73d..2a41d2b583 100644 --- a/script/search/find-indexable-pages.js +++ b/script/search/find-indexable-pages.js @@ -1,7 +1,7 @@ #!/usr/bin/env node import { loadPages } from '../../lib/page-data.js' -export default async function findIndexablePages() { +export default async function findIndexablePages(match = '') { const allPages = await loadPages() const indexablePages = allPages // exclude hidden pages @@ -10,6 +10,7 @@ export default async function findIndexablePages() { .filter((page) => !page.parentProduct || !page.parentProduct.wip || page.parentProduct.hidden) // exclude absolute home page (e.g. /en or /ja) .filter((page) => page.relativePath !== 'index.md') + .filter((page) => !match || page.relativePath.includes(match)) console.log('total pages', allPages.length) console.log('indexable pages', indexablePages.length) diff --git a/script/search/parse-page-sections-into-records.js b/script/search/parse-page-sections-into-records.js index 4ab6897d1e..0404d6a7b8 100644 --- a/script/search/parse-page-sections-into-records.js +++ b/script/search/parse-page-sections-into-records.js @@ -1,4 +1,6 @@ #!/usr/bin/env node +import { render } from 'cheerio-to-text' + import { maxContentLength } from '../../lib/search/config.js' // This module takes cheerio page object and divides it into sections @@ -67,7 +69,7 @@ export default function parsePageSectionsIntoRecords(page) { // pages that yields some decent content to be searched on, because // when you view these pages in a browser, there's clearly text there. if ($root.length > 0) { - body = getAllText($root) + body = render($root) } if (!body && !intro) { @@ -95,43 +97,3 @@ export default function parsePageSectionsIntoRecords(page) { topics, } } - -function getAllText($root) { - const inlineElements = new Set( - `a,abbr,acronym,audio,b,bdi,bdo,big,br,button,canvas,cite,code,data, - datalist,del,dfn,em,embed,i,iframe,img,input,ins,kbd,label,map,mark, - meter,noscript,object,output,picture,progress,q,ruby,s,samp,script, - select,slot,small,span,strong,sub,sup,svg,template,textarea,time, - tt,u,var,video,wbr` - .split(',') - .map((s) => s.trim()) - ) - - const walkTree = (node, callback, index = 0, level = 0) => { - callback(node, index, level) - for (let i = 0; i < (node.children || []).length; i++) { - walkTree(node.children[i], callback, i, ++level) - level-- - } - } - - const fragments = [] - - walkTree($root[0], (element) => { - if (element.name === 'body') return - - if (element.type === 'text') { - const parentElement = element.parent || {} - const previousElement = element.prev || {} - let { data } = element - if (data.trim()) { - if (!inlineElements.has(parentElement.name) && !inlineElements.has(previousElement.name)) { - data = `\n${data}` - } - fragments.push(data) - } - } - }) - - return fragments.join('').trim() -} diff --git a/script/search/sync-search-indices.js b/script/search/sync-search-indices.js index 6c7a60c495..715a5f1d7f 100755 --- a/script/search/sync-search-indices.js +++ b/script/search/sync-search-indices.js @@ -50,6 +50,7 @@ program // any Lunr indexes. .option('--no-lunr-index', `Do not generate a Lunr index, just the records file (default false)`) .option('--no-markers', 'Do not print a marker for each parsed document') + .option('--filter ', 'Filter to only do pages that match this string') .parse(process.argv) main(program.opts()) @@ -131,6 +132,7 @@ async function main(opts) { const config = { noMarkers: !opts.markers, + filter: opts.filter, } const options = { diff --git a/script/search/sync.js b/script/search/sync.js index aaa2e56bf6..304fd8edb2 100644 --- a/script/search/sync.js +++ b/script/search/sync.js @@ -40,7 +40,7 @@ export default async function syncSearchIndexes({ ) // Exclude WIP pages, hidden pages, index pages, etc - const indexablePages = await findIndexablePages() + const indexablePages = await findIndexablePages(config.filter) const redirects = {} indexablePages.forEach((page) => { const href = page.relativePath.replace('index.md', '').replace('.md', '') diff --git a/tests/unit/search/fixtures/page-with-heading-and-paragraph-no-whitespace.html b/tests/unit/search/fixtures/page-with-heading-and-paragraph-no-whitespace.html index c46b88c799..cf4911cff6 100644 --- a/tests/unit/search/fixtures/page-with-heading-and-paragraph-no-whitespace.html +++ b/tests/unit/search/fixtures/page-with-heading-and-paragraph-no-whitespace.html @@ -20,4 +20,6 @@

  • Changing your primary email address

    You can change the email address associated with your personal account at any time.

    • +

    Paragraph.

    + diff --git a/tests/unit/search/parse-page-sections-into-records.js b/tests/unit/search/parse-page-sections-into-records.js index eff36a033d..f4ff34c259 100644 --- a/tests/unit/search/parse-page-sections-into-records.js +++ b/tests/unit/search/parse-page-sections-into-records.js @@ -119,5 +119,7 @@ describe('search parsePageSectionsIntoRecords module', () => { // But note also that it would also concatenate the text of the heading // with the text of the paragraph without a whitespace in between. expect(record.content.includes('email addressYou can set')).toBeFalsy() + // Make sure that inline elements are still together. + expect(record.content).toMatch(/Paragraph\./) }) })