diff --git a/lib/data-directory.js b/lib/data-directory.js index d72abcbf55..851a8fb7a4 100644 --- a/lib/data-directory.js +++ b/lib/data-directory.js @@ -2,10 +2,13 @@ const assert = require('assert') const fs = require('fs').promises const path = require('path') const walk = require('walk-sync') +const { mapLimit } = require('async') const yaml = require('js-yaml') const { isRegExp, set } = require('lodash') const filenameToKey = require('./filename-to-key') +const FILE_READ_LIMIT = 100 + module.exports = async function dataDirectory (dir, opts = {}) { const defaultOpts = { preprocess: (content) => { return content }, @@ -31,7 +34,7 @@ module.exports = async function dataDirectory (dir, opts = {}) { const data = {} // find YAML and Markdown files in the given directory, recursively - await Promise.all(walk(dir, { includeBasePath: true }) + const filenames = walk(dir, { includeBasePath: true }) .filter(filename => { // ignore files that match any of ignorePatterns regexes if (opts.ignorePatterns.some(pattern => pattern.test(filename))) return false @@ -39,7 +42,10 @@ module.exports = async function dataDirectory (dir, opts = {}) { // ignore files that don't have a whitelisted file extension return opts.extensions.includes(path.extname(filename).toLowerCase()) }) - .map(async filename => { + await mapLimit( + filenames, + FILE_READ_LIMIT, + async filename => { // derive `foo.bar.baz` object key from `foo/bar/baz.yml` filename const key = filenameToKey(path.relative(dir, filename)) const extension = path.extname(filename).toLowerCase() @@ -62,7 +68,8 @@ module.exports = async function dataDirectory (dir, opts = {}) { set(data, key, fileContent) break } - })) + } + ) return data } diff --git a/lib/pages.js b/lib/pages.js index 4db70e7dd5..e1d2bec57b 100644 --- a/lib/pages.js +++ b/lib/pages.js @@ -3,7 +3,8 @@ const walk = require('walk-sync').entries const Page = require('./page') const languages = require('./languages') const { mapLimit, filterLimit } = require('async') -const FILE_READ_LIMIT = 500 + +const FILE_READ_LIMIT = 100 async function loadPageList () { const pageList = [] diff --git a/lib/path-utils.js b/lib/path-utils.js index 02e75aab1d..54845166f9 100644 --- a/lib/path-utils.js +++ b/lib/path-utils.js @@ -6,6 +6,8 @@ const allProducts = require('./all-products') const allVersions = require('./all-versions') const { getNewVersionedPath } = require('./old-versions-utils') +const supportedVersions = new Set(Object.keys(allVersions)) + // construct appropriate versioned path for any given HREF function getVersionedPathWithoutLanguage (href, version) { // start clean without language code or trailing slash @@ -24,7 +26,7 @@ function getVersionedPathWithoutLanguage (href, version) { // if the version found is not a currently supported version... let productObjectFromPath - if (!Object.keys(allVersions).includes(versionFromPath)) { + if (!supportedVersions.has(versionFromPath)) { // first check if the first segment is instead a current product; // example: /admin/foo or /desktop/foo productObjectFromPath = allProducts[versionFromPath] @@ -71,11 +73,7 @@ function getPathWithLanguage (href, languageCode) { // remove the language from the given HREF // /articles/foo -> /en/articles/foo function getPathWithoutLanguage (href) { - const newHref = href.match(patterns.hasLanguageCode) - ? '/' + href.split('/').slice(2).join('/') - : href - - return slash(newHref) + return slash(href.replace(patterns.hasLanguageCode, '/')) } function getPathWithoutVersion (href) {