From 0f685115f7cafd0084d41afcf86fec368bff272d Mon Sep 17 00:00:00 2001 From: Kevin Heis Date: Tue, 15 Dec 2020 14:04:41 -0800 Subject: [PATCH] Revert file reads from async to sync (#16979) * Revert file reads from async to sync * Async funness * Fix warm server timers Co-authored-by: Chiedo John --- lib/data-directory.js | 14 +++++--------- lib/page.js | 10 +++++----- lib/pages.js | 24 +++++++++--------------- lib/warm-server.js | 6 +++--- package-lock.json | 15 ++++++++------- package.json | 2 +- 6 files changed, 31 insertions(+), 40 deletions(-) diff --git a/lib/data-directory.js b/lib/data-directory.js index 17bb7c60b4..845452591f 100644 --- a/lib/data-directory.js +++ b/lib/data-directory.js @@ -1,15 +1,12 @@ const assert = require('assert') -const fs = require('fs').promises +const fs = require('fs') const path = require('path') const walk = require('walk-sync') -const { mapLimit } = require('async') const yaml = require('js-yaml') const { isRegExp, set } = require('lodash') const filenameToKey = require('./filename-to-key') -const FILE_READ_LIMIT = 100 - -module.exports = async function dataDirectory (dir, opts = {}) { +module.exports = function dataDirectory (dir, opts = {}) { const defaultOpts = { preprocess: (content) => { return content }, ignorePatterns: [/README\.md$/i], @@ -42,10 +39,9 @@ module.exports = async function dataDirectory (dir, opts = {}) { // ignore files that don't have a whitelisted file extension return opts.extensions.includes(path.extname(filename).toLowerCase()) }) - const files = await mapLimit( - filenames, - FILE_READ_LIMIT, - async filename => [filename, await fs.readFile(filename, 'utf8')] + + const files = filenames.map( + filename => [filename, fs.readFileSync(filename, 'utf8')] ) files.forEach(([filename, fileContent]) => { // derive `foo.bar.baz` object key from `foo/bar/baz.yml` filename diff --git a/lib/page.js b/lib/page.js index 2471debe2a..5ff8b9d32d 100644 --- a/lib/page.js +++ b/lib/page.js @@ -1,5 +1,5 @@ const assert = require('assert') -const fs = require('fs').promises +const fs = require('fs') const path = require('path') const cheerio = require('cheerio') const patterns = require('./patterns') @@ -23,13 +23,13 @@ const slash = require('slash') const statsd = require('./statsd') class Page { - static async init (opts) { - opts = await Page.read(opts) + static init (opts) { + opts = Page.read(opts) if (!opts) return return new Page(opts) } - static async read (opts) { + static read (opts) { assert(opts.relativePath, 'relativePath is required') assert(opts.basePath, 'basePath is required') @@ -39,7 +39,7 @@ class Page { // Per https://nodejs.org/api/fs.html#fs_fs_exists_path_callback // its better to read and handle errors than to check access/stats first try { - const raw = await fs.readFile(fullPath, 'utf8') + const raw = fs.readFileSync(fullPath, 'utf8') return { ...opts, relativePath, fullPath, raw } } catch (err) { if (err.code === 'ENOENT') return false diff --git a/lib/pages.js b/lib/pages.js index b3449ee92c..7a90065e62 100644 --- a/lib/pages.js +++ b/lib/pages.js @@ -2,21 +2,16 @@ const path = require('path') const walk = require('walk-sync').entries const Page = require('./page') const languages = require('./languages') -const { mapLimit } = require('async') -const FILE_READ_LIMIT = 100 - -async function loadPageList () { +function loadPageList () { // load english pages const englishPath = path.join(__dirname, '..', languages.en.dir, 'content') const englishPaths = walk(englishPath, { globs: ['**/*.md'], ignore: ['**/README.md'] }) - const englishPages = await mapLimit( - englishPaths, - FILE_READ_LIMIT, - async opts => await Page.read({ + const englishPages = englishPaths.map( + opts => Page.read({ ...opts, languageCode: languages.en.code }) @@ -34,11 +29,10 @@ async function loadPageList () { })) }) .flat() - const localizedPages = await mapLimit( - localizedPaths, - FILE_READ_LIMIT, - async ({ basePath, relativePath, languageCode }) => - await Page.read({ basePath, relativePath, languageCode }) + + const localizedPages = localizedPaths.map( + ({ basePath, relativePath, languageCode }) => + Page.read({ basePath, relativePath, languageCode }) ) // Build out the list of prepared pages @@ -64,8 +58,8 @@ function createMapFromArray (pageList) { return pageMap } -async function loadPageMap (pageList) { - const pages = pageList || await loadPageList() +function loadPageMap (pageList) { + const pages = pageList || loadPageList() return createMapFromArray(pages) } diff --git a/lib/warm-server.js b/lib/warm-server.js index 36cf5e7416..cb5e03837c 100644 --- a/lib/warm-server.js +++ b/lib/warm-server.js @@ -7,10 +7,10 @@ const loadSiteTree = require('./site-tree') // Instrument these functions so that // it's wrapped in a timer that reports to Datadog const dog = { - loadPages: statsd.asyncTimer(loadPages, 'load_pages'), - loadPageMap: statsd.asyncTimer(loadPageMap, 'load_page_map'), + loadPages: statsd.timer(loadPages, 'load_pages'), + loadPageMap: statsd.timer(loadPageMap, 'load_page_map'), loadRedirects: statsd.asyncTimer(loadRedirects, 'load_redirects'), - loadSiteData: statsd.asyncTimer(loadSiteData, 'load_site_data'), + loadSiteData: statsd.timer(loadSiteData, 'load_site_data'), loadSiteTree: statsd.asyncTimer(loadSiteTree, 'load_site_tree') } diff --git a/package-lock.json b/package-lock.json index 53465f4809..7b0505168a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5271,7 +5271,7 @@ }, "agentkeepalive": { "version": "2.2.0", - "resolved": "http://registry.npmjs.org/agentkeepalive/-/agentkeepalive-2.2.0.tgz", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-2.2.0.tgz", "integrity": "sha1-xdG9SxKQCPEWPyNvhuX66iAm4u8=" }, "aggregate-error": { @@ -5417,7 +5417,7 @@ "argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha1-vNZ5HqWuCXJeF+WtmIE0zUCz2RE=", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", "requires": { "sprintf-js": "~1.0.2" } @@ -5547,7 +5547,8 @@ "async": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", - "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==" + "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==", + "dev": true }, "async-each": { "version": "1.0.3", @@ -6794,7 +6795,7 @@ }, "brfs": { "version": "1.6.1", - "resolved": "http://registry.npmjs.org/brfs/-/brfs-1.6.1.tgz", + "resolved": "https://registry.npmjs.org/brfs/-/brfs-1.6.1.tgz", "integrity": "sha512-OfZpABRQQf+Xsmju8XE9bDjs+uU4vLREGolP7bDgcpsI17QREyZ4Bl+2KLxxx1kCgA0fAIhKQBaBYh+PEcCqYQ==", "requires": { "quote-stream": "^1.0.1", @@ -9421,7 +9422,7 @@ "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", - "integrity": "sha1-tKxAZIEH/c3PriQvQovqihTU8b8=", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", "requires": { "is-arrayish": "^0.2.1" } @@ -12559,7 +12560,7 @@ "dependencies": { "mkdirp": { "version": "0.3.0", - "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz", "integrity": "sha1-G79asbqCevI1dRQ0kEJkVfSB/h4=" }, "nopt": { @@ -17685,7 +17686,7 @@ }, "magic-string": { "version": "0.22.5", - "resolved": "http://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.22.5.tgz", "integrity": "sha512-oreip9rJZkzvA8Qzk9HFs8fZGF/u7H/gtrE8EN6RjKJ9kh2HlC+yQ2QezifqTZfGyiuAV0dRv5a+y/8gBb1m9w==", "requires": { "vlq": "^0.2.2" diff --git a/package.json b/package.json index 77b1e6913a..8f064dac0f 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "@primer/css": "^15.1.0", "@primer/octicons": "^11.0.0", "algoliasearch": "^3.35.1", - "async": "^3.2.0", "babel-loader": "^8.1.0", "babel-preset-env": "^1.7.0", "browser-date-formatter": "^3.0.3", @@ -95,6 +94,7 @@ "devDependencies": { "@actions/core": "^1.2.6", "ajv": "^6.11.0", + "async": "^3.2.0", "await-sleep": "0.0.1", "aws-sdk": "^2.610.0", "babel-eslint": "^10.1.0",