57 lines
2.2 KiB
JavaScript
57 lines
2.2 KiB
JavaScript
#!/usr/bin/env node
|
|
import domwaiter from 'domwaiter'
|
|
import eventToPromise from 'event-to-promise'
|
|
import chalk from 'chalk'
|
|
import parsePageSectionsIntoRecords from './parse-page-sections-into-records.js'
|
|
import languages from '../../lib/languages.js'
|
|
const pageMarker = chalk.green('|')
|
|
const recordMarker = chalk.grey('.')
|
|
const port = 4002
|
|
|
|
export default async function buildRecords(indexName, indexablePages, pageVersion, languageCode) {
|
|
console.log(`\n\nBuilding records for index '${indexName}' (${languages[languageCode].name})`)
|
|
const records = []
|
|
const pages = indexablePages
|
|
// exclude pages that are not in the current language
|
|
.filter((page) => page.languageCode === languageCode)
|
|
// exclude pages that don't have a permalink for the current product version
|
|
.filter((page) => page.permalinks.some((permalink) => permalink.pageVersion === pageVersion))
|
|
|
|
// Find the approve permalink for the given language and GitHub product variant (dotcom v enterprise)
|
|
const permalinks = pages
|
|
.map((page) => {
|
|
return page.permalinks.find((permalink) => {
|
|
return permalink.languageCode === languageCode && permalink.pageVersion === pageVersion
|
|
})
|
|
})
|
|
.map((permalink) => {
|
|
permalink.url = `http://localhost:${port}${permalink.href}`
|
|
return permalink
|
|
})
|
|
|
|
console.log('indexable pages', indexablePages.length)
|
|
console.log('pages in index', pages.length)
|
|
console.log('permalinks in index', permalinks.length)
|
|
console.log(pageMarker, 'denotes pages')
|
|
console.log(recordMarker, 'denotes records derived from sections of pages')
|
|
|
|
const waiter = domwaiter(permalinks, { maxConcurrent: 200, minTime: 5 })
|
|
.on('page', (page) => {
|
|
process.stdout.write(pageMarker)
|
|
const newRecords = parsePageSectionsIntoRecords(page.href, page.$)
|
|
if (!newRecords.length) {
|
|
console.log(chalk.red(`\nno records found: ${page.href}`))
|
|
}
|
|
process.stdout.write(recordMarker.repeat(newRecords.length))
|
|
records.push(...newRecords)
|
|
})
|
|
.on('error', (err) => {
|
|
console.error(err)
|
|
})
|
|
|
|
return eventToPromise(waiter, 'done').then(() => {
|
|
console.log('\nrecords in index: ', records.length)
|
|
return records
|
|
})
|
|
}
|