add new link checking code
This commit is contained in:
Sarah Schneider
255
tests/helpers/links-checker.js
Normal file
255
tests/helpers/links-checker.js
Normal file
@@ -0,0 +1,255 @@
|
||||
const cheerio = require('cheerio')
|
||||
const { union, uniq } = require('lodash')
|
||||
const fs = require('fs')
|
||||
const path = require('path')
|
||||
|
||||
const { getVersionStringFromPath } = require('../../lib/path-utils')
|
||||
const patterns = require('../../lib/patterns')
|
||||
const { deprecated, latest } = require('../../lib/enterprise-server-releases')
|
||||
const findPageInVersion = require('../../lib/find-page-in-version')
|
||||
const nonEnterpriseDefaultVersion = require('../../lib/non-enterprise-default-version')
|
||||
const rest = require('../../middleware/contextualizers/rest')
|
||||
const graphql = require('../../middleware/contextualizers/graphql')
|
||||
const contextualize = require('../../middleware/context')
|
||||
// TODO enable when release notes are live
|
||||
// const releaseNotes = require('../../middleware/contextualizers/enterprise-release-notes')
|
||||
|
||||
class LinksChecker {
|
||||
constructor (opts = { languageCode: 'en', internalHrefPrefixes: ['/', '#'] }) {
|
||||
Object.assign(this, { ...opts })
|
||||
|
||||
// Some caching mechanism so we do not load pages unnecessarily,
|
||||
// nor check links that have been checked
|
||||
this.pageCache = new Map()
|
||||
this.checkedLinksCache = new Set()
|
||||
|
||||
// stores images to check all at once in a Map:
|
||||
// imageSrc => {
|
||||
// "usedBy": [version:path, ...]
|
||||
// }
|
||||
this.imagesToCheck = new Map()
|
||||
|
||||
// Stores broken images in a Map, formatted the same way as imagesToCheck
|
||||
this.brokenImages = new Map()
|
||||
|
||||
// Stores broken links in a Map in the format of:
|
||||
// link => {
|
||||
// linkedFrom: [ version:filePath, ... ]
|
||||
// }, ...
|
||||
this.brokenLinks = new Map()
|
||||
|
||||
// stores anchor links to check all at once in a Map:
|
||||
// version:filePath => {
|
||||
// '#anchor-link' : {
|
||||
// linkedFrom: ['url1', 'url2']
|
||||
// },
|
||||
// '#anchor-link2': {...}
|
||||
// }
|
||||
this.anchorLinksToCheck = new Map()
|
||||
|
||||
// Stores broken anchors in a Map, formatted the same way as anchorLinksToCheck
|
||||
this.brokenAnchors = new Map()
|
||||
}
|
||||
|
||||
async setRenderedPageObj (pathCacheKey, context, reRender = false) {
|
||||
if (this.pageCache.has(pathCacheKey) && !reRender) return
|
||||
let pageHTML = await context.page.render(context)
|
||||
|
||||
// handle special pre-rendered snowflake
|
||||
if (context.page.relativePath.endsWith('graphql/reference/objects.md')) {
|
||||
pageHTML += context.graphql.prerenderedObjectsForCurrentVersion.html
|
||||
}
|
||||
|
||||
const pageObj = cheerio.load(pageHTML, { xmlMode: true })
|
||||
this.pageCache.set(pathCacheKey, pageObj)
|
||||
}
|
||||
|
||||
async getRenderedPageObj (pathCacheKey, context) {
|
||||
if (!this.pageCache.has(pathCacheKey)) {
|
||||
if (context) {
|
||||
await this.setRenderedPageObj(pathCacheKey, context)
|
||||
} else {
|
||||
console.error('cannot find pre-rendered page, and does not have enough context to render one.')
|
||||
}
|
||||
}
|
||||
return this.pageCache.get(pathCacheKey)
|
||||
}
|
||||
|
||||
addAnchorForLater (pagePath, anchor, linkedFrom) {
|
||||
const anchorsInPath = this.anchorLinksToCheck.get(pagePath) || {}
|
||||
const anchorLink = anchorsInPath[anchor] ? anchorsInPath[anchor] : { linkedFrom: [] }
|
||||
anchorLink.linkedFrom = union(anchorLink.linkedFrom, [linkedFrom])
|
||||
anchorsInPath[anchor] = anchorLink
|
||||
this.anchorLinksToCheck.set(pagePath, anchorsInPath)
|
||||
}
|
||||
|
||||
addImagesForLater (images, pagePath) {
|
||||
uniq(images).forEach(imageSrc => {
|
||||
const imageUsage = this.imagesToCheck.get(imageSrc) || { usedBy: [] }
|
||||
imageUsage.usedBy = union(imageUsage.usedBy, [pagePath])
|
||||
this.imagesToCheck.set(imageSrc, imageUsage)
|
||||
})
|
||||
}
|
||||
|
||||
async checkPage (context, checkExternalAnchors) {
|
||||
const path = context.relativePath
|
||||
const version = context.currentVersion
|
||||
|
||||
const pathCacheKey = `${version}:${path}`
|
||||
const $ = await this.getRenderedPageObj(pathCacheKey, context)
|
||||
|
||||
const imageSrcs = $('img[src^="/assets"]').map((i, el) => $(el).attr('src')).toArray()
|
||||
|
||||
this.addImagesForLater(imageSrcs, pathCacheKey)
|
||||
|
||||
for (const href of this.internalHrefPrefixes) {
|
||||
const internalLinks = $(`a[href^="${href}"]`).get()
|
||||
|
||||
for (const internalLink of internalLinks) {
|
||||
const href = $(internalLink).attr('href')
|
||||
|
||||
let [link, anchor] = href.split('#')
|
||||
// remove trailing slash
|
||||
link = link.replace(patterns.trailingSlash, '$1')
|
||||
|
||||
// if it's an external link and has been checked before, skip
|
||||
if (link && this.checkedLinksCache.has(link)) {
|
||||
// if it's been determined this link is broken, add to the linkedFrom field
|
||||
if (this.brokenLinks.has(link)) {
|
||||
const brokenLink = this.brokenLinks.get(link)
|
||||
brokenLink.linkedFrom = union(brokenLink.linkedFrom, [pathCacheKey])
|
||||
this.brokenLinks.set(link, brokenLink)
|
||||
}
|
||||
if (!anchor) continue
|
||||
}
|
||||
|
||||
// if it's an internal anchor (e.g., #foo), save for later
|
||||
if (anchor && !link) {
|
||||
// ignore anchors that are autogenerated from headings
|
||||
if (anchor === $(internalLink).parent().attr('id')) continue
|
||||
this.addAnchorForLater(pathCacheKey, anchor, 'same page')
|
||||
continue
|
||||
}
|
||||
|
||||
//------ BEGIN ONEOFF EXCLUSIONS -------///
|
||||
// skip GraphQL public schema paths (these are checked by separate tests)
|
||||
if (link.startsWith('/public/') && link.endsWith('.graphql')) continue
|
||||
|
||||
// skip links that start with /assets/images, as these are not in the pages collection
|
||||
// and /assets/images paths should be checked during the image check
|
||||
if (link.startsWith('/assets/images')) continue
|
||||
|
||||
// skip rare hardcoded links to old GHE versions
|
||||
// these paths will always be in the old versioned format
|
||||
// example: /enterprise/11.10.340/admin/articles/upgrading-to-the-latest-release
|
||||
const gheVersionInLink = link.match(patterns.getEnterpriseVersionNumber)
|
||||
if (gheVersionInLink && deprecated.includes(gheVersionInLink[1])) continue
|
||||
//------ END ONEOFF EXCLUSIONS -------///
|
||||
|
||||
// look for linked page
|
||||
const versionFromHref = getVersionStringFromPath(link)
|
||||
const linkedPage = findPageInVersion(link, context.pages, context.redirects, this.languageCode, versionFromHref)
|
||||
this.checkedLinksCache.add(link)
|
||||
|
||||
if (!linkedPage) {
|
||||
this.brokenLinks.set(link, { linkedFrom: [pathCacheKey] })
|
||||
continue
|
||||
}
|
||||
|
||||
// if we're not checking external anchors, we're done
|
||||
if (!checkExternalAnchors) {
|
||||
continue
|
||||
}
|
||||
|
||||
// find the permalink for the current version
|
||||
const linkedPagePermalink = linkedPage.permalinks.find(permalink => permalink.pageVersion === version)
|
||||
|
||||
if (linkedPagePermalink) {
|
||||
const linkedPageContext = await buildPathContext(context, linkedPage, linkedPagePermalink)
|
||||
|
||||
if (anchor) {
|
||||
await this.setRenderedPageObj(`${version}:${linkedPage.relativePath}`, linkedPageContext)
|
||||
this.addAnchorForLater(`${version}:${linkedPage.relativePath}`, anchor, pathCacheKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async checkAnchors () {
|
||||
for await (const [pathCacheKey, anchors] of this.anchorLinksToCheck) {
|
||||
const $ = await this.getRenderedPageObj(pathCacheKey)
|
||||
for (const anchorText in anchors) {
|
||||
const matchingHeadings = $(`[id="${anchorText}"], [name="${anchorText}"]`)
|
||||
if (matchingHeadings.length === 0) {
|
||||
const brokenAnchorPath = this.brokenAnchors.get(pathCacheKey) || {}
|
||||
brokenAnchorPath[anchorText] = anchors[anchorText]
|
||||
this.brokenAnchors.set(pathCacheKey, brokenAnchorPath)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
getBrokenLinks () {
|
||||
return this.brokenLinks
|
||||
}
|
||||
|
||||
async getBrokenAnchors () {
|
||||
await this.checkAnchors()
|
||||
return this.brokenAnchors
|
||||
}
|
||||
|
||||
async getBrokenImages () {
|
||||
for await (const [imageSrc, imageUsage] of this.imagesToCheck) {
|
||||
try {
|
||||
await fs.promises.access(path.join(process.cwd(), imageSrc))
|
||||
} catch (e) {
|
||||
this.brokenImages.set(imageSrc, imageUsage)
|
||||
}
|
||||
}
|
||||
return this.brokenImages
|
||||
}
|
||||
}
|
||||
|
||||
// this function is async because the middleware functions are likely async
|
||||
async function applyMiddleware (middleware, req) {
|
||||
return middleware(req, null, () => {})
|
||||
}
|
||||
|
||||
async function buildInitialContext () {
|
||||
const req = {
|
||||
path: '/en',
|
||||
language: 'en',
|
||||
query: {}
|
||||
}
|
||||
await applyMiddleware(contextualize, req)
|
||||
return req.context
|
||||
}
|
||||
|
||||
async function buildPathContext (context, page, permalink) {
|
||||
const pathContext = {
|
||||
page,
|
||||
currentVersion: permalink.pageVersion,
|
||||
relativePath: permalink.relativePath
|
||||
}
|
||||
|
||||
const req = {
|
||||
path: permalink.href,
|
||||
context: Object.assign({}, context, pathContext),
|
||||
language: 'en',
|
||||
query: {}
|
||||
}
|
||||
|
||||
await applyMiddleware(rest, req)
|
||||
await applyMiddleware(graphql, req)
|
||||
// TODO enable when release notes are live
|
||||
// await applyMiddleware(releaseNotes, req)
|
||||
|
||||
return Object.assign({}, context, req.context)
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
LinksChecker,
|
||||
buildPathContext,
|
||||
buildInitialContext
|
||||
}
|
||||
@@ -1,113 +1,46 @@
|
||||
const cheerio = require('cheerio')
|
||||
const loadPages = require('../../lib/pages')
|
||||
const loadSiteData = require('../../lib/site-data')
|
||||
const getApplicableVersions = require('../../lib/get-applicable-versions')
|
||||
const renderContent = require('../../lib/render-content')
|
||||
const checkImages = require('../../lib/check-images')
|
||||
const checkLinks = require('../../lib/check-links')
|
||||
const enterpriseServerVersions = Object.keys(require('../../lib/all-versions'))
|
||||
.filter(version => version.startsWith('enterprise-server@'))
|
||||
const flat = require('flat')
|
||||
const { last } = require('lodash')
|
||||
|
||||
// english only for now
|
||||
const { LinksChecker, buildInitialContext, buildPathContext, updateContextPerPath } = require('../helpers/links-checker')
|
||||
const languageCode = 'en'
|
||||
|
||||
const context = { currentLanguage: languageCode }
|
||||
|
||||
const loadRedirects = require('../../lib/redirects/precompile')
|
||||
// TODO set to true when we're ready to report and fix broken anchors
|
||||
const checkExternalAnchors = false
|
||||
|
||||
describe('page rendering', () => {
|
||||
jest.setTimeout(1000 * 1000)
|
||||
|
||||
const brokenImages = {}
|
||||
const brokenAnchors = {}
|
||||
const brokenLinks = {}
|
||||
const linksChecker = new LinksChecker()
|
||||
|
||||
let pages, siteData, redirects
|
||||
beforeAll(async (done) => {
|
||||
pages = await loadPages()
|
||||
siteData = await loadSiteData()
|
||||
redirects = await loadRedirects(pages)
|
||||
// fetch context.pages, context.redirects, etc.
|
||||
// we only want to build these one time
|
||||
const context = await buildInitialContext()
|
||||
|
||||
context.pages = pages
|
||||
context.site = siteData[languageCode].site
|
||||
context.redirects = redirects
|
||||
|
||||
let checkedLinks = {}
|
||||
let checkedImages = {}
|
||||
|
||||
const englishPages = pages
|
||||
const englishPages = context.pages
|
||||
.filter(page => page.languageCode === languageCode)
|
||||
// ignore developers content, to be checked separately
|
||||
.filter(page => !page.relativePath.match(/^(rest|graphql|developers)/))
|
||||
|
||||
for (const page of englishPages) {
|
||||
// skip map topics because they have no content of their own
|
||||
if (page.mapTopic) continue
|
||||
|
||||
const brokenImagesPerPage = {}
|
||||
const brokenAnchorsPerPage = {}
|
||||
const brokenLinksPerPage = {}
|
||||
|
||||
// get an array of the pages product versions
|
||||
const pageVersions = getApplicableVersions(page.versions, page.relativePath)
|
||||
|
||||
for (const pageVersion of pageVersions) {
|
||||
// attach page-specific properties to context
|
||||
page.version = pageVersion
|
||||
context.page = page
|
||||
context.currentVersion = pageVersion
|
||||
context.enterpriseServerVersions = enterpriseServerVersions
|
||||
|
||||
// collect elements of the page that may contain links
|
||||
const pageContent = page.intro + page.permissions + page.markdown
|
||||
|
||||
// renderContent is much faster than page.render, even though we later have to run
|
||||
// rewriteLocalLinks in check-images and rewriteAssetPathsToS3 in check-links
|
||||
const pageHtml = await renderContent(pageContent, context)
|
||||
const $ = cheerio.load(pageHtml, { xmlMode: true })
|
||||
|
||||
// check images
|
||||
const { brokenImages: brokenImagesPerVersion, checkedImageCache } = await checkImages($, pageVersion, page.relativePath, checkedImages)
|
||||
if (brokenImagesPerVersion.length) brokenImagesPerPage[pageVersion] = brokenImagesPerVersion
|
||||
checkedImages = checkedImageCache
|
||||
|
||||
// check anchors and links
|
||||
const { brokenLinks: brokenLinksPerVersion, checkedLinkCache } = await checkLinks($, page, context, pageVersion, checkedLinks)
|
||||
if (brokenLinksPerVersion.anchors.length) brokenAnchorsPerPage[pageVersion] = brokenLinksPerVersion.anchors
|
||||
if (brokenLinksPerVersion.links.length) brokenLinksPerPage[pageVersion] = brokenLinksPerVersion.links
|
||||
checkedLinks = checkedLinkCache
|
||||
for (const permalink of page.permalinks) {
|
||||
const pathContext = await buildPathContext(context, page, permalink)
|
||||
await linksChecker.checkPage(pathContext, checkExternalAnchors)
|
||||
}
|
||||
|
||||
if (Object.keys(brokenImagesPerPage).length) brokenImages[page.fullPath] = brokenImagesPerPage
|
||||
if (Object.keys(brokenAnchorsPerPage).length) brokenAnchors[page.fullPath] = brokenAnchorsPerPage
|
||||
if (Object.keys(brokenLinksPerPage).length) brokenLinks[page.fullPath] = brokenLinksPerPage
|
||||
}
|
||||
|
||||
done()
|
||||
})
|
||||
|
||||
test('every page has image references that can be resolved', async () => {
|
||||
const numbrokenImages = getNumBrokenItems(brokenImages)
|
||||
expect(numbrokenImages, `Found ${numbrokenImages} total broken images: ${JSON.stringify(brokenImages, null, 2)}`).toBe(0)
|
||||
const result = await linksChecker.getBrokenImages()
|
||||
expect(result.size, `Found ${result.size} total broken images: ${JSON.stringify([...result], null, 2)}`).toBe(0)
|
||||
})
|
||||
|
||||
test('every page has links with anchors that can be resolved', async () => {
|
||||
const numbrokenAnchors = getNumBrokenItems(brokenAnchors)
|
||||
expect(numbrokenAnchors, `Found ${numbrokenAnchors} total broken anchors: ${JSON.stringify(brokenAnchors, null, 2)}`).toBe(0)
|
||||
// When ready to unskip this,
|
||||
test.skip('every page has links with anchors that can be resolved', async () => {
|
||||
const result = await linksChecker.getBrokenAnchors()
|
||||
const numBrokenAnchors = [...result].reduce((accumulator, [path, anchors]) => accumulator + Object.keys(anchors).length, 0)
|
||||
expect(numBrokenAnchors, `Found ${numBrokenAnchors} total broken anchors in ${result.size} pages: ${JSON.stringify([...result], null, 2)}`).toBe(0)
|
||||
})
|
||||
|
||||
test('every page has links that can be resolved', async () => {
|
||||
const numbrokenLinks = getNumBrokenItems(brokenLinks)
|
||||
expect(numbrokenLinks, `Found ${numbrokenLinks} total broken links: ${JSON.stringify(brokenLinks, null, 2)}`).toBe(0)
|
||||
test('every page has links that can be resolved', () => {
|
||||
const result = linksChecker.getBrokenLinks()
|
||||
expect(result.size, `Found ${result.size} total broken links: ${JSON.stringify([...result], null, 2)}`).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
// count all the nested items
|
||||
function getNumBrokenItems (items) {
|
||||
// filter for entries like this:
|
||||
// '/article-path-here.md.dotcom.1.broken link': '/en/articles/foo',
|
||||
return Object.keys(flat(items))
|
||||
.filter(key => last(key.split('.')).includes('broken'))
|
||||
.length
|
||||
}
|
||||
})
|
||||
Reference in New Issue
Block a user