jprdonnelly/docs
1
0
Fork 0
mirror of synced 2025-12-22 11:26:57 -05:00
Code Issues Projects Releases Wiki Activity

update middleware and tests around robots.txt exclusions of hidden/EA content

Browse Source
This commit is contained in:
Sarah Schneider
2020-10-22 10:07:20 -04:00
parent 07d090ec19
commit d290d40ffc
2 changed files with 52 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
middleware/robots.js
View File

@@ -27,11 +27,13 @@ module.exports = function (req, res, next) {
defaultResponse = defaultResponse.concat(`\nDisallow: /${language.code}\nDisallow: /${language.code}/*\n`) defaultResponse = defaultResponse.concat(`\nDisallow: /${language.code}\nDisallow: /${language.code}/*\n`)
}) })
// Disallow crawling of WIP products // Disallow crawling of WIP or early access products
Object.values(products) Object.values(products)
.filter(product => product.wip) .filter(product => product.wip || product.hidden)
.forEach(product => { .forEach(product => {
defaultResponse = defaultResponse.concat(`\nDisallow: /*${product.href}\nDisallow: /*/enterprise/*/user${product.href}`) product.versions.forEach(version => {
defaultResponse = defaultResponse.concat(`\nDisallow: /*/${version}/${product.id}\nDisallow: /*/${version}/${product.id}/*\n`)
})
}) })
return res.send(defaultResponse) return res.send(defaultResponse)

70
tests/rendering/robots-txt.js
View File

@@ -12,22 +12,22 @@ describe('robots.txt', () => {
let res, robots let res, robots
beforeAll(async (done) => { beforeAll(async (done) => {
res = await get('/robots.txt') res = await get('/robots.txt')
robots = robotsParser('https://help.github.com/robots.txt', res.text) robots = robotsParser('https://docs.github.com/robots.txt', res.text)
done() done()
}) })
it('allows indexing of the homepage and English content', async () => { it('allows indexing of the homepage and English content', async () => {
expect(robots.isAllowed('https://help.github.com/')).toBe(true) expect(robots.isAllowed('https://docs.github.com/')).toBe(true)
expect(robots.isAllowed('https://help.github.com/en')).toBe(true) expect(robots.isAllowed('https://docs.github.com/en')).toBe(true)
expect(robots.isAllowed('https://help.github.com/en/articles/verifying-your-email-address')).toBe(true) expect(robots.isAllowed('https://docs.github.com/en/articles/verifying-your-email-address')).toBe(true)
}) })
it('allows indexing of generally available localized content', async () => { it('allows indexing of generally available localized content', async () => {
Object.values(languages) Object.values(languages)
.filter(language => !language.wip) .filter(language => !language.wip)
.forEach(language => { .forEach(language => {
expect(robots.isAllowed(`https://help.github.com/${language.code}`)).toBe(true) expect(robots.isAllowed(`https://docs.github.com/${language.code}`)).toBe(true)
expect(robots.isAllowed(`https://help.github.com/${language.code}/articles/verifying-your-email-address`)).toBe(true) expect(robots.isAllowed(`https://docs.github.com/${language.code}/articles/verifying-your-email-address`)).toBe(true)
}) })
}) })
@@ -35,8 +35,8 @@ describe('robots.txt', () => {
Object.values(languages) Object.values(languages)
.filter(language => language.wip) .filter(language => language.wip)
.forEach(language => { .forEach(language => {
expect(robots.isAllowed(`https://help.github.com/${language.code}`)).toBe(false) expect(robots.isAllowed(`https://docs.github.com/${language.code}`)).toBe(false)
expect(robots.isAllowed(`https://help.github.com/${language.code}/articles/verifying-your-email-address`)).toBe(false) expect(robots.isAllowed(`https://docs.github.com/${language.code}/articles/verifying-your-email-address`)).toBe(false)
}) })
}) })
@@ -61,18 +61,18 @@ describe('robots.txt', () => {
const { href } = products[id] const { href } = products[id]
const blockedPaths = [ const blockedPaths = [
// English // English
`https://help.github.com/en${href}`, `https://docs.github.com/en${href}`,
`https://help.github.com/en${href}/overview`, `https://docs.github.com/en${href}/overview`,
`https://help.github.com/en${href}/overview/intro`, `https://docs.github.com/en${href}/overview/intro`,
`https://help.github.com/en/enterprise/${enterpriseServerReleases.latest}/user${href}`, `https://docs.github.com/en/enterprise/${enterpriseServerReleases.latest}/user${href}`,
`https://help.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user${href}`, `https://docs.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user${href}`,
// Japanese // Japanese
`https://help.github.com/ja${href}`, `https://docs.github.com/ja${href}`,
`https://help.github.com/ja${href}/overview`, `https://docs.github.com/ja${href}/overview`,
`https://help.github.com/ja${href}/overview/intro`, `https://docs.github.com/ja${href}/overview/intro`,
`https://help.github.com/ja/enterprise/${enterpriseServerReleases.latest}/user${href}`, `https://docs.github.com/ja/enterprise/${enterpriseServerReleases.latest}/user${href}`,
`https://help.github.com/ja/enterprise/${enterpriseServerReleases.oldestSupported}/user${href}` `https://docs.github.com/ja/enterprise/${enterpriseServerReleases.oldestSupported}/user${href}`
] ]
blockedPaths.forEach(path => { blockedPaths.forEach(path => {
@@ -81,12 +81,36 @@ describe('robots.txt', () => {
}) })
}) })
it('disallows indexing of early access "hidden" products', async () => {
const hiddenProductIds = Object.values(products)
.filter(product => product.hidden)
.map(product => product.id)
hiddenProductIds.forEach(id => {
const { versions } = products[id]
const blockedPaths = versions.map(version => {
return [
// English
`https://docs.github.com/en/${version}/${id}`,
`https://docs.github.com/en/${version}/${id}/some-early-access-article`,
// Japanese
`https://docs.github.com/ja/${version}/${id}`,
`https://docs.github.com/ja/${version}/${id}/some-early-access-article`
]
}).flat()
blockedPaths.forEach(path => {
expect(robots.isAllowed(path)).toBe(false)
})
})
})
it('allows indexing of non-WIP products', async () => { it('allows indexing of non-WIP products', async () => {
expect('actions' in products).toBe(true) expect('actions' in products).toBe(true)
expect(robots.isAllowed('https://help.github.com/en/actions')).toBe(true) expect(robots.isAllowed('https://docs.github.com/en/actions')).toBe(true)
expect(robots.isAllowed('https://help.github.com/en/actions/overview')).toBe(true) expect(robots.isAllowed('https://docs.github.com/en/actions/overview')).toBe(true)
expect(robots.isAllowed('https://help.github.com/en/actions/overview/intro')).toBe(true) expect(robots.isAllowed('https://docs.github.com/en/actions/overview/intro')).toBe(true)
expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.latest}/user/actions`)).toBe(true) expect(robots.isAllowed(`https://docs.github.com/en/enterprise/${enterpriseServerReleases.latest}/user/actions`)).toBe(true)
expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user/actions`)).toBe(true) expect(robots.isAllowed(`https://docs.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user/actions`)).toBe(true)
}) })
}) })