jprdonnelly/docs
1
0
Fork 0
mirror of synced 2025-12-19 18:10:59 -05:00
Code Issues Projects Releases Wiki Activity

Instruct search engines to not crawl archived versions (#16292)

Browse Source 
* Instruct search engines to not crawl archived versions

* Update middleware/robots.js

Co-authored-by: Sarah Schneider <sarahs@users.noreply.github.com>

* Add test

* Fix spooky bug

Co-authored-by: Chiedo <chiedo@users.noreply.github.com>
Co-authored-by: Sarah Schneider <sarahs@users.noreply.github.com>
This commit is contained in:
Chiedo John
2020-11-03 08:23:22 -05:00
committed by GitHub
parent 23732ffe46
commit 7924aea2f8
2 changed files with 30 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
middleware/robots.js
View File

@@ -1,5 +1,6 @@
const languages = require('../lib/languages') const languages = require('../lib/languages')
const products = require('../lib/all-products') const products = require('../lib/all-products')
const { deprecated } = require('../lib/enterprise-server-releases.js')
let defaultResponse = 'User-agent: *' let defaultResponse = 'User-agent: *'
@@ -34,5 +35,13 @@ module.exports = function (req, res, next) {
defaultResponse = defaultResponse.concat(`\nDisallow: /*${product.href}\nDisallow: /*/enterprise/*/user${product.href}`) defaultResponse = defaultResponse.concat(`\nDisallow: /*${product.href}\nDisallow: /*/enterprise/*/user${product.href}`)
}) })
// Disallow crawling of Deprecated enterprise versions
deprecated
.forEach(version => {
defaultResponse = defaultResponse
.concat(`\nDisallow: /*/enterprise-server@${version}/*`)
.concat(`\nDisallow: /*/enterprise/${version}/*`)
})
return res.send(defaultResponse) return res.send(defaultResponse)
} }

21
tests/rendering/robots-txt.js
View File

@@ -89,4 +89,25 @@ describe('robots.txt', () => {
expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.latest}/user/actions`)).toBe(true) expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.latest}/user/actions`)).toBe(true)
expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user/actions`)).toBe(true) expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user/actions`)).toBe(true)
}) })
it('disallows indexing of deprecated enterprise releases', async () => {
enterpriseServerReleases.deprecated.forEach(version => {
const blockedPaths = [
// English
`https://help.github.com/en/enterprise-server@${version}/actions`,
`https://help.github.com/en/enterprise/${version}/actions`,
`https://help.github.com/en/enterprise-server@${version}/actions/overview`,
`https://help.github.com/en/enterprise/${version}/actions/overview`,
// Japanese
`https://help.github.com/ja/enterprise-server@${version}/actions`,
`https://help.github.com/ja/enterprise/${version}/actions`,
`https://help.github.com/ja/enterprise-server@${version}/actions/overview`,
`https://help.github.com/ja/enterprise/${version}/actions/overview`
]
blockedPaths.forEach(path => {
expect(robots.isAllowed(path)).toBe(false)
})
})
})
}) })