jprdonnelly/docs
1
0
Fork 0
mirror of synced 2025-12-22 11:26:57 -05:00
Code Issues Projects Releases Wiki Activity

update middleware and tests around robots.txt exclusions of hidden/EA content

Browse Source
This commit is contained in:
Sarah Schneider
2020-10-22 10:07:20 -04:00
parent 07d090ec19
commit d290d40ffc
2 changed files with 52 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
middleware/robots.js
View File

@@ -27,11 +27,13 @@ module.exports = function (req, res, next) {
defaultResponse = defaultResponse.concat(`\nDisallow: /${language.code}\nDisallow: /${language.code}/*\n`)
})
// Disallow crawling of WIP products
// Disallow crawling of WIP or early access products
Object.values(products)
.filter(product => product.wip)
.filter(product => product.wip || product.hidden)
.forEach(product => {
defaultResponse = defaultResponse.concat(`\nDisallow: /*${product.href}\nDisallow: /*/enterprise/*/user${product.href}`)
product.versions.forEach(version => {
defaultResponse = defaultResponse.concat(`\nDisallow: /*/${version}/${product.id}\nDisallow: /*/${version}/${product.id}/*\n`)
})
})
return res.send(defaultResponse)