From 71f94334765124d752e2432286a5ff59525a8366 Mon Sep 17 00:00:00 2001 From: Grace Park Date: Tue, 23 Nov 2021 10:56:05 -0800 Subject: [PATCH] Robots update (#23046) * update to allow only if docs.github.com * update if * update for test --- middleware/robots.js | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/middleware/robots.js b/middleware/robots.js index ac89bee965..53fa6f8abc 100644 --- a/middleware/robots.js +++ b/middleware/robots.js @@ -8,14 +8,10 @@ export default function robots(req, res, next) { res.type('text/plain') - // remove subdomain from host - // docs-internal-12345--branch-name.herokuapp.com -> herokuapp.com - const rootDomain = req.hostname.split('.').slice(1).join('.') - - // prevent crawlers from indexing staging apps - if (rootDomain === 'herokuapp.com') { - return res.send(disallowAll) + // only include robots.txt when it's our production domain and adding localhost for robots-txt.js test + if (req.hostname === 'docs.github.com' || req.hostname === '127.0.0.1') { + return res.send(defaultResponse) } - return res.send(defaultResponse) + return res.send(disallowAll) }