18 lines
470 B
JavaScript
18 lines
470 B
JavaScript
const defaultResponse = 'User-agent: *'
|
|
|
|
const disallowAll = `User-agent: *
|
|
Disallow: /`
|
|
|
|
export default function robots(req, res, next) {
|
|
if (req.path !== '/robots.txt') return next()
|
|
|
|
res.type('text/plain')
|
|
|
|
// only include robots.txt when it's our production domain and adding localhost for robots-txt.js test
|
|
if (req.hostname === 'docs.github.com' || req.hostname === '127.0.0.1') {
|
|
return res.send(defaultResponse)
|
|
}
|
|
|
|
return res.send(disallowAll)
|
|
}
|