Cache robots.txt longer (#36907)

middleware/robots.js

@@ -1,3 +1,5 @@
+import { defaultCacheControl } from './cache-control.js'
+
 const defaultResponse = 'User-agent: *'
 
 const disallowAll = `User-agent: *
@@ -8,6 +10,8 @@ export default function robots(req, res, next) {
 
   res.type('text/plain')
 
+  defaultCacheControl(res)
+
   // only include robots.txt when it's our production domain and adding localhost for robots-txt.js test
   if (req.hostname === 'docs.github.com' || req.hostname === '127.0.0.1') {
     return res.send(defaultResponse)