update middleware and tests around robots.txt exclusions of hidden/EA content

middleware/robots.js

@@ -27,11 +27,13 @@ module.exports = function (req, res, next) {
       defaultResponse = defaultResponse.concat(`\nDisallow: /${language.code}\nDisallow: /${language.code}/*\n`)
     })
 
-  // Disallow crawling of WIP products
+  // Disallow crawling of WIP or early access products
   Object.values(products)
-    .filter(product => product.wip)
+    .filter(product => product.wip || product.hidden)
     .forEach(product => {
-      defaultResponse = defaultResponse.concat(`\nDisallow: /*${product.href}\nDisallow: /*/enterprise/*/user${product.href}`)
+      product.versions.forEach(version => {
+        defaultResponse = defaultResponse.concat(`\nDisallow: /*/${version}/${product.id}\nDisallow: /*/${version}/${product.id}/*\n`)
+      })
     })
 
   return res.send(defaultResponse)

tests/rendering/robots-txt.js

@@ -12,22 +12,22 @@ describe('robots.txt', () => {
   let res, robots
   beforeAll(async (done) => {
     res = await get('/robots.txt')
-    robots = robotsParser('https://help.github.com/robots.txt', res.text)
+    robots = robotsParser('https://docs.github.com/robots.txt', res.text)
     done()
   })
 
   it('allows indexing of the homepage and English content', async () => {
-    expect(robots.isAllowed('https://help.github.com/')).toBe(true)
-    expect(robots.isAllowed('https://help.github.com/en')).toBe(true)
-    expect(robots.isAllowed('https://help.github.com/en/articles/verifying-your-email-address')).toBe(true)
+    expect(robots.isAllowed('https://docs.github.com/')).toBe(true)
+    expect(robots.isAllowed('https://docs.github.com/en')).toBe(true)
+    expect(robots.isAllowed('https://docs.github.com/en/articles/verifying-your-email-address')).toBe(true)
   })
 
   it('allows indexing of generally available localized content', async () => {
     Object.values(languages)
       .filter(language => !language.wip)
       .forEach(language => {
-        expect(robots.isAllowed(`https://help.github.com/${language.code}`)).toBe(true)
-        expect(robots.isAllowed(`https://help.github.com/${language.code}/articles/verifying-your-email-address`)).toBe(true)
+        expect(robots.isAllowed(`https://docs.github.com/${language.code}`)).toBe(true)
+        expect(robots.isAllowed(`https://docs.github.com/${language.code}/articles/verifying-your-email-address`)).toBe(true)
       })
   })
 
@@ -35,8 +35,8 @@ describe('robots.txt', () => {
     Object.values(languages)
       .filter(language => language.wip)
       .forEach(language => {
-        expect(robots.isAllowed(`https://help.github.com/${language.code}`)).toBe(false)
-        expect(robots.isAllowed(`https://help.github.com/${language.code}/articles/verifying-your-email-address`)).toBe(false)
+        expect(robots.isAllowed(`https://docs.github.com/${language.code}`)).toBe(false)
+        expect(robots.isAllowed(`https://docs.github.com/${language.code}/articles/verifying-your-email-address`)).toBe(false)
       })
   })
 
@@ -61,18 +61,18 @@ describe('robots.txt', () => {
       const { href } = products[id]
       const blockedPaths = [
         // English
-        `https://help.github.com/en${href}`,
-        `https://help.github.com/en${href}/overview`,
-        `https://help.github.com/en${href}/overview/intro`,
-        `https://help.github.com/en/enterprise/${enterpriseServerReleases.latest}/user${href}`,
-        `https://help.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user${href}`,
+        `https://docs.github.com/en${href}`,
+        `https://docs.github.com/en${href}/overview`,
+        `https://docs.github.com/en${href}/overview/intro`,
+        `https://docs.github.com/en/enterprise/${enterpriseServerReleases.latest}/user${href}`,
+        `https://docs.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user${href}`,
 
         // Japanese
-        `https://help.github.com/ja${href}`,
-        `https://help.github.com/ja${href}/overview`,
-        `https://help.github.com/ja${href}/overview/intro`,
-        `https://help.github.com/ja/enterprise/${enterpriseServerReleases.latest}/user${href}`,
-        `https://help.github.com/ja/enterprise/${enterpriseServerReleases.oldestSupported}/user${href}`
+        `https://docs.github.com/ja${href}`,
+        `https://docs.github.com/ja${href}/overview`,
+        `https://docs.github.com/ja${href}/overview/intro`,
+        `https://docs.github.com/ja/enterprise/${enterpriseServerReleases.latest}/user${href}`,
+        `https://docs.github.com/ja/enterprise/${enterpriseServerReleases.oldestSupported}/user${href}`
       ]
 
       blockedPaths.forEach(path => {
@@ -81,12 +81,36 @@ describe('robots.txt', () => {
     })
   })
 
+  it('disallows indexing of early access "hidden" products', async () => {
+    const hiddenProductIds = Object.values(products)
+      .filter(product => product.hidden)
+      .map(product => product.id)
+
+    hiddenProductIds.forEach(id => {
+      const { versions } = products[id]
+      const blockedPaths = versions.map(version => {
+        return [
+          // English
+          `https://docs.github.com/en/${version}/${id}`,
+          `https://docs.github.com/en/${version}/${id}/some-early-access-article`,
+          // Japanese
+          `https://docs.github.com/ja/${version}/${id}`,
+          `https://docs.github.com/ja/${version}/${id}/some-early-access-article`
+        ]
+      }).flat()
+
+      blockedPaths.forEach(path => {
+        expect(robots.isAllowed(path)).toBe(false)
+      })
+    })
+  })
+
   it('allows indexing of non-WIP products', async () => {
     expect('actions' in products).toBe(true)
-    expect(robots.isAllowed('https://help.github.com/en/actions')).toBe(true)
-    expect(robots.isAllowed('https://help.github.com/en/actions/overview')).toBe(true)
-    expect(robots.isAllowed('https://help.github.com/en/actions/overview/intro')).toBe(true)
-    expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.latest}/user/actions`)).toBe(true)
-    expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user/actions`)).toBe(true)
+    expect(robots.isAllowed('https://docs.github.com/en/actions')).toBe(true)
+    expect(robots.isAllowed('https://docs.github.com/en/actions/overview')).toBe(true)
+    expect(robots.isAllowed('https://docs.github.com/en/actions/overview/intro')).toBe(true)
+    expect(robots.isAllowed(`https://docs.github.com/en/enterprise/${enterpriseServerReleases.latest}/user/actions`)).toBe(true)
+    expect(robots.isAllowed(`https://docs.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user/actions`)).toBe(true)
   })
 })