Block indexing not crawling (#17044)

* Block indexing instead of crawling

* Lint

* Update deprecated-enterprise-versions.js

* Combine loops

middleware/robots.js

@@ -1,33 +1,4 @@
-const languages = require('../lib/languages')
-const products = require('../lib/all-products')
-const { deprecated } = require('../lib/enterprise-server-releases.js')
-
-let defaultResponse = 'User-agent: *'
-
-// Disallow crawling of WIP localized content
-Object.values(languages)
-  .filter(language => language.wip)
-  .forEach(language => {
-    defaultResponse = defaultResponse.concat(`\nDisallow: /${language.code}\nDisallow: /${language.code}/*\n`)
-  })
-
-// Disallow crawling of WIP products
-Object.values(products)
-  .filter(product => product.wip || product.hidden)
-  .forEach(product => {
-    defaultResponse = defaultResponse.concat(`\nDisallow: /*${product.href}`)
-    product.versions.forEach(version => {
-      defaultResponse = defaultResponse.concat(`\nDisallow: /*${version}/${product.id}`)
-    })
-  })
-
-// Disallow crawling of Deprecated enterprise versions
-deprecated
-  .forEach(version => {
-    defaultResponse = defaultResponse
-      .concat(`\nDisallow: /*/enterprise-server@${version}/*`)
-      .concat(`\nDisallow: /*/enterprise/${version}/*`)
-  })
+const defaultResponse = 'User-agent: *'
 
 const disallowAll = `User-agent: *
 Disallow: /`