diff --git a/.env.example b/.env.example index 20c941977c..e8896feab8 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,3 @@ ALGOLIA_API_KEY= ALGOLIA_APPLICATION_ID= ALLOW_TRANSLATION_COMMITS= -EARLY_ACCESS_HOSTNAME= -EARLY_ACCESS_SHARED_SECRET= -GITHUB_TOKEN= \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/improve-existing-docs.md b/.github/ISSUE_TEMPLATE/improve-existing-docs.md index 6b15af3a21..26d822b9a4 100644 --- a/.github/ISSUE_TEMPLATE/improve-existing-docs.md +++ b/.github/ISSUE_TEMPLATE/improve-existing-docs.md @@ -7,13 +7,13 @@ labels: assignees: '' --- diff --git a/lib/algolia/find-indexable-pages.js b/lib/algolia/find-indexable-pages.js index d3fb24408f..4d897e0100 100644 --- a/lib/algolia/find-indexable-pages.js +++ b/lib/algolia/find-indexable-pages.js @@ -3,8 +3,10 @@ const loadPages = require('../pages') module.exports = async function findIndexablePages () { const allPages = await loadPages() const indexablePages = allPages - // exclude pages that are part of WIP products - .filter(page => !page.parentProduct || !page.parentProduct.wip) + // exclude hidden pages + .filter(page => !page.hidden) + // exclude pages that are part of WIP or hidden products + .filter(page => !page.parentProduct || !page.parentProduct.wip || page.parentProduct.hidden) // exclude index homepages .filter(page => !page.relativePath.endsWith('index.md')) diff --git a/lib/all-products.js b/lib/all-products.js index b56dc4bee2..f65a8a11be 100644 --- a/lib/all-products.js +++ b/lib/all-products.js @@ -14,14 +14,32 @@ const productsYml = yaml.load(fs.readFileSync(productsFile, 'utf8')) const sortedProductIds = productsYml.productsInOrder const contentProductIds = fs.readdirSync(contentDir, { withFileTypes: true }) + .map(entry => { + // `fs.readdir` provides file entries based on `fs.lstat`, which doesn't + // resolve symbolic links to their target file/directory. We need to take + // an extra step here to resolve the Early Access symlinked directory. + const { name } = entry + if (entry.isSymbolicLink()) { + entry = fs.statSync(path.join(contentDir, entry.name)) + entry.name = name + } + return entry + }) .filter(entry => entry.isDirectory()) .map(entry => entry.name) -assert(difference(sortedProductIds, contentProductIds).length === 0) -assert(difference(contentProductIds, sortedProductIds).length === 0) +// require the content/ list to match the list in data/products.yml, +// with the exception of content/early-access, which lives in a separate private repo +const publicContentProductIds = contentProductIds.filter(id => id !== 'early-access') +assert(difference(sortedProductIds, publicContentProductIds).length === 0) +assert(difference(publicContentProductIds, sortedProductIds).length === 0) const internalProducts = {} +// add optional early access content dir to sorted products list if present +const earlyAccessId = contentProductIds.find(id => id === 'early-access') +if (earlyAccessId) sortedProductIds.push(earlyAccessId) + sortedProductIds.forEach(productId => { const relPath = productId const dir = slash(path.join('content', relPath)) @@ -36,7 +54,8 @@ sortedProductIds.forEach(productId => { href, dir, toc, - wip: data.wip || false + wip: data.wip || false, + hidden: data.hidden || false } internalProducts[productId].versions = applicableVersions diff --git a/lib/authenticate-to-aws.js b/lib/authenticate-to-aws.js index c3e89506f4..f6c1f54ab0 100644 --- a/lib/authenticate-to-aws.js +++ b/lib/authenticate-to-aws.js @@ -6,7 +6,7 @@ const s3ConfigPath = path.join(homedir, '.s3cfg') // check for config files if (!(fs.existsSync(awsCredsPath) || fs.existsSync(s3ConfigPath))) { - console.error('You need to set up awssume and s3cmd. Follow the steps at https://github.com/github/product-documentation/blob/master/doc-team-workflows/workflow-information-for-all-writers/setting-up-awssume-and-s3cmd.md') + console.error('You need to set up awssume and s3cmd. Follow the steps at docs-content/doc-team-workflows/workflow-information-for-all-writers/setting-up-awssume-and-s3cmd.md') process.exit(1) } diff --git a/lib/enterprise-server-releases.js b/lib/enterprise-server-releases.js index 0e90882998..1c1ffcaae8 100644 --- a/lib/enterprise-server-releases.js +++ b/lib/enterprise-server-releases.js @@ -1,7 +1,7 @@ const versionSatisfiesRange = require('./version-satisfies-range') // GHES Release Lifecycle Dates: -// https://github.com/github/enterprise-releases/blob/master/docs/supported-versions.md#release-lifecycle-dates +// enterprise-releases/docs/supported-versions.md#release-lifecycle-dates const dates = require('../lib/enterprise-dates.json') const supported = [ diff --git a/lib/fetch-early-access-paths.js b/lib/fetch-early-access-paths.js deleted file mode 100644 index 6cc6b86e02..0000000000 --- a/lib/fetch-early-access-paths.js +++ /dev/null @@ -1,33 +0,0 @@ -// This module loads an array of Early Access page paths from EARLY_ACCESS_HOSTNAME -// -// See also middleware/early-acces-proxy.js which fetches Early Access docs from the obscured remote host - -require('dotenv').config() - -const got = require('got') -const isURL = require('is-url') - -module.exports = async function fetchEarlyAccessPaths () { - let url - if (process.env.NODE_ENV === 'test') return [] - - if (!isURL(process.env.EARLY_ACCESS_HOSTNAME)) { - console.log('EARLY_ACCESS_HOSTNAME is not defined; skipping fetching early access paths') - return [] - } - - try { - url = `${process.env.EARLY_ACCESS_HOSTNAME}/early-access-paths.json` - const { body } = await got(url, { - json: true, - timeout: 3000, - headers: { - 'early-access-shared-secret': process.env.EARLY_ACCESS_SHARED_SECRET - } - }) - return body - } catch (err) { - console.error('Unable to fetch early-access-paths.json from', url, err) - return [] - } -} diff --git a/lib/frontmatter.js b/lib/frontmatter.js index 1870f11612..9e9e3a0438 100644 --- a/lib/frontmatter.js +++ b/lib/frontmatter.js @@ -38,11 +38,9 @@ const schema = { mapTopic: { type: 'boolean' }, - // The `hidden` frontmatter property is no longer used, but leaving it here - // with an enum of `[false]` will help us catch any possible regressions. + // allow hidden articles under `early-access` hidden: { - type: 'boolean', - enum: [false] + type: 'boolean' }, layout: { type: ['string', 'boolean'], @@ -90,6 +88,11 @@ const schema = { } } }, + // Show in `product-landing.html` + product_video: { + type: 'string', + format: 'url' + }, interactive: { type: 'boolean' } @@ -109,7 +112,7 @@ function frontmatter (markdown, opts = {}) { const defaults = { schema, validateKeyNames: true, - validateKeyOrder: false // TODO: enable this once we've sorted all the keys. See https://github.com/github/docs-internal/issues/9658 + validateKeyOrder: false // TODO: enable this once we've sorted all the keys. See issue 9658 } return parse(markdown, Object.assign({}, defaults, opts)) diff --git a/lib/page.js b/lib/page.js index 9df106f71e..c282f1792f 100644 --- a/lib/page.js +++ b/lib/page.js @@ -33,7 +33,7 @@ class Page { this.fullPath = slash(path.join(this.basePath, this.relativePath)) this.raw = fs.readFileSync(this.fullPath, 'utf8') - // TODO remove this when https://github.com/github/crowdin-support/issues/66 has been resolved + // TODO remove this when crowdin-support issue 66 has been resolved if (this.languageCode !== 'en' && this.raw.includes(': verdadero')) { this.raw = this.raw.replace(': verdadero', ': true') } diff --git a/lib/patterns.js b/lib/patterns.js index 44a621ea0c..1184486b75 100644 --- a/lib/patterns.js +++ b/lib/patterns.js @@ -25,7 +25,7 @@ module.exports = { searchPath: /\/search(?:\/)?(\?)/, ymd: /^\d{4}-\d{2}-\d{2}$/, hasLiquid: /[{{][{%]/, - dataReference: /{% ?data\s(?:reusables|variables|ui)\..*?%}/gm, + dataReference: /{% ?data\s(?:early-access\.)?(?:reusables|variables|ui)\..*?%}/gm, imagePath: /\/?assets\/images\/.*?\.(png|svg|gif|pdf|ico|jpg|jpeg)/gi, homepagePath: /^\/\w{2}$/, // /en, /ja, /cn multipleSlashes: /^\/{2,}/, diff --git a/lib/warm-server.js b/lib/warm-server.js index 9d538891dd..7141f5ff66 100644 --- a/lib/warm-server.js +++ b/lib/warm-server.js @@ -1,15 +1,14 @@ const statsd = require('./statsd') -const fetchEarlyAccessPaths = require('./fetch-early-access-paths') const loadPages = require('./pages') const loadRedirects = require('./redirects/precompile') const loadSiteData = require('./site-data') const loadSiteTree = require('./site-tree') // For local caching -let pages, site, redirects, siteTree, earlyAccessPaths +let pages, site, redirects, siteTree function isFullyWarmed () { - return Boolean(pages && site && earlyAccessPaths && redirects && siteTree) + return Boolean(pages && site && redirects && siteTree) } function getWarmedCache () { @@ -17,8 +16,7 @@ function getWarmedCache () { pages, site, redirects, - siteTree, - earlyAccessPaths + siteTree } } @@ -29,12 +27,11 @@ async function warmServer () { console.log('Priming context information...') } - if (!pages || !site || !earlyAccessPaths) { + if (!pages || !site) { // Promise.all is used to load multiple things in parallel - [pages, site, earlyAccessPaths] = await Promise.all([ + [pages, site] = await Promise.all([ pages || loadPages(), - site || loadSiteData(), - earlyAccessPaths || fetchEarlyAccessPaths() + site || loadSiteData() ]) } diff --git a/middleware/archived-enterprise-versions-assets.js b/middleware/archived-enterprise-versions-assets.js index f72f579994..22362e5a83 100644 --- a/middleware/archived-enterprise-versions-assets.js +++ b/middleware/archived-enterprise-versions-assets.js @@ -5,7 +5,7 @@ const got = require('got') // This module handles requests for the CSS and JS assets for // deprecated GitHub Enterprise versions by routing them to static content in -// https://github.com/github/help-docs-archived-enterprise-versions +// help-docs-archived-enterprise-versions // // See also ./archived-enterprise-versions.js for non-CSS/JS paths diff --git a/middleware/archived-enterprise-versions.js b/middleware/archived-enterprise-versions.js index bf52678060..c2c9f6a7ad 100644 --- a/middleware/archived-enterprise-versions.js +++ b/middleware/archived-enterprise-versions.js @@ -8,8 +8,7 @@ const got = require('got') const findPage = require('../lib/find-page') // This module handles requests for deprecated GitHub Enterprise versions -// by routing them to static content in -// https://github.com/github/help-docs-archived-enterprise-versions +// by routing them to static content in help-docs-archived-enterprise-versions module.exports = async (req, res, next) => { const { isArchived, requestedVersion } = isArchivedVersion(req) diff --git a/middleware/context.js b/middleware/context.js index ec9915f421..54fbbdcbe7 100644 --- a/middleware/context.js +++ b/middleware/context.js @@ -2,7 +2,7 @@ const languages = require('../lib/languages') const enterpriseServerReleases = require('../lib/enterprise-server-releases') const allVersions = require('../lib/all-versions') const allProducts = require('../lib/all-products') -const activeProducts = Object.values(allProducts).filter(product => !product.wip) +const activeProducts = Object.values(allProducts).filter(product => !product.wip && !product.hidden) const { getVersionStringFromPath, getProductStringFromPath, getPathWithoutLanguage } = require('../lib/path-utils') const productNames = require('../lib/product-names') const warmServer = require('../lib/warm-server') @@ -12,7 +12,7 @@ const featureFlags = Object.keys(require('../feature-flags')) // Note that additional middleware in middleware/index.js adds to this context object module.exports = async function contextualize (req, res, next) { // Ensure that we load some data only once on first request - const { site, redirects, pages, siteTree, earlyAccessPaths } = await warmServer() + const { site, redirects, pages, siteTree } = await warmServer() req.context = {} // make feature flag environment variables accessible in layouts @@ -33,7 +33,6 @@ module.exports = async function contextualize (req, res, next) { req.context.currentPath = req.path req.context.query = req.query req.context.languages = languages - req.context.earlyAccessPaths = earlyAccessPaths req.context.productNames = productNames req.context.enterpriseServerReleases = enterpriseServerReleases req.context.enterpriseServerVersions = Object.keys(allVersions).filter(version => version.startsWith('enterprise-server@')) diff --git a/middleware/contextualizers/early-access-links.js b/middleware/contextualizers/early-access-links.js new file mode 100644 index 0000000000..4a63a3cd7a --- /dev/null +++ b/middleware/contextualizers/early-access-links.js @@ -0,0 +1,27 @@ +module.exports = function earlyAccessContext (req, res, next) { + if (process.env.NODE_ENV === 'production') { + return next(404) + } + + // Get a list of all hidden pages per version + const earlyAccessPageLinks = req.context.pages + .filter(page => page.hidden) + // Do not include early access landing page + .filter(page => page.relativePath !== 'early-access/index.md') + // Create Markdown links + .map(page => { + return page.permalinks.map(permalink => `- [${permalink.title}](${permalink.href})`) + }) + .flat() + // Get links for the current version + .filter(link => link.includes(req.context.currentVersion)) + .sort() + + // Add to the rendering context + // This is only used in the separate EA repo on local development + req.context.earlyAccessPageLinks = earlyAccessPageLinks.length + ? earlyAccessPageLinks.join('\n') + : '_None for this version!_' + + return next() +} diff --git a/middleware/csp.js b/middleware/csp.js index 774cef31c4..a081f0490c 100644 --- a/middleware/csp.js +++ b/middleware/csp.js @@ -34,7 +34,8 @@ module.exports = contentSecurityPolicy({ ], frameSrc: [ // exceptions for GraphQL Explorer 'https://graphql-explorer.githubapp.com', // production env - 'http://localhost:3000' // development env + 'http://localhost:3000', // development env + 'https://www.youtube-nocookie.com' ], styleSrc: [ "'self'", diff --git a/middleware/early-access-paths.js b/middleware/early-access-paths.js deleted file mode 100644 index 164780f1ac..0000000000 --- a/middleware/early-access-paths.js +++ /dev/null @@ -1,33 +0,0 @@ -const { chain } = require('lodash') -let paths - -// This middleware finds all pages with `hidden: true` frontmatter -// and responds with a JSON array of all requests paths (and redirects) that lead to those pages. - -// Requesting this path from EARLY_ACCESS_HOSTNAME will respond with an array of Early Access paths. -// Requesting this path from docs.github.com (production) will respond with an empty array (no Early Access paths). - -module.exports = async (req, res, next) => { - if (req.path !== '/early-access-paths.json') return next() - - if ( - !req.headers || - !req.headers['early-access-shared-secret'] || - req.headers['early-access-shared-secret'] !== process.env.EARLY_ACCESS_SHARED_SECRET - ) { - return res.status(401).send({ error: '401 Unauthorized' }) - } - - paths = paths || chain(req.context.pages) - .filter(page => page.hidden && page.languageCode === 'en') - .map(page => { - const permalinks = page.permalinks.map(permalink => permalink.href) - const redirects = Object.keys(page.redirects) - return permalinks.concat(redirects) - }) - .flatten() - .uniq() - .value() - - return res.json(paths) -} diff --git a/middleware/early-access-proxy.js b/middleware/early-access-proxy.js deleted file mode 100644 index 5df20e2228..0000000000 --- a/middleware/early-access-proxy.js +++ /dev/null @@ -1,25 +0,0 @@ -// This module serves requests to Early Access content from a hidden proxy host (EARLY_ACCESS_HOSTNAME). -// Paths to this content are fetched in the warmServer module at startup. - -const got = require('got') -const isURL = require('is-url') - -module.exports = async (req, res, next) => { - if ( - isURL(process.env.EARLY_ACCESS_HOSTNAME) && - req.context && - req.context.earlyAccessPaths && - req.context.earlyAccessPaths.includes(req.path) - ) { - try { - const proxyURL = `${process.env.EARLY_ACCESS_HOSTNAME}${req.path}` - const proxiedRes = await got(proxyURL) - res.set('content-type', proxiedRes.headers['content-type']) - res.send(proxiedRes.body) - } catch (err) { - next() - } - } else { - next() - } -} diff --git a/middleware/index.js b/middleware/index.js index 7c7d7f672e..1d3958a96f 100644 --- a/middleware/index.js +++ b/middleware/index.js @@ -62,8 +62,7 @@ module.exports = function (app) { app.use('/csrf', require('./csrf-route')) app.use(require('./archived-enterprise-versions')) app.use(require('./robots')) - app.use(require('./early-access-paths')) - app.use(require('./early-access-proxy')) + app.use(/(\/.*)?\/early-access$/, require('./contextualizers/early-access-links')) app.use(require('./categories-for-support-team')) app.use(require('./loaderio-verification')) app.get('/_500', asyncMiddleware(require('./trigger-error'))) diff --git a/middleware/render-page.js b/middleware/render-page.js index a5bf166e97..904744bfae 100644 --- a/middleware/render-page.js +++ b/middleware/render-page.js @@ -1,5 +1,4 @@ const { get } = require('lodash') -const env = require('lil-env-thing') const { liquid } = require('../lib/render-content') const patterns = require('../lib/patterns') const layouts = require('../lib/layouts') @@ -66,7 +65,7 @@ module.exports = async function renderPage (req, res, next) { } // `?json` query param for debugging request context - if ('json' in req.query && !env.production) { + if ('json' in req.query && process.env.NODE_ENV !== 'production') { if (req.query.json.length > 1) { // deep reference: ?json=page.permalinks return res.json(get(context, req.query.json)) diff --git a/middleware/robots.js b/middleware/robots.js index 603a592282..d00986cc85 100644 --- a/middleware/robots.js +++ b/middleware/robots.js @@ -13,9 +13,12 @@ Object.values(languages) // Disallow crawling of WIP products Object.values(products) - .filter(product => product.wip) + .filter(product => product.wip || product.hidden) .forEach(product => { - defaultResponse = defaultResponse.concat(`\nDisallow: /*${product.href}\nDisallow: /*/enterprise/*/user${product.href}`) + defaultResponse = defaultResponse.concat(`\nDisallow: /*${product.href}`) + product.versions.forEach(version => { + defaultResponse = defaultResponse.concat(`\nDisallow: /*${version}/${product.id}`) + }) }) // Disallow crawling of Deprecated enterprise versions diff --git a/ownership.yaml b/ownership.yaml index c476d609a8..2475138f10 100644 --- a/ownership.yaml +++ b/ownership.yaml @@ -8,7 +8,7 @@ ownership: team: github/docs-engineering maintainer: zeke exec_sponsor: danaiszuul - product_manager: jwargo + product_manager: simpsoka mention: github/docs-engineering qos: critical dependencies: [] diff --git a/package-lock.json b/package-lock.json index 0f31a4b62d..b33963cf8b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17412,11 +17412,6 @@ "type-check": "~0.3.2" } }, - "lil-env-thing": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/lil-env-thing/-/lil-env-thing-1.0.0.tgz", - "integrity": "sha1-etQmBiG/M1rR6HE1d5s15vFmxns=" - }, "lines-and-columns": { "version": "1.1.6", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.1.6.tgz", @@ -17898,11 +17893,11 @@ "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==" }, "resolve": { - "version": "1.18.1", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.18.1.tgz", - "integrity": "sha512-lDfCPaMKfOJXjy0dPayzPdF1phampNWr3qFCjAu+rw/qbQmr5jWH5xN2hwh9QKfw9E5v4hwV7A+jrCmL8yjjqA==", + "version": "1.19.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.19.0.tgz", + "integrity": "sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg==", "requires": { - "is-core-module": "^2.0.0", + "is-core-module": "^2.1.0", "path-parse": "^1.0.6" } }, @@ -17912,9 +17907,9 @@ "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==" }, "type-fest": { - "version": "0.18.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.18.0.tgz", - "integrity": "sha512-fbDukFPnJBdn2eZ3RR+5mK2slHLFd6gYHY7jna1KWWy4Yr4XysHuCdXRzy+RiG/HwG4WJat00vdC2UHky5eKiQ==" + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.18.1.tgz", + "integrity": "sha512-OIAYXk8+ISY+qTOwkHtKqzAuxchoMiD9Udx+FSGQDuiRR+PJKJHc2NJAXlbhkGwTt/4/nKZxELY1w3ReWOL8mw==" }, "yallist": { "version": "4.0.0", diff --git a/package.json b/package.json index d12a883c3f..a0663ce8fb 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,6 @@ "is-url": "^1.2.4", "js-cookie": "^2.2.1", "js-yaml": "^3.14.0", - "lil-env-thing": "^1.0.0", "linkinator": "^2.2.2", "liquid": "^5.1.0", "lodash": "^4.17.19", @@ -169,7 +168,8 @@ "check-deps": "node script/check-deps.js", "prevent-pushes-to-main": "node script/prevent-pushes-to-main.js", "pa11y-ci": "pa11y-ci", - "pa11y-test": "start-server-and-test browser-test-server 4001 pa11y-ci" + "pa11y-test": "start-server-and-test browser-test-server 4001 pa11y-ci", + "heroku-postbuild": "node script/early-access/clone-for-build.js && npm run build" }, "engines": { "node": "12 - 14" diff --git a/script/README.md b/script/README.md index ceae876753..e6e425048e 100644 --- a/script/README.md +++ b/script/README.md @@ -73,7 +73,7 @@ This script is run automatically when you run the server locally. It checks whet ### [`check-s3-images.js`](check-s3-images.js) -Run this script in your branch to check whether any images referenced in content are not in an expected S3 bucket. You will need to authenticate to S3 via `awssume` to use this script. Instructions for the one-time setup are [here](https://github.com/github/product-documentation/blob/master/doc-team-workflows/workflow-information-for-all-writers/setting-up-awssume-and-s3cmd.md). +Run this script in your branch to check whether any images referenced in content are not in an expected S3 bucket. You will need to authenticate to S3 via `awssume` to use this script. --- @@ -304,14 +304,14 @@ This script is run as a git precommit hook (installed by husky after npm install ### [`purge-fastly`](purge-fastly) -Run this script to manually purge the [Fastly cache](https://github.com/github/docs-internal#fastly-cdn). Note this script requires a `FASTLY_SERVICE_ID` and `FASTLY_TOKEN` in your `.env` file. +Run this script to manually purge the Fastly cache. Note this script requires a `FASTLY_SERVICE_ID` and `FASTLY_TOKEN` in your `.env` file. --- ### [`purge-fastly-by-url.js`](purge-fastly-by-url.js) -Run this script to manually purge the [Fastly cache](https://github.com/github/docs-internal#fastly-cdn) for all language variants of a single URL or for a batch of URLs in a file. This script does not require authentication. +Run this script to manually purge the Fastly cache for all language variants of a single URL or for a batch of URLs in a file. This script does not require authentication. --- @@ -362,11 +362,11 @@ Examples: reset a single translated file using a relative path: $ script/reset-translated-file.js translations/es-XL/content/actions/index.md -reset a single translated file using a full path: $ script/reset-translated-file.js /Users/z/git/github/docs-internal/translations/es-XL/content/actions/index.md +reset a single translated file using a full path: $ script/reset-translated-file.js /Users/z/git/github/docs/translations/es-XL/content/actions/index.md reset all language variants of a single English file (using a relative path): $ script/reset-translated-file.js content/actions/index.md $ script/reset-translated-file.js data/ui.yml -reset all language variants of a single English file (using a full path): $ script/reset-translated-file.js /Users/z/git/github/docs-internal/content/desktop/index.md $ script/reset-translated-file.js /Users/z/git/github/docs-internal/data/ui.yml +reset all language variants of a single English file (using a full path): $ script/reset-translated-file.js /Users/z/git/github/docs/content/desktop/index.md $ script/reset-translated-file.js /Users/z/git/github/docs/data/ui.yml --- @@ -422,7 +422,7 @@ Starts the local development server with all of the available languages enabled. ### [`standardize-frontmatter-order.js`](standardize-frontmatter-order.js) -Run this script to standardize frontmatter fields in all content files, per the order decided in https://github.com/github/docs-internal/issues/9658#issuecomment-485536265. +Run this script to standardize frontmatter fields in all content files. --- @@ -443,7 +443,7 @@ List all the TODOs in our JavaScript files and stylesheets. ### [`update-enterprise-dates.js`](update-enterprise-dates.js) -Run this script during Enterprise releases and deprecations. It uses the GitHub API to get dates from [`enterprise-releases`](https://github.com/github/enterprise-releases/blob/master/releases.json) and updates `lib/enterprise-dates.json`. The help site uses this JSON to display dates at the top of some Enterprise versions. +Run this script during Enterprise releases and deprecations. It uses the GitHub API to get dates from `enterprise-releases` and updates `lib/enterprise-dates.json`. The help site uses this JSON to display dates at the top of some Enterprise versions. This script requires that you have a GitHub Personal Access Token in a `.env` file. If you don't have a token, get one [here](https://github.com/settings/tokens/new?scopes=repo&description=docs-dev). If you don't have an `.env` file in your docs checkout, run this command in Terminal: @@ -465,7 +465,7 @@ This script crawls the script directory, hooks on special comment markers in eac ### [`update-s3cmd-config.js`](update-s3cmd-config.js) -This script is used by other scripts to update temporary AWS credentials and authenticate to S3. See docs at [Setting up awssume and S3cmd](https://github.com/github/product-documentation/tree/master/doc-team-workflows/workflow-information-for-all-writers/setting-up-awssume-and-s3cmd.md). +This script is used by other scripts to update temporary AWS credentials and authenticate to S3. --- diff --git a/script/check-s3-images.js b/script/check-s3-images.js index b548ecf3e9..1d20c4dde2 100755 --- a/script/check-s3-images.js +++ b/script/check-s3-images.js @@ -22,7 +22,7 @@ const versionsToCheck = Object.keys(allVersions) // // Run this script in your branch to check whether any images referenced in content are // not in an expected S3 bucket. You will need to authenticate to S3 via `awssume` to use this script. -// Instructions for the one-time setup are [here](https://github.com/github/product-documentation/blob/master/doc-team-workflows/workflow-information-for-all-writers/setting-up-awssume-and-s3cmd.md). +// Instructions for the one-time setup are at docs-content/doc-team-workflows/workflow-information-for-all-writers/setting-up-awssume-and-s3cmd.md // // [end-readme] diff --git a/script/early-access/clone-for-build.js b/script/early-access/clone-for-build.js new file mode 100755 index 0000000000..a1b61faa91 --- /dev/null +++ b/script/early-access/clone-for-build.js @@ -0,0 +1,130 @@ +#!/usr/bin/env node + +// [start-readme] +// +// This script is run as a postbuild script during staging and deployments on Heroku. It clones a branch +// in the early-access repo that matches the current branch in the docs repo; if one can't be found, it +// clones the `main` branch. +// +// [end-readme] + +require('dotenv').config() +const { + DOCUBOT_REPO_PAT, + HEROKU_PRODUCTION_APP, + GIT_BRANCH // Set by the deployer with the name of the docs-internal branch +} = process.env + +// Exit if PAT is not found +if (!DOCUBOT_REPO_PAT) { + console.log('Skipping early access, not authorized') + process.exit(0) +} + +const { execSync } = require('child_process') +const rimraf = require('rimraf').sync +const fs = require('fs') +const path = require('path') +const os = require('os') +const EA_PRODUCTION_BRANCH = 'main' + +// If a branch name is not provided in the environment, attempt to get +// the local branch name; or default to 'main' +let currentBranch = (GIT_BRANCH || '').replace(/^refs\/heads\//, '') +if (!currentBranch) { + try { + currentBranch = execSync('git branch --show-current').toString() + } catch (err) { + // Ignore but log + console.warn('Error checking for local branch:', err.message) + } +} +if (!currentBranch) { + currentBranch = EA_PRODUCTION_BRANCH +} + +// Early Access details +const earlyAccessOwner = 'github' +const earlyAccessRepoName = 'docs-early-access' +const earlyAccessDirName = 'early-access' +const earlyAccessFullRepo = `https://${DOCUBOT_REPO_PAT}@github.com/${earlyAccessOwner}/${earlyAccessRepoName}` + +const earlyAccessCloningParentDir = os.tmpdir() +const earlyAccessCloningDir = path.join(earlyAccessCloningParentDir, earlyAccessRepoName) + +const destinationDirNames = ['content', 'data', 'assets/images'] +const destinationDirsMap = destinationDirNames + .reduce( + (map, dirName) => { + map[dirName] = path.join(process.cwd(), dirName, earlyAccessDirName) + return map + }, + {} + ) + +// Production vs. staging environment +// TODO test that this works as expected +const environment = HEROKU_PRODUCTION_APP ? 'production' : 'staging' + +// Early access branch to clone +let earlyAccessBranch = HEROKU_PRODUCTION_APP ? EA_PRODUCTION_BRANCH : currentBranch + +// Confirm that the branch exists in the remote +let branchExists = execSync(`git ls-remote --heads ${earlyAccessFullRepo} ${earlyAccessBranch}`).toString() + +// If the branch did NOT exist, try checking for the default branch instead +if (!branchExists && earlyAccessBranch !== EA_PRODUCTION_BRANCH) { + console.warn(`The branch '${earlyAccessBranch}' was not found in ${earlyAccessOwner}/${earlyAccessRepoName}!`) + console.warn(`Attempting the default branch ${EA_PRODUCTION_BRANCH} instead...`) + + earlyAccessBranch = EA_PRODUCTION_BRANCH + branchExists = execSync(`git ls-remote --heads ${earlyAccessFullRepo} ${earlyAccessBranch}`).toString() +} + +// If no suitable branch was found, bail out now +if (!branchExists) { + console.error(`The branch '${earlyAccessBranch}' was not found in ${earlyAccessOwner}/${earlyAccessRepoName}!`) + console.error('Exiting!') + process.exit(1) +} + +// Remove any previously cloned copies of the early access repo +rimraf(earlyAccessCloningDir) + +// Clone the repo +console.log(`Setting up: ${earlyAccessCloningDir}`) +execSync( + `git clone --single-branch --branch ${earlyAccessBranch} ${earlyAccessFullRepo} ${earlyAccessRepoName}`, + { + cwd: earlyAccessCloningParentDir + } +) +console.log(`Using early-access ${environment} branch: '${earlyAccessBranch}'`) + +// Remove all existing early access directories from this repo +destinationDirNames.forEach(key => rimraf(destinationDirsMap[key])) + +// Move the latest early access source directories into this repo +destinationDirNames.forEach((dirName) => { + const sourceDir = path.join(earlyAccessCloningDir, dirName) + const destDir = destinationDirsMap[dirName] + + // If the source directory doesn't exist, skip it + if (!fs.existsSync(sourceDir)) { + console.warn(`Early access directory '${dirName}' does not exist. Skipping...`) + return + } + + // Move the directory from the cloned source to the destination + fs.renameSync(sourceDir, destDir) + + // Confirm the newly moved directory exist + if (fs.existsSync(destDir)) { + console.log(`Successfully moved early access directory '${dirName}' into this repo`) + } else { + throw new Error(`Failed to move early access directory '${dirName}'!`) + } +}) + +// Remove the source content again for good hygiene +rimraf(earlyAccessCloningDir) diff --git a/script/early-access/clone-locally b/script/early-access/clone-locally new file mode 100755 index 0000000000..5cd35d69c2 --- /dev/null +++ b/script/early-access/clone-locally @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# [start-readme] +# +# This script is run on a writer's machine to begin developing Early Access content locally. +# +# [end-readme] + +# Go up a directory +pushd .. > /dev/null + +if [ -d "docs-early-access" ]; then + echo "A 'docs-early-access' directory already exists! Try script/early-access/feature-branch.js." + popd > /dev/null + exit 0 +fi + +# Clone the repo +git clone git@github.com:github/docs-early-access.git + +# Go back to the previous working directory +popd > /dev/null + +# Symlink the local docs-early-access repo into this repo +node script/early-access/symlink-from-local-repo.js -p ../docs-early-access + +echo -e '\nDone!' \ No newline at end of file diff --git a/script/early-access/symlink-from-local-repo.js b/script/early-access/symlink-from-local-repo.js new file mode 100755 index 0000000000..1e62274267 --- /dev/null +++ b/script/early-access/symlink-from-local-repo.js @@ -0,0 +1,106 @@ +#!/usr/bin/env node + +// [start-readme] +// +// This script is run on a writer's machine while developing Early Access content locally. +// You must pass the script the location of your local copy of +// the `github/docs-early-access` git repo as the first argument. +// +// [end-readme] + +const rimraf = require('rimraf').sync +const fs = require('fs') +const path = require('path') +const program = require('commander') + +// Early Access details +const earlyAccessRepo = 'docs-early-access' +const earlyAccessDirName = 'early-access' +const earlyAccessRepoUrl = `https://github.com/github/${earlyAccessRepo}` + +program + .description(`Create or destroy symlinks to your local "${earlyAccessRepo}" repository.`) + .option('-p, --path-to-early-access-repo ', `path to a local checkout of ${earlyAccessRepoUrl}`) + .option('-u, --unlink', 'remove the symlinks') + .parse(process.argv) + +const { pathToEarlyAccessRepo, unlink } = program + +if (!pathToEarlyAccessRepo && !unlink) { + throw new Error('Must provide either `--path-to-early-access-repo ` or `--unlink`') +} + +let earlyAccessLocalRepoDir + +// If creating symlinks, run some extra validation +if (!unlink && pathToEarlyAccessRepo) { + earlyAccessLocalRepoDir = path.resolve(process.cwd(), pathToEarlyAccessRepo) + + let dirStats + try { + dirStats = fs.statSync(earlyAccessLocalRepoDir) + } catch (err) { + dirStats = null + } + + if (!dirStats) { + throw new Error(`The local "${earlyAccessRepo}" repo directory does not exist:`, earlyAccessLocalRepoDir) + } + if (dirStats && !dirStats.isDirectory()) { + throw new Error(`A non-directory entry exists at the local "${earlyAccessRepo}" repo directory location:`, earlyAccessLocalRepoDir) + } +} + +const destinationDirNames = ['content', 'data', 'assets/images'] +const destinationDirsMap = destinationDirNames + .reduce( + (map, dirName) => { + map[dirName] = path.join(process.cwd(), dirName, earlyAccessDirName) + return map + }, + {} + ) + +// Remove all existing early access directories from this repo +destinationDirNames.forEach((dirName) => { + const destDir = destinationDirsMap[dirName] + rimraf(destDir) + console.log(`- Removed symlink for early access directory '${dirName}' from this repo`) +}) + +// If removing symlinks, just stop here! +if (unlink) { + process.exit(0) +} + +// +// Otherwise, keep going... +// + +// Move the latest early access source directories into this repo +destinationDirNames.forEach((dirName) => { + const sourceDir = path.join(earlyAccessLocalRepoDir, dirName) + const destDir = destinationDirsMap[dirName] + + // If the source directory doesn't exist, skip it + if (!fs.existsSync(sourceDir)) { + console.warn(`Early access directory '${dirName}' does not exist. Skipping...`) + return + } + + // Create a symbolic link to the directory + fs.symlinkSync(sourceDir, destDir, 'junction') + + // Confirm the newly moved directory exist + if (!fs.existsSync(destDir)) { + throw new Error(`Failed to symlink early access directory '${dirName}'!`) + } + if (!fs.lstatSync(destDir).isSymbolicLink()) { + throw new Error(`The early access directory '${dirName}' entry is not a symbolic link!`) + } + if (!fs.statSync(destDir).isDirectory()) { + throw new Error(`The early access directory '${dirName}' entry's symbolic link does not refer to a directory!`) + } + + console.log(`+ Added symlink for early access directory '${dirName}' into this repo`) +}) diff --git a/script/early-access/update-data-and-image-paths.js b/script/early-access/update-data-and-image-paths.js new file mode 100755 index 0000000000..7fbb2446ec --- /dev/null +++ b/script/early-access/update-data-and-image-paths.js @@ -0,0 +1,159 @@ +#!/usr/bin/env node + +// [start-readme] +// +// This script is run on a writer's machine while developing Early Access content locally. It +// updates the data and image paths to either include `early-access` or remove it. +// +// [end-readme] + +const fs = require('fs') +const path = require('path') +const program = require('commander') +const walk = require('walk-sync') +const { escapeRegExp, last } = require('lodash') +const yaml = require('js-yaml') +const patterns = require('../../lib/patterns') +const earlyAccessContent = path.posix.join(process.cwd(), 'content/early-access') +const earlyAccessData = path.posix.join(process.cwd(), 'data/early-access') +const earlyAccessImages = path.posix.join(process.cwd(), 'assets/images/early-access') + +program + .description('Update data and image paths.') + .option('-p, --path-to-early-access-content-file ', 'Path to a specific content file. Defaults to all Early Access content files if not provided.') + .option('-a, --add', 'Add "early-access" to data and image paths.') + .option('-r, --remove', 'Remove "early-access" from data and image paths.') + .parse(process.argv) + +if (!(program.add || program.remove)) { + console.error('Error! Must specify either `--add` or `--remove`.') + process.exit(1) +} + +let earlyAccessContentAndDataFiles +if (program.pathToEarlyAccessContentFile) { + earlyAccessContentAndDataFiles = path.posix.join(process.cwd(), program.pathToEarlyAccessContentFile) + + if (!fs.existsSync(earlyAccessContentAndDataFiles)) { + console.error(`Error! ${program.pathToEarlyAccessContentFile} can't be found. Make sure the path starts with 'content/early-access'.`) + process.exit(1) + } + earlyAccessContentAndDataFiles = [earlyAccessContentAndDataFiles] +} else { + // Gather the EA content and data files + earlyAccessContentAndDataFiles = walk(earlyAccessContent, { includeBasePath: true, directories: false }) + .concat(walk(earlyAccessData, { includeBasePath: true, directories: false })) +} + +// Update the EA content and data files +earlyAccessContentAndDataFiles + .forEach(file => { + const oldContents = fs.readFileSync(file, 'utf8') + + // Get all the data references in each file that exist in data/early-access + const dataRefs = (oldContents.match(patterns.dataReference) || []) + .filter(dataRef => dataRef.includes('variables') ? checkVariable(dataRef) : checkReusable(dataRef)) + + // Get all the image references in each file that exist in assets/images/early-access + const imageRefs = (oldContents.match(patterns.imagePath) || []) + .filter(imageRef => checkImage(imageRef)) + + const replacements = {} + + if (program.add) { + dataRefs + // Since we're adding early-access to the path, filter for those that do not already include it + .filter(dataRef => !dataRef.includes('data early-access.')) + // Add to the { oldRef: newRef } replacements object + .forEach(dataRef => { + replacements[dataRef] = dataRef.replace(/({% data )(.*)/, '$1early-access.$2') + }) + + imageRefs + // Since we're adding early-access to the path, filter for those that do not already include it + .filter(imageRef => !imageRef.split('/').includes('early-access')) + // Add to the { oldRef: newRef } replacements object + .forEach(imageRef => { + replacements[imageRef] = imageRef.replace('/assets/images/', '/assets/images/early-access/') + }) + } + + if (program.remove) { + dataRefs + // Since we're removing early-access from the path, filter for those that include it + .filter(dataRef => dataRef.includes('{% data early-access.')) + // Add to the { oldRef: newRef } replacements object + .forEach(dataRef => { + replacements[dataRef] = dataRef.replace('early-access.', '') + }) + + imageRefs + // Since we're removing early-access from the path, filter for those that include it + .filter(imageRef => imageRef.split('/').includes('early-access')) + // Add to the { oldRef: newRef } replacements object + .forEach(imageRef => { + replacements[imageRef] = imageRef.replace('/assets/images/early-access/', '/assets/images/') + }) + } + + // Return early if nothing to replace + if (!Object.keys(replacements).length) { + return + } + + // Make the replacement in the content + let newContents = oldContents + Object.entries(replacements).forEach(([oldRef, newRef]) => { + newContents = newContents.replace(new RegExp(escapeRegExp(oldRef), 'g'), newRef) + }) + + // Write the updated content + fs.writeFileSync(file, newContents) + }) + +console.log('Done! Run "git status" in your docs-early-access checkout to see the changes.\n') + +function checkVariable (dataRef) { + // Get the data filepath from the data reference, + // where the data reference looks like: {% data variables.foo.bar %} + // and the data filepath looks like: data/variables/foo.yml with key of 'bar'. + const variablePathArray = dataRef.match(/{% data (.*?) %}/)[1].split('.') + // If early access is part of the path, remove it (since the path below already includes it) + .filter(n => n !== 'early-access') + + // Given a string `variables.foo.bar` split into an array, we want the last segment 'bar', which is the variable key. + // Then pop 'bar' off the array because it's not really part of the filepath. + // The filepath we want is `variables/foo.yml`. + const variableKey = last(variablePathArray); variablePathArray.pop() + const variablePath = path.posix.join(earlyAccessData, `${variablePathArray.join('/')}.yml`) + + // If the variable file doesn't exist in data/early-access, exclude it + if (!fs.existsSync(variablePath)) return false + + // If the variable file exists but doesn't have the referenced key, exclude it + const variableFileContent = yaml.safeLoad(fs.readFileSync(variablePath, 'utf8')) + return variableFileContent[variableKey] +} + +function checkReusable (dataRef) { + // Get the data filepath from the data reference, + // where the data reference looks like: {% data reusables.foo.bar %} + // and the data filepath looks like: data/reusables/foo/bar.md. + const reusablePath = dataRef.match(/{% data (.*?) %}/)[1].split('.') + // If early access is part of the path, remove it (since the path below already includes it) + .filter(n => n !== 'early-access') + .join('/') + + // If the reusable file doesn't exist in data/early-access, exclude it + return fs.existsSync(`${path.posix.join(earlyAccessData, reusablePath)}.md`) +} + +function checkImage (imageRef) { + const imagePath = imageRef + .replace('/assets/images/', '') + // If early access is part of the path, remove it (since the path below already includes it) + .replace('early-access', '') + + // If the image file doesn't exist in assets/images/early-access, exclude it + return fs.existsSync(path.posix.join(earlyAccessImages, imagePath)) +} diff --git a/script/enterprise-server-deprecations/archive-version.js b/script/enterprise-server-deprecations/archive-version.js index 633cc3fd4d..f3a4d9766a 100755 --- a/script/enterprise-server-deprecations/archive-version.js +++ b/script/enterprise-server-deprecations/archive-version.js @@ -180,7 +180,6 @@ async function createRedirectPages (permalinks, pages, finalDirectory) { console.log('done creating redirect files!\n') } -// prior art: https://github.com/github/help-docs-archived-enterprise-versions/blob/master/2.12/user/leave-a-repo/index.html // redirect html files already exist in <=2.12 because these versions were deprecated on the old static site function getRedirectHtml (newPath) { return ` diff --git a/script/graphql/README.md b/script/graphql/README.md index 4588357efd..989ed3abc9 100644 --- a/script/graphql/README.md +++ b/script/graphql/README.md @@ -9,4 +9,4 @@ These scripts update the [static JSON files](../../lib/graphql/static) used to render GraphQL docs. See the [`lib/graphql/README`](../../lib/graphql/README.md) for more info. -**Note**: The changelog script pulls content from [the internal-developer repo](https://github.com/github/internal-developer.github.com/tree/master/content/v4/changelog). It relies on [graphql-docs automation](https://github.com/github/graphql-docs/blob/master/lib/graphql_docs/update_internal_developer/change_log.rb) running daily to update the changelog files in internal-developer. +**Note**: The changelog script pulls content from the internal-developer repo. It relies on graphql-docs automation running daily to update the changelog files in internal-developer. diff --git a/script/graphql/build-changelog.js b/script/graphql/build-changelog.js index fc67d5f558..ad4aaa22a5 100644 --- a/script/graphql/build-changelog.js +++ b/script/graphql/build-changelog.js @@ -127,7 +127,7 @@ function cleanPreviewTitle (title) { /** * Turn the given title into an HTML-ready anchor. - * (ported from https://github.com/github/graphql-docs/blob/master/lib/graphql_docs/update_internal_developer/change_log.rb#L281) + * (ported from graphql-docs/lib/graphql_docs/update_internal_developer/change_log.rb#L281) * @param {string} [previewTitle] * @return {string} */ @@ -155,7 +155,7 @@ function cleanMessagesFromChanges (changes) { * Split `changesToReport` into two parts, * one for changes in the main schema, * and another for changes that are under preview. - * (Ported from https://github.com/github/graphql-docs/blob/7e6a5ccbf13cc7d875fee65527b25bc49e886b41/lib/graphql_docs/update_internal_developer/change_log.rb#L230) + * (Ported from /graphql-docs/lib/graphql_docs/update_internal_developer/change_log.rb#L230) * @param {Array} changesToReport * @param {object} previews * @return {object} @@ -203,7 +203,7 @@ function segmentPreviewChanges (changesToReport, previews) { // Deprecations are covered by "upcoming changes." // By listing the changes explicitly here, we can make sure that, // if the library changes, we don't miss publishing anything that we mean to. -// This was originally ported from https://github.com/github/graphql-docs/blob/7e6a5ccbf13cc7d875fee65527b25bc49e886b41/lib/graphql_docs/update_internal_developer/change_log.rb#L35-L103 +// This was originally ported from graphql-docs/lib/graphql_docs/update_internal_developer/change_log.rb#L35-L103 const CHANGES_TO_REPORT = [ ChangeType.FieldArgumentDefaultChanged, ChangeType.FieldArgumentTypeChanged, diff --git a/script/graphql/utils/remove-hidden-schema-members.rb b/script/graphql/utils/remove-hidden-schema-members.rb index ec0d967214..aee919f80a 100755 --- a/script/graphql/utils/remove-hidden-schema-members.rb +++ b/script/graphql/utils/remove-hidden-schema-members.rb @@ -8,7 +8,7 @@ if ARGV.empty? exit 1 end -# borrowed from https://github.com/github/graphql-docs/blob/master/lib/graphql_docs/update_internal_developer/idl.rb +# borrowed from graphql-docs/lib/graphql_docs/update_internal_developer/idl.rb class Printer < GraphQL::Language::DocumentFromSchemaDefinition def build_object_type_node(object_type) apply_directives_to_node(object_type, super) diff --git a/script/purge-fastly b/script/purge-fastly index daaaec670c..ac9587ea07 100755 --- a/script/purge-fastly +++ b/script/purge-fastly @@ -2,7 +2,7 @@ # [start-readme] # -# Run this script to manually purge the [Fastly cache](https://github.com/github/docs-internal#fastly-cdn). +# Run this script to manually purge the Fastly cache. # Note this script requires a `FASTLY_SERVICE_ID` and `FASTLY_TOKEN` in your `.env` file. # # [end-readme] diff --git a/script/purge-fastly-by-url.js b/script/purge-fastly-by-url.js index d1d4732dbd..069255084c 100755 --- a/script/purge-fastly-by-url.js +++ b/script/purge-fastly-by-url.js @@ -9,7 +9,7 @@ const { getPathWithoutLanguage } = require('../lib/path-utils') // [start-readme] // -// Run this script to manually purge the [Fastly cache](https://github.com/github/docs-internal#fastly-cdn) +// Run this script to manually purge the Fastly cache // for all language variants of a single URL or for a batch of URLs in a file. This script does // not require authentication. // diff --git a/script/standardize-frontmatter-order.js b/script/standardize-frontmatter-order.js index 4d2c65e309..74e68ed862 100755 --- a/script/standardize-frontmatter-order.js +++ b/script/standardize-frontmatter-order.js @@ -14,7 +14,15 @@ const contentFiles = walk(contentDir, { includeBasePath: true }) // [start-readme] // // Run this script to standardize frontmatter fields in all content files, -// per the order decided in https://github.com/github/docs-internal/issues/9658#issuecomment-485536265. +// per the order: +// - title +// - intro +// - product callout +// - productVersion +// - map topic status +// - hidden status +// - layout +// - redirect // // [end-readme] diff --git a/script/update-enterprise-dates.js b/script/update-enterprise-dates.js index a82bc538aa..9f3c67f972 100755 --- a/script/update-enterprise-dates.js +++ b/script/update-enterprise-dates.js @@ -9,7 +9,7 @@ const jsonFile = require(filename) // [start-readme] // // Run this script during Enterprise releases and deprecations. -// It uses the GitHub API to get dates from [`enterprise-releases`](https://github.com/github/enterprise-releases/blob/master/releases.json) and updates `lib/enterprise-dates.json`. +// It uses the GitHub API to get dates from enterprise-releases and updates `lib/enterprise-dates.json`. // The help site uses this JSON to display dates at the top of some Enterprise versions. // // This script requires that you have a GitHub Personal Access Token in a `.env` file. @@ -26,8 +26,7 @@ const jsonFile = require(filename) main() -// GHE Release Lifecycle Dates: -// https://github.com/github/enterprise-releases/blob/master/releases.json +// GHE Release Lifecycle Dates async function main () { let raw try { @@ -38,7 +37,7 @@ async function main () { } const json = prepareData(raw) if (json === prettify(jsonFile)) { - console.log('This repo is already in sync with https://github.com/github/enterprise-releases/blob/master/releases.json!') + console.log('This repo is already in sync with enterprise-releases!') } else { fs.writeFileSync(filename, json, 'utf8') console.log(`${filename} has been updated!`) diff --git a/script/update-s3cmd-config.js b/script/update-s3cmd-config.js index ce6bb6adb1..2103a330cc 100755 --- a/script/update-s3cmd-config.js +++ b/script/update-s3cmd-config.js @@ -5,7 +5,6 @@ const authenticateToAWS = require('../lib/authenticate-to-aws.js') // [start-readme] // // This script is used by other scripts to update temporary AWS credentials and authenticate to S3. -// See docs at [Setting up awssume and S3cmd](https://github.com/github/product-documentation/tree/master/doc-team-workflows/workflow-information-for-all-writers/setting-up-awssume-and-s3cmd.md). // // [end-readme] diff --git a/tests/content/category-pages.js b/tests/content/category-pages.js index 7f7d77137a..702c2d4c8c 100644 --- a/tests/content/category-pages.js +++ b/tests/content/category-pages.js @@ -25,7 +25,7 @@ describe('category pages', () => { const walkOptions = { globs: ['*/index.md', 'enterprise/*/index.md'], - ignore: ['{rest,graphql,developers}/**', 'enterprise/index.md', '**/articles/**'], + ignore: ['{rest,graphql,developers}/**', 'enterprise/index.md', '**/articles/**', 'early-access/**'], directories: false, includeBasePath: true } diff --git a/tests/content/crowdin-config.js b/tests/content/crowdin-config.js index 2bbb509482..220f4cfc35 100644 --- a/tests/content/crowdin-config.js +++ b/tests/content/crowdin-config.js @@ -1,9 +1,46 @@ const config = require('../../lib/crowdin-config').read() +const loadPages = require('../../lib/pages') +const ignoredPagePaths = config.files[0].ignore +const ignoredDataPaths = config.files[2].ignore describe('crowdin.yml config file', () => { + let pages + beforeAll(async (done) => { + pages = await loadPages() + done() + }) + test('has expected file stucture', async () => { expect(config.files.length).toBe(3) expect(config.files[0].source).toBe('/content/**/*.md') expect(config.files[0].ignore).toContain('/content/README.md') }) + + test('ignores all Early Access paths', async () => { + expect(ignoredPagePaths).toContain('/content/early-access') + expect(ignoredDataPaths).toContain('/data/early-access') + }) + + test('ignores all hidden pages', async () => { + const hiddenPages = pages + .filter(page => page.hidden && page.languageCode === 'en') + .map(page => `/content/${page.relativePath}`) + const overlooked = hiddenPages.filter(page => !isIgnored(page, ignoredPagePaths)) + const message = `Found some hidden pages that are not yet excluded from localization. + Please copy and paste the lines below into the \`ignore\` section of /crowdin.yml: \n\n"${overlooked.join('",\n"')}"` + + // This may not be true anymore given the separation of Early Access docs + // expect(hiddenPages.length).toBeGreaterThan(0) + expect(ignoredPagePaths.length).toBeGreaterThan(0) + expect(overlooked, message).toHaveLength(0) + }) }) + +// file is ignored if its exact filename in the list, +// or if it's within an ignored directory +function isIgnored (filename, ignoredPagePaths) { + return ignoredPagePaths.some(ignoredPath => { + const isDirectory = !ignoredPath.endsWith('.md') + return ignoredPath === filename || (isDirectory && filename.startsWith(ignoredPath)) + }) +} diff --git a/tests/content/lint-files.js b/tests/content/lint-files.js index 4755ad72cf..ae6ba18312 100644 --- a/tests/content/lint-files.js +++ b/tests/content/lint-files.js @@ -4,6 +4,7 @@ const fs = require('fs') const walk = require('walk-sync') const { zip } = require('lodash') const yaml = require('js-yaml') +const frontmatter = require('../../lib/frontmatter') const languages = require('../../lib/languages') const { tags } = require('../../lib/liquid-tags/extended-markdown') const ghesReleaseNotesSchema = require('../../lib/release-notes-schema') @@ -66,6 +67,18 @@ const languageLinkRegex = new RegExp(`(?=^|[^\\]]\\s*)\\[[^\\]]+\\](?::\\n?[ \\t // - [link text](/github/site-policy/enterprise/2.2/admin/blah) const versionLinkRegEx = /(?=^|[^\]]\s*)\[[^\]]+\](?::\n?[ \t]+|\s*\()(?:(?:https?:\/\/(?:help|docs|developer)\.github\.com)?\/enterprise\/\d+(\.\d+)+(?:\/[^)\s]*)?)(?:\)|\s+|$)/gm +// Things matched by this RegExp: +// - [link text](/early-access/github/blah) +// - [link text] (https://docs.github.com/early-access/github/blah) +// - [link-definition-ref]: http://help.github.com/early-access/github/blah +// - etc. +// +// Things intentionally NOT matched by this RegExp: +// - [Node.js](https://nodejs.org/early-access/) +// - etc. +// +const earlyAccessLinkRegex = /(?=^|[^\]]\s*)\[[^\]]+\](?::\n?[ \t]+|\s*\()(?:(?:https?:\/\/(?:help|docs|developer)\.github\.com)?\/early-access(?:\/[^)\s]*)?)(?:\)|\s+|$)/gm + // - [link text](https://docs.github.com/github/blah) // - [link text] (https://help.github.com/github/blah) // - [link-definition-ref]: http://developer.github.com/v3/ @@ -79,6 +92,33 @@ const versionLinkRegEx = /(?=^|[^\]]\s*)\[[^\]]+\](?::\n?[ \t]+|\s*\()(?:(?:http // const domainLinkRegex = /(?=^|[^\]]\s*)\[[^\]]+\](?::\n?[ \t]+|\s*\()(?:https?:)?\/\/(?:help|docs|developer)\.github\.com(?!\/changes\/)[^)\s]*(?:\)|\s+|$)/gm +// Things matched by this RegExp: +// - ![image text](/assets/images/early-access/github/blah.gif) +// - ![image text] (https://docs.github.com/assets/images/early-access/github/blah.gif) +// - [image-definition-ref]: http://help.github.com/assets/images/early-access/github/blah.gif +// - [link text](/assets/images/early-access/github/blah.gif) +// - etc. +// +// Things intentionally NOT matched by this RegExp: +// - [Node.js](https://nodejs.org/assets/images/early-access/blah.gif) +// - etc. +// +const earlyAccessImageRegex = /(?=^|[^\]]\s*)\[[^\]]+\](?::\n?[ \t]+|\s*\()(?:(?:https?:\/\/(?:help|docs|developer)\.github\.com)?\/assets\/images\/early-access(?:\/[^)\s]*)?)(?:\)|\s+|$)/gm + +// Things matched by this RegExp: +// - ![image text](/assets/early-access/images/github/blah.gif) +// - ![image text] (https://docs.github.com/images/early-access/github/blah.gif) +// - [image-definition-ref]: http://help.github.com/assets/early-access/github/blah.gif +// - [link text](/early-access/assets/images/github/blah.gif) +// - [link text](/early-access/images/github/blah.gif) +// - etc. +// +// Things intentionally NOT matched by this RegExp: +// - [Node.js](https://nodejs.org/assets/early-access/images/blah.gif) +// - etc. +// +const badEarlyAccessImageRegex = /(?=^|[^\]]\s*)\[[^\]]+\](?::\n?[ \t]+|\s*\()(?:(?:https?:\/\/(?:help|docs|developer)\.github\.com)?\/(?:(?:assets|images)\/early-access|early-access\/(?:assets|images))(?:\/[^)\s]*)?)(?:\)|\s+|$)/gm + // {{ site.data.example.pizza }} const oldVariableRegex = /{{\s*?site\.data\..*?}}/g @@ -98,6 +138,9 @@ const relativeArticleLinkErrorText = 'Found unexpected relative article links:' const languageLinkErrorText = 'Found article links with hard-coded language codes:' const versionLinkErrorText = 'Found article links with hard-coded version numbers:' const domainLinkErrorText = 'Found article links with hard-coded domain names:' +const earlyAccessLinkErrorText = 'Found article links leaking Early Access docs:' +const earlyAccessImageErrorText = 'Found article images/links leaking Early Access images:' +const badEarlyAccessImageErrorText = 'Found article images/links leaking incorrect Early Access images:' const oldVariableErrorText = 'Found article uses old {{ site.data... }} syntax. Use {% data example.data.string %} instead!' const oldOcticonErrorText = 'Found octicon variables with the old {{ octicon-name }} syntax. Use {% octicon "name" %} instead!' const oldExtendedMarkdownErrorText = 'Found extended markdown tags with the old {{#note}} syntax. Use {% note %}/{% endnote %} instead!' @@ -121,10 +164,19 @@ describe('lint-files', () => { describe.each([...contentMarkdownTuples, ...reusableMarkdownTuples])( 'in "%s"', (markdownRelPath, markdownAbsPath) => { - let content + let content, isHidden, isEarlyAccess beforeAll(async () => { - content = await fs.promises.readFile(markdownAbsPath, 'utf8') + const fileContents = await fs.promises.readFile(markdownAbsPath, 'utf8') + const { data, content: bodyContent } = frontmatter(fileContents) + + content = bodyContent + isHidden = data.hidden === true + isEarlyAccess = markdownRelPath.split('/').includes('early-access') + }) + + test('hidden docs must be Early Access', async () => { + expect(isHidden).toBe(isEarlyAccess) }) test('relative URLs must start with "/"', async () => { @@ -171,6 +223,14 @@ describe('lint-files', () => { if (match === '[Contribution guidelines for this project](docs/CONTRIBUTING.md)') { return false } + } else if (markdownRelPath === 'content/early-access/github/enforcing-best-practices-with-github-policies/constraints.md') { + if (match === '[a-z]([a-z]|-)') { + return false + } + } else if (markdownRelPath === 'content/early-access/github/enforcing-best-practices-with-github-policies/overview.md') { + if (match === '[A-Z]([a-z]|-)') { + return false + } } return true }) @@ -206,6 +266,32 @@ describe('lint-files', () => { expect(matches.length, errorMessage).toBe(0) }) + test('must not leak Early Access doc URLs', async () => { + // Only execute for docs that are NOT Early Access + if (!isEarlyAccess) { + const matches = (content.match(earlyAccessLinkRegex) || []) + const errorMessage = formatLinkError(earlyAccessLinkErrorText, matches) + expect(matches.length, errorMessage).toBe(0) + } + }) + + test('must not leak Early Access image URLs', async () => { + // Only execute for docs that are NOT Early Access + if (!isEarlyAccess) { + const matches = (content.match(earlyAccessImageRegex) || []) + const errorMessage = formatLinkError(earlyAccessImageErrorText, matches) + expect(matches.length, errorMessage).toBe(0) + } + }) + + test('must have correctly formatted Early Access image URLs', async () => { + // Execute for ALL docs (not just Early Access) to ensure non-EA docs + // are not leaking incorrectly formatted EA image URLs + const matches = (content.match(badEarlyAccessImageRegex) || []) + const errorMessage = formatLinkError(badEarlyAccessImageErrorText, matches) + expect(matches.length, errorMessage).toBe(0) + }) + test('does not use old site.data variable syntax', async () => { const matches = (content.match(oldVariableRegex) || []) const matchesWithExample = matches.map(match => { @@ -248,17 +334,19 @@ describe('lint-files', () => { } const variableYamlAbsPaths = walk(variablesDir, yamlWalkOptions).sort() - const variableYamlRelPaths = variableYamlAbsPaths.map(p => path.relative(rootDir, p)) + const variableYamlRelPaths = variableYamlAbsPaths.map(p => slash(path.relative(rootDir, p))) const variableYamlTuples = zip(variableYamlRelPaths, variableYamlAbsPaths) describe.each(variableYamlTuples)( 'in "%s"', (yamlRelPath, yamlAbsPath) => { - let dictionary + let dictionary, isEarlyAccess beforeAll(async () => { const fileContents = await fs.promises.readFile(yamlAbsPath, 'utf8') dictionary = yaml.safeLoad(fileContents, { filename: yamlRelPath }) + + isEarlyAccess = yamlRelPath.split('/').includes('early-access') }) test('relative URLs must start with "/"', async () => { @@ -321,6 +409,59 @@ describe('lint-files', () => { expect(matches.length, errorMessage).toBe(0) }) + test('must not leak Early Access doc URLs', async () => { + // Only execute for docs that are NOT Early Access + if (!isEarlyAccess) { + const matches = [] + + for (const [key, content] of Object.entries(dictionary)) { + if (typeof content !== 'string') continue + const valMatches = (content.match(earlyAccessLinkRegex) || []) + if (valMatches.length > 0) { + matches.push(...valMatches.map((match) => `Key "${key}": ${match}`)) + } + } + + const errorMessage = formatLinkError(earlyAccessLinkErrorText, matches) + expect(matches.length, errorMessage).toBe(0) + } + }) + + test('must not leak Early Access image URLs', async () => { + // Only execute for docs that are NOT Early Access + if (!isEarlyAccess) { + const matches = [] + + for (const [key, content] of Object.entries(dictionary)) { + if (typeof content !== 'string') continue + const valMatches = (content.match(earlyAccessImageRegex) || []) + if (valMatches.length > 0) { + matches.push(...valMatches.map((match) => `Key "${key}": ${match}`)) + } + } + + const errorMessage = formatLinkError(earlyAccessImageErrorText, matches) + expect(matches.length, errorMessage).toBe(0) + } + }) + + test('must have correctly formatted Early Access image URLs', async () => { + // Execute for ALL docs (not just Early Access) to ensure non-EA docs + // are not leaking incorrectly formatted EA image URLs + const matches = [] + + for (const [key, content] of Object.entries(dictionary)) { + if (typeof content !== 'string') continue + const valMatches = (content.match(badEarlyAccessImageRegex) || []) + if (valMatches.length > 0) { + matches.push(...valMatches.map((match) => `Key "${key}": ${match}`)) + } + } + + const errorMessage = formatLinkError(badEarlyAccessImageErrorText, matches) + expect(matches.length, errorMessage).toBe(0) + }) + test('does not use old site.data variable syntax', async () => { const matches = [] diff --git a/tests/content/site-data-references.js b/tests/content/site-data-references.js index c35d6044f1..30f16950ca 100644 --- a/tests/content/site-data-references.js +++ b/tests/content/site-data-references.js @@ -1,13 +1,14 @@ -const { isEqual, get, uniqBy } = require('lodash') +const { isEqual, get, uniqWith } = require('lodash') const loadSiteData = require('../../lib/site-data') const loadPages = require('../../lib/pages') const getDataReferences = require('../../lib/get-liquid-data-references') +const frontmatter = require('@github-docs/frontmatter') const fs = require('fs') const path = require('path') describe('data references', () => { - let data - let pages + let data, pages + beforeAll(async (done) => { data = await loadSiteData() pages = await loadPages() @@ -20,15 +21,34 @@ describe('data references', () => { expect(pages.length).toBeGreaterThan(0) pages.forEach(page => { + const file = path.join('content', page.relativePath) const pageRefs = getDataReferences(page.markdown) pageRefs.forEach(key => { const value = get(data.en, key) - const file = path.join('content', page.relativePath) if (typeof value !== 'string') errors.push({ key, value, file }) }) }) - errors = uniqBy(errors, isEqual) // remove duplicates + errors = uniqWith(errors, isEqual) // remove duplicates + expect(errors.length, JSON.stringify(errors, null, 2)).toBe(0) + }) + + test('every data reference found in metadata of English content files is defined and has a value', () => { + let errors = [] + expect(pages.length).toBeGreaterThan(0) + + pages.forEach(page => { + const metadataFile = path.join('content', page.relativePath) + const fileContents = fs.readFileSync(path.join(__dirname, '../..', metadataFile)) + const { data: metadata } = frontmatter(fileContents, { filepath: page.fullPath }) + const metadataRefs = getDataReferences(JSON.stringify(metadata)) + metadataRefs.forEach(key => { + const value = get(data.en, key) + if (typeof value !== 'string') errors.push({ key, value, metadataFile }) + }) + }) + + errors = uniqWith(errors, isEqual) // remove duplicates expect(errors.length, JSON.stringify(errors, null, 2)).toBe(0) }) @@ -39,17 +59,18 @@ describe('data references', () => { expect(reusables.length).toBeGreaterThan(0) reusables.forEach(reusablesPerFile => { + let reusableFile = path.join(__dirname, '../../data/reusables/', getFilenameByValue(allReusables, reusablesPerFile)) + reusableFile = getFilepath(reusableFile) + const reusableRefs = getDataReferences(JSON.stringify(reusablesPerFile)) reusableRefs.forEach(key => { const value = get(data.en, key) - let reusableFile = path.join(__dirname, '../../data/reusables/', getFilenameByValue(allReusables, reusablesPerFile)) - reusableFile = getFilepath(reusableFile) if (typeof value !== 'string') errors.push({ key, value, reusableFile }) }) }) - errors = uniqBy(errors, isEqual) // remove duplicates + errors = uniqWith(errors, isEqual) // remove duplicates expect(errors.length, JSON.stringify(errors, null, 2)).toBe(0) }) @@ -60,17 +81,18 @@ describe('data references', () => { expect(variables.length).toBeGreaterThan(0) variables.forEach(variablesPerFile => { + let variableFile = path.join(__dirname, '../../data/variables/', getFilenameByValue(allVariables, variablesPerFile)) + variableFile = getFilepath(variableFile) + const variableRefs = getDataReferences(JSON.stringify(variablesPerFile)) variableRefs.forEach(key => { const value = get(data.en, key) - let variableFile = path.join(__dirname, '../../data/variables/', getFilenameByValue(allVariables, variablesPerFile)) - variableFile = getFilepath(variableFile) if (typeof value !== 'string') errors.push({ key, value, variableFile }) }) }) - errors = uniqBy(errors, isEqual) // remove duplicates + errors = uniqWith(errors, isEqual) // remove duplicates expect(errors.length, JSON.stringify(errors, null, 2)).toBe(0) }) }) diff --git a/tests/meta/repository-references.js b/tests/meta/repository-references.js new file mode 100644 index 0000000000..9df0e07888 --- /dev/null +++ b/tests/meta/repository-references.js @@ -0,0 +1,62 @@ +const walkSync = require('walk-sync') +const fs = require('fs').promises + +const REPO_REGEXP = /\/\/github\.com\/github\/(?!docs[/'"\n])([\w-.]+)/gi + +// These are a list of known public repositories in the GitHub organization +const ALLOW_LIST = new Set([ + 'site-policy', + 'roadmap', + 'linguist', + 'super-linter', + 'backup-utils', + 'codeql-action-sync-tool', + 'codeql-action', + 'platform-samples', + 'github-services', + 'explore', + 'markup', + 'hubot', + 'VisualStudio', + 'codeql', + 'gitignore', + 'feedback', + 'semantic', + 'git-lfs', + 'git-sizer', + 'dmca', + 'gov-takedowns', + 'janky', + 'rest-api-description', + 'smimesign', + 'tweetsodium', + 'choosealicense.com' +]) + +describe('check for repository references', () => { + const filenames = walkSync(process.cwd(), { + directories: false, + ignore: [ + '.algolia-cache', + '.git', + 'dist', + 'node_modules', + 'translations', + 'lib/rest/**/*.json', + 'lib/webhooks/**/*.json', + 'ownership.yaml', + 'docs/index.yaml', + 'lib/excluded-links.js', + 'content/early-access', + 'data/early-access' + ] + }) + + test.each(filenames)('in file %s', async (filename) => { + const file = await fs.readFile(filename, 'utf8') + const matches = Array.from(file.matchAll(REPO_REGEXP)) + .map(([, repoName]) => repoName) + .filter(repoName => !ALLOW_LIST.has(repoName)) + expect(matches).toHaveLength(0) + }) +}) diff --git a/tests/rendering/early-access-paths.js b/tests/rendering/early-access-paths.js deleted file mode 100644 index d597b54b9d..0000000000 --- a/tests/rendering/early-access-paths.js +++ /dev/null @@ -1,64 +0,0 @@ -const MockExpressResponse = require('mock-express-response') -const middleware = require('../../middleware/early-access-paths') - -describe('GET /early-access-paths.json', () => { - beforeEach(() => { - delete process.env['early-access-shared-secret'] - }) - - test('responds with 401 if shared secret is missing', async () => { - const req = { - path: '/early-access-paths.json', - headers: {} - } - const res = new MockExpressResponse() - const next = jest.fn() - await middleware(req, res, next) - - expect(res._getJSON()).toEqual({ error: '401 Unauthorized' }) - }) - - test('responds with an array of hidden paths', async () => { - process.env.EARLY_ACCESS_SHARED_SECRET = 'bananas' - - const req = { - path: '/early-access-paths.json', - headers: { - 'early-access-shared-secret': 'bananas' - }, - context: { - pages: [ - { - hidden: true, - languageCode: 'en', - permalinks: [ - { href: '/some-hidden-page' } - ], - redirects: { - '/old-hidden-page': '/new-hidden-page' - } - }, - { - hidden: false, - languageCode: 'en' - } - ] - } - } - const res = new MockExpressResponse() - const next = jest.fn() - await middleware(req, res, next) - - expect(res._getJSON()).toEqual(['/some-hidden-page', '/old-hidden-page']) - }) - - test('ignores requests to other paths', async () => { - const req = { - path: '/not-early-access' - } - const res = new MockExpressResponse() - const next = jest.fn() - await middleware(req, res, next) - expect(next).toHaveBeenCalled() - }) -}) diff --git a/tests/rendering/early-access-proxy.js b/tests/rendering/early-access-proxy.js deleted file mode 100644 index e0e60ee287..0000000000 --- a/tests/rendering/early-access-proxy.js +++ /dev/null @@ -1,80 +0,0 @@ - -const middleware = require('../../middleware/early-access-proxy') -const nock = require('nock') -const MockExpressResponse = require('mock-express-response') - -describe('Early Access middleware', () => { - const OLD_EARLY_ACCESS_HOSTNAME = process.env.EARLY_ACCESS_HOSTNAME - - beforeAll(() => { - process.env.EARLY_ACCESS_HOSTNAME = 'https://secret-website.com' - }) - - afterAll(() => { - process.env.EARLY_ACCESS_HOSTNAME = OLD_EARLY_ACCESS_HOSTNAME - }) - - const baseReq = { - context: { - earlyAccessPaths: ['/alpha-product/foo', '/beta-product/bar', '/baz'] - } - } - - test('are proxied from an obscured host', async () => { - const mock = nock('https://secret-website.com') - .get('/alpha-product/foo') - .reply(200, 'yay here is your proxied content', { 'content-type': 'text/html' }) - const req = { ...baseReq, path: '/alpha-product/foo' } - const res = new MockExpressResponse() - const next = jest.fn() - await middleware(req, res, next) - expect(mock.isDone()).toBe(true) - expect(res._getString()).toBe('yay here is your proxied content') - }) - - test('follows redirects', async () => { - const mock = nock('https://secret-website.com') - .get('/alpha-product/foo') - .reply(301, undefined, { Location: 'https://secret-website.com/alpha-product/foo2' }) - .get('/alpha-product/foo2') - .reply(200, 'yay you survived the redirect', { 'content-type': 'text/html' }) - const req = { ...baseReq, path: '/alpha-product/foo' } - const res = new MockExpressResponse() - const next = jest.fn() - await middleware(req, res, next) - expect(mock.isDone()).toBe(true) - expect(res._getString()).toBe('yay you survived the redirect') - }) - - test('calls next() if no redirect is found', async () => { - const req = { ...baseReq, path: '/en' } - const res = new MockExpressResponse() - const next = jest.fn() - await middleware(req, res, next) - expect(next).toHaveBeenCalled() - }) - - test('calls next() if proxy request respond with 404', async () => { - const mock = nock('https://secret-website.com') - .get('/beta-product/bar') - .reply(404, 'no dice', { 'content-type': 'text/html' }) - const req = { ...baseReq, path: '/beta-product/bar' } - const res = new MockExpressResponse() - const next = jest.fn() - await middleware(req, res, next) - expect(mock.isDone()).toBe(true) - expect(next).toHaveBeenCalled() - }) - - test('calls next() if proxy request responds with 500', async () => { - const mock = nock('https://secret-website.com') - .get('/beta-product/bar') - .reply(500, 'no dice', { 'content-type': 'text/html' }) - const req = { ...baseReq, path: '/beta-product/bar' } - const res = new MockExpressResponse() - const next = jest.fn() - await middleware(req, res, next) - expect(mock.isDone()).toBe(true) - expect(next).toHaveBeenCalled() - }) -}) diff --git a/tests/rendering/robots-txt.js b/tests/rendering/robots-txt.js index e5a073747e..6bd7854269 100644 --- a/tests/rendering/robots-txt.js +++ b/tests/rendering/robots-txt.js @@ -12,22 +12,22 @@ describe('robots.txt', () => { let res, robots beforeAll(async (done) => { res = await get('/robots.txt') - robots = robotsParser('https://help.github.com/robots.txt', res.text) + robots = robotsParser('https://docs.github.com/robots.txt', res.text) done() }) it('allows indexing of the homepage and English content', async () => { - expect(robots.isAllowed('https://help.github.com/')).toBe(true) - expect(robots.isAllowed('https://help.github.com/en')).toBe(true) - expect(robots.isAllowed('https://help.github.com/en/articles/verifying-your-email-address')).toBe(true) + expect(robots.isAllowed('https://docs.github.com/')).toBe(true) + expect(robots.isAllowed('https://docs.github.com/en')).toBe(true) + expect(robots.isAllowed('https://docs.github.com/en/articles/verifying-your-email-address')).toBe(true) }) it('allows indexing of generally available localized content', async () => { Object.values(languages) .filter(language => !language.wip) .forEach(language => { - expect(robots.isAllowed(`https://help.github.com/${language.code}`)).toBe(true) - expect(robots.isAllowed(`https://help.github.com/${language.code}/articles/verifying-your-email-address`)).toBe(true) + expect(robots.isAllowed(`https://docs.github.com/${language.code}`)).toBe(true) + expect(robots.isAllowed(`https://docs.github.com/${language.code}/articles/verifying-your-email-address`)).toBe(true) }) }) @@ -35,8 +35,8 @@ describe('robots.txt', () => { Object.values(languages) .filter(language => language.wip) .forEach(language => { - expect(robots.isAllowed(`https://help.github.com/${language.code}`)).toBe(false) - expect(robots.isAllowed(`https://help.github.com/${language.code}/articles/verifying-your-email-address`)).toBe(false) + expect(robots.isAllowed(`https://docs.github.com/${language.code}`)).toBe(false) + expect(robots.isAllowed(`https://docs.github.com/${language.code}/articles/verifying-your-email-address`)).toBe(false) }) }) @@ -61,18 +61,18 @@ describe('robots.txt', () => { const { href } = products[id] const blockedPaths = [ // English - `https://help.github.com/en${href}`, - `https://help.github.com/en${href}/overview`, - `https://help.github.com/en${href}/overview/intro`, - `https://help.github.com/en/enterprise/${enterpriseServerReleases.latest}/user${href}`, - `https://help.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user${href}`, + `https://docs.github.com/en${href}`, + `https://docs.github.com/en${href}/overview`, + `https://docs.github.com/en${href}/overview/intro`, + `https://docs.github.com/en/enterprise/${enterpriseServerReleases.latest}/user${href}`, + `https://docs.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user${href}`, // Japanese - `https://help.github.com/ja${href}`, - `https://help.github.com/ja${href}/overview`, - `https://help.github.com/ja${href}/overview/intro`, - `https://help.github.com/ja/enterprise/${enterpriseServerReleases.latest}/user${href}`, - `https://help.github.com/ja/enterprise/${enterpriseServerReleases.oldestSupported}/user${href}` + `https://docs.github.com/ja${href}`, + `https://docs.github.com/ja${href}/overview`, + `https://docs.github.com/ja${href}/overview/intro`, + `https://docs.github.com/ja/enterprise/${enterpriseServerReleases.latest}/user${href}`, + `https://docs.github.com/ja/enterprise/${enterpriseServerReleases.oldestSupported}/user${href}` ] blockedPaths.forEach(path => { @@ -81,28 +81,52 @@ describe('robots.txt', () => { }) }) + it('disallows indexing of early access "hidden" products', async () => { + const hiddenProductIds = Object.values(products) + .filter(product => product.hidden) + .map(product => product.id) + + hiddenProductIds.forEach(id => { + const { versions } = products[id] + const blockedPaths = versions.map(version => { + return [ + // English + `https://docs.github.com/en/${version}/${id}`, + `https://docs.github.com/en/${version}/${id}/some-early-access-article`, + // Japanese + `https://docs.github.com/ja/${version}/${id}`, + `https://docs.github.com/ja/${version}/${id}/some-early-access-article` + ] + }).flat() + + blockedPaths.forEach(path => { + expect(robots.isAllowed(path)).toBe(false) + }) + }) + }) + it('allows indexing of non-WIP products', async () => { expect('actions' in products).toBe(true) - expect(robots.isAllowed('https://help.github.com/en/actions')).toBe(true) - expect(robots.isAllowed('https://help.github.com/en/actions/overview')).toBe(true) - expect(robots.isAllowed('https://help.github.com/en/actions/overview/intro')).toBe(true) - expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.latest}/user/actions`)).toBe(true) - expect(robots.isAllowed(`https://help.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user/actions`)).toBe(true) + expect(robots.isAllowed('https://docs.github.com/en/actions')).toBe(true) + expect(robots.isAllowed('https://docs.github.com/en/actions/overview')).toBe(true) + expect(robots.isAllowed('https://docs.github.com/en/actions/overview/intro')).toBe(true) + expect(robots.isAllowed(`https://docs.github.com/en/enterprise/${enterpriseServerReleases.latest}/user/actions`)).toBe(true) + expect(robots.isAllowed(`https://docs.github.com/en/enterprise/${enterpriseServerReleases.oldestSupported}/user/actions`)).toBe(true) }) it('disallows indexing of deprecated enterprise releases', async () => { enterpriseServerReleases.deprecated.forEach(version => { const blockedPaths = [ // English - `https://help.github.com/en/enterprise-server@${version}/actions`, - `https://help.github.com/en/enterprise/${version}/actions`, - `https://help.github.com/en/enterprise-server@${version}/actions/overview`, - `https://help.github.com/en/enterprise/${version}/actions/overview`, + `https://docs.github.com/en/enterprise-server@${version}/actions`, + `https://docs.github.com/en/enterprise/${version}/actions`, + `https://docs.github.com/en/enterprise-server@${version}/actions/overview`, + `https://docs.github.com/en/enterprise/${version}/actions/overview`, // Japanese - `https://help.github.com/ja/enterprise-server@${version}/actions`, - `https://help.github.com/ja/enterprise/${version}/actions`, - `https://help.github.com/ja/enterprise-server@${version}/actions/overview`, - `https://help.github.com/ja/enterprise/${version}/actions/overview` + `https://docs.github.com/ja/enterprise-server@${version}/actions`, + `https://docs.github.com/ja/enterprise/${version}/actions`, + `https://docs.github.com/ja/enterprise-server@${version}/actions/overview`, + `https://docs.github.com/ja/enterprise/${version}/actions/overview` ] blockedPaths.forEach(path => { diff --git a/tests/rendering/server.js b/tests/rendering/server.js index bf6b16b3b2..a83c73abe7 100644 --- a/tests/rendering/server.js +++ b/tests/rendering/server.js @@ -3,6 +3,7 @@ const enterpriseServerReleases = require('../../lib/enterprise-server-releases') const { get, getDOM, head } = require('../helpers/supertest') const path = require('path') const nonEnterpriseDefaultVersion = require('../../lib/non-enterprise-default-version') +const loadPages = require('../../lib/pages') describe('server', () => { jest.setTimeout(60 * 1000) @@ -90,7 +91,7 @@ describe('server', () => { expect($.res.statusCode).toBe(400) }) - // see https://github.com/github/docs-internal/issues/12427 + // see issue 12427 test('renders a 404 for leading slashes', async () => { let $ = await getDOM('//foo.com/enterprise') expect($('h1').text()).toBe('Ooops!') @@ -130,7 +131,7 @@ describe('server', () => { expect($('div.permissions-statement').text()).toContain('GitHub Pages site') }) - // see https://github.com/github/docs-internal/issues/9678 + // see issue 9678 test('does not use cached intros in map topics', async () => { let $ = await getDOM('/en/github/importing-your-projects-to-github/importing-a-git-repository-using-the-command-line') const articleIntro = $('.lead-mktg').text() @@ -355,6 +356,46 @@ describe('server', () => { }) }) + describe.skip('Early Access articles', () => { + let hiddenPageHrefs, hiddenPages + + beforeAll(async (done) => { + const $ = await getDOM('/early-access') + hiddenPageHrefs = $('#article-contents ul > li > a').map((i, el) => $(el).attr('href')).get() + + const allPages = await loadPages() + hiddenPages = allPages.filter(page => page.languageCode === 'en' && page.hidden) + + done() + }) + + test('exist in the set of English pages', async () => { + expect(hiddenPages.length).toBeGreaterThan(0) + }) + + test('are listed at /early-access', async () => { + expect(hiddenPageHrefs.length).toBeGreaterThan(0) + }) + + test('are not listed at /early-access in production', async () => { + const oldNodeEnv = process.env.NODE_ENV + process.env.NODE_ENV = 'production' + const res = await get('/early-access', { followRedirects: true }) + process.env.NODE_ENV = oldNodeEnv + expect(res.statusCode).toBe(404) + }) + + test('have noindex meta tags', async () => { + const $ = await getDOM(hiddenPageHrefs[0]) + expect($('meta[content="noindex"]').length).toBe(1) + }) + + test('public articles do not have noindex meta tags', async () => { + const $ = await getDOM('/en/articles/set-up-git') + expect($('meta[content="noindex"]').length).toBe(0) + }) + }) + describe('redirects', () => { test('redirects old articles to their English URL', async () => { const res = await get('/articles/deleting-a-team') diff --git a/tests/rendering/sidebar.js b/tests/rendering/sidebar.js index 51afa72bb0..813018b8a9 100644 --- a/tests/rendering/sidebar.js +++ b/tests/rendering/sidebar.js @@ -36,4 +36,9 @@ describe('sidebar', () => { expect($('.sidebar .is-current-page').length).toBe(1) expect($('.sidebar .is-current-page a').attr('href')).toContain(url) }) + + test('does not display Early Access as a product', async () => { + expect($homePage('.sidebar li.sidebar-product[title*="Early"]').length).toBe(0) + expect($homePage('.sidebar li.sidebar-product[title*="early"]').length).toBe(0) + }) }) diff --git a/tests/unit/early-access.js b/tests/unit/early-access.js new file mode 100644 index 0000000000..f8ad0b3b88 --- /dev/null +++ b/tests/unit/early-access.js @@ -0,0 +1,23 @@ +const fs = require('fs') +const path = require('path') + +const { GITHUB_ACTIONS, GITHUB_REPOSITORY } = process.env +const runningActionsOnInternalRepo = GITHUB_ACTIONS === 'true' && GITHUB_REPOSITORY === 'github/docs-internal' +const testViaActionsOnly = runningActionsOnInternalRepo ? test : test.skip + +describe('cloning early-access', () => { + testViaActionsOnly('the content directory exists', async () => { + const eaContentDir = path.join(process.cwd(), 'content/early-access') + expect(fs.existsSync(eaContentDir)).toBe(true) + }) + + testViaActionsOnly('the data directory exists', async () => { + const eaContentDir = path.join(process.cwd(), 'data/early-access') + expect(fs.existsSync(eaContentDir)).toBe(true) + }) + + testViaActionsOnly('the assets/images directory exists', async () => { + const eaContentDir = path.join(process.cwd(), 'assets/images/early-access') + expect(fs.existsSync(eaContentDir)).toBe(true) + }) +}) diff --git a/translations/de-DE/content/rest/overview/api-previews.md b/translations/de-DE/content/rest/overview/api-previews.md index 892472c4de..e4d6a1e383 100644 --- a/translations/de-DE/content/rest/overview/api-previews.md +++ b/translations/de-DE/content/rest/overview/api-previews.md @@ -2,7 +2,6 @@ title: API previews intro: You can use API previews to try out new features and provide feedback before these features become official. redirect_from: - - /early-access/ - /v3/previews versions: free-pro-team: '*' diff --git a/translations/es-XL/content/rest/overview/api-previews.md b/translations/es-XL/content/rest/overview/api-previews.md index a3658b02f9..2953fef157 100644 --- a/translations/es-XL/content/rest/overview/api-previews.md +++ b/translations/es-XL/content/rest/overview/api-previews.md @@ -2,7 +2,6 @@ title: Vistas previas de la API intro: Puedes utilizar las vistas previas de la API para probar características nuevas y proporcionar retroalimentación antes de que dichas características se hagan oficiales. redirect_from: - - /early-access/ - /v3/previews versions: free-pro-team: '*' diff --git a/translations/ja-JP/content/rest/overview/api-previews.md b/translations/ja-JP/content/rest/overview/api-previews.md index a97c3408f9..3b221ce743 100644 --- a/translations/ja-JP/content/rest/overview/api-previews.md +++ b/translations/ja-JP/content/rest/overview/api-previews.md @@ -2,7 +2,6 @@ title: API プレビュー intro: API プレビューを使用して新機能を試し、これらの機能が正式なものになる前にフィードバックを提供できます。 redirect_from: - - /early-access/ - /v3/previews versions: free-pro-team: '*' diff --git a/translations/ko-KR/content/rest/overview/api-previews.md b/translations/ko-KR/content/rest/overview/api-previews.md index 904af72be7..3cdc9b7033 100644 --- a/translations/ko-KR/content/rest/overview/api-previews.md +++ b/translations/ko-KR/content/rest/overview/api-previews.md @@ -2,7 +2,6 @@ title: API previews intro: You can use API previews to try out new features and provide feedback before these features become official. redirect_from: - - /early-access/ - /v3/previews versions: free-pro-team: '*' diff --git a/translations/pt-BR/content/rest/overview/api-previews.md b/translations/pt-BR/content/rest/overview/api-previews.md index 18e13102ea..672a4051f1 100644 --- a/translations/pt-BR/content/rest/overview/api-previews.md +++ b/translations/pt-BR/content/rest/overview/api-previews.md @@ -2,7 +2,6 @@ title: Pré-visualizações da API intro: Você pode usar pré-visualizações da API para testar novos recursos e fornecer feedback antes que estes recursos se tornem oficiais. redirect_from: - - /early-access/ - /v3/previews versions: free-pro-team: '*' diff --git a/translations/ru-RU/content/rest/overview/api-previews.md b/translations/ru-RU/content/rest/overview/api-previews.md index 1806d8eef0..d103d50a56 100644 --- a/translations/ru-RU/content/rest/overview/api-previews.md +++ b/translations/ru-RU/content/rest/overview/api-previews.md @@ -2,7 +2,6 @@ title: API previews intro: You can use API previews to try out new features and provide feedback before these features become official. redirect_from: - - /early-access/ - /v3/previews versions: free-pro-team: '*' diff --git a/translations/zh-CN/content/rest/overview/api-previews.md b/translations/zh-CN/content/rest/overview/api-previews.md index cab5d0f4cb..9324186a13 100644 --- a/translations/zh-CN/content/rest/overview/api-previews.md +++ b/translations/zh-CN/content/rest/overview/api-previews.md @@ -2,7 +2,6 @@ title: API 预览 intro: 您可以使用 API 预览来试用新功能并在这些功能正式发布之前提供反馈。 redirect_from: - - /early-access/ - /v3/previews versions: free-pro-team: '*'