From b6bd838ab3723b1f3951bf3692596da396849cf7 Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Tue, 2 Jan 2024 11:52:20 -0500 Subject: [PATCH 1/2] Test to check all .introLinks frontmatter (#48483) --- content/apps/index.md | 2 +- content/copilot/index.md | 7 ++-- content/packages/index.md | 6 ++-- .../tests/lint-frontmatter-links.js | 32 +++++++++++++++++++ src/links/lib/update-internal-links.js | 5 +++ 5 files changed, 45 insertions(+), 7 deletions(-) diff --git a/content/apps/index.md b/content/apps/index.md index f13b8c9d06..28c4b8b798 100644 --- a/content/apps/index.md +++ b/content/apps/index.md @@ -4,7 +4,7 @@ shortTitle: Apps intro: 'Go deeper with {% data variables.product.prodname_dotcom %} by integrating with our APIs and webhooks, customizing your {% data variables.product.prodname_dotcom %} workflow, and building and sharing apps with the community.' introLinks: overview: /apps/overview - quickstart: /apps/creating-github-apps/guides/quickstart + quickstart: /apps/creating-github-apps/writing-code-for-a-github-app/quickstart layout: product-landing featuredLinks: startHere: diff --git a/content/copilot/index.md b/content/copilot/index.md index b56d941ede..bb36fcf334 100644 --- a/content/copilot/index.md +++ b/content/copilot/index.md @@ -1,13 +1,13 @@ --- -title: "{% data variables.product.prodname_copilot %} documentation" -shortTitle: "{% data variables.product.prodname_copilot %}" +title: '{% data variables.product.prodname_copilot %} documentation' +shortTitle: '{% data variables.product.prodname_copilot %}' intro: 'You can use {% data variables.product.prodname_copilot %} to get autocomplete-style suggestions from an AI pair programmer as you code.' redirect_from: - /github/copilot changelog: label: copilot introLinks: - overview: /copilot/overview-of-github-copilot/about-github-copilot + overview: /copilot/overview-of-github-copilot/about-github-copilot-individual quickstart: /copilot/quickstart featuredLinks: startHere: @@ -32,3 +32,4 @@ children: topics: - Copilot --- + diff --git a/content/packages/index.md b/content/packages/index.md index c25b88d74a..63f05c4f27 100644 --- a/content/packages/index.md +++ b/content/packages/index.md @@ -1,10 +1,10 @@ --- -title: "{% data variables.product.prodname_registry %} documentation" -shortTitle: "{% data variables.product.prodname_registry %}" +title: '{% data variables.product.prodname_registry %} documentation' +shortTitle: '{% data variables.product.prodname_registry %}' intro: 'Learn to safely publish and consume packages, store your packages alongside your code, and share your packages privately with your team or publicly with the open source community. You can also automate your packages with {% data variables.product.prodname_actions %}.' introLinks: quickstart: /packages/quickstart - reference: /packages/manage-packages + reference: /packages/learn-github-packages featuredLinks: startHere: - /packages/learn-github-packages diff --git a/src/content-linter/tests/lint-frontmatter-links.js b/src/content-linter/tests/lint-frontmatter-links.js index 452332a32a..968788ffab 100644 --- a/src/content-linter/tests/lint-frontmatter-links.js +++ b/src/content-linter/tests/lint-frontmatter-links.js @@ -6,6 +6,9 @@ const pageList = await loadPages(undefined, ['en']) const pages = await loadPageMap(pageList) const redirects = await loadRedirects(pageList) +const liquidElsif = /{%\s*elsif/ +const containsLiquidElseIf = (text) => liquidElsif.test(text) + describe('front matter', () => { function makeCustomErrorMessage(page, trouble, key) { let customErrorMessage = `In the front matter of ${page.relativePath} ` @@ -33,6 +36,7 @@ describe('front matter', () => { } // Test content with .includeGuides front matter + const pagesWithIncludeGuides = pageList.filter((page) => page.includeGuides) test.each(pagesWithIncludeGuides)( '$relativePath .includeGuides have pristine links', @@ -79,4 +83,32 @@ describe('front matter', () => { expect(trouble.length, customErrorMessage).toEqual(0) }, ) + + // Test content with .introLinks front matter + + const pagesWithIntroLinks = pageList.filter((page) => page.introLinks) + test.each(pagesWithIntroLinks)('$relativePath .introLinks have pristine links', async (page) => { + const redirectsContext = { redirects, pages } + + const trouble = [] + for (const linksRaw of Object.values(page.introLinks)) { + const links = Array.isArray(linksRaw) ? linksRaw : [linksRaw] + trouble.push( + ...links + // At the present, we're not able to check when the URI + // contains an `elsif` Liquid tag. So just skip them. + .filter((uri) => !containsLiquidElseIf(uri)) + // On /en/enterprise-cloud@latest/admin we have, + // + // try_ghec_for_free: '{% ifversion ghec %}https://github.com/account/enterprises/new{% endif %}' + // + // Ignore those too. + .filter((uri) => !uri.includes('https://')) + .map((uri, i) => checkURL(uri, i, redirectsContext)) + .filter(Boolean), + ) + } + const customErrorMessage = makeCustomErrorMessage(page, trouble, 'introLinks') + expect(trouble.length, customErrorMessage).toEqual(0) + }) }) diff --git a/src/links/lib/update-internal-links.js b/src/links/lib/update-internal-links.js index 41dddafe56..ce193c1281 100644 --- a/src/links/lib/update-internal-links.js +++ b/src/links/lib/update-internal-links.js @@ -134,6 +134,11 @@ async function updateFile(file, context, opts) { newData[key][group] = better } } + } else if (typeof thing === 'string' && thing.startsWith('/')) { + const better = getNewFrontmatterLinkList([thing], context, opts, file, rawContent) + if (!equalArray(better, [thing])) { + newData[key][group] = better[0] + } } } } From 1ac44b81e50ec292363b6ae00da4c248fefcd676 Mon Sep 17 00:00:00 2001 From: Peter Bengtsson Date: Tue, 2 Jan 2024 11:58:17 -0500 Subject: [PATCH 2/2] Shield from junk path ends (#48481) --- .../middleware/handle-invalid-paths.js | 22 +++++++++++++++++-- src/shielding/tests/shielding.js | 7 +++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/shielding/middleware/handle-invalid-paths.js b/src/shielding/middleware/handle-invalid-paths.js index 13ce9f0446..0e67ecec2e 100644 --- a/src/shielding/middleware/handle-invalid-paths.js +++ b/src/shielding/middleware/handle-invalid-paths.js @@ -1,13 +1,25 @@ import { defaultCacheControl } from '#src/frame/middleware/cache-control.js' -// When a *whole* path is considerered junk. +// We'll check if the current request path is one of these, or ends with +// one of these. +// These are clearly intentional "guesses" made by some sort of +// pen-testing bot. +const JUNK_ENDS = [ + '/package.json', + '/package-lock.json', + '/etc/passwd', + '/Gemfile', + '/Gemfile.lock', + '/WEB-INF/web.xml', + '/WEB-INF/web.xml%C0%80.jsp', +] const JUNK_PATHS = new Set([ + ...JUNK_ENDS, '/env', '/xmlrpc.php', '/wp-login.php', '/README.md', '/server.js', - '/package.json', '/.git', '/_next', ]) @@ -22,6 +34,12 @@ const JUNK_BASENAMES = new Set([ function isJunkPath(path) { if (JUNK_PATHS.has(path)) return true + for (const junkPath of JUNK_ENDS) { + if (path.endsWith(junkPath)) { + return true + } + } + const basename = path.split('/').pop() // E.g. `/billing/.env.local` or `/billing/.env_sample` if (/^\.env(.|_)[\w.]+/.test(basename)) return true diff --git a/src/shielding/tests/shielding.js b/src/shielding/tests/shielding.js index 8be51dab58..98b7d56544 100644 --- a/src/shielding/tests/shielding.js +++ b/src/shielding/tests/shielding.js @@ -1,5 +1,4 @@ import { SURROGATE_ENUMS } from '#src/frame/middleware/set-fastly-surrogate-key.js' -// import { setFastlySurrogateKey } from '#src/frame/middleware/set-fastly-surrogate-key.js' import { get } from '#src/tests/helpers/e2etest.js' describe('honeypotting', () => { @@ -46,6 +45,12 @@ describe('junk paths', () => { const res = await get('/_next') expect(res.statusCode).toBe(404) }) + + test('with a starting /en/ but with a junk end', async () => { + const res = await get('/en/package-lock.json') + expect(res.statusCode).toBe(404) + expect(res.headers['content-type']).toMatch('text/plain') + }) }) describe('rate limiting', () => {