{
context,
}
- await contextualize(req, res, next)
+ await contextualize(req as ExtendedRequest, res as Response, next)
await shortVersions(req, res, next)
req.context.page = page
await features(req, res, next)
diff --git a/src/dev-toc/generate.js b/src/dev-toc/generate.js
index 8bdcbb0125..52aa65f812 100755
--- a/src/dev-toc/generate.js
+++ b/src/dev-toc/generate.js
@@ -7,7 +7,7 @@ import { program } from 'commander'
import fpt from '#src/versions/lib/non-enterprise-default-version.js'
import { allVersionKeys } from '#src/versions/lib/all-versions.js'
import { liquid } from '#src/content-render/index.js'
-import contextualize from '#src/frame/middleware/context/context.js'
+import contextualize from '#src/frame/middleware/context/context'
const layoutFilename = path.posix.join(process.cwd(), 'src/dev-toc/layout.html')
const layout = fs.readFileSync(layoutFilename, 'utf8')
diff --git a/src/frame/middleware/context/context.js b/src/frame/middleware/context/context.ts
similarity index 77%
rename from src/frame/middleware/context/context.js
rename to src/frame/middleware/context/context.ts
index 01c6117883..dbf3c8a6cd 100644
--- a/src/frame/middleware/context/context.js
+++ b/src/frame/middleware/context/context.ts
@@ -1,18 +1,22 @@
-import languages from '#src/languages/lib/languages.js'
-import enterpriseServerReleases from '#src/versions/lib/enterprise-server-releases.js'
-import { allVersions } from '#src/versions/lib/all-versions.js'
-import { productMap } from '#src/products/lib/all-products.js'
+import type { NextFunction, Response } from 'express'
+
+import type { ExtendedRequest, Context } from '@/types'
+
+import languages from '@/languages/lib/languages.js'
+import enterpriseServerReleases from '@/versions/lib/enterprise-server-releases.js'
+import { allVersions } from '@/versions/lib/all-versions.js'
+import { productMap } from '@/products/lib/all-products.js'
import {
getVersionStringFromPath,
getProductStringFromPath,
getCategoryStringFromPath,
getPathWithoutLanguage,
getPathWithoutVersion,
-} from '#src/frame/lib/path-utils.js'
-import productNames from '#src/products/lib/product-names.js'
-import warmServer from '#src/frame/lib/warm-server.js'
-import nonEnterpriseDefaultVersion from '#src/versions/lib/non-enterprise-default-version.js'
-import { getDataByLanguage, getUIDataMerged } from '#src/data-directory/lib/get-data.js'
+} from '@/frame/lib/path-utils.js'
+import productNames from '@/products/lib/product-names.js'
+import warmServer from '@/frame/lib/warm-server.js'
+import nonEnterpriseDefaultVersion from '@/versions/lib/non-enterprise-default-version.js'
+import { getDataByLanguage, getUIDataMerged } from '@/data-directory/lib/get-data.js'
// This doesn't change just because the request changes, so compute it once.
const enterpriseServerVersions = Object.keys(allVersions).filter((version) =>
@@ -21,18 +25,24 @@ const enterpriseServerVersions = Object.keys(allVersions).filter((version) =>
// Supply all route handlers with a baseline `req.context` object
// Note that additional middleware in middleware/index.js adds to this context object
-export default async function contextualize(req, res, next) {
+export default async function contextualize(
+ req: ExtendedRequest,
+ res: Response,
+ next: NextFunction,
+) {
// Ensure that we load some data only once on first request
- const { redirects, siteTree, pages: pageMap } = await warmServer()
+ const { redirects, siteTree, pages: pageMap } = await warmServer([])
- req.context = {}
+ const context: Context = {}
+ req.context = context
req.context.process = { env: {} }
// define each context property explicitly for code-search friendliness
// e.g. searches for "req.context.page" will include results from this file
req.context.currentLanguage = req.language
req.context.userLanguage = req.userLanguage
- req.context.currentVersion = getVersionStringFromPath(req.pagePath)
+ req.context.currentVersion = getVersionStringFromPath(req.pagePath) as string
+
req.context.currentVersionObj = allVersions[req.context.currentVersion]
req.context.currentProduct = getProductStringFromPath(req.pagePath)
req.context.currentCategory = getCategoryStringFromPath(req.pagePath)
@@ -79,8 +89,8 @@ export default async function contextualize(req, res, next) {
if (!page) {
throw new Error("The 'page' has not been put into the context yet.")
}
- const enPath = context.currentPath.replace(`/${page.languageCode}`, '/en')
- const enPage = context.pages[enPath]
+ const enPath = context.currentPath!.replace(`/${page.languageCode}`, '/en')
+ const enPage = context.pages![enPath]
if (!enPage) {
throw new Error(`Unable to find equivalent English page by the path '${enPath}'`)
}
diff --git a/src/frame/middleware/index.ts b/src/frame/middleware/index.ts
index 0c2c1b6a65..74801504c3 100644
--- a/src/frame/middleware/index.ts
+++ b/src/frame/middleware/index.ts
@@ -19,7 +19,7 @@ import handleErrors from '@/observability/middleware/handle-errors'
import handleNextDataPath from './handle-next-data-path'
import detectLanguage from '@/languages/middleware/detect-language'
import reloadTree from './reload-tree'
-import context from './context/context.js'
+import context from './context/context'
import shortVersions from '@/versions/middleware/short-versions.js'
import languageCodeRedirects from '@/redirects/middleware/language-code-redirects.js'
import handleRedirects from '@/redirects/middleware/handle-redirects.js'
diff --git a/src/links/lib/README.md b/src/links/lib/README.md
index a831ea39d5..96e9001c7c 100644
--- a/src/links/lib/README.md
+++ b/src/links/lib/README.md
@@ -20,7 +20,7 @@ If the action finds any broken links, it opens an internal issue for the Docs Co
curl -Lso /dev/null -w "%{http_code}\n" URL
- A `200` response is success.
+ A `200` response is success.
- You can see a comprehensive list of HTTP response codes [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status).
- For external links that now `404` or have otherwise gone missing entirely, you may be able to use the [Wayback Machine](https://web.archive.org) to see the page before it went offline.
@@ -39,4 +39,4 @@ Before you decide whether to exclude a link from the daily link checker, you sho
- Has it has been flagged as a broken link for more than a week, but the URL works when a real user opens it in their browser?
- Has the URL been available for more than 3 months? You can check using the [Wayback Machine](https://web.archive.org).
-If you are confident that the URL for the article should work for real users, then you can open a pull request to add it to the `src/links/lib/excluded-links.js` file.
+If you are confident that the URL for the article should work for real users, then you can open a pull request to add it to the `src/links/lib/excluded-links.ts` file.
diff --git a/src/links/lib/excluded-links.js b/src/links/lib/excluded-links.ts
similarity index 96%
rename from src/links/lib/excluded-links.js
rename to src/links/lib/excluded-links.ts
index b3210adba2..f4b1e3d3ba 100644
--- a/src/links/lib/excluded-links.js
+++ b/src/links/lib/excluded-links.ts
@@ -8,7 +8,9 @@
/* eslint-disable prefer-regex-literals */
-export default [
+type ExcludedLink = string | RegExp
+
+const excludedLinks: ExcludedLink[] = [
// Skip GitHub search links.
// E.g. https://github.com/search?foo=bar
regex('https://github.com/search?'),
@@ -79,6 +81,8 @@ export default [
'https://packages.ubuntu.com/search?keywords=netcat&searchon=names',
]
+export default excludedLinks
+
// Return a regular expression from a URL string that matches the URL
// as a base. It's basically shorthand for "URL.startsWith(BASE_URL)"
// but as a RegExp object.
@@ -88,6 +92,6 @@ export default [
// true
// > regex('https://github.com').test('otherhttps://github.com/page')
// false
-function regex(url) {
+function regex(url: string) {
return new RegExp('^' + url.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
}
diff --git a/src/links/lib/validate-docs-urls.ts b/src/links/lib/validate-docs-urls.ts
index d9319bd85b..f4c3450042 100644
--- a/src/links/lib/validate-docs-urls.ts
+++ b/src/links/lib/validate-docs-urls.ts
@@ -1,14 +1,15 @@
+import type { Response } from 'express'
import cheerio from 'cheerio'
import warmServer from '@/frame/lib/warm-server.js'
import { liquid } from '@/content-render/index.js'
import shortVersions from '@/versions/middleware/short-versions.js'
-import contextualize from '@/frame/middleware/context/context.js'
+import contextualize from '@/frame/middleware/context/context'
import features from '@/versions/middleware/features.js'
import findPage from '@/frame/middleware/find-page.js'
import { createMinimalProcessor } from '@/content-render/unified/processor.js'
import getRedirect from '@/redirects/lib/get-redirect.js'
-import type { Page } from '@/types'
+import type { ExtendedRequest, Page } from '@/types'
export type DocsUrls = {
[identifier: string]: string
@@ -116,7 +117,7 @@ async function renderInnerHTML(page: Page, permalink: Permalink) {
// Here it just exists for the sake of TypeScript.
context: {},
}
- await contextualize(req, res, next)
+ await contextualize(req as ExtendedRequest, res as Response, next)
await shortVersions(req, res, next)
await findPage(req, res, next)
await features(req, res, next)
diff --git a/src/links/scripts/action-injections.js b/src/links/scripts/action-injections.ts
similarity index 63%
rename from src/links/scripts/action-injections.js
rename to src/links/scripts/action-injections.ts
index c4837de49d..f6f150cdda 100644
--- a/src/links/scripts/action-injections.js
+++ b/src/links/scripts/action-injections.ts
@@ -7,21 +7,29 @@ import fs from 'fs'
import path from 'path'
import chalk from 'chalk'
-import github from '#src/workflows/github.js'
+import github from '@/workflows/github.js'
+export type CoreInject = {
+ info: (message: string) => void
+ debug: (message: string) => void
+ warning: (message: string) => void
+ error: (message: string) => void
+ setOutput: (name: string, value: any) => void
+ setFailed: (message: string) => void
+}
// Directs core logging to console
-export function getCoreInject(debug) {
+export function getCoreInject(debug: boolean): CoreInject {
return {
info: console.log,
- debug: (message) => (debug ? console.warn(chalk.blue(message)) : {}),
- warning: (message) => console.warn(chalk.yellow(message)),
+ debug: (message: string) => (debug ? console.warn(chalk.blue(message)) : {}),
+ warning: (message: string) => console.warn(chalk.yellow(message)),
error: console.error,
- setOutput: (name, value) => {
+ setOutput: (name: string, value: any) => {
if (debug) {
console.log(`Output "${name}" set to: "${value}"`)
}
},
- setFailed: (message) => {
+ setFailed: (message: string) => {
if (debug) {
console.log('setFailed called.')
}
@@ -36,8 +44,8 @@ const logsPath = path.join(cwd, '..', '..', 'logs')
if (!fs.existsSync(logsPath)) {
fs.mkdirSync(logsPath)
}
-export function getUploadArtifactInject(debug) {
- return (name, contents) => {
+export function getUploadArtifactInject(debug: boolean) {
+ return (name: string, contents: string) => {
const logFilename = path.join(logsPath, `${new Date().toISOString().substr(0, 16)}-${name}`)
if (debug) {
fs.writeFileSync(logFilename, contents)
diff --git a/src/links/scripts/rendered-content-link-checker-cli.js b/src/links/scripts/rendered-content-link-checker-cli.ts
similarity index 92%
rename from src/links/scripts/rendered-content-link-checker-cli.js
rename to src/links/scripts/rendered-content-link-checker-cli.ts
index f9ce83f0dd..8d6b836f7a 100755
--- a/src/links/scripts/rendered-content-link-checker-cli.js
+++ b/src/links/scripts/rendered-content-link-checker-cli.ts
@@ -10,10 +10,10 @@
import fs from 'fs'
import path from 'path'
import { program, Option, InvalidArgumentError } from 'commander'
-import renderedContentLinkChecker from '#src/links/scripts/rendered-content-link-checker.js'
-import { getCoreInject, getUploadArtifactInject } from '#src/links/scripts/action-injections.js'
-import { allVersions } from '#src/versions/lib/all-versions.js'
-import github from '#src/workflows/github.js'
+import renderedContentLinkChecker from './rendered-content-link-checker'
+import { getCoreInject, getUploadArtifactInject } from '@/links/scripts/action-injections.js'
+import { allVersions } from '@/versions/lib/all-versions.js'
+import github from '@/workflows/github.js'
const STATIC_PREFIXES = {
assets: path.resolve('assets'),
@@ -115,7 +115,7 @@ program
return resolvedPath
},
)
- .arguments('[files...]', 'Specific files to check')
+ .arguments('[files...]')
.parse(process.argv)
const opts = program.opts()
diff --git a/src/links/scripts/rendered-content-link-checker.js b/src/links/scripts/rendered-content-link-checker.ts
similarity index 84%
rename from src/links/scripts/rendered-content-link-checker.js
rename to src/links/scripts/rendered-content-link-checker.ts
index 4945908340..7d3142123d 100755
--- a/src/links/scripts/rendered-content-link-checker.js
+++ b/src/links/scripts/rendered-content-link-checker.ts
@@ -2,30 +2,85 @@
import fs from 'fs'
import path from 'path'
-import cheerio from 'cheerio'
+
+import cheerio, { type CheerioAPI, type Element } from 'cheerio'
import coreLib from '@actions/core'
import got, { RequestError } from 'got'
import chalk from 'chalk'
-import { Low } from 'lowdb'
-import { JSONFile } from 'lowdb/node'
+import { JSONFilePreset } from 'lowdb/node'
+import { type Octokit } from '@octokit/rest'
+import type { Response } from 'express'
-import shortVersions from '#src/versions/middleware/short-versions.js'
-import contextualize from '#src/frame/middleware/context/context.js'
-import features from '#src/versions/middleware/features.js'
-import getRedirect from '#src/redirects/lib/get-redirect.js'
-import warmServer from '#src/frame/lib/warm-server.js'
-import { liquid } from '#src/content-render/index.js'
-import { deprecated } from '#src/versions/lib/enterprise-server-releases.js'
-import excludedLinks from '#src/links/lib/excluded-links.js'
-import { getEnvInputs, boolEnvVar } from '#src/workflows/get-env-inputs.js'
+import type { ExtendedRequest, Page, Permalink, Context } from '@/types'
+import shortVersions from '@/versions/middleware/short-versions.js'
+import contextualize from '@/frame/middleware/context/context'
+import features from '@/versions/middleware/features.js'
+import getRedirect from '@/redirects/lib/get-redirect.js'
+import warmServer from '@/frame/lib/warm-server.js'
+import { liquid } from '@/content-render/index.js'
+import { deprecated } from '@/versions/lib/enterprise-server-releases.js'
+import excludedLinks from '@/links/lib/excluded-links.js'
+import { getEnvInputs, boolEnvVar } from '@/workflows/get-env-inputs.js'
import { debugTimeEnd, debugTimeStart } from './debug-time-taken.js'
import { uploadArtifact as uploadArtifactLib } from './upload-artifact.js'
-import github from '#src/workflows/github.js'
-import { getActionContext } from '#src/workflows/action-context.js'
-import { createMinimalProcessor } from '#src/content-render/unified/processor.js'
-import { createReportIssue, linkReports } from '#src/workflows/issue-report.js'
+import github from '@/workflows/github.js'
+import { getActionContext } from '@/workflows/action-context.js'
+import { createMinimalProcessor } from '@/content-render/unified/processor.js'
+import { createReportIssue, linkReports } from '@/workflows/issue-report.js'
+import { type CoreInject } from '@/links/scripts/action-injections.js'
-const STATIC_PREFIXES = {
+type Flaw = {
+ WARNING?: string
+ CRITICAL?: string
+ isExternal?: boolean
+}
+
+type LinkFlaw = {
+ page: Page
+ permalink: Permalink
+ href?: string
+ url?: string
+ text?: string
+ src: string
+ flaw: Flaw
+}
+
+// type Core = CoreInject
+
+type Redirects = Record
+type PageMap = Record
+
+type UploadArtifact = (name: string, message: string) => void
+
+type Options = {
+ level?: string
+ files?: string[]
+ random?: boolean
+ language?: string | string[]
+ filter?: string[]
+ version?: string | string[]
+ max?: number
+ linkReports?: boolean
+ actionUrl?: string
+ verbose?: boolean
+ checkExternalLinks?: boolean
+ createReport?: boolean
+ failOnFlaw?: boolean
+ shouldComment?: boolean
+ reportRepository?: string
+ reportAuthor?: string
+ reportLabel?: string
+ checkAnchors?: boolean
+ checkImages?: boolean
+ patient?: boolean
+ externalServerErrorsAsWarning?: string
+ verboseUrl?: string
+ bail?: boolean
+ commentLimitToExternalLinks?: boolean
+ actionContext?: any
+}
+
+const STATIC_PREFIXES: Record = {
assets: path.resolve('assets'),
public: path.resolve(path.join('src', 'graphql', 'data')),
}
@@ -44,8 +99,22 @@ const EXTERNAL_LINK_CHECKER_MAX_AGE_MS =
const EXTERNAL_LINK_CHECKER_DB =
process.env.EXTERNAL_LINK_CHECKER_DB || 'external-link-checker-db.json'
-const adapter = new JSONFile(EXTERNAL_LINK_CHECKER_DB)
-const externalLinkCheckerDB = new Low(adapter, { urls: {} })
+// const adapter = new JSONFile(EXTERNAL_LINK_CHECKER_DB)
+type Data = {
+ urls: {
+ [url: string]: {
+ timestamp: number
+ result: {
+ ok: boolean
+ statusCode: number
+ }
+ }
+ }
+}
+const defaultData: Data = { urls: {} }
+const externalLinkCheckerDB = await JSONFilePreset(EXTERNAL_LINK_CHECKER_DB, defaultData)
+
+type DBType = typeof externalLinkCheckerDB
// Given a number and a percentage, return the same number with a *percentage*
// max change of making a bit larger or smaller.
@@ -54,7 +123,7 @@ const externalLinkCheckerDB = new Low(adapter, { urls: {} })
// numbers from the day it started which means that they don't ALL expire
// on the same day but start to expire in a bit of a "random pattern" so
// you don't get all or nothing.
-function jitter(base, percentage) {
+function jitter(base: number, percentage: number) {
const r = percentage / 100
const negative = Math.random() > 0.5 ? -1 : 1
return base + base * Math.random() * r * negative
@@ -65,11 +134,12 @@ function jitter(base, percentage) {
// check.
function linksToSkipFactory() {
const set = new Set(excludedLinks.filter((regexOrURL) => typeof regexOrURL === 'string'))
- const regexes = excludedLinks.filter((regexOrURL) => regexOrURL instanceof RegExp)
- return (href) => set.has(href) || regexes.some((regex) => regex.test(href))
+ // This `... as RegExp` because TypeScript can't (currently) understand the filtering.
+ const regexes = excludedLinks.filter((regexOrURL) => regexOrURL instanceof RegExp) as RegExp[]
+ return (href: string) => set.has(href) || regexes.some((regex) => regex.test(href))
}
-const linksToSkip = linksToSkipFactory(excludedLinks)
+const linksToSkip = linksToSkipFactory()
const CONTENT_ROOT = path.resolve('content')
@@ -105,7 +175,7 @@ if (import.meta.url.endsWith(process.argv[1])) {
}
}
- const opts = {
+ const opts: Options = {
level: LEVEL,
files,
verbose: true,
@@ -134,7 +204,7 @@ if (import.meta.url.endsWith(process.argv[1])) {
getEnvInputs(['GITHUB_TOKEN'])
}
- main(coreLib, octokit, uploadArtifactLib, opts, {})
+ main(coreLib, octokit, uploadArtifactLib, opts)
}
/*
@@ -173,7 +243,13 @@ if (import.meta.url.endsWith(process.argv[1])) {
* versions {Array} - only certain pages' versions (e.g. )
*
*/
-async function main(core, octokit, uploadArtifact, opts = {}) {
+
+async function main(
+ core: any,
+ octokit: Octokit,
+ uploadArtifact: UploadArtifact,
+ opts: Options = {},
+) {
const {
level = 'warning',
files = [],
@@ -201,7 +277,7 @@ async function main(core, octokit, uploadArtifact, opts = {}) {
// If we'd manually do the same operations that `warmServer()` does
// here (e.g. `loadPageMap()`), we'd end up having to do it all over
// again, the next time `contextualize()` is called.
- const { redirects, pages: pageMap, pageList } = await warmServer()
+ const { redirects, pages: pageMap, pageList } = await warmServer([])
if (files.length) {
core.debug(`Limitting to files list: ${files.join(', ')}`)
@@ -265,7 +341,7 @@ async function main(core, octokit, uploadArtifact, opts = {}) {
debugTimeStart(core, 'processPages')
const t0 = new Date().getTime()
const flawsGroups = await Promise.all(
- pages.map((page) =>
+ pages.map((page: Page) =>
processPage(core, page, pageMap, redirects, opts, externalLinkCheckerDB, versions),
),
)
@@ -288,7 +364,9 @@ async function main(core, octokit, uploadArtifact, opts = {}) {
const uniqueHrefs = new Set(flaws.map((flaw) => flaw.href))
if (flaws.length > 0) {
- await uploadJsonFlawsArtifact(uploadArtifact, flaws, opts)
+ await uploadJsonFlawsArtifact(uploadArtifact, flaws, {
+ verboseUrl: opts.verboseUrl,
+ })
core.info(`All flaws written to artifact log.`)
if (createReport) {
core.info(`Creating issue for flaws...`)
@@ -344,7 +422,7 @@ async function main(core, octokit, uploadArtifact, opts = {}) {
}
}
-async function commentOnPR(core, octokit, flaws, opts) {
+async function commentOnPR(core: CoreInject, octokit: Octokit, flaws: LinkFlaw[], opts: Options) {
const { actionContext = {} } = opts
const { owner, repo } = actionContext
const pullNumber = actionContext?.pull_request?.number
@@ -364,7 +442,7 @@ async function commentOnPR(core, octokit, flaws, opts) {
})
let previousCommentId
for (const { body, id } of data) {
- if (body.includes(findAgainSymbol)) {
+ if (body && body.includes(findAgainSymbol)) {
previousCommentId = id
}
}
@@ -412,12 +490,22 @@ async function commentOnPR(core, octokit, flaws, opts) {
}
}
-function flawIssueDisplay(flaws, opts, mentionExternalExclusionList = true) {
+function flawIssueDisplay(flaws: LinkFlaw[], opts: Options, mentionExternalExclusionList = true) {
let output = ''
let flawsToDisplay = 0
+ type LinkFlawWithPermalink = {
+ // page?: Page
+ // permalink?: Permalink
+ href?: string
+ url?: string
+ text?: string
+ src: string
+ flaw: Flaw
+ permalinkHrefs: string[]
+ }
// Group broken links for each page
- const hrefsOnPageGroup = {}
+ const hrefsOnPageGroup: Record> = {}
for (const { page, permalink, href, text, src, flaw } of flaws) {
// When we don't want to include external links in PR comments
if (opts.commentLimitToExternalLinks && !flaw.isExternal) {
@@ -473,7 +561,7 @@ function flawIssueDisplay(flaws, opts, mentionExternalExclusionList = true) {
if (mentionExternalExclusionList) {
output +=
'\n\n---\n\nIf any link reported in this issue is not actually broken ' +
- 'and repeatedly shows up on reports, consider making a PR that adds it as an exception to `src/links/lib/excluded-links.js`. ' +
+ 'and repeatedly shows up on reports, consider making a PR that adds it as an exception to `src/links/lib/excluded-links.ts`. ' +
'For more information, see [Fixing broken links in GitHub user docs](https://github.com/github/docs/blob/main/src/links/lib/README.md).'
}
@@ -482,7 +570,7 @@ function flawIssueDisplay(flaws, opts, mentionExternalExclusionList = true) {
}links found in [this](${opts.actionUrl}) workflow.\n${output}`
}
-function printGlobalCacheHitRatio(core) {
+function printGlobalCacheHitRatio(core: CoreInject) {
const hits = globalCacheHitCount
const misses = globalCacheMissCount
// It could be that the files that were tested didn't have a single
@@ -498,9 +586,15 @@ function printGlobalCacheHitRatio(core) {
}
}
-function getPages(pageList, languages, filters, files, max) {
+function getPages(
+ pageList: Page[],
+ languages: string[],
+ filters: string[],
+ files: string[],
+ max: number | undefined,
+) {
return pageList
- .filter((page) => {
+ .filter((page: Page) => {
if (languages.length && !languages.includes(page.languageCode)) {
return false
}
@@ -537,7 +631,15 @@ function getPages(pageList, languages, filters, files, max) {
.slice(0, max ? Math.min(max, pageList.length) : pageList.length)
}
-async function processPage(core, page, pageMap, redirects, opts, db, versions) {
+async function processPage(
+ core: CoreInject,
+ page: Page,
+ pageMap: PageMap,
+ redirects: Redirects,
+ opts: Options,
+ db: DBType,
+ versions: string[],
+) {
const { verbose, verboseUrl, bail } = opts
const allFlawsEach = await Promise.all(
page.permalinks
@@ -567,7 +669,15 @@ async function processPage(core, page, pageMap, redirects, opts, db, versions) {
return allFlaws
}
-async function processPermalink(core, permalink, page, pageMap, redirects, opts, db) {
+async function processPermalink(
+ core: any,
+ permalink: Permalink,
+ page: Page,
+ pageMap: PageMap,
+ redirects: Redirects,
+ opts: Options,
+ db: DBType,
+) {
const {
level = 'critical',
checkAnchors,
@@ -587,12 +697,12 @@ async function processPermalink(core, permalink, page, pageMap, redirects, opts,
throw error
}
const $ = cheerio.load(html, { xmlMode: true })
- const flaws = []
- const links = []
+ const flaws: LinkFlaw[] = []
+ const links: Element[] = []
$('a[href]').each((i, link) => {
links.push(link)
})
- const newFlaws = await Promise.all(
+ const newFlaws: LinkFlaw[] = await Promise.all(
links.map(async (link) => {
const { href } = link.attribs
@@ -615,7 +725,8 @@ async function processPermalink(core, permalink, page, pageMap, redirects, opts,
checkAnchors,
checkExternalLinks,
externalServerErrorsAsWarning,
- { verbose, patient, permalink },
+ permalink,
+ { verbose, patient },
db,
)
@@ -657,7 +768,7 @@ async function processPermalink(core, permalink, page, pageMap, redirects, opts,
return globalImageSrcCheckCache.get(src)
}
- const flaw = checkImageSrc(src, $)
+ const flaw = checkImageSrc(src)
globalImageSrcCheckCache.set(src, flaw)
@@ -674,12 +785,19 @@ async function processPermalink(core, permalink, page, pageMap, redirects, opts,
}
async function uploadJsonFlawsArtifact(
- uploadArtifact,
- flaws,
- { verboseUrl = null } = {},
+ uploadArtifact: UploadArtifact,
+ flaws: LinkFlaw[],
+ { verboseUrl = null }: { verboseUrl?: string | null } = {},
artifactName = 'all-rendered-link-flaws.json',
) {
- const printableFlaws = {}
+ type PrintableLinkFlaw = {
+ href?: string
+ url?: string
+ text?: string
+ src?: string
+ flaw?: Flaw
+ }
+ const printableFlaws: Record = {}
for (const { page, permalink, href, text, src, flaw } of flaws) {
const fullPath = prettyFullPath(page.fullPath)
@@ -703,7 +821,11 @@ async function uploadJsonFlawsArtifact(
return uploadArtifact(artifactName, message)
}
-function printFlaws(core, flaws, { verboseUrl = null } = {}) {
+function printFlaws(
+ core: CoreInject,
+ flaws: LinkFlaw[],
+ { verboseUrl }: { verboseUrl?: string | undefined } = {},
+) {
let previousPage = null
let previousPermalink = null
@@ -743,7 +865,7 @@ function printFlaws(core, flaws, { verboseUrl = null } = {}) {
// `vi` or `ls` or `code` but if we display it relative to `cwd()` you
// can still paste it to the next command but it's not taking up so much
// space.
-function prettyFullPath(fullPath) {
+function prettyFullPath(fullPath: string) {
return path.relative(process.cwd(), fullPath)
}
@@ -753,17 +875,18 @@ let globalCacheHitCount = 0
let globalCacheMissCount = 0
async function checkHrefLink(
- core,
- href,
- $,
- redirects,
- pageMap,
+ core: any,
+ href: string,
+ $: CheerioAPI,
+ redirects: Redirects,
+ pageMap: PageMap,
checkAnchors = false,
checkExternalLinks = false,
- externalServerErrorsAsWarning = false,
- { verbose = false, patient = false, permalink } = {},
- db = null,
-) {
+ externalServerErrorsAsWarning: string | undefined | null = null,
+ permalink: Permalink,
+ { verbose = false, patient = false }: { verbose?: boolean; patient?: boolean } = {},
+ db: DBType | null = null,
+): Promise {
// this function handles hrefs in all the following forms:
// same article links:
@@ -860,9 +983,10 @@ async function checkHrefLink(
}
return { WARNING: 'Links with a trailing / will always redirect' }
} else {
- if (pathname.split('/')[1] in STATIC_PREFIXES) {
+ const firstPart = pathname.split('/')[1]
+ if (STATIC_PREFIXES[firstPart]) {
const staticFilePath = path.join(
- STATIC_PREFIXES[pathname.split('/')[1]],
+ STATIC_PREFIXES[firstPart],
pathname.split(path.sep).slice(2).join(path.sep),
)
if (!fs.existsSync(staticFilePath)) {
@@ -914,7 +1038,7 @@ async function checkHrefLink(
// simply try again later.
// However, an `ETIMEDOUT` means it could work but it didn't this time but
// might if we try again a different hour or day.
-function isTemporaryRequestError(requestError) {
+function isTemporaryRequestError(requestError: string | undefined) {
if (typeof requestError === 'string') {
// See https://betterstack.com/community/guides/scaling-nodejs/nodejs-errors/
// for a definition of each one.
@@ -927,7 +1051,12 @@ function isTemporaryRequestError(requestError) {
// Can't do this memoization within the checkExternalURL because it can
// return a Promise since it already collates multiple URLs under the
// same cache key.
-async function checkExternalURLCached(core, href, { verbose, patient }, db) {
+async function checkExternalURLCached(
+ core: CoreInject,
+ href: string,
+ { verbose, patient }: { verbose?: boolean; patient?: boolean },
+ db: DBType | null,
+) {
const cacheMaxAge = EXTERNAL_LINK_CHECKER_MAX_AGE_MS
const now = new Date().getTime()
const url = href.split('#')[0]
@@ -968,7 +1097,11 @@ async function checkExternalURLCached(core, href, { verbose, patient }, db) {
}
const _fetchCache = new Map()
-async function checkExternalURL(core, url, { verbose = false, patient = false } = {}) {
+async function checkExternalURL(
+ core: CoreInject,
+ url: string,
+ { verbose = false, patient = false } = {},
+) {
if (!url.startsWith('https://')) throw new Error('Invalid URL')
const cleanURL = url.split('#')[0]
if (!_fetchCache.has(cleanURL)) {
@@ -977,7 +1110,7 @@ async function checkExternalURL(core, url, { verbose = false, patient = false }
return _fetchCache.get(cleanURL)
}
-const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
// Global for recording which domains we get rate-limited on.
// For example, if you got rate limited on `something.github.com/foo`
@@ -985,7 +1118,11 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
// it's good to know to now bother yet.
const _rateLimitedDomains = new Map()
-async function innerFetch(core, url, config = {}) {
+async function innerFetch(
+ core: CoreInject,
+ url: string,
+ config: { verbose?: boolean; useGET?: boolean; patient?: boolean; retries?: number } = {},
+) {
const { verbose, useGET, patient } = config
const { hostname } = new URL(url)
@@ -1039,7 +1176,10 @@ async function innerFetch(core, url, config = {}) {
if (r.statusCode === 429) {
let sleepTime = Math.min(
60_000,
- Math.max(10_000, getRetryAfterSleep(r.headers['retry-after'])),
+ Math.max(
+ 10_000,
+ r.headers['retry-after'] ? getRetryAfterSleep(r.headers['retry-after']) : 1_000,
+ ),
)
// Sprinkle a little jitter so it doesn't all start again all
// at the same time
@@ -1081,16 +1221,17 @@ async function innerFetch(core, url, config = {}) {
}
// Return number of milliseconds from a `Retry-After` header value
-function getRetryAfterSleep(headerValue) {
+function getRetryAfterSleep(headerValue: string) {
if (!headerValue) return 0
let ms = Math.round(parseFloat(headerValue) * 1000)
if (isNaN(ms)) {
- ms = Math.max(0, new Date(headerValue) - new Date())
+ const nextDate = new Date(headerValue)
+ ms = Math.max(0, nextDate.getTime() - new Date().getTime())
}
return ms
}
-function checkImageSrc(src, $) {
+function checkImageSrc(src: string) {
if (!src.startsWith('/') && !src.startsWith('http')) {
return { CRITICAL: 'Image path is not absolute. Should start with a /' }
}
@@ -1115,7 +1256,7 @@ function checkImageSrc(src, $) {
}
}
-function summarizeFlaws(core, flaws) {
+function summarizeFlaws(core: CoreInject, flaws: LinkFlaw[]) {
if (flaws.length) {
core.info(
chalk.bold(
@@ -1127,7 +1268,7 @@ function summarizeFlaws(core, flaws) {
}
}
-function summarizeCounts(core, pages, tookSeconds) {
+function summarizeCounts(core: CoreInject, pages: Page[], tookSeconds: number) {
const count = pages.map((page) => page.permalinks.length).reduce((a, b) => a + b, 0)
core.info(
`Tested ${count.toLocaleString()} permalinks across ${pages.length.toLocaleString()} pages`,
@@ -1139,7 +1280,7 @@ function summarizeCounts(core, pages, tookSeconds) {
core.info(`~${pagesPerSecond.toFixed(1)} pages per second.`)
}
-function shuffle(array) {
+function shuffle(array: any[]) {
let currentIndex = array.length
let randomIndex
@@ -1156,19 +1297,21 @@ function shuffle(array) {
return array
}
-async function renderInnerHTML(page, permalink) {
+async function renderInnerHTML(page: Page, permalink: Permalink) {
const next = () => {}
const res = {}
const pagePath = permalink.href
+ const context: Context = {}
const req = {
path: pagePath,
language: permalink.languageCode,
pagePath,
cookies: {},
+ context,
}
// This will create and set `req.context = {...}`
- await contextualize(req, res, next)
+ await contextualize(req as ExtendedRequest, res as Response, next)
await shortVersions(req, res, next)
req.context.page = page
await features(req, res, next)
diff --git a/src/pageinfo/middleware.ts b/src/pageinfo/middleware.ts
index 6341ed3be2..19d9a70579 100644
--- a/src/pageinfo/middleware.ts
+++ b/src/pageinfo/middleware.ts
@@ -11,7 +11,7 @@ import {
makeLanguageSurrogateKey,
} from '@/frame/middleware/set-fastly-surrogate-key.js'
import shortVersions from '@/versions/middleware/short-versions.js'
-import contextualize from '@/frame/middleware/context/context.js'
+import contextualize from '@/frame/middleware/context/context'
import features from '@/versions/middleware/features.js'
import getRedirect from '@/redirects/lib/get-redirect.js'
import { isArchivedVersionByPath } from '@/archives/lib/is-archived-version.js'
@@ -130,7 +130,7 @@ export async function getPageInfo(page: Page, pathname: string) {
}
const next = () => {}
const res = {}
- await contextualize(renderingReq, res, next)
+ await contextualize(renderingReq as ExtendedRequest, res as Response, next)
await shortVersions(renderingReq, res, next)
renderingReq.context.page = page
await features(renderingReq, res, next)
diff --git a/src/products/lib/product-names.js b/src/products/lib/product-names.ts
similarity index 50%
rename from src/products/lib/product-names.js
rename to src/products/lib/product-names.ts
index dab91fc17e..a9001e235e 100644
--- a/src/products/lib/product-names.js
+++ b/src/products/lib/product-names.ts
@@ -1,6 +1,7 @@
-import enterpriseServerReleases from '#src/versions/lib/enterprise-server-releases.js'
+import type { ProductNames } from '@/types'
+import enterpriseServerReleases from '@/versions/lib/enterprise-server-releases.js'
-const productNames = {
+const productNames: ProductNames = {
dotcom: 'GitHub.com',
}
diff --git a/src/products/tests/product-names.js b/src/products/tests/product-names.ts
similarity index 85%
rename from src/products/tests/product-names.js
rename to src/products/tests/product-names.ts
index ca41bea4c5..de99b87596 100644
--- a/src/products/tests/product-names.js
+++ b/src/products/tests/product-names.ts
@@ -1,6 +1,6 @@
import { describe, expect, test } from 'vitest'
-import productNames from '#src/products/lib/product-names.js'
+import productNames from '@/products/lib/product-names'
describe('productNames module', () => {
test('is an object with product codes as keys and human-friendly names as values', () => {
diff --git a/src/redirects/README.md b/src/redirects/README.md
index eeb1ab304a..40f473ad2d 100644
--- a/src/redirects/README.md
+++ b/src/redirects/README.md
@@ -22,7 +22,7 @@ The results comprise the `page.redirects` object, whose keys are always only the
Sometimes it contains the specific plan/version (e.g. `/enterprise-server@3.0/v3/integrations` to `enterprise-server@3.0/developers/apps`) and sometimes it's just the plain path
(e.g. `/articles/viewing-your-repositorys-workflows` to `/actions/monitoring-and-troubleshooting-workflows`)
-All of the above are merged into a global redirects object. This object gets added to `req.context` via `src/frame/middleware/context/context.js` and is made accessible on every request.
+All of the above are merged into a global redirects object. This object gets added to `req.context` via `src/frame/middleware/context/context.ts` and is made accessible on every request.
In the `handle-redirects.js` middleware, the language part of the URL is
removed, looked up, and if matched to something, redirects with language
diff --git a/src/types.ts b/src/types.ts
index 5059086f04..4ff46d4d59 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -1,5 +1,7 @@
import type { Request } from 'express'
+import type enterpriseServerReleases from '@/versions/lib/enterprise-server-releases.d.ts'
+
// Throughout our codebase we "extend" the Request object by attaching
// things to it. For example `req.context = { currentCategory: 'foo' }`.
// This type aims to match all the custom things we do to requests
@@ -12,15 +14,64 @@ export type ExtendedRequest = Request & {
// Add more properties here as needed
}
+type Product = {
+ id: string
+ name: string
+ href: string
+ dir: string
+ toc: string
+ wip: boolean
+ hidden: boolean
+ versions: string[]
+}
+
+type ProductMap = {
+ [key: string]: Product
+}
+
+export type ProductNames = {
+ [shortName: string]: string
+}
+
+type Redirects = {
+ [key: string]: string
+}
+
export type Context = {
currentCategory?: string
error?: Error
siteTree?: SiteTree
pages?: Record
- redirects?: Record
+ productMap?: ProductMap
+ redirects?: Redirects
currentLanguage?: string
+ userLanguage?: string
+ currentPath?: string
+ allVersions?: AllVersions
+ currentPathWithoutLanguage?: string
+ currentArticle?: string
+ query?: Record
+ relativePath?: string
page?: Page
+ enPage?: Page
+ productNames?: ProductNames
currentVersion?: string
+ process?: { env: {} }
+ site?: {
+ data: {
+ ui: any
+ }
+ }
+ currentVersionObj?: Version
+ currentProduct?: string
+ getEnglishPage?: (ctx: Context) => Page
+ getDottedData?: (dottedPath: string) => any
+ initialRestVersioningReleaseDate?: string
+ initialRestVersioningReleaseDateLong?: string
+ nonEnterpriseDefaultVersion?: string
+ enterpriseServerVersions?: string[]
+ enterpriseServerReleases?: typeof enterpriseServerReleases
+ languages?: Languages
}
type Language = {
@@ -56,6 +107,8 @@ export type Page = {
title: string
shortTitle?: string
intro: string
+ rawIntro?: string
+ rawPermissions?: string
languageCode: string
documentType: string
renderProp: (prop: string, context: any, opts: any) => Promise
@@ -89,9 +142,33 @@ export type UnversionLanguageTree = {
export type Site = {
pages: Record
- redirects: Record
+ redirects: Redirects
unversionedTree: UnversionLanguageTree
siteTree: SiteTree
pageList: Page[]
pageMap: Record
}
+
+export type Version = {
+ version: string
+ versionTitle: string
+ latestVersion: string
+ currentRelease: string
+ openApiVersionName: string
+ miscVersionName: string
+ apiVersions: string[]
+ latestApiVersion: string
+ plan: string
+ planTitle: string
+ shortName: string
+ releases: string[]
+ latestRelease: string
+ hasNumberedReleases: boolean
+ openApiBaseName: string
+ miscBaseName: string
+ nonEnterpriseDefault?: boolean
+}
+
+export type AllVersions = {
+ [name: string]: Version
+}
diff --git a/src/versions/lib/all-versions.d.ts b/src/versions/lib/all-versions.d.ts
new file mode 100644
index 0000000000..ea2ad0058c
--- /dev/null
+++ b/src/versions/lib/all-versions.d.ts
@@ -0,0 +1,13 @@
+import type { AllVersions } from '@/types'
+
+export const allVersionKeys: string[]
+
+export const allVersionShortnames: Record
+
+export declare function isApiVersioned(version: string): boolean
+
+export declare function getDocsVersion(openApiVersion: string): string
+
+export declare function getOpenApiVersion(version: string): string
+
+export const allVersions: AllVersions
diff --git a/src/versions/lib/enterprise-server-releases.d.ts b/src/versions/lib/enterprise-server-releases.d.ts
new file mode 100644
index 0000000000..0cde233385
--- /dev/null
+++ b/src/versions/lib/enterprise-server-releases.d.ts
@@ -0,0 +1,80 @@
+type Dates = {
+ [key: string]: {
+ releaseDate: string
+ deprecationDate: string
+ }
+}
+
+export const dates: Dates
+
+export const next: string
+
+export const nextNext: string
+
+export const supported: string[]
+
+export const releaseCandidate: null | string
+
+export const deprecatedWithFunctionalRedirects: string[]
+
+export const deprecated: string[]
+
+export const legacyAssetVersions: string[]
+
+export const firstReleaseStoredInBlobStorage: string
+
+export const all: string[]
+export const latest: string
+export const latestStable: string
+export const oldestSupported: string
+export const nextDeprecationDate: string
+export const isOldestReleaseDeprecated: boolean
+export const deprecatedOnNewSite: string[]
+
+export const firstVersionDeprecatedOnNewSite: string
+export const lastVersionWithoutArchivedRedirectsFile: string
+export const lastReleaseWithLegacyFormat: string
+export const deprecatedReleasesWithLegacyFormat: string[]
+
+export const deprecatedReleasesWithNewFormat: string[]
+
+export const deprecatedReleasesOnDeveloperSite: string[]
+
+export const firstReleaseNote: string
+export const firstRestoredAdminGuides: string
+
+export declare function findReleaseNumberIndex(releaseNum: number): number
+export declare function getNextReleaseNumber(releaseNum: number): string
+export declare function getPreviousReleaseNumber(releaseNum: number): string
+
+const allExports = {
+ dates,
+ next,
+ nextNext,
+ supported,
+ releaseCandidate,
+ deprecatedWithFunctionalRedirects,
+ deprecated,
+ legacyAssetVersions,
+ firstReleaseStoredInBlobStorage,
+ all,
+ latest,
+ latestStable,
+ oldestSupported,
+ nextDeprecationDate,
+ isOldestReleaseDeprecated,
+ deprecatedOnNewSite,
+ firstVersionDeprecatedOnNewSite,
+ lastVersionWithoutArchivedRedirectsFile,
+ lastReleaseWithLegacyFormat,
+ deprecatedReleasesWithLegacyFormat,
+ deprecatedReleasesWithNewFormat,
+ deprecatedReleasesOnDeveloperSite,
+ firstReleaseNote,
+ firstRestoredAdminGuides,
+ findReleaseNumberIndex,
+ getNextReleaseNumber,
+ getPreviousReleaseNumber,
+}
+
+export default allExports