227
eslint.config.js
Normal file
227
eslint.config.js
Normal file
@@ -0,0 +1,227 @@
|
||||
import js from '@eslint/js'
|
||||
import tseslint from '@typescript-eslint/eslint-plugin'
|
||||
import tsParser from '@typescript-eslint/parser'
|
||||
import github from 'eslint-plugin-github'
|
||||
import importPlugin from 'eslint-plugin-import'
|
||||
import jsxA11y from 'eslint-plugin-jsx-a11y'
|
||||
import primerReact from 'eslint-plugin-primer-react'
|
||||
import eslintComments from 'eslint-plugin-eslint-comments'
|
||||
import i18nText from 'eslint-plugin-i18n-text'
|
||||
import filenames from 'eslint-plugin-filenames'
|
||||
import noOnlyTests from 'eslint-plugin-no-only-tests'
|
||||
import prettierPlugin from 'eslint-plugin-prettier'
|
||||
import prettier from 'eslint-config-prettier'
|
||||
import globals from 'globals'
|
||||
|
||||
export default [
|
||||
// JavaScript and MJS files configuration
|
||||
{
|
||||
files: ['**/*.{js,mjs}'],
|
||||
languageOptions: {
|
||||
ecmaVersion: 2022,
|
||||
sourceType: 'module',
|
||||
globals: {
|
||||
...globals.browser,
|
||||
...globals.node,
|
||||
...globals.commonjs,
|
||||
...globals.es2020,
|
||||
},
|
||||
parserOptions: {
|
||||
requireConfigFile: false,
|
||||
},
|
||||
},
|
||||
settings: {
|
||||
'import/resolver': {
|
||||
typescript: true,
|
||||
node: true,
|
||||
},
|
||||
},
|
||||
plugins: {
|
||||
github,
|
||||
import: importPlugin,
|
||||
'eslint-comments': eslintComments,
|
||||
'i18n-text': i18nText,
|
||||
filenames,
|
||||
'no-only-tests': noOnlyTests,
|
||||
prettier: prettierPlugin,
|
||||
},
|
||||
rules: {
|
||||
// ESLint recommended rules
|
||||
...js.configs.recommended.rules,
|
||||
|
||||
// GitHub plugin recommended rules
|
||||
...github.configs.recommended.rules,
|
||||
|
||||
// Import plugin error rules
|
||||
...importPlugin.configs.errors.rules,
|
||||
|
||||
// JavaScript-specific overrides
|
||||
'import/no-extraneous-dependencies': [
|
||||
'error',
|
||||
{
|
||||
packageDir: '.',
|
||||
},
|
||||
],
|
||||
'import/extensions': 'off',
|
||||
'no-console': 'off',
|
||||
camelcase: 'off',
|
||||
'no-shadow': 'off',
|
||||
'prefer-template': 'off',
|
||||
'no-constant-condition': 'off',
|
||||
'no-unused-vars': 'off',
|
||||
'import/no-named-as-default-member': 'off',
|
||||
'one-var': 'off',
|
||||
'import/no-namespace': 'off',
|
||||
'import/no-anonymous-default-export': 'off',
|
||||
'object-shorthand': 'off',
|
||||
'no-empty': 'off',
|
||||
'prefer-const': 'off',
|
||||
'import/no-named-as-default': 'off',
|
||||
'no-useless-concat': 'off',
|
||||
'func-style': 'off',
|
||||
|
||||
// Disable GitHub plugin rules that were disabled in original config
|
||||
'github/array-foreach': 'off',
|
||||
'github/no-then': 'off',
|
||||
|
||||
// Disable rules that might not exist or cause issues initially
|
||||
'i18n-text/no-en': 'off',
|
||||
'filenames/match-regex': 'off',
|
||||
'eslint-comments/no-use': 'off',
|
||||
'eslint-comments/no-unused-disable': 'off',
|
||||
'eslint-comments/no-unlimited-disable': 'off',
|
||||
|
||||
// Disable new ESLint 9 rules that are causing issues
|
||||
'no-constant-binary-expression': 'off',
|
||||
},
|
||||
},
|
||||
|
||||
// TypeScript and TSX files configuration
|
||||
{
|
||||
files: ['**/*.{ts,tsx}'],
|
||||
languageOptions: {
|
||||
parser: tsParser,
|
||||
ecmaVersion: 2022,
|
||||
sourceType: 'module',
|
||||
globals: {
|
||||
...globals.browser,
|
||||
...globals.node,
|
||||
...globals.commonjs,
|
||||
...globals.es2020,
|
||||
},
|
||||
parserOptions: {
|
||||
requireConfigFile: false,
|
||||
},
|
||||
},
|
||||
settings: {
|
||||
'import/resolver': {
|
||||
typescript: true,
|
||||
node: true,
|
||||
},
|
||||
},
|
||||
plugins: {
|
||||
github,
|
||||
import: importPlugin,
|
||||
'eslint-comments': eslintComments,
|
||||
'i18n-text': i18nText,
|
||||
filenames,
|
||||
'no-only-tests': noOnlyTests,
|
||||
prettier: prettierPlugin,
|
||||
'@typescript-eslint': tseslint,
|
||||
'primer-react': primerReact,
|
||||
'jsx-a11y': jsxA11y,
|
||||
},
|
||||
rules: {
|
||||
// ESLint recommended rules
|
||||
...js.configs.recommended.rules,
|
||||
|
||||
// GitHub plugin recommended rules
|
||||
...github.configs.recommended.rules,
|
||||
|
||||
// Import plugin error rules
|
||||
...importPlugin.configs.errors.rules,
|
||||
|
||||
// TypeScript ESLint recommended rules
|
||||
...tseslint.configs.recommended.rules,
|
||||
|
||||
// Primer React recommended rules
|
||||
...primerReact.configs.recommended.rules,
|
||||
|
||||
// JSX A11y recommended rules
|
||||
...jsxA11y.configs.recommended.rules,
|
||||
|
||||
// TypeScript-specific overrides
|
||||
'import/no-extraneous-dependencies': [
|
||||
'error',
|
||||
{
|
||||
packageDir: '.',
|
||||
},
|
||||
],
|
||||
'import/extensions': 'off',
|
||||
'no-console': 'off',
|
||||
camelcase: 'off',
|
||||
'no-shadow': 'off',
|
||||
'prefer-template': 'off',
|
||||
'no-constant-condition': 'off',
|
||||
'no-unused-vars': 'off',
|
||||
'no-undef': 'off',
|
||||
'no-use-before-define': 'off',
|
||||
'no-redeclare': 'off', // Allow function overloads in TypeScript
|
||||
'import/no-named-as-default-member': 'off',
|
||||
'one-var': 'off',
|
||||
'import/no-namespace': 'off',
|
||||
'import/no-anonymous-default-export': 'off',
|
||||
'object-shorthand': 'off',
|
||||
'no-empty': 'off',
|
||||
'prefer-const': 'off',
|
||||
'import/no-named-as-default': 'off',
|
||||
'no-useless-concat': 'off',
|
||||
'func-style': 'off',
|
||||
|
||||
// TypeScript ESLint specific rules
|
||||
'@typescript-eslint/no-unused-vars': 'error',
|
||||
|
||||
// Disable GitHub plugin rules that were disabled in original config
|
||||
'github/array-foreach': 'off',
|
||||
'github/no-then': 'off',
|
||||
|
||||
// Disable rules that might not exist or cause issues initially
|
||||
'i18n-text/no-en': 'off',
|
||||
'filenames/match-regex': 'off',
|
||||
'eslint-comments/no-use': 'off',
|
||||
'eslint-comments/no-unused-disable': 'off',
|
||||
'eslint-comments/no-unlimited-disable': 'off',
|
||||
|
||||
// Disable new ESLint 9 rules that are causing issues
|
||||
'no-constant-binary-expression': 'off',
|
||||
|
||||
// Disable stricter TypeScript rules initially
|
||||
'@typescript-eslint/no-explicit-any': 'off',
|
||||
'@typescript-eslint/no-unused-expressions': 'off',
|
||||
'@typescript-eslint/ban-ts-comment': 'off',
|
||||
'@typescript-eslint/no-wrapper-object-types': 'off',
|
||||
'@typescript-eslint/no-non-null-asserted-optional-chain': 'off',
|
||||
'@typescript-eslint/no-unsafe-function-type': 'off',
|
||||
'@typescript-eslint/no-empty-object-type': 'off',
|
||||
'@typescript-eslint/prefer-as-const': 'off',
|
||||
|
||||
// React/JSX specific rules
|
||||
'jsx-a11y/no-onchange': 'off',
|
||||
},
|
||||
},
|
||||
|
||||
// Ignored patterns
|
||||
{
|
||||
ignores: [
|
||||
'tmp/*',
|
||||
'.next/',
|
||||
'src/bookmarklets/*',
|
||||
'rest-api-description/',
|
||||
'docs-internal-data/',
|
||||
'src/code-scanning/scripts/generate-code-scanning-query-list.ts',
|
||||
],
|
||||
},
|
||||
|
||||
// Prettier config (should be last to override formatting rules)
|
||||
prettier,
|
||||
]
|
||||
2018
package-lock.json
generated
2018
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
132
package.json
132
package.json
@@ -27,7 +27,7 @@
|
||||
"clone-translations": "./src/languages/scripts/clone-translations.sh",
|
||||
"cmp-files": "tsx src/workflows/cmp-files.ts",
|
||||
"content-changes-table-comment": "tsx src/workflows/content-changes-table-comment.ts",
|
||||
"copy-fixture-data": "tsx src/tests/scripts/copy-fixture-data.js",
|
||||
"copy-fixture-data": "tsx src/tests/scripts/copy-fixture-data.ts",
|
||||
"count-translation-corruptions": "tsx src/languages/scripts/count-translation-corruptions.ts",
|
||||
"create-enterprise-issue": "tsx src/ghes-releases/scripts/create-enterprise-issue.ts",
|
||||
"debug": "cross-env NODE_ENV=development ENABLED_LANGUAGES=en nodemon --inspect src/frame/server.ts",
|
||||
@@ -58,16 +58,16 @@
|
||||
"index-test-fixtures": "./src/search/scripts/index-test-fixtures.sh",
|
||||
"labeler": "tsx .github/actions/labeler/labeler.ts",
|
||||
"lint": "eslint '**/*.{js,mjs,ts,tsx}'",
|
||||
"lint-content": "tsx src/content-linter/scripts/lint-content.js",
|
||||
"lint-translation": "vitest src/content-linter/tests/lint-files.js",
|
||||
"lint-content": "tsx src/content-linter/scripts/lint-content.ts",
|
||||
"lint-translation": "vitest src/content-linter/tests/lint-files.ts",
|
||||
"liquid-markdown-tables": "tsx src/tools/scripts/liquid-markdown-tables/index.ts",
|
||||
"generate-article-api-docs": "tsx src/article-api/scripts/generate-api-docs.ts",
|
||||
"generate-code-scanning-query-list": "tsx src/code-scanning/scripts/generate-code-scanning-query-list.ts",
|
||||
"generate-content-linter-docs": "tsx src/content-linter/scripts/generate-docs.ts",
|
||||
"move-content": "tsx src/content-render/scripts/move-content.js",
|
||||
"openapi-docs": "tsx src/rest/docs.js",
|
||||
"move-content": "tsx src/content-render/scripts/move-content.ts",
|
||||
"openapi-docs": "tsx src/rest/docs.ts",
|
||||
"playwright-test": "playwright test --config src/fixtures/playwright.config.ts --project=\"Google Chrome\"",
|
||||
"lint-report": "tsx src/content-linter/scripts/lint-report.js",
|
||||
"lint-report": "tsx src/content-linter/scripts/lint-report.ts",
|
||||
"postinstall": "cp package-lock.json .installed.package-lock.json && echo \"Updated .installed.package-lock.json\" # see husky/post-checkout and husky/post-merge",
|
||||
"precompute-pageinfo": "tsx src/article-api/scripts/precompute-pageinfo.ts",
|
||||
"prepare": "husky src/workflows/husky",
|
||||
@@ -75,7 +75,7 @@
|
||||
"prettier-check": "prettier -c \"**/*.{ts,tsx,js,mjs,scss,yml,yaml}\"",
|
||||
"prevent-pushes-to-main": "tsx src/workflows/prevent-pushes-to-main.ts",
|
||||
"purge-fastly-edge-cache": "tsx src/workflows/purge-fastly-edge-cache.ts",
|
||||
"purge-fastly-edge-cache-per-language": "tsx src/languages/scripts/purge-fastly-edge-cache-per-language.js",
|
||||
"purge-fastly-edge-cache-per-language": "tsx src/languages/scripts/purge-fastly-edge-cache-per-language.ts",
|
||||
"readability-report": "tsx src/workflows/experimental/readability-report.ts",
|
||||
"ready-for-docs-review": "tsx src/workflows/ready-for-docs-review.ts",
|
||||
"release-banner": "tsx src/ghes-releases/scripts/release-banner.ts",
|
||||
@@ -83,15 +83,15 @@
|
||||
"reusables": "tsx src/content-render/scripts/reusables-cli.ts",
|
||||
"rendered-content-link-checker": "tsx src/links/scripts/rendered-content-link-checker.ts",
|
||||
"rendered-content-link-checker-cli": "tsx src/links/scripts/rendered-content-link-checker-cli.ts",
|
||||
"rest-dev": "tsx src/rest/scripts/update-files.js",
|
||||
"rest-dev": "tsx src/rest/scripts/update-files.ts",
|
||||
"show-action-deps": "echo 'Action Dependencies:' && rg '^[\\s|-]*(uses:.*)$' .github -I -N --no-heading -r '$1$2' | sort | uniq | cut -c 7-",
|
||||
"start": "cross-env NODE_ENV=development ENABLED_LANGUAGES=en nodemon src/frame/server.ts",
|
||||
"start-all-languages": "cross-env NODE_ENV=development tsx src/frame/server.ts",
|
||||
"start-for-playwright": "cross-env ROOT=src/fixtures/fixtures TRANSLATIONS_FIXTURE_ROOT=src/fixtures/fixtures/translations ENABLED_LANGUAGES=en,ja NODE_ENV=test tsx src/frame/server.ts",
|
||||
"symlink-from-local-repo": "tsx src/early-access/scripts/symlink-from-local-repo.ts",
|
||||
"sync-audit-log": "tsx src/audit-logs/scripts/sync.ts",
|
||||
"sync-codeql-cli": "tsx src/codeql-cli/scripts/sync.js",
|
||||
"sync-graphql": "tsx src/graphql/scripts/sync.js",
|
||||
"sync-codeql-cli": "tsx src/codeql-cli/scripts/sync.ts",
|
||||
"sync-graphql": "tsx src/graphql/scripts/sync.ts",
|
||||
"sync-rest": "tsx src/rest/scripts/update-files.ts",
|
||||
"sync-secret-scanning": "tsx src/secret-scanning/scripts/sync.ts",
|
||||
"sync-webhooks": "npx tsx src/rest/scripts/update-files.ts -o webhooks",
|
||||
@@ -154,101 +154,10 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"eslintConfig": {
|
||||
"env": {
|
||||
"browser": true,
|
||||
"commonjs": true,
|
||||
"es2020": true,
|
||||
"node": true
|
||||
},
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 2022,
|
||||
"requireConfigFile": "false",
|
||||
"sourceType": "module"
|
||||
},
|
||||
"settings": {
|
||||
"import/resolver": {
|
||||
"typescript": true,
|
||||
"node": true
|
||||
}
|
||||
},
|
||||
"extends": [
|
||||
"eslint:recommended",
|
||||
"plugin:github/recommended",
|
||||
"plugin:import/errors",
|
||||
"prettier"
|
||||
],
|
||||
"rules": {
|
||||
"import/no-extraneous-dependencies": [
|
||||
"error",
|
||||
{
|
||||
"packageDir": "."
|
||||
}
|
||||
],
|
||||
"import/extensions": "off",
|
||||
"no-console": "off",
|
||||
"github/array-foreach": "off",
|
||||
"camelcase": "off",
|
||||
"i18n-text/no-en": "off",
|
||||
"no-shadow": "off",
|
||||
"prefer-template": "off",
|
||||
"filenames/match-regex": "off",
|
||||
"no-constant-condition": "off",
|
||||
"no-unused-vars": "off",
|
||||
"github/no-then": "off",
|
||||
"import/no-named-as-default-member": "off",
|
||||
"one-var": "off",
|
||||
"import/no-namespace": "off",
|
||||
"import/no-anonymous-default-export": "off",
|
||||
"object-shorthand": "off",
|
||||
"eslint-comments/no-use": "off",
|
||||
"no-empty": "off",
|
||||
"prefer-const": "off",
|
||||
"import/no-named-as-default": "off",
|
||||
"eslint-comments/no-unused-disable": "off",
|
||||
"no-useless-concat": "off",
|
||||
"func-style": "off",
|
||||
"eslint-comments/no-unlimited-disable": "off"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"files": [
|
||||
"**/*.tsx",
|
||||
"**/*.ts"
|
||||
],
|
||||
"parser": "@typescript-eslint/parser",
|
||||
"plugins": [
|
||||
"@typescript-eslint",
|
||||
"primer-react",
|
||||
"jsx-a11y"
|
||||
],
|
||||
"extends": [
|
||||
"plugin:primer-react/recommended",
|
||||
"plugin:jsx-a11y/recommended"
|
||||
],
|
||||
"rules": {
|
||||
"camelcase": "off",
|
||||
"no-undef": "off",
|
||||
"no-unused-vars": "off",
|
||||
"no-use-before-define": "off",
|
||||
"@typescript-eslint/no-unused-vars": "error",
|
||||
"jsx-a11y/no-onchange": "off"
|
||||
}
|
||||
}
|
||||
],
|
||||
"ignorePatterns": [
|
||||
"tmp/*",
|
||||
"!/.*",
|
||||
"/.next/",
|
||||
"src/bookmarklets/*",
|
||||
"rest-api-description/",
|
||||
"docs-internal-data/",
|
||||
"src/code-scanning/scripts/generate-code-scanning-query-list.ts"
|
||||
]
|
||||
},
|
||||
"dependencies": {
|
||||
"@elastic/elasticsearch": "8.13.1",
|
||||
"@github/failbot": "0.8.3",
|
||||
"@gr2m/gray-matter": "4.0.3-with-pr-137",
|
||||
"@horizon-rs/language-guesser": "0.1.1",
|
||||
"@octokit/plugin-retry": "8.0.1",
|
||||
"@octokit/request-error": "7.0.0",
|
||||
@@ -282,7 +191,6 @@
|
||||
"github-slugger": "^2.0.0",
|
||||
"glob": "11.0.2",
|
||||
"got": "^14.4.7",
|
||||
"@gr2m/gray-matter": "4.0.3-with-pr-137",
|
||||
"hast-util-from-parse5": "^8.0.3",
|
||||
"hast-util-to-string": "^3.0.1",
|
||||
"hastscript": "^9.0.1",
|
||||
@@ -343,6 +251,7 @@
|
||||
"@actions/core": "^1.10.1",
|
||||
"@actions/github": "^6.0.0",
|
||||
"@axe-core/playwright": "^4.10.1",
|
||||
"@eslint/js": "^9.33.0",
|
||||
"@github/markdownlint-github": "^0.6.3",
|
||||
"@graphql-inspector/core": "^6.1.0",
|
||||
"@graphql-tools/load": "^8.0.19",
|
||||
@@ -375,14 +284,21 @@
|
||||
"cross-env": "^10.0.0",
|
||||
"csp-parse": "0.0.2",
|
||||
"csv-parse": "6.1.0",
|
||||
"eslint": "^8.57.1",
|
||||
"eslint-config-prettier": "^10.0.3",
|
||||
"eslint": "^9.33.0",
|
||||
"eslint-config-prettier": "^10.1.8",
|
||||
"eslint-import-resolver-typescript": "^4.4.2",
|
||||
"eslint-plugin-github": "^5.0.2",
|
||||
"eslint-plugin-import": "^2.30.0",
|
||||
"eslint-plugin-escompat": "^3.11.4",
|
||||
"eslint-plugin-eslint-comments": "^3.2.0",
|
||||
"eslint-plugin-filenames": "^1.3.2",
|
||||
"eslint-plugin-github": "^6.0.0",
|
||||
"eslint-plugin-i18n-text": "^1.0.1",
|
||||
"eslint-plugin-import": "^2.32.0",
|
||||
"eslint-plugin-jsx-a11y": "^6.10.2",
|
||||
"eslint-plugin-primer-react": "^7.0.2",
|
||||
"eslint-plugin-no-only-tests": "^3.3.0",
|
||||
"eslint-plugin-prettier": "^5.5.4",
|
||||
"eslint-plugin-primer-react": "^8.0.0",
|
||||
"event-to-promise": "^0.8.0",
|
||||
"globals": "^16.3.0",
|
||||
"graphql": "^16.9.0",
|
||||
"http-status-code": "^2.1.0",
|
||||
"husky": "^9.1.7",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import got from 'got'
|
||||
import { fetchWithRetry } from '@/frame/lib/fetch-utils'
|
||||
import type { Response, NextFunction } from 'express'
|
||||
|
||||
import patterns from '@/frame/lib/patterns'
|
||||
@@ -85,11 +85,31 @@ export default async function archivedEnterpriseVersionsAssets(
|
||||
|
||||
const proxyPath = `https://github.github.com/docs-ghes-${requestedVersion}${assetPath}`
|
||||
try {
|
||||
const r = await got(proxyPath)
|
||||
const r = await fetchWithRetry(
|
||||
proxyPath,
|
||||
{},
|
||||
{
|
||||
retries: 0,
|
||||
throwHttpErrors: true,
|
||||
},
|
||||
)
|
||||
|
||||
const body = await r.arrayBuffer()
|
||||
|
||||
res.set('accept-ranges', 'bytes')
|
||||
res.set('content-type', r.headers['content-type'])
|
||||
res.set('content-length', r.headers['content-length'])
|
||||
const contentType = r.headers.get('content-type')
|
||||
if (contentType) {
|
||||
// Match got's behavior by adding charset=utf-8 to SVG files
|
||||
if (contentType === 'image/svg+xml') {
|
||||
res.set('content-type', `${contentType}; charset=utf-8`)
|
||||
} else {
|
||||
res.set('content-type', contentType)
|
||||
}
|
||||
}
|
||||
const contentLength = r.headers.get('content-length')
|
||||
if (contentLength) {
|
||||
res.set('content-length', contentLength)
|
||||
}
|
||||
res.set('x-is-archived', 'true')
|
||||
res.set('x-robots-tag', 'noindex')
|
||||
|
||||
@@ -98,7 +118,7 @@ export default async function archivedEnterpriseVersionsAssets(
|
||||
archivedCacheControl(res)
|
||||
setFastlySurrogateKey(res, SURROGATE_ENUMS.MANUAL)
|
||||
|
||||
return res.send(r.body)
|
||||
return res.send(Buffer.from(body))
|
||||
} catch (err) {
|
||||
// Primarily for the developers working on tests that mock
|
||||
// requests. If you don't set up `nock` correctly, you might
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { Response, NextFunction } from 'express'
|
||||
import got from 'got'
|
||||
import { fetchWithRetry } from '@/frame/lib/fetch-utils'
|
||||
|
||||
import statsd from '@/observability/lib/statsd'
|
||||
import {
|
||||
@@ -190,11 +190,15 @@ export default async function archivedEnterpriseVersions(
|
||||
}
|
||||
// Retrieve the page from the archived repo
|
||||
const doGet = () =>
|
||||
got(getProxyPath(req.path, requestedVersion), {
|
||||
throwHttpErrors: false,
|
||||
retry: retryConfiguration,
|
||||
timeout: timeoutConfiguration,
|
||||
})
|
||||
fetchWithRetry(
|
||||
getProxyPath(req.path, requestedVersion),
|
||||
{},
|
||||
{
|
||||
retries: retryConfiguration.limit,
|
||||
timeout: timeoutConfiguration.response,
|
||||
throwHttpErrors: false,
|
||||
},
|
||||
)
|
||||
|
||||
const statsdTags = [`version:${requestedVersion}`]
|
||||
const r = await statsd.asyncTimer(doGet, 'archive_enterprise_proxy', [
|
||||
@@ -202,7 +206,8 @@ export default async function archivedEnterpriseVersions(
|
||||
`path:${req.path}`,
|
||||
])()
|
||||
|
||||
if (r.statusCode === 200) {
|
||||
if (r.status === 200) {
|
||||
const body = await r.text()
|
||||
const [, withoutLanguagePath] = splitByLanguage(req.path)
|
||||
const isDeveloperPage = withoutLanguagePath?.startsWith(
|
||||
`/enterprise/${requestedVersion}/developer`,
|
||||
@@ -210,13 +215,13 @@ export default async function archivedEnterpriseVersions(
|
||||
res.set('x-robots-tag', 'noindex')
|
||||
|
||||
// make stubbed redirect files (which exist in versions <2.13) redirect with a 301
|
||||
const staticRedirect = r.body.match(patterns.staticRedirect)
|
||||
const staticRedirect = body.match(patterns.staticRedirect)
|
||||
if (staticRedirect) {
|
||||
cacheAggressively(res)
|
||||
return res.redirect(redirectCode, staticRedirect[1])
|
||||
}
|
||||
|
||||
res.set('content-type', r.headers['content-type'])
|
||||
res.set('content-type', r.headers.get('content-type') || '')
|
||||
|
||||
cacheAggressively(res)
|
||||
|
||||
@@ -230,7 +235,7 @@ export default async function archivedEnterpriseVersions(
|
||||
// `x-host` is a custom header set by Fastly.
|
||||
// GLB automatically deletes the `x-forwarded-host` header.
|
||||
const host = req.get('x-host') || req.get('x-forwarded-host') || req.get('host')
|
||||
r.body = r.body
|
||||
let modifiedBody = body
|
||||
.replaceAll(
|
||||
`${OLD_AZURE_BLOB_ENTERPRISE_DIR}/${requestedVersion}/assets/cb-`,
|
||||
`${ENTERPRISE_GH_PAGES_URL_PREFIX}${requestedVersion}/assets/cb-`,
|
||||
@@ -239,6 +244,8 @@ export default async function archivedEnterpriseVersions(
|
||||
`${OLD_AZURE_BLOB_ENTERPRISE_DIR}/${requestedVersion}/`,
|
||||
`${req.protocol}://${host}/enterprise-server@${requestedVersion}/`,
|
||||
)
|
||||
|
||||
return res.send(modifiedBody)
|
||||
}
|
||||
|
||||
// Releases 3.1 and lower were previously hosted in the
|
||||
@@ -247,23 +254,42 @@ export default async function archivedEnterpriseVersions(
|
||||
// The image paths all need to be updated to reference the images in the
|
||||
// new archived enterprise repo's root assets directory.
|
||||
if (versionSatisfiesRange(requestedVersion, `<${firstReleaseStoredInBlobStorage}`)) {
|
||||
r.body = r.body.replaceAll(
|
||||
let modifiedBody = body.replaceAll(
|
||||
`${OLD_GITHUB_IMAGES_ENTERPRISE_DIR}/${requestedVersion}`,
|
||||
`${ENTERPRISE_GH_PAGES_URL_PREFIX}${requestedVersion}`,
|
||||
)
|
||||
if (versionSatisfiesRange(requestedVersion, '<=2.18') && isDeveloperPage) {
|
||||
r.body = r.body.replaceAll(
|
||||
modifiedBody = modifiedBody.replaceAll(
|
||||
`${OLD_DEVELOPER_SITE_CONTAINER}/${requestedVersion}`,
|
||||
`${ENTERPRISE_GH_PAGES_URL_PREFIX}${requestedVersion}/developer`,
|
||||
)
|
||||
// Update all hrefs to add /developer to the path
|
||||
r.body = r.body.replaceAll(
|
||||
modifiedBody = modifiedBody.replaceAll(
|
||||
`="/enterprise/${requestedVersion}`,
|
||||
`="/enterprise/${requestedVersion}/developer`,
|
||||
)
|
||||
// The changelog is the only thing remaining on developer.github.com
|
||||
r.body = r.body.replaceAll('href="/changes', 'href="https://developer.github.com/changes')
|
||||
modifiedBody = modifiedBody.replaceAll(
|
||||
'href="/changes',
|
||||
'href="https://developer.github.com/changes',
|
||||
)
|
||||
}
|
||||
|
||||
// Continue with remaining replacements
|
||||
modifiedBody = modifiedBody.replaceAll(
|
||||
/="(\.\.\/)*assets/g,
|
||||
`="${ENTERPRISE_GH_PAGES_URL_PREFIX}${requestedVersion}/assets`,
|
||||
)
|
||||
|
||||
// Fix broken hrefs on the 2.16 landing page
|
||||
if (requestedVersion === '2.16' && req.path === '/en/enterprise/2.16') {
|
||||
modifiedBody = modifiedBody.replaceAll('ref="/en/enterprise', 'ref="/en/enterprise/2.16')
|
||||
}
|
||||
|
||||
// Remove the search results container from the page
|
||||
modifiedBody = modifiedBody.replaceAll('<div id="search-results-container"></div>', '')
|
||||
|
||||
return res.send(modifiedBody)
|
||||
}
|
||||
|
||||
// In all releases, some assets were incorrectly scraped and contain
|
||||
@@ -275,21 +301,21 @@ export default async function archivedEnterpriseVersions(
|
||||
// We want to update the URLs in the format
|
||||
// "../../../../../../assets/" to prefix the assets directory with the
|
||||
// new archived enterprise repo URL.
|
||||
r.body = r.body.replaceAll(
|
||||
let modifiedBody = body.replaceAll(
|
||||
/="(\.\.\/)*assets/g,
|
||||
`="${ENTERPRISE_GH_PAGES_URL_PREFIX}${requestedVersion}/assets`,
|
||||
)
|
||||
|
||||
// Fix broken hrefs on the 2.16 landing page
|
||||
if (requestedVersion === '2.16' && req.path === '/en/enterprise/2.16') {
|
||||
r.body = r.body.replaceAll('ref="/en/enterprise', 'ref="/en/enterprise/2.16')
|
||||
modifiedBody = modifiedBody.replaceAll('ref="/en/enterprise', 'ref="/en/enterprise/2.16')
|
||||
}
|
||||
|
||||
// Remove the search results container from the page, which removes a white
|
||||
// box that prevents clicking on page links
|
||||
r.body = r.body.replaceAll('<div id="search-results-container"></div>', '')
|
||||
modifiedBody = modifiedBody.replaceAll('<div id="search-results-container"></div>', '')
|
||||
|
||||
return res.send(r.body)
|
||||
return res.send(modifiedBody)
|
||||
}
|
||||
// In releases 2.13 - 2.17, we lost access to frontmatter redirects
|
||||
// during the archival process. This workaround finds potentially
|
||||
|
||||
@@ -29,7 +29,7 @@ async function main() {
|
||||
includeBasePath: true,
|
||||
globs: ['**/*.md'],
|
||||
})
|
||||
const cliMarkdownContents = {}
|
||||
const cliMarkdownContents: Record<string, { data: any; content: string }> = {}
|
||||
|
||||
for (const file of markdownFiles) {
|
||||
const sourceContent = await readFile(file, 'utf8')
|
||||
@@ -83,7 +83,7 @@ async function setupEnvironment() {
|
||||
|
||||
// copy the raw rst files to the temp directory and convert them
|
||||
// to Markdownusing pandoc
|
||||
async function rstToMarkdown(sourceDirectory) {
|
||||
async function rstToMarkdown(sourceDirectory: string) {
|
||||
const sourceFiles = walk(sourceDirectory, {
|
||||
includeBasePath: true,
|
||||
globs: ['**/*.rst'],
|
||||
@@ -1,3 +1,4 @@
|
||||
// @ts-nocheck
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import { execSync } from 'child_process'
|
||||
@@ -258,7 +259,7 @@ async function main() {
|
||||
}
|
||||
|
||||
const fixableFiles = Object.entries(formattedResults)
|
||||
.filter(([_, results]) => results.some((result) => result.fixable))
|
||||
.filter(([, results]) => results.some((result) => result.fixable))
|
||||
.map(([file]) => file)
|
||||
if (fixableFiles.length) {
|
||||
console.log('') // Just for some whitespace before the next message
|
||||
@@ -692,7 +693,7 @@ function isOptionsValid() {
|
||||
for (const path of paths) {
|
||||
try {
|
||||
fs.statSync(path)
|
||||
} catch (err) {
|
||||
} catch {
|
||||
if ('paths'.includes(path)) {
|
||||
console.log('error: did you mean --paths')
|
||||
} else {
|
||||
@@ -10,20 +10,21 @@ import { reportingConfig } from '@/content-linter/style/github-docs'
|
||||
// GitHub issue body size limit is ~65k characters, so we'll use 60k as a safe limit
|
||||
const MAX_ISSUE_BODY_SIZE = 60000
|
||||
|
||||
interface LintFlaw {
|
||||
severity: string
|
||||
ruleNames: string[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Determines if a lint result should be included in the automated report
|
||||
* @param {Object} flaw - The lint result object
|
||||
* @param {string} flaw.severity - 'error' or 'warning'
|
||||
* @param {string[]} flaw.ruleNames - Array of rule names for this flaw
|
||||
* @returns {boolean} - True if this flaw should be included in the report
|
||||
*/
|
||||
function shouldIncludeInReport(flaw) {
|
||||
function shouldIncludeInReport(flaw: LintFlaw): boolean {
|
||||
if (!flaw.ruleNames || !Array.isArray(flaw.ruleNames)) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if any rule name is in the exclude list
|
||||
const hasExcludedRule = flaw.ruleNames.some((ruleName) =>
|
||||
const hasExcludedRule = flaw.ruleNames.some((ruleName: string) =>
|
||||
reportingConfig.excludeRules.includes(ruleName),
|
||||
)
|
||||
if (hasExcludedRule) {
|
||||
@@ -36,7 +37,7 @@ function shouldIncludeInReport(flaw) {
|
||||
}
|
||||
|
||||
// Check if any rule name is in the include list
|
||||
const hasIncludedRule = flaw.ruleNames.some((ruleName) =>
|
||||
const hasIncludedRule = flaw.ruleNames.some((ruleName: string) =>
|
||||
reportingConfig.includeRules.includes(ruleName),
|
||||
)
|
||||
if (hasIncludedRule) {
|
||||
@@ -88,9 +89,9 @@ async function main() {
|
||||
const parsedResults = JSON.parse(lintResults)
|
||||
|
||||
// Filter results based on reporting configuration
|
||||
const filteredResults = {}
|
||||
const filteredResults: Record<string, LintFlaw[]> = {}
|
||||
for (const [file, flaws] of Object.entries(parsedResults)) {
|
||||
const filteredFlaws = flaws.filter(shouldIncludeInReport)
|
||||
const filteredFlaws = (flaws as LintFlaw[]).filter(shouldIncludeInReport)
|
||||
|
||||
// Only include files that have remaining flaws after filtering
|
||||
if (filteredFlaws.length > 0) {
|
||||
@@ -127,8 +128,8 @@ async function main() {
|
||||
octokit,
|
||||
reportTitle: `Content linting issues requiring attention`,
|
||||
reportBody,
|
||||
reportRepository: REPORT_REPOSITORY,
|
||||
reportLabel: REPORT_LABEL,
|
||||
reportRepository: REPORT_REPOSITORY!,
|
||||
reportLabel: REPORT_LABEL!,
|
||||
}
|
||||
|
||||
await createReportIssue(reportProps)
|
||||
@@ -137,9 +138,9 @@ async function main() {
|
||||
core,
|
||||
octokit,
|
||||
newReport: await createReportIssue(reportProps),
|
||||
reportRepository: REPORT_REPOSITORY,
|
||||
reportAuthor: REPORT_AUTHOR,
|
||||
reportLabel: REPORT_LABEL,
|
||||
reportRepository: REPORT_REPOSITORY!,
|
||||
reportAuthor: REPORT_AUTHOR!,
|
||||
reportLabel: REPORT_LABEL!,
|
||||
}
|
||||
|
||||
await linkReports(linkProps)
|
||||
@@ -1,3 +1,4 @@
|
||||
// @ts-nocheck
|
||||
import { fileURLToPath } from 'url'
|
||||
import path from 'path'
|
||||
import yaml from 'js-yaml'
|
||||
@@ -1,3 +1,4 @@
|
||||
// @ts-nocheck
|
||||
// [start-readme]
|
||||
//
|
||||
// Use this script to help you move or rename a single file or a folder. The script will move or rename the file or folder for you, update relevant `children` in the index.md file(s), and add a `redirect_from` to frontmatter in the renamed file(s). Note: You will still need to manually update the `title` if necessary.
|
||||
@@ -507,7 +508,7 @@ function editFiles(files, updateParent, opts) {
|
||||
|
||||
// Add contentType frontmatter to moved files
|
||||
if (files.length > 0) {
|
||||
const filePaths = files.map(([oldPath, newPath, oldHref, newHref]) => newPath)
|
||||
const filePaths = files.map(([, newPath]) => newPath)
|
||||
try {
|
||||
const cmd = ['run', 'add-content-type', '--', '--paths', ...filePaths]
|
||||
const result = execFileSync('npm', cmd, { cwd: process.cwd(), encoding: 'utf8' })
|
||||
@@ -1,8 +1,6 @@
|
||||
import path from 'path'
|
||||
import { escapeRegExp } from 'lodash-es'
|
||||
|
||||
/* eslint-disable prefer-regex-literals */
|
||||
|
||||
// slash at the beginning of a filename
|
||||
const leadingPathSeparator = new RegExp(`^${escapeRegExp(path.sep)}`)
|
||||
const windowsLeadingPathSeparator = new RegExp('^/')
|
||||
|
||||
@@ -44,7 +44,7 @@ describe('get-data', () => {
|
||||
const jaTranslationsRoot = path.join(dd.root, 'translations', 'ja-JP')
|
||||
fs.mkdirSync(jaTranslationsRoot, { recursive: true })
|
||||
languages.ja.dir = jaTranslationsRoot
|
||||
new DataDirectory( // eslint-disable-line no-new
|
||||
new DataDirectory(
|
||||
{
|
||||
data: {
|
||||
ui: {
|
||||
@@ -242,7 +242,7 @@ describe('get-data on corrupt translations', () => {
|
||||
const jaTranslationsRoot = path.join(dd.root, 'translations', 'ja-JP')
|
||||
fs.mkdirSync(jaTranslationsRoot, { recursive: true })
|
||||
languages.ja.dir = jaTranslationsRoot
|
||||
new DataDirectory( // eslint-disable-line no-new
|
||||
new DataDirectory(
|
||||
{
|
||||
data: {
|
||||
variables: {
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
/* eslint-disable camelcase */
|
||||
import Cookies from '@/frame/components/lib/cookies'
|
||||
import { parseUserAgent } from './user-agent'
|
||||
import { Router } from 'next/router'
|
||||
|
||||
@@ -3,7 +3,7 @@ import { allVersionKeys } from '@/versions/lib/all-versions'
|
||||
import { productIds } from '@/products/lib/all-products'
|
||||
import { allTools } from '@/tools/lib/all-tools'
|
||||
|
||||
const versionPattern = '^\\d+(\\.\\d+)?(\\.\\d+)?$' // eslint-disable-line
|
||||
const versionPattern = '^\\d+(\\.\\d+)?(\\.\\d+)?$'
|
||||
|
||||
const context = {
|
||||
type: 'object',
|
||||
|
||||
@@ -12,11 +12,8 @@ type UseQueryParamReturn<T extends string | boolean> = {
|
||||
}
|
||||
|
||||
// Overloads so we can use this for a boolean or string query param
|
||||
// eslint-disable-next-line no-redeclare
|
||||
export function useQueryParam(queryParamKey: string, isBoolean: true): UseQueryParamReturn<boolean>
|
||||
// eslint-disable-next-line no-redeclare
|
||||
export function useQueryParam(queryParamKey: string, isBoolean?: false): UseQueryParamReturn<string>
|
||||
// eslint-disable-next-line no-redeclare
|
||||
export function useQueryParam(
|
||||
queryParamKey: string,
|
||||
isBoolean?: boolean,
|
||||
|
||||
113
src/frame/lib/fetch-utils.ts
Normal file
113
src/frame/lib/fetch-utils.ts
Normal file
@@ -0,0 +1,113 @@
|
||||
/**
|
||||
* Utility functions for fetch with retry and timeout functionality
|
||||
* to replace got library functionality
|
||||
*/
|
||||
|
||||
export interface FetchWithRetryOptions {
|
||||
retries?: number
|
||||
retryDelay?: number
|
||||
timeout?: number
|
||||
throwHttpErrors?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Default retry delay calculation matching got's behavior:
|
||||
* sleep = 1000 * Math.pow(2, retry - 1) + Math.random() * 100
|
||||
*/
|
||||
function calculateDefaultDelay(attempt: number): number {
|
||||
return 1000 * Math.pow(2, attempt - 1) + Math.random() * 100
|
||||
}
|
||||
|
||||
/**
|
||||
* Sleep for a given number of milliseconds
|
||||
*/
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch with timeout support
|
||||
*/
|
||||
async function fetchWithTimeout(
|
||||
url: string | URL,
|
||||
init?: RequestInit,
|
||||
timeout?: number,
|
||||
): Promise<Response> {
|
||||
if (!timeout) {
|
||||
return fetch(url, init)
|
||||
}
|
||||
|
||||
const controller = new AbortController()
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeout)
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
...init,
|
||||
signal: controller.signal,
|
||||
})
|
||||
clearTimeout(timeoutId)
|
||||
return response
|
||||
} catch (error) {
|
||||
clearTimeout(timeoutId)
|
||||
if (error instanceof Error && error.name === 'AbortError') {
|
||||
throw new Error(`Request timed out after ${timeout}ms`)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch with retry logic matching got's behavior
|
||||
*/
|
||||
export async function fetchWithRetry(
|
||||
url: string | URL,
|
||||
init?: RequestInit,
|
||||
options: FetchWithRetryOptions = {},
|
||||
): Promise<Response> {
|
||||
const { retries = 0, timeout, throwHttpErrors = true } = options
|
||||
|
||||
let lastError: Error | null = null
|
||||
|
||||
for (let attempt = 0; attempt <= retries; attempt++) {
|
||||
try {
|
||||
const response = await fetchWithTimeout(url, init, timeout)
|
||||
|
||||
// Check if we should retry based on status code
|
||||
if (response.status >= 500 && attempt < retries) {
|
||||
lastError = new Error(`HTTP ${response.status}: ${response.statusText}`)
|
||||
const delay = calculateDefaultDelay(attempt + 1)
|
||||
await sleep(delay)
|
||||
continue
|
||||
}
|
||||
|
||||
// If throwHttpErrors is true and status indicates an error
|
||||
if (throwHttpErrors && !response.ok && response.status >= 400) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`)
|
||||
}
|
||||
|
||||
return response
|
||||
} catch (error) {
|
||||
lastError = error instanceof Error ? error : new Error(String(error))
|
||||
|
||||
// Don't retry on the last attempt
|
||||
if (attempt === retries) {
|
||||
throw lastError
|
||||
}
|
||||
|
||||
// Don't retry on client errors (4xx) unless it's specific ones
|
||||
if (
|
||||
error instanceof Error &&
|
||||
error.message.includes('HTTP 4') &&
|
||||
!error.message.includes('HTTP 429')
|
||||
) {
|
||||
throw lastError
|
||||
}
|
||||
|
||||
// Calculate delay and wait before retry
|
||||
const delay = calculateDefaultDelay(attempt + 1)
|
||||
await sleep(delay)
|
||||
}
|
||||
}
|
||||
|
||||
throw lastError || new Error('Maximum retries exceeded')
|
||||
}
|
||||
@@ -358,7 +358,6 @@ const semverRange = {
|
||||
format: 'semver',
|
||||
// This is JSON pointer syntax with ajv so we can specify the bad version
|
||||
// in the error message.
|
||||
// eslint-disable-next-line no-template-curly-in-string
|
||||
errorMessage: 'Must be a valid SemVer range: ${0}',
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import path from 'path'
|
||||
import fs from 'fs'
|
||||
import crypto from 'crypto'
|
||||
|
||||
import got from 'got'
|
||||
import { fetchWithRetry } from './fetch-utils'
|
||||
import statsd from '@/observability/lib/statsd'
|
||||
|
||||
// The only reason this is exported is for the sake of the unit tests'
|
||||
@@ -65,23 +65,36 @@ export default async function getRemoteJSON(url, config) {
|
||||
}
|
||||
|
||||
if (!foundOnDisk) {
|
||||
// got will, by default, follow redirects and it will throw if the ultimate
|
||||
// fetch will, by default, follow redirects and fetchWithRetry will throw if the ultimate
|
||||
// response is not a 2xx.
|
||||
// But it's possible that the page is a 200 OK but it's just not a JSON
|
||||
// page at all. Then we can't assume we can deserialize it.
|
||||
const res = await got(url, config)
|
||||
if (!res.headers['content-type'].startsWith('application/json')) {
|
||||
throw new Error(
|
||||
`Fetching '${url}' resulted in a non-JSON response (${res.headers['content-type']})`,
|
||||
)
|
||||
const retries = config?.retry?.limit || 0
|
||||
const timeout = config?.timeout?.response
|
||||
|
||||
const res = await fetchWithRetry(
|
||||
url,
|
||||
{},
|
||||
{
|
||||
retries,
|
||||
timeout,
|
||||
throwHttpErrors: true,
|
||||
},
|
||||
)
|
||||
|
||||
const contentType = res.headers.get('content-type')
|
||||
if (!contentType || !contentType.startsWith('application/json')) {
|
||||
throw new Error(`Fetching '${url}' resulted in a non-JSON response (${contentType})`)
|
||||
}
|
||||
cache.set(cacheKey, JSON.parse(res.body))
|
||||
|
||||
const body = await res.text()
|
||||
cache.set(cacheKey, JSON.parse(body))
|
||||
|
||||
// Only write to disk for testing and local review.
|
||||
// In production, we never write to disk. Only in-memory.
|
||||
if (!inProd) {
|
||||
fs.mkdirSync(path.dirname(onDisk), { recursive: true })
|
||||
fs.writeFileSync(onDisk, res.body, 'utf-8')
|
||||
fs.writeFileSync(onDisk, body, 'utf-8')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
// @ts-nocheck
|
||||
import fs from 'fs/promises'
|
||||
import { appendFileSync } from 'fs'
|
||||
import path from 'path'
|
||||
@@ -1,12 +1,10 @@
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
|
||||
// eslint-disable-next-line import/named
|
||||
import { visit, Test } from 'unist-util-visit'
|
||||
import { fromMarkdown } from 'mdast-util-from-markdown'
|
||||
import { toMarkdown } from 'mdast-util-to-markdown'
|
||||
import yaml from 'js-yaml'
|
||||
// eslint-disable-next-line import/no-unresolved
|
||||
import { type Node, type Nodes, type Definition, type Link } from 'mdast'
|
||||
|
||||
import frontmatter from '@/frame/lib/read-frontmatter'
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
import fs from 'fs/promises'
|
||||
|
||||
import got, { RequestError } from 'got'
|
||||
import { fetchWithRetry } from '@/frame/lib/fetch-utils'
|
||||
import { program } from 'commander'
|
||||
|
||||
import { getContents, getPathsWithMatchingStrings } from '@/workflows/git-utils'
|
||||
@@ -188,18 +188,22 @@ async function main(opts: MainOptions, args: string[]) {
|
||||
await Promise.all(
|
||||
slice.map(async ({ linkPath, file }) => {
|
||||
// This isn't necessary but if it can't be constructed, it'll
|
||||
// fail in quite a nice way and not "blame got".
|
||||
// fail in quite a nice way and not "blame fetch".
|
||||
const url = new URL(BASE_URL + linkPath)
|
||||
try {
|
||||
await got.head(url.href, {
|
||||
retry: retryConfiguration,
|
||||
timeout: timeoutConfiguration,
|
||||
})
|
||||
await fetchWithRetry(
|
||||
url.href,
|
||||
{ method: 'HEAD' },
|
||||
{
|
||||
retries: retryConfiguration.limit,
|
||||
timeout: timeoutConfiguration.request,
|
||||
throwHttpErrors: true,
|
||||
},
|
||||
)
|
||||
} catch (error) {
|
||||
if (error instanceof RequestError) {
|
||||
if (error instanceof Error) {
|
||||
brokenLinks.push({ linkPath, file })
|
||||
} else {
|
||||
console.warn(`URL when it threw: ${url}`)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import path from 'path'
|
||||
|
||||
import cheerio from 'cheerio'
|
||||
import coreLib from '@actions/core'
|
||||
import got, { RequestError } from 'got'
|
||||
import { fetchWithRetry } from '@/frame/lib/fetch-utils'
|
||||
import chalk from 'chalk'
|
||||
import { JSONFilePreset } from 'lowdb/node'
|
||||
import { type Octokit } from '@octokit/rest'
|
||||
@@ -1166,31 +1166,35 @@ async function innerFetch(
|
||||
}
|
||||
|
||||
const retries = config.retries || 0
|
||||
const httpFunction = useGET ? got.get : got.head
|
||||
const method = useGET ? 'GET' : 'HEAD'
|
||||
|
||||
if (verbose) core.info(`External URL ${useGET ? 'GET' : 'HEAD'}: ${url} (retries: ${retries})`)
|
||||
if (verbose) core.info(`External URL ${method}: ${url} (retries: ${retries})`)
|
||||
try {
|
||||
const r = await httpFunction(url, {
|
||||
headers,
|
||||
throwHttpErrors: false,
|
||||
retry,
|
||||
timeout,
|
||||
})
|
||||
const r = await fetchWithRetry(
|
||||
url,
|
||||
{
|
||||
method,
|
||||
headers,
|
||||
},
|
||||
{
|
||||
retries: retry.limit,
|
||||
timeout: timeout.request,
|
||||
throwHttpErrors: false,
|
||||
},
|
||||
)
|
||||
if (verbose) {
|
||||
core.info(
|
||||
`External URL ${useGET ? 'GET' : 'HEAD'} ${url}: ${r.statusCode} (retries: ${retries})`,
|
||||
)
|
||||
core.info(`External URL ${method} ${url}: ${r.status} (retries: ${retries})`)
|
||||
}
|
||||
|
||||
// If we get rate limited, remember that this hostname is now all
|
||||
// rate limited. And sleep for the number of seconds that the
|
||||
// `retry-after` header indicated.
|
||||
if (r.statusCode === 429) {
|
||||
if (r.status === 429) {
|
||||
let sleepTime = Math.min(
|
||||
60_000,
|
||||
Math.max(
|
||||
10_000,
|
||||
r.headers['retry-after'] ? getRetryAfterSleep(r.headers['retry-after']) : 1_000,
|
||||
r.headers.get('retry-after') ? getRetryAfterSleep(r.headers.get('retry-after')) : 1_000,
|
||||
),
|
||||
)
|
||||
// Sprinkle a little jitter so it doesn't all start again all
|
||||
@@ -1214,17 +1218,17 @@ async function innerFetch(
|
||||
|
||||
// Perhaps the server doesn't support HEAD requests.
|
||||
// If so, try again with a regular GET.
|
||||
if ((r.statusCode === 405 || r.statusCode === 404 || r.statusCode === 403) && !useGET) {
|
||||
if ((r.status === 405 || r.status === 404 || r.status === 403) && !useGET) {
|
||||
return innerFetch(core, url, Object.assign({}, config, { useGET: true }))
|
||||
}
|
||||
if (verbose) {
|
||||
core.info((r.ok ? chalk.green : chalk.red)(`${r.statusCode} on ${url}`))
|
||||
core.info((r.ok ? chalk.green : chalk.red)(`${r.status} on ${url}`))
|
||||
}
|
||||
return { ok: r.ok, statusCode: r.statusCode }
|
||||
return { ok: r.ok, statusCode: r.status }
|
||||
} catch (err) {
|
||||
if (err instanceof RequestError) {
|
||||
if (err instanceof Error) {
|
||||
if (verbose) {
|
||||
core.info(chalk.yellow(`RequestError (${err.message}) on ${url}`))
|
||||
core.info(chalk.yellow(`Request Error (${err.message}) on ${url}`))
|
||||
}
|
||||
return { ok: false, requestError: err.message }
|
||||
}
|
||||
@@ -1233,7 +1237,7 @@ async function innerFetch(
|
||||
}
|
||||
|
||||
// Return number of milliseconds from a `Retry-After` header value
|
||||
function getRetryAfterSleep(headerValue: string) {
|
||||
function getRetryAfterSleep(headerValue: string | null) {
|
||||
if (!headerValue) return 0
|
||||
let ms = Math.round(parseFloat(headerValue) * 1000)
|
||||
if (isNaN(ms)) {
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
/* eslint-disable import/no-extraneous-dependencies */
|
||||
import fs from 'fs'
|
||||
|
||||
/* Writes string to file to be uploaded as an action artifact.
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
/* eslint-disable no-invalid-this */
|
||||
/* eslint-disable prettier/prettier */
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
||||
import { getAutomaticRequestLogger } from '@/observability/logger/middleware/get-automatic-request-logger'
|
||||
import type { Request, Response, NextFunction } from 'express'
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
/* eslint-disable prettier/prettier */
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
||||
import { createLogger } from '@/observability/logger'
|
||||
import { initLoggerContext, updateLoggerContext } from '@/observability/logger/lib/logger-context'
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
/* eslint-disable prettier/prettier */
|
||||
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'
|
||||
import { createLogger } from '@/observability/logger'
|
||||
|
||||
|
||||
@@ -3,11 +3,13 @@ import { readFile } from 'fs/promises'
|
||||
import { allVersions } from '@/versions/lib/all-versions'
|
||||
|
||||
// Translate the docs versioning nomenclature back to the OpenAPI names
|
||||
const invertedVersionMapping = JSON.parse(await readFile('src/rest/lib/config.json')).versionMapping
|
||||
const versionMapping = {}
|
||||
const invertedVersionMapping = JSON.parse(
|
||||
await readFile('src/rest/lib/config.json', 'utf8'),
|
||||
).versionMapping
|
||||
const versionMapping: Record<string, string> = {}
|
||||
Object.assign(
|
||||
versionMapping,
|
||||
...Object.entries(invertedVersionMapping).map(([a, b]) => ({ [b]: a })),
|
||||
...Object.entries(invertedVersionMapping).map(([a, b]) => ({ [b as string]: a })),
|
||||
)
|
||||
const openApiVersions = Object.values(allVersions)
|
||||
.map((version) => version.openApiVersionName)
|
||||
@@ -3,6 +3,17 @@ import dereferenceJsonSchema from 'dereference-json-schema'
|
||||
import { existsSync } from 'fs'
|
||||
import { readFile, readdir } from 'fs/promises'
|
||||
|
||||
// OpenAPI 3.0 schema interface with the properties we need to access
|
||||
// The dereference-json-schema library returns a DereferencedJSONSchema type
|
||||
// but the actual object contains OpenAPI-specific properties that aren't in that type
|
||||
interface OpenAPISchema {
|
||||
openapi?: string
|
||||
info?: any
|
||||
servers?: any[]
|
||||
paths?: Record<string, Record<string, any>>
|
||||
[key: string]: any
|
||||
}
|
||||
|
||||
export const MODELS_GATEWAY_ROOT = 'models-gateway'
|
||||
const MODELS_GATEWAY_PATH = 'docs/api'
|
||||
|
||||
@@ -40,14 +51,16 @@ export async function injectModelsSchema(schema: any, schemaName: string): Promi
|
||||
const deferencedYaml = dereferenceJsonSchema.dereferenceSync(loadedYaml)
|
||||
|
||||
// Copy over top-level OpenAPI fields
|
||||
schema.openapi = schema.openapi || deferencedYaml.openapi
|
||||
schema.info = schema.info || deferencedYaml.info
|
||||
schema.servers = schema.servers || deferencedYaml.servers
|
||||
// Cast to OpenAPISchema because dereference-json-schema doesn't include OpenAPI-specific properties in its type
|
||||
const openApiYaml = deferencedYaml as OpenAPISchema
|
||||
schema.openapi = schema.openapi || openApiYaml.openapi
|
||||
schema.info = schema.info || openApiYaml.info
|
||||
schema.servers = schema.servers || openApiYaml.servers
|
||||
|
||||
// Process each path and operation in the YAML
|
||||
for (const path of Object.keys(deferencedYaml.paths)) {
|
||||
for (const operation of Object.keys(deferencedYaml.paths[path])) {
|
||||
const operationObject = deferencedYaml.paths[path][operation]
|
||||
for (const path of Object.keys(openApiYaml.paths || {})) {
|
||||
for (const operation of Object.keys(openApiYaml.paths![path])) {
|
||||
const operationObject = openApiYaml.paths![path][operation]
|
||||
|
||||
// Use values from the YAML where possible
|
||||
const name = operationObject.summary || ''
|
||||
@@ -85,8 +98,8 @@ export async function injectModelsSchema(schema: any, schemaName: string): Promi
|
||||
|
||||
// Preserve operation-level servers if present
|
||||
// !Needed! to use models.github.ai instead of api.github.com
|
||||
if (deferencedYaml.servers) {
|
||||
enhancedOperation.servers = deferencedYaml.servers
|
||||
if (openApiYaml.servers) {
|
||||
enhancedOperation.servers = openApiYaml.servers
|
||||
}
|
||||
|
||||
// Add the enhanced operation to the schema
|
||||
|
||||
@@ -37,7 +37,7 @@ program
|
||||
|
||||
main(program.opts())
|
||||
|
||||
async function main(opts) {
|
||||
async function main(opts: { check?: boolean; dryRun?: boolean; verbose?: boolean }) {
|
||||
let errors = 0
|
||||
for (const file of MANDATORY_FILES) {
|
||||
const source = fs.readFileSync(file, 'utf-8')
|
||||
@@ -54,7 +54,7 @@ async function main(opts) {
|
||||
} else if (opts.verbose) {
|
||||
console.log(`The file ${chalk.green(destination)} is up-to-date 🥰`)
|
||||
}
|
||||
} catch (error) {
|
||||
} catch (error: any) {
|
||||
if (error.code === 'ENOENT') {
|
||||
console.warn(`The file ${chalk.red(destination)} does not exist`)
|
||||
errors++
|
||||
@@ -71,7 +71,7 @@ async function main(opts) {
|
||||
}
|
||||
continue
|
||||
}
|
||||
} catch (error) {
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') throw error
|
||||
}
|
||||
if (!opts.dryRun) {
|
||||
Reference in New Issue
Block a user