jprdonnelly/docs
1
0
Fork 0
mirror of synced 2025-12-20 18:36:31 -05:00
Code Issues Projects Releases Wiki Activity

filter OpenAPI dereferenced files (#32112)

Browse Source 
Co-authored-by: Grace Park <gracepark@github.com>
This commit is contained in:
Rachael Sewell
2022-11-04 16:27:39 -07:00
committed by GitHub
parent ed771c88be
commit 738c16f6fc
10 changed files with 657 additions and 268 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
script/rest/update-files.js
View File

@@ -12,12 +12,12 @@ import { program } from 'commander'
import { execSync } from 'child_process' import { execSync } from 'child_process'
import mkdirp from 'mkdirp' import mkdirp from 'mkdirp'
import rimraf from 'rimraf' import rimraf from 'rimraf'
import yaml from 'js-yaml'
import { decorate } from './utils/decorator.js' import { decorate } from './utils/decorator.js'
import { getSchemas } from './utils/get-openapi-schemas.js'
const TEMP_DOCS_DIR = path.join(process.cwd(), 'openapiTmp') const TEMP_DOCS_DIR = path.join(process.cwd(), 'openapiTmp')
const DEREFERENCED_DIR = path.join(process.cwd(), 'lib/rest/static/dereferenced') const DOCS_DEREF_OPENAPI_DIR = path.join(process.cwd(), 'lib/rest/static/dereferenced')
const GITHUB_REP_DIR = path.join(process.cwd(), '../github') const GITHUB_REP_DIR = path.join(process.cwd(), '../github')
const OPEN_API_RELEASES_DIR = path.join(GITHUB_REP_DIR, '/app/api/description/config/releases') const OPEN_API_RELEASES_DIR = path.join(GITHUB_REP_DIR, '/app/api/description/config/releases')
@@ -36,6 +36,7 @@ program
.parse(process.argv) .parse(process.argv)
const { decorateOnly, versions, includeUnpublished, includeDeprecated } = program.opts() const { decorateOnly, versions, includeUnpublished, includeDeprecated } = program.opts()
const versionsArray = versions ? versions.split(' ') : []
await validateInputParameters() await validateInputParameters()
@@ -45,7 +46,9 @@ async function main() {
// When the input parameter type is decorate-only, use the // When the input parameter type is decorate-only, use the
// `github/docs-internal` repo to generate a list of schema files. // `github/docs-internal` repo to generate a list of schema files.
// Otherwise, use the `github/github` list of config files // Otherwise, use the `github/github` list of config files
const schemas = decorateOnly ? await readdir(DEREFERENCED_DIR) : await getSchemas() const schemas = decorateOnly
? await readdir(DOCS_DEREF_OPENAPI_DIR)
: await getSchemas(OPEN_API_RELEASES_DIR, includeDeprecated, includeUnpublished, versionsArray)
// Generate the dereferenced OpenAPI schema files // Generate the dereferenced OpenAPI schema files
if (!decorateOnly) { if (!decorateOnly) {
@@ -53,7 +56,6 @@ async function main() {
} }
// Decorate the dereferenced files in a format ingestible by docs.github.com // Decorate the dereferenced files in a format ingestible by docs.github.com
await decorate(schemas) await decorate(schemas)
console.log( console.log(
'\n🏁 The static REST API files are now up-to-date with your local `github/github` checkout. To revert uncommitted changes, run `git checkout lib/rest/static/*`.\n\n' '\n🏁 The static REST API files are now up-to-date with your local `github/github` checkout. To revert uncommitted changes, run `git checkout lib/rest/static/*`.\n\n'
) )
@@ -95,7 +97,7 @@ async function getBundledFiles(schemas) {
} }
execSync( execSync(
`find ${TEMP_DOCS_DIR} -type f -name "*deref.json" -exec mv '{}' ${DEREFERENCED_DIR} ';'` `find ${TEMP_DOCS_DIR} -type f -name "*deref.json" -exec mv '{}' ${DOCS_DEREF_OPENAPI_DIR} ';'`
) )
rimraf.sync(TEMP_DOCS_DIR) rimraf.sync(TEMP_DOCS_DIR)
@@ -105,53 +107,13 @@ async function getBundledFiles(schemas) {
// name of the `github/github` checkout. A CI test // name of the `github/github` checkout. A CI test
// checks the version and fails if it's not a semantic version. // checks the version and fails if it's not a semantic version.
for (const filename of schemas) { for (const filename of schemas) {
const schema = JSON.parse(await readFile(path.join(DEREFERENCED_DIR, filename))) const schema = JSON.parse(await readFile(path.join(DOCS_DEREF_OPENAPI_DIR, filename)))
schema.info.version = `${githubBranch} !!DEVELOPMENT MODE - DO NOT MERGE!!` schema.info.version = `${githubBranch} !!DEVELOPMENT MODE - DO NOT MERGE!!`
await writeFile(path.join(DEREFERENCED_DIR, filename), JSON.stringify(schema, null, 2)) await writeFile(path.join(DOCS_DEREF_OPENAPI_DIR, filename), JSON.stringify(schema, null, 2))
} }
} }
// Gets the full list of unpublished, deprecated, and active schemas
// from the github/github repo
async function getSchemas() {
const openAPIConfigs = await readdir(OPEN_API_RELEASES_DIR)
const unpublished = []
const deprecated = []
const currentReleases = []
// The file content in the `github/github` repo is YAML before it is
// bundled into JSON.
for (const file of openAPIConfigs) {
const newFileName = `${path.basename(file, 'yaml')}deref.json`
const content = await readFile(path.join(OPEN_API_RELEASES_DIR, file), 'utf8')
const yamlContent = yaml.load(content)
if (!yamlContent.published) {
unpublished.push(newFileName)
}
if (yamlContent.deprecated) {
deprecated.push(newFileName)
}
if (!yamlContent.deprecated && yamlContent.published) {
currentReleases.push(newFileName)
}
}
const allSchemas = { currentReleases, unpublished, deprecated }
if (versions) {
await validateVersionsOptions(allSchemas)
return versions.map((elem) => `${elem}.deref.json`)
}
const schemas = allSchemas.currentReleases
if (includeUnpublished) {
schemas.push(...allSchemas.unpublished)
}
if (includeDeprecated) {
schemas.push(...allSchemas.deprecated)
}
return schemas
}
async function getBundlerOptions() { async function getBundlerOptions() {
let includeParams = [] let includeParams = []
@@ -188,18 +150,3 @@ async function validateInputParameters() {
} }
} }
} }
async function validateVersionsOptions(schemas) {
// Validate individual versions provided
versions.forEach((version) => {
if (
schemas.deprecated.includes(`${version}.deref.json`) ||
schemas.unpublished.includes(`${version}.deref.json`)
) {
const errorMsg = `🛑 This script doesn't support generating individual deprecated or unpublished schemas. Please reach out to #docs-engineering if this is a use case that you need.`
throw new Error(errorMsg)
} else if (!schemas.currentReleases.includes(`${version}.deref.json`)) {
throw new Error(`🛑 The version (${version}) you specified is not valid.`)
}
})
}

496
script/rest/utils/decorator.js
View File

@@ -2,230 +2,233 @@ import { existsSync, mkdirSync } from 'fs'
import { readFile, writeFile } from 'fs/promises' import { readFile, writeFile } from 'fs/promises'
import path from 'path' import path from 'path'
import { slug } from 'github-slugger' import { slug } from 'github-slugger'
import rimraf from 'rimraf'
import { allVersions } from '../../../lib/all-versions.js'
import { categoriesWithoutSubcategories } from '../../../lib/rest/index.js' import { categoriesWithoutSubcategories } from '../../../lib/rest/index.js'
import getOperations, { getWebhooks } from './get-operations.js' import getOperations, { getWebhooks } from './get-operations.js'
const appsStaticPath = path.join(process.cwd(), 'lib/rest/static/apps') const ENABLED_APPS = 'lib/rest/static/apps/enabled-for-apps.json'
const restDecoratedPath = path.join(process.cwd(), 'lib/rest/static/decorated') const STATIC_REDIRECTS = 'lib/redirects/static/client-side-rest-api-redirects.json'
const webhooksDecoratedPath = path.join(process.cwd(), 'lib/webhooks/static/decorated') const REST_DECORATED_DIR = 'lib/rest/static/decorated'
const dereferencedPath = path.join(process.cwd(), 'lib/rest/static/dereferenced') const WEBHOOK_DECORATED_DIR = 'lib/webhooks/static/decorated'
const REST_DEREFERENCED_DIR = 'lib/rest/static/dereferenced'
export async function decorate(schemas) { export async function decorate(schemas) {
console.log('\n🎄 Decorating the OpenAPI schema files in lib/rest/static/dereferenced.\n') console.log('\n🎄 Decorating the OpenAPI schema files in lib/rest/static/dereferenced.\n')
const dereferencedSchemas = {} const { restSchemas, webhookSchemas } = await getOpenApiSchemaFiles(schemas)
for (const filename of schemas) { const webhookOperations = await getWebhookOperations(webhookSchemas)
const schema = JSON.parse(await readFile(path.join(dereferencedPath, filename))) await createStaticWebhookFiles(webhookOperations)
const key = filename.replace('.deref.json', '') const restOperations = await getRestOperations(restSchemas)
dereferencedSchemas[key] = schema await createStaticRestFiles(restOperations)
} }
const operationsEnabledForGitHubApps = {} async function getRestOperations(restSchemas) {
const clientSideRedirects = await getCategoryOverrideRedirects() console.log('⏭️ Start generating static REST files\n')
const restSchemaData = await getDereferencedFiles(restSchemas)
for (const [schemaName, schema] of Object.entries(dereferencedSchemas)) { const restOperations = {}
for (const [schemaName, schema] of Object.entries(restSchemaData)) {
try { try {
// get all of the operations and wehbooks for a particular version of the openapi // get all of the operations and wehbooks for a particular version of the openapi
const operations = await getOperations(schema) const operations = await getOperations(schema)
const webhooks = await getWebhooks(schema)
// process each operation and webhook, asynchronously rendering markdown and stuff // process each operation and webhook, asynchronously rendering markdown and stuff
await Promise.all(operations.map((operation) => operation.process())) if (operations.length) {
await Promise.all(webhooks.map((webhook) => webhook.process())) console.log(`...processing ${schemaName} rest operations`)
await Promise.all(operations.map((operation) => operation.process()))
// For each rest operation that doesn't have an override defined restOperations[schemaName] = operations
// in script/rest/utils/rest-api-overrides.json,
// add a client-side redirect
operations.forEach((operation) => {
// A handful of operations don't have external docs properties
const externalDocs = operation.getExternalDocs()
if (!externalDocs) {
return
}
const oldUrl = `/rest${
externalDocs.url.replace('/rest/reference', '/rest').split('/rest')[1]
}`
if (!(oldUrl in clientSideRedirects)) {
// There are some operations that aren't nested in the sidebar
// For these, don't need to add a client-side redirect, the
// frontmatter redirect will handle it for us.
if (categoriesWithoutSubcategories.includes(operation.category)) {
return
}
const anchor = oldUrl.split('#')[1]
const subcategory = operation.subcategory
// If there is no subcategory, a new page with the same name as the
// category was created. That page name may change going forward.
const redirectTo = subcategory
? `/rest/${operation.category}/${subcategory}#${anchor}`
: `/rest/${operation.category}/${operation.category}#${anchor}`
clientSideRedirects[oldUrl] = redirectTo
}
// There are a lot of section headings that we'll want to redirect too,
// now that subcategories are on their own page. For example,
// /rest/reference/actions#artifacts should redirect to
// /rest/actions/artifacts
if (operation.subcategory) {
const sectionRedirectFrom = `/rest/${operation.category}#${operation.subcategory}`
const sectionRedirectTo = `/rest/${operation.category}/${operation.subcategory}`
if (!(sectionRedirectFrom in clientSideRedirects)) {
clientSideRedirects[sectionRedirectFrom] = sectionRedirectTo
}
}
})
const categories = [...new Set(operations.map((operation) => operation.category))].sort()
// Orders the operations by their category and subcategories.
// All operations must have a category, but operations don't need
// a subcategory. When no subcategory is present, the subcategory
// property is an empty string ('').
/*
Example:
{
[category]: {
'': {
"description": "",
"operations": []
},
[subcategory sorted alphabetically]: {
"description": "",
"operations": []
}
}
}
*/
const operationsByCategory = {}
categories.forEach((category) => {
operationsByCategory[category] = {}
const categoryOperations = operations.filter((operation) => operation.category === category)
categoryOperations
.filter((operation) => !operation.subcategory)
.map((operation) => (operation.subcategory = operation.category))
const subcategories = [
...new Set(categoryOperations.map((operation) => operation.subcategory)),
].sort()
// the first item should be the item that has no subcategory
// e.g., when the subcategory = category
const firstItemIndex = subcategories.indexOf(category)
if (firstItemIndex > -1) {
const firstItem = subcategories.splice(firstItemIndex, 1)[0]
subcategories.unshift(firstItem)
}
subcategories.forEach((subcategory) => {
operationsByCategory[category][subcategory] = {}
const subcategoryOperations = categoryOperations.filter(
(operation) => operation.subcategory === subcategory
)
operationsByCategory[category][subcategory] = subcategoryOperations
})
})
// Create a map of webhooks (e.g. check_run, issues, release) to the
// webhook's actions (e.g. created, deleted, etc.).
//
// Some webhooks like the ping webhook have no action types -- in cases
// like this we set a default action of 'default'.
//
// Example:
/*
{
'branch-protection-rule': {
created: Webhook {
descriptionHtml: '<p>A branch protection rule was created.</p>',
summaryHtml: '<p>This event occurs when there is activity relating to branch protection rules. For more information, see "<a href="https://docs.github.com/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/about-protected-branches">About protected branches</a>." For information about the Branch protection APIs, see <a href="https://docs.github.com/graphql/reference/objects#branchprotectionrule">the GraphQL documentation</a> and <a href="https://docs.github.com/rest/branches/branch-protection">the REST API documentation</a>.</p>\n' +
'<p>In order to install this event on a GitHub App, the app must have <code>read-only</code> access on repositories administration.</p>',
bodyParameters: [Array],
availability: [Array],
action: 'created',
category: 'branch-protection-rule'
},
deleted: Webhook {
descriptionHtml: '<p>A branch protection rule was deleted.</p>',
summaryHtml: '<p>This event occurs when there is activity relating to branch protection rules. For more information, see "<a href="https://docs.github.com/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/about-protected-branches">About protected branches</a>." For information about the Branch protection APIs, see <a href="https://docs.github.com/graphql/reference/objects#branchprotectionrule">the GraphQL documentation</a> and <a href="https://docs.github.com/rest/branches/branch-protection">the REST API documentation</a>.</p>\n' +
'<p>In order to install this event on a GitHub App, the app must have <code>read-only</code> access on repositories administration.</p>',
bodyParameters: [Array],
availability: [Array],
action: 'deleted',
category: 'branch-protection-rule'
},
...
}
*/
const categorizedWebhooks = {}
webhooks.forEach((webhook) => {
if (!webhook.action) webhook.action = 'default'
if (categorizedWebhooks[webhook.category]) {
categorizedWebhooks[webhook.category][webhook.action] = webhook
} else {
categorizedWebhooks[webhook.category] = {}
categorizedWebhooks[webhook.category][webhook.action] = webhook
}
})
const webhooksFilename = path
.join(webhooksDecoratedPath, `${schemaName}.json`)
.replace('.deref', '')
const restFilename = path.join(restDecoratedPath, `${schemaName}.json`).replace('.deref', '')
// write processed operations to disk
await writeFile(restFilename, JSON.stringify(operationsByCategory, null, 2))
console.log('Wrote', path.relative(process.cwd(), restFilename))
if (Object.keys(categorizedWebhooks).length > 0) {
if (!existsSync(webhooksDecoratedPath)) {
mkdirSync(webhooksDecoratedPath)
}
await writeFile(webhooksFilename, JSON.stringify(categorizedWebhooks, null, 2))
console.log('Wrote', path.relative(process.cwd(), webhooksFilename))
}
// Create the enabled-for-apps.json file used for
// https://docs.github.com/en/rest/overview/endpoints-available-for-github-apps
operationsEnabledForGitHubApps[schemaName] = {}
for (const category of categories) {
const categoryOperations = operations.filter((operation) => operation.category === category)
// This is a collection of operations that have `enabledForGitHubApps = true`
// It's grouped by resource title to make rendering easier
operationsEnabledForGitHubApps[schemaName][category] = categoryOperations
.filter((operation) => operation.enabledForGitHubApps)
.map((operation) => ({
slug: slug(operation.title),
subcategory: operation.subcategory,
verb: operation.verb,
requestPath: operation.requestPath,
}))
} }
} catch (error) { } catch (error) {
console.error(error) throw new Error(
console.log(
"🐛 Whoops! It looks like the decorator script wasn't able to parse the dereferenced schema. A recent change may not yet be supported by the decorator. Please reach out in the #docs-engineering slack channel for help." "🐛 Whoops! It looks like the decorator script wasn't able to parse the dereferenced schema. A recent change may not yet be supported by the decorator. Please reach out in the #docs-engineering slack channel for help."
) )
process.exit(1)
} }
} }
await writeFile( return restOperations
path.join(appsStaticPath, 'enabled-for-apps.json'), }
JSON.stringify(operationsEnabledForGitHubApps, null, 2)
) async function getWebhookOperations(webhookSchemas) {
console.log('Wrote', path.relative(process.cwd(), `${appsStaticPath}/enabled-for-apps.json`)) console.log('⏭️ Start generating static webhook files\n')
await writeFile( const webhookSchemaData = getDereferencedFiles(webhookSchemas)
'lib/redirects/static/client-side-rest-api-redirects.json', const webhookOperations = {}
JSON.stringify(clientSideRedirects, null, 2), for (const [schemaName, schema] of Object.entries(webhookSchemaData)) {
'utf8' try {
) const webhooks = await getWebhooks(schema)
console.log( if (webhooks.length) {
'Wrote', console.log(`...processing ${schemaName} webhook operations`)
path.relative(process.cwd(), `lib/redirects/static/client-side-rest-api-redirects.json`) await Promise.all(webhooks.map((webhook) => webhook.process()))
) webhookOperations[schemaName] = webhooks
}
} catch (error) {
throw new Error(
"🐛 Whoops! It looks like the decorator script wasn't able to parse the dereferenced schema. A recent change may not yet be supported by the decorator. Please reach out in the #docs-engineering slack channel for help."
)
}
}
return webhookOperations
}
async function createStaticRestFiles(restOperations) {
rimraf.sync(`${REST_DECORATED_DIR}/*`)
const operationsEnabledForGitHubApps = {}
const clientSideRedirects = await getCategoryOverrideRedirects()
for (const schemaName in restOperations) {
const operations = restOperations[schemaName]
await addRestClientSideRedirects(operations, clientSideRedirects)
const categories = [...new Set(operations.map((operation) => operation.category))].sort()
// Orders the operations by their category and subcategories.
// All operations must have a category, but operations don't need
// a subcategory. When no subcategory is present, the subcategory
// property is an empty string ('').
/*
Example:
{
[category]: {
'': {
"description": "",
"operations": []
},
[subcategory sorted alphabetically]: {
"description": "",
"operations": []
}
}
}
*/
const operationsByCategory = {}
categories.forEach((category) => {
operationsByCategory[category] = {}
const categoryOperations = operations.filter((operation) => operation.category === category)
categoryOperations
.filter((operation) => !operation.subcategory)
.map((operation) => (operation.subcategory = operation.category))
const subcategories = [
...new Set(categoryOperations.map((operation) => operation.subcategory)),
].sort()
// the first item should be the item that has no subcategory
// e.g., when the subcategory = category
const firstItemIndex = subcategories.indexOf(category)
if (firstItemIndex > -1) {
const firstItem = subcategories.splice(firstItemIndex, 1)[0]
subcategories.unshift(firstItem)
}
subcategories.forEach((subcategory) => {
operationsByCategory[category][subcategory] = {}
const subcategoryOperations = categoryOperations.filter(
(operation) => operation.subcategory === subcategory
)
operationsByCategory[category][subcategory] = subcategoryOperations
})
})
const restFilename = path.join(REST_DECORATED_DIR, `${schemaName}.json`).replace('.deref', '')
// write processed operations to disk
await writeFile(restFilename, JSON.stringify(operationsByCategory, null, 2))
console.log('Wrote', path.relative(process.cwd(), restFilename))
// Create the enabled-for-apps.json file used for
// https://docs.github.com/en/rest/overview/endpoints-available-for-github-apps
operationsEnabledForGitHubApps[schemaName] = {}
for (const category of categories) {
const categoryOperations = operations.filter((operation) => operation.category === category)
// This is a collection of operations that have `enabledForGitHubApps = true`
// It's grouped by resource title to make rendering easier
operationsEnabledForGitHubApps[schemaName][category] = categoryOperations
.filter((operation) => operation.enabledForGitHubApps)
.map((operation) => ({
slug: slug(operation.title),
subcategory: operation.subcategory,
verb: operation.verb,
requestPath: operation.requestPath,
}))
}
}
await writeFile(ENABLED_APPS, JSON.stringify(operationsEnabledForGitHubApps, null, 2))
console.log('Wrote', ENABLED_APPS)
await writeFile(STATIC_REDIRECTS, JSON.stringify(clientSideRedirects, null, 2), 'utf8')
console.log('Wrote', STATIC_REDIRECTS)
}
async function getDereferencedFiles(schemas) {
const schemaData = {}
for (const filename of schemas) {
const file = path.join(REST_DEREFERENCED_DIR, `${filename}.deref.json`)
const schema = JSON.parse(await readFile(file))
schemaData[filename] = schema
}
return schemaData
}
async function createStaticWebhookFiles(webhookSchemas) {
if (!Object.keys(webhookSchemas).length) {
console.log(
'🟡 No webhooks exist in the dereferenced files. No static webhook files will be generated.\n'
)
return
}
rimraf.sync(`${WEBHOOK_DECORATED_DIR}/*`)
// Create a map of webhooks (e.g. check_run, issues, release) to the
// webhook's actions (e.g. created, deleted, etc.).
//
// Some webhooks like the ping webhook have no action types -- in cases
// like this we set a default action of 'default'.
//
// Example:
/*
{
'branch-protection-rule': {
created: Webhook {
descriptionHtml: '<p>A branch protection rule was created.</p>',
summaryHtml: '<p>This event occurs when there is activity relating to branch protection rules. For more information, see "<a href="https://docs.github.com/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/about-protected-branches">About protected branches</a>." For information about the Branch protection APIs, see <a href="https://docs.github.com/graphql/reference/objects#branchprotectionrule">the GraphQL documentation</a> and <a href="https://docs.github.com/rest/branches/branch-protection">the REST API documentation</a>.</p>\n' +
'<p>In order to install this event on a GitHub App, the app must have <code>read-only</code> access on repositories administration.</p>',
bodyParameters: [Array],
availability: [Array],
action: 'created',
category: 'branch-protection-rule'
},
deleted: Webhook {
descriptionHtml: '<p>A branch protection rule was deleted.</p>',
summaryHtml: '<p>This event occurs when there is activity relating to branch protection rules. For more information, see "<a href="https://docs.github.com/repositories/configuring-branches-and-merges-in-your-repository/defining-the-mergeability-of-pull-requests/about-protected-branches">About protected branches</a>." For information about the Branch protection APIs, see <a href="https://docs.github.com/graphql/reference/objects#branchprotectionrule">the GraphQL documentation</a> and <a href="https://docs.github.com/rest/branches/branch-protection">the REST API documentation</a>.</p>\n' +
'<p>In order to install this event on a GitHub App, the app must have <code>read-only</code> access on repositories administration.</p>',
bodyParameters: [Array],
availability: [Array],
action: 'deleted',
category: 'branch-protection-rule'
},
...
}
*/
const categorizedWebhooks = {}
for (const [schemaName, webhooks] of Object.entries(webhookSchemas)) {
webhooks.forEach((webhook) => {
if (!webhook.action) webhook.action = 'default'
if (categorizedWebhooks[webhook.category]) {
categorizedWebhooks[webhook.category][webhook.action] = webhook
} else {
categorizedWebhooks[webhook.category] = {}
categorizedWebhooks[webhook.category][webhook.action] = webhook
}
})
const webhooksFilename = path
.join(WEBHOOK_DECORATED_DIR, `${schemaName}.json`)
.replace('.deref', '')
if (Object.keys(categorizedWebhooks).length > 0) {
if (!existsSync(WEBHOOK_DECORATED_DIR)) {
mkdirSync(WEBHOOK_DECORATED_DIR)
}
await writeFile(webhooksFilename, JSON.stringify(categorizedWebhooks, null, 2))
console.log('Wrote', path.relative(process.cwd(), webhooksFilename))
}
}
} }
async function getCategoryOverrideRedirects() { async function getCategoryOverrideRedirects() {
@@ -250,3 +253,84 @@ async function getCategoryOverrideRedirects() {
} }
return redirects return redirects
} }
async function addRestClientSideRedirects(operations, clientSideRedirects) {
// For each rest operation that doesn't have an override defined
// in script/rest/utils/rest-api-overrides.json,
// add a client-side redirect
operations.forEach((operation) => {
// A handful of operations don't have external docs properties
const externalDocs = operation.getExternalDocs()
if (!externalDocs) {
return
}
const oldUrl = `/rest${externalDocs.url.replace('/rest/reference', '/rest').split('/rest')[1]}`
if (!(oldUrl in clientSideRedirects)) {
// There are some operations that aren't nested in the sidebar
// For these, don't need to add a client-side redirect, the
// frontmatter redirect will handle it for us.
if (categoriesWithoutSubcategories.includes(operation.category)) {
return
}
const anchor = oldUrl.split('#')[1]
const subcategory = operation.subcategory
// If there is no subcategory, a new page with the same name as the
// category was created. That page name may change going forward.
const redirectTo = subcategory
? `/rest/${operation.category}/${subcategory}#${anchor}`
: `/rest/${operation.category}/${operation.category}#${anchor}`
clientSideRedirects[oldUrl] = redirectTo
}
// There are a lot of section headings that we'll want to redirect too,
// now that subcategories are on their own page. For example,
// /rest/reference/actions#artifacts should redirect to
// /rest/actions/artifacts
if (operation.subcategory) {
const sectionRedirectFrom = `/rest/${operation.category}#${operation.subcategory}`
const sectionRedirectTo = `/rest/${operation.category}/${operation.subcategory}`
if (!(sectionRedirectFrom in clientSideRedirects)) {
clientSideRedirects[sectionRedirectFrom] = sectionRedirectTo
}
}
})
}
export async function getOpenApiSchemaFiles(schemas) {
const webhookSchemas = []
const restSchemas = []
// All of the schema releases that we store in allVersions
// Ex: 'api.github.com', 'ghec', 'ghes-3.6', 'ghes-3.5',
// 'ghes-3.4', 'ghes-3.3', 'ghes-3.2', 'github.ae'
const openApiVersions = Object.keys(allVersions).map(
(elem) => allVersions[elem].openApiVersionName
)
// The full list of dereferened OpenAPI schemas received from
// bundling the OpenAPI in github/github
const schemaBaseNames = schemas.map((schema) => path.basename(schema, '.deref.json'))
for (const schema of schemaBaseNames) {
// catches all of the schemas that are not
// calendar date versioned. Ex: ghec, ghes-3.7, and api.github.com
if (openApiVersions.includes(schema)) {
webhookSchemas.push(schema)
// Non-calendar date schemas could also match the calendar date versioned
// counterpart.
// Ex: api.github.com would match api.github.com and
// api.github.com.2022-09-09
const filteredMatches = schemaBaseNames.filter((elem) => elem.includes(schema))
// If there is only one match then there are no calendar date counterparts
// and this is the only schema for this plan and release.
if (filteredMatches.length === 1) {
restSchemas.push(schema)
}
// catches all of the calendar date versioned schemas in the
// format api.github.com.<year>-<month>-<day>
} else {
restSchemas.push(schema)
}
}
return { restSchemas, webhookSchemas }
}

80
script/rest/utils/get-openapi-schemas.js Normal file
View File

@@ -0,0 +1,80 @@
import { readFile, readdir } from 'fs/promises'
import yaml from 'js-yaml'
import path from 'path'
import { allVersions } from '../../../lib/all-versions.js'
// Gets the full list of unpublished + active, deprecated + active,
// or active schemas from the github/github repo
// `openApiReleaseDir` is the path to the `app/api/description/config/releases`
// directory in `github/github`
// You can also specify getting specific versions of schemas.
export async function getSchemas(
openApiReleaseDir,
includeDeprecated = false,
includeUnpublished = false,
versions = []
) {
const openAPIConfigs = await readdir(openApiReleaseDir)
const unpublished = []
const deprecated = []
const currentReleases = []
// The file content in the `github/github` repo is YAML before it is
// bundled into JSON.
for (const file of openAPIConfigs) {
const fileBaseName = path.basename(file, '.yaml')
const newFileName = `${fileBaseName}.deref.json`
const content = await readFile(path.join(openApiReleaseDir, file), 'utf8')
const yamlContent = yaml.load(content)
const isDeprecatedInDocs = !Object.keys(allVersions).find(
(version) => allVersions[version].openApiVersionName === fileBaseName
)
if (!yamlContent.published) {
unpublished.push(newFileName)
}
// If it's deprecated, it must have been published at some point in the past
// This checks if the schema is deprecated in github/github and
// github/docs-internal. Sometimes deprecating in github/github lags
// behind deprecating in github/docs-internal a few days
if (
(yamlContent.deprecated && yamlContent.published) ||
(isDeprecatedInDocs && yamlContent.published)
) {
deprecated.push(newFileName)
}
if (!yamlContent.deprecated && !isDeprecatedInDocs && yamlContent.published) {
currentReleases.push(newFileName)
}
}
const allSchemas = { currentReleases, unpublished, deprecated }
if (versions.length) {
await validateVersionsOptions(allSchemas, versions)
return versions.map((elem) => `${elem}.deref.json`)
}
const schemas = allSchemas.currentReleases
if (includeUnpublished) {
schemas.push(...allSchemas.unpublished)
}
if (includeDeprecated) {
schemas.push(...allSchemas.deprecated)
}
return schemas
}
async function validateVersionsOptions(schemas, versions) {
// Validate individual versions provided
versions.forEach((version) => {
if (
schemas.deprecated.includes(`${version}.deref.json`) ||
schemas.unpublished.includes(`${version}.deref.json`)
) {
const errorMsg = `🛑 This script doesn't support generating individual deprecated or unpublished schemas. Please reach out to #docs-engineering if this is a use case that you need.`
throw new Error(errorMsg)
} else if (!schemas.currentReleases.includes(`${version}.deref.json`)) {
throw new Error(`🛑 The version (${version}) you specified is not valid.`)
}
})
}

19
tests/fixtures/openapi-release-configs/api.github.com.yaml vendored Normal file
View File

@@ -0,0 +1,19 @@
---
published: true
deprecated: false
variables:
externalDocsUrl: 'https://docs.github.com'
apiName: 'GitHub'
patch:
- op: add
path: /servers
value:
- url: https://api.github.com
- op: add
path: /externalDocs
value:
description: GitHub v3 REST API
url: https://docs.github.com/rest/
- op: add
path: /info/x-github-plan
value: api.github.com

31
tests/fixtures/openapi-release-configs/ghec.yaml vendored Normal file
View File

@@ -0,0 +1,31 @@
---
published: true
deprecated: false
variables:
externalDocsUrl: 'https://docs.github.com/enterprise-cloud@latest/'
apiName: 'GitHub Enterprise Cloud'
patch:
- op: add
path: /servers
value:
- url: https://api.github.com
- op: add
path: /externalDocs
value:
description: GitHub Enterprise Cloud REST API
url: https://docs.github.com/enterprise-cloud@latest/rest/
- op: add
path: /tags/-
value:
name: oidc
description: Endpoints to manage GitHub OIDC configuration using the REST API.
x-displayName: OIDC
- op: add
path: /tags/-
value:
name: scim
description: Provisioning of GitHub organization membership for SCIM-enabled providers.
x-displayName: SCIM
- op: add
path: /info/x-github-plan
value: ghec

30
tests/fixtures/openapi-release-configs/ghes-3.2.yaml vendored Normal file
View File

@@ -0,0 +1,30 @@
---
published: true
deprecated: false
variables:
externalDocsUrl: https://docs.github.com/enterprise-server@3.2
ghesVersion: '3.2'
apiName: GitHub Enterprise Server
patch:
- op: add
path: /servers
value:
- url: '{protocol}://{hostname}/api/v3'
variables:
hostname:
description: Self-hosted Enterprise Server or Enterprise Cloud hostname
default: HOSTNAME
protocol:
description: Self-hosted Enterprise Server or Enterprise Cloud protocol
default: http
- op: add
path: /externalDocs
value:
description: GitHub Enterprise Developer Docs
url: 'https://docs.github.com/enterprise-server@3.2/rest/'
- op: add
path: /info/x-github-plan
value: ghes
- op: add
path: /info/x-github-release
value: 3.2

42
tests/fixtures/openapi-release-configs/ghes-3.7.yaml vendored Normal file
View File

@@ -0,0 +1,42 @@
---
published: true
deprecated: false
variables:
externalDocsUrl: https://docs.github.com/enterprise-server@3.7
ghesVersion: '3.7'
apiName: GitHub Enterprise Server
patch:
- op: add
path: /servers
value:
- url: '{protocol}://{hostname}/api/v3'
variables:
hostname:
description: Self-hosted Enterprise Server hostname
default: HOSTNAME
protocol:
description: Self-hosted Enterprise Server protocol
default: http
- op: add
path: /externalDocs
value:
description: GitHub Enterprise Developer Docs
url: 'https://docs.github.com/enterprise-server@3.7/rest/'
- op: add
path: /tags/-
value:
name: oidc
description: Endpoints to manage GitHub OIDC configuration using the REST API.
x-displayName: OIDC
- op: add
path: /tags/-
value:
name: scim
description: Provisioning of GitHub organization membership for SCIM-enabled providers.
x-displayName: SCIM
- op: add
path: /info/x-github-plan
value: ghes
- op: add
path: /info/x-github-release
value: 3.7

42
tests/fixtures/openapi-release-configs/ghes-3.8.yaml vendored Normal file
View File

@@ -0,0 +1,42 @@
---
published: false
deprecated: false
variables:
externalDocsUrl: https://docs.github.com/enterprise-server@3.8
ghesVersion: '3.8'
apiName: GitHub Enterprise Server
patch:
- op: add
path: /servers
value:
- url: '{protocol}://{hostname}/api/v3'
variables:
hostname:
description: Self-hosted Enterprise Server hostname
default: HOSTNAME
protocol:
description: Self-hosted Enterprise Server protocol
default: http
- op: add
path: /externalDocs
value:
description: GitHub Enterprise Developer Docs
url: 'https://docs.github.com/enterprise-server@3.8/rest/'
- op: add
path: /tags/-
value:
name: oidc
description: Endpoints to manage GitHub OIDC configuration using the REST API.
x-displayName: OIDC
- op: add
path: /tags/-
value:
name: scim
description: Provisioning of GitHub organization membership for SCIM-enabled providers.
x-displayName: SCIM
- op: add
path: /info/x-github-plan
value: ghes
- op: add
path: /info/x-github-release
value: 3.8

23
tests/fixtures/openapi-release-configs/github.ae.yaml vendored Normal file
View File

@@ -0,0 +1,23 @@
---
published: true
deprecated: false
variables:
externalDocsUrl: https://docs.github.com/github-ae@latest
apiName: GitHub AE
patch:
- op: add
path: /servers
value:
- url: 'https://{hostname}/api/v3'
variables:
hostname:
description: Self-hosted GitHub AE hostname
default: HOSTNAME
- op: add
path: /externalDocs
value:
description: GitHub AE Developer Docs
url: 'https://docs.github.com/github-ae@latest/rest/'
- op: add
path: /info/x-github-plan
value: github.ae

91
tests/unit/openapi-decorator.js Normal file
View File

@@ -0,0 +1,91 @@
import { describe, expect } from '@jest/globals'
import path from 'path'
import { getOpenApiSchemaFiles } from '../../script/rest/utils/decorator.js'
import { getSchemas } from '../../script/rest/utils/get-openapi-schemas.js'
import { allVersions } from '../../lib/all-versions.js'
const supportedReleases = Object.keys(allVersions).map(
(version) => allVersions[version].openApiVersionName
)
describe('decorated static files are generated correctly from dereferenced openapi files', () => {
// If there is a request with no request body parameters and all of
// the responses have no content, then we can create a docs
// example for just status codes below 300. All other status codes will
// be listed in the status code table in the docs.
test('webhook schema list should not include calendar date versions', async () => {
const schemas = [
'api.github.com.2022-08-09.deref.json',
'api.github.com.2022-10-09.deref.json',
'api.github.com.2022-11-09.deref.json',
'ghec.2022-09-09.deref.json',
...supportedReleases,
]
const expectedRestSchemas = [
'api.github.com.2022-08-09',
'api.github.com.2022-10-09',
'api.github.com.2022-11-09',
'ghec.2022-09-09',
...supportedReleases.filter((release) => release !== 'ghec' && release !== 'api.github.com'),
]
const { restSchemas, webhookSchemas } = await getOpenApiSchemaFiles(schemas)
expect(restSchemas.sort()).toEqual(expectedRestSchemas.sort())
expect(webhookSchemas.sort()).toEqual(supportedReleases.sort())
})
test('deprecated schemas in docs-internal are not included', async () => {
const myPath = path.join(process.cwd(), 'tests/fixtures/openapi-release-configs')
const currentReleaseSchemas = await getSchemas(myPath)
const expectedSchemas = [
'api.github.com.deref.json',
'ghec.deref.json',
'ghes-3.7.deref.json',
'github.ae.deref.json',
]
expect(currentReleaseSchemas.sort()).toEqual(expectedSchemas.sort())
})
test('deprecated schemas in docs-internal are included when deprecated option is enabled', async () => {
const myPath = path.join(process.cwd(), 'tests/fixtures/openapi-release-configs')
const deprecatedSchemas = await getSchemas(myPath, true)
const expectedSchemas = [
'api.github.com.deref.json',
'ghec.deref.json',
'ghes-3.7.deref.json',
'github.ae.deref.json',
'ghes-3.2.deref.json',
]
expect(deprecatedSchemas.sort()).toEqual(expectedSchemas.sort())
})
test('unpublished schemas in github/github are included when unpublished option is enabled', async () => {
const myPath = path.join(process.cwd(), 'tests/fixtures/openapi-release-configs')
const unpublishedSchemas = await getSchemas(myPath, false, true)
const expectedSchemas = [
'api.github.com.deref.json',
'ghec.deref.json',
'ghes-3.7.deref.json',
'github.ae.deref.json',
'ghes-3.8.deref.json',
]
expect(unpublishedSchemas.sort()).toEqual(expectedSchemas.sort())
})
test('specifying specific openapi versions is successful', async () => {
const myPath = path.join(process.cwd(), 'tests/fixtures/openapi-release-configs')
const versionSchemas = await getSchemas(myPath, false, false, ['ghec', 'ghes-3.7'])
const expectedSchemas = ['ghec.deref.json', 'ghes-3.7.deref.json']
expect(versionSchemas.sort()).toEqual(expectedSchemas.sort())
})
test('specifying deprecated or unpublished versions fails', async () => {
const myPath = path.join(process.cwd(), 'tests/fixtures/openapi-release-configs')
// const testError = async () => {
// await getSchemas(myPath, false, false, ['ghes-3.8', 'ghes-3.2'])
// }
// await expect(await testError()).toThrow(Error)
await expect(getSchemas(myPath, false, false, ['ghes-3.8', 'ghes-3.2'])).rejects.toThrow()
})
})