diff --git a/script/rest/update-files.js b/script/rest/update-files.js index 386898667f..5eee8eace8 100755 --- a/script/rest/update-files.js +++ b/script/rest/update-files.js @@ -12,12 +12,12 @@ import { program } from 'commander' import { execSync } from 'child_process' import mkdirp from 'mkdirp' import rimraf from 'rimraf' -import yaml from 'js-yaml' import { decorate } from './utils/decorator.js' +import { getSchemas } from './utils/get-openapi-schemas.js' const TEMP_DOCS_DIR = path.join(process.cwd(), 'openapiTmp') -const DEREFERENCED_DIR = path.join(process.cwd(), 'lib/rest/static/dereferenced') +const DOCS_DEREF_OPENAPI_DIR = path.join(process.cwd(), 'lib/rest/static/dereferenced') const GITHUB_REP_DIR = path.join(process.cwd(), '../github') const OPEN_API_RELEASES_DIR = path.join(GITHUB_REP_DIR, '/app/api/description/config/releases') @@ -36,6 +36,7 @@ program .parse(process.argv) const { decorateOnly, versions, includeUnpublished, includeDeprecated } = program.opts() +const versionsArray = versions ? versions.split(' ') : [] await validateInputParameters() @@ -45,7 +46,9 @@ async function main() { // When the input parameter type is decorate-only, use the // `github/docs-internal` repo to generate a list of schema files. // Otherwise, use the `github/github` list of config files - const schemas = decorateOnly ? await readdir(DEREFERENCED_DIR) : await getSchemas() + const schemas = decorateOnly + ? await readdir(DOCS_DEREF_OPENAPI_DIR) + : await getSchemas(OPEN_API_RELEASES_DIR, includeDeprecated, includeUnpublished, versionsArray) // Generate the dereferenced OpenAPI schema files if (!decorateOnly) { @@ -53,7 +56,6 @@ async function main() { } // Decorate the dereferenced files in a format ingestible by docs.github.com await decorate(schemas) - console.log( '\nšŸ The static REST API files are now up-to-date with your local `github/github` checkout. To revert uncommitted changes, run `git checkout lib/rest/static/*`.\n\n' ) @@ -95,7 +97,7 @@ async function getBundledFiles(schemas) { } execSync( - `find ${TEMP_DOCS_DIR} -type f -name "*deref.json" -exec mv '{}' ${DEREFERENCED_DIR} ';'` + `find ${TEMP_DOCS_DIR} -type f -name "*deref.json" -exec mv '{}' ${DOCS_DEREF_OPENAPI_DIR} ';'` ) rimraf.sync(TEMP_DOCS_DIR) @@ -105,53 +107,13 @@ async function getBundledFiles(schemas) { // name of the `github/github` checkout. A CI test // checks the version and fails if it's not a semantic version. for (const filename of schemas) { - const schema = JSON.parse(await readFile(path.join(DEREFERENCED_DIR, filename))) + const schema = JSON.parse(await readFile(path.join(DOCS_DEREF_OPENAPI_DIR, filename))) schema.info.version = `${githubBranch} !!DEVELOPMENT MODE - DO NOT MERGE!!` - await writeFile(path.join(DEREFERENCED_DIR, filename), JSON.stringify(schema, null, 2)) + await writeFile(path.join(DOCS_DEREF_OPENAPI_DIR, filename), JSON.stringify(schema, null, 2)) } } -// Gets the full list of unpublished, deprecated, and active schemas -// from the github/github repo -async function getSchemas() { - const openAPIConfigs = await readdir(OPEN_API_RELEASES_DIR) - const unpublished = [] - const deprecated = [] - const currentReleases = [] - - // The file content in the `github/github` repo is YAML before it is - // bundled into JSON. - for (const file of openAPIConfigs) { - const newFileName = `${path.basename(file, 'yaml')}deref.json` - const content = await readFile(path.join(OPEN_API_RELEASES_DIR, file), 'utf8') - const yamlContent = yaml.load(content) - if (!yamlContent.published) { - unpublished.push(newFileName) - } - if (yamlContent.deprecated) { - deprecated.push(newFileName) - } - if (!yamlContent.deprecated && yamlContent.published) { - currentReleases.push(newFileName) - } - } - - const allSchemas = { currentReleases, unpublished, deprecated } - if (versions) { - await validateVersionsOptions(allSchemas) - return versions.map((elem) => `${elem}.deref.json`) - } - const schemas = allSchemas.currentReleases - if (includeUnpublished) { - schemas.push(...allSchemas.unpublished) - } - if (includeDeprecated) { - schemas.push(...allSchemas.deprecated) - } - return schemas -} - async function getBundlerOptions() { let includeParams = [] @@ -188,18 +150,3 @@ async function validateInputParameters() { } } } - -async function validateVersionsOptions(schemas) { - // Validate individual versions provided - versions.forEach((version) => { - if ( - schemas.deprecated.includes(`${version}.deref.json`) || - schemas.unpublished.includes(`${version}.deref.json`) - ) { - const errorMsg = `šŸ›‘ This script doesn't support generating individual deprecated or unpublished schemas. Please reach out to #docs-engineering if this is a use case that you need.` - throw new Error(errorMsg) - } else if (!schemas.currentReleases.includes(`${version}.deref.json`)) { - throw new Error(`šŸ›‘ The version (${version}) you specified is not valid.`) - } - }) -} diff --git a/script/rest/utils/decorator.js b/script/rest/utils/decorator.js index d00598ac37..b3c856addc 100644 --- a/script/rest/utils/decorator.js +++ b/script/rest/utils/decorator.js @@ -2,230 +2,233 @@ import { existsSync, mkdirSync } from 'fs' import { readFile, writeFile } from 'fs/promises' import path from 'path' import { slug } from 'github-slugger' +import rimraf from 'rimraf' +import { allVersions } from '../../../lib/all-versions.js' import { categoriesWithoutSubcategories } from '../../../lib/rest/index.js' import getOperations, { getWebhooks } from './get-operations.js' -const appsStaticPath = path.join(process.cwd(), 'lib/rest/static/apps') -const restDecoratedPath = path.join(process.cwd(), 'lib/rest/static/decorated') -const webhooksDecoratedPath = path.join(process.cwd(), 'lib/webhooks/static/decorated') -const dereferencedPath = path.join(process.cwd(), 'lib/rest/static/dereferenced') +const ENABLED_APPS = 'lib/rest/static/apps/enabled-for-apps.json' +const STATIC_REDIRECTS = 'lib/redirects/static/client-side-rest-api-redirects.json' +const REST_DECORATED_DIR = 'lib/rest/static/decorated' +const WEBHOOK_DECORATED_DIR = 'lib/webhooks/static/decorated' +const REST_DEREFERENCED_DIR = 'lib/rest/static/dereferenced' export async function decorate(schemas) { console.log('\nšŸŽ„ Decorating the OpenAPI schema files in lib/rest/static/dereferenced.\n') - const dereferencedSchemas = {} - for (const filename of schemas) { - const schema = JSON.parse(await readFile(path.join(dereferencedPath, filename))) - const key = filename.replace('.deref.json', '') - dereferencedSchemas[key] = schema - } + const { restSchemas, webhookSchemas } = await getOpenApiSchemaFiles(schemas) + const webhookOperations = await getWebhookOperations(webhookSchemas) + await createStaticWebhookFiles(webhookOperations) + const restOperations = await getRestOperations(restSchemas) + await createStaticRestFiles(restOperations) +} - const operationsEnabledForGitHubApps = {} - const clientSideRedirects = await getCategoryOverrideRedirects() - - for (const [schemaName, schema] of Object.entries(dereferencedSchemas)) { +async function getRestOperations(restSchemas) { + console.log('ā­ļø Start generating static REST files\n') + const restSchemaData = await getDereferencedFiles(restSchemas) + const restOperations = {} + for (const [schemaName, schema] of Object.entries(restSchemaData)) { try { // get all of the operations and wehbooks for a particular version of the openapi const operations = await getOperations(schema) - const webhooks = await getWebhooks(schema) // process each operation and webhook, asynchronously rendering markdown and stuff - await Promise.all(operations.map((operation) => operation.process())) - await Promise.all(webhooks.map((webhook) => webhook.process())) - - // For each rest operation that doesn't have an override defined - // in script/rest/utils/rest-api-overrides.json, - // add a client-side redirect - operations.forEach((operation) => { - // A handful of operations don't have external docs properties - const externalDocs = operation.getExternalDocs() - if (!externalDocs) { - return - } - const oldUrl = `/rest${ - externalDocs.url.replace('/rest/reference', '/rest').split('/rest')[1] - }` - - if (!(oldUrl in clientSideRedirects)) { - // There are some operations that aren't nested in the sidebar - // For these, don't need to add a client-side redirect, the - // frontmatter redirect will handle it for us. - if (categoriesWithoutSubcategories.includes(operation.category)) { - return - } - const anchor = oldUrl.split('#')[1] - const subcategory = operation.subcategory - - // If there is no subcategory, a new page with the same name as the - // category was created. That page name may change going forward. - const redirectTo = subcategory - ? `/rest/${operation.category}/${subcategory}#${anchor}` - : `/rest/${operation.category}/${operation.category}#${anchor}` - clientSideRedirects[oldUrl] = redirectTo - } - - // There are a lot of section headings that we'll want to redirect too, - // now that subcategories are on their own page. For example, - // /rest/reference/actions#artifacts should redirect to - // /rest/actions/artifacts - if (operation.subcategory) { - const sectionRedirectFrom = `/rest/${operation.category}#${operation.subcategory}` - const sectionRedirectTo = `/rest/${operation.category}/${operation.subcategory}` - if (!(sectionRedirectFrom in clientSideRedirects)) { - clientSideRedirects[sectionRedirectFrom] = sectionRedirectTo - } - } - }) - - const categories = [...new Set(operations.map((operation) => operation.category))].sort() - - // Orders the operations by their category and subcategories. - // All operations must have a category, but operations don't need - // a subcategory. When no subcategory is present, the subcategory - // property is an empty string (''). - /* - Example: - { - [category]: { - '': { - "description": "", - "operations": [] - }, - [subcategory sorted alphabetically]: { - "description": "", - "operations": [] - } - } - } - */ - const operationsByCategory = {} - categories.forEach((category) => { - operationsByCategory[category] = {} - const categoryOperations = operations.filter((operation) => operation.category === category) - categoryOperations - .filter((operation) => !operation.subcategory) - .map((operation) => (operation.subcategory = operation.category)) - - const subcategories = [ - ...new Set(categoryOperations.map((operation) => operation.subcategory)), - ].sort() - // the first item should be the item that has no subcategory - // e.g., when the subcategory = category - const firstItemIndex = subcategories.indexOf(category) - if (firstItemIndex > -1) { - const firstItem = subcategories.splice(firstItemIndex, 1)[0] - subcategories.unshift(firstItem) - } - - subcategories.forEach((subcategory) => { - operationsByCategory[category][subcategory] = {} - - const subcategoryOperations = categoryOperations.filter( - (operation) => operation.subcategory === subcategory - ) - - operationsByCategory[category][subcategory] = subcategoryOperations - }) - }) - - // Create a map of webhooks (e.g. check_run, issues, release) to the - // webhook's actions (e.g. created, deleted, etc.). - // - // Some webhooks like the ping webhook have no action types -- in cases - // like this we set a default action of 'default'. - // - // Example: - /* - { - 'branch-protection-rule': { - created: Webhook { - descriptionHtml: '

A branch protection rule was created.

', - summaryHtml: '

This event occurs when there is activity relating to branch protection rules. For more information, see "About protected branches." For information about the Branch protection APIs, see the GraphQL documentation and the REST API documentation.

\n' + - '

In order to install this event on a GitHub App, the app must have read-only access on repositories administration.

', - bodyParameters: [Array], - availability: [Array], - action: 'created', - category: 'branch-protection-rule' - }, - deleted: Webhook { - descriptionHtml: '

A branch protection rule was deleted.

', - summaryHtml: '

This event occurs when there is activity relating to branch protection rules. For more information, see "About protected branches." For information about the Branch protection APIs, see the GraphQL documentation and the REST API documentation.

\n' + - '

In order to install this event on a GitHub App, the app must have read-only access on repositories administration.

', - bodyParameters: [Array], - availability: [Array], - action: 'deleted', - category: 'branch-protection-rule' - }, - ... - } - */ - const categorizedWebhooks = {} - - webhooks.forEach((webhook) => { - if (!webhook.action) webhook.action = 'default' - - if (categorizedWebhooks[webhook.category]) { - categorizedWebhooks[webhook.category][webhook.action] = webhook - } else { - categorizedWebhooks[webhook.category] = {} - categorizedWebhooks[webhook.category][webhook.action] = webhook - } - }) - - const webhooksFilename = path - .join(webhooksDecoratedPath, `${schemaName}.json`) - .replace('.deref', '') - const restFilename = path.join(restDecoratedPath, `${schemaName}.json`).replace('.deref', '') - - // write processed operations to disk - await writeFile(restFilename, JSON.stringify(operationsByCategory, null, 2)) - console.log('Wrote', path.relative(process.cwd(), restFilename)) - if (Object.keys(categorizedWebhooks).length > 0) { - if (!existsSync(webhooksDecoratedPath)) { - mkdirSync(webhooksDecoratedPath) - } - await writeFile(webhooksFilename, JSON.stringify(categorizedWebhooks, null, 2)) - console.log('Wrote', path.relative(process.cwd(), webhooksFilename)) - } - - // Create the enabled-for-apps.json file used for - // https://docs.github.com/en/rest/overview/endpoints-available-for-github-apps - operationsEnabledForGitHubApps[schemaName] = {} - for (const category of categories) { - const categoryOperations = operations.filter((operation) => operation.category === category) - - // This is a collection of operations that have `enabledForGitHubApps = true` - // It's grouped by resource title to make rendering easier - operationsEnabledForGitHubApps[schemaName][category] = categoryOperations - .filter((operation) => operation.enabledForGitHubApps) - .map((operation) => ({ - slug: slug(operation.title), - subcategory: operation.subcategory, - verb: operation.verb, - requestPath: operation.requestPath, - })) + if (operations.length) { + console.log(`...processing ${schemaName} rest operations`) + await Promise.all(operations.map((operation) => operation.process())) + restOperations[schemaName] = operations } } catch (error) { - console.error(error) - console.log( + throw new Error( "šŸ› Whoops! It looks like the decorator script wasn't able to parse the dereferenced schema. A recent change may not yet be supported by the decorator. Please reach out in the #docs-engineering slack channel for help." ) - process.exit(1) } } - await writeFile( - path.join(appsStaticPath, 'enabled-for-apps.json'), - JSON.stringify(operationsEnabledForGitHubApps, null, 2) - ) - console.log('Wrote', path.relative(process.cwd(), `${appsStaticPath}/enabled-for-apps.json`)) - await writeFile( - 'lib/redirects/static/client-side-rest-api-redirects.json', - JSON.stringify(clientSideRedirects, null, 2), - 'utf8' - ) - console.log( - 'Wrote', - path.relative(process.cwd(), `lib/redirects/static/client-side-rest-api-redirects.json`) - ) + return restOperations +} + +async function getWebhookOperations(webhookSchemas) { + console.log('ā­ļø Start generating static webhook files\n') + const webhookSchemaData = getDereferencedFiles(webhookSchemas) + const webhookOperations = {} + for (const [schemaName, schema] of Object.entries(webhookSchemaData)) { + try { + const webhooks = await getWebhooks(schema) + if (webhooks.length) { + console.log(`...processing ${schemaName} webhook operations`) + await Promise.all(webhooks.map((webhook) => webhook.process())) + webhookOperations[schemaName] = webhooks + } + } catch (error) { + throw new Error( + "šŸ› Whoops! It looks like the decorator script wasn't able to parse the dereferenced schema. A recent change may not yet be supported by the decorator. Please reach out in the #docs-engineering slack channel for help." + ) + } + } + return webhookOperations +} + +async function createStaticRestFiles(restOperations) { + rimraf.sync(`${REST_DECORATED_DIR}/*`) + const operationsEnabledForGitHubApps = {} + const clientSideRedirects = await getCategoryOverrideRedirects() + for (const schemaName in restOperations) { + const operations = restOperations[schemaName] + await addRestClientSideRedirects(operations, clientSideRedirects) + + const categories = [...new Set(operations.map((operation) => operation.category))].sort() + + // Orders the operations by their category and subcategories. + // All operations must have a category, but operations don't need + // a subcategory. When no subcategory is present, the subcategory + // property is an empty string (''). + /* + Example: + { + [category]: { + '': { + "description": "", + "operations": [] + }, + [subcategory sorted alphabetically]: { + "description": "", + "operations": [] + } + } + } + */ + const operationsByCategory = {} + categories.forEach((category) => { + operationsByCategory[category] = {} + const categoryOperations = operations.filter((operation) => operation.category === category) + categoryOperations + .filter((operation) => !operation.subcategory) + .map((operation) => (operation.subcategory = operation.category)) + + const subcategories = [ + ...new Set(categoryOperations.map((operation) => operation.subcategory)), + ].sort() + // the first item should be the item that has no subcategory + // e.g., when the subcategory = category + const firstItemIndex = subcategories.indexOf(category) + if (firstItemIndex > -1) { + const firstItem = subcategories.splice(firstItemIndex, 1)[0] + subcategories.unshift(firstItem) + } + + subcategories.forEach((subcategory) => { + operationsByCategory[category][subcategory] = {} + + const subcategoryOperations = categoryOperations.filter( + (operation) => operation.subcategory === subcategory + ) + + operationsByCategory[category][subcategory] = subcategoryOperations + }) + }) + + const restFilename = path.join(REST_DECORATED_DIR, `${schemaName}.json`).replace('.deref', '') + + // write processed operations to disk + await writeFile(restFilename, JSON.stringify(operationsByCategory, null, 2)) + console.log('Wrote', path.relative(process.cwd(), restFilename)) + + // Create the enabled-for-apps.json file used for + // https://docs.github.com/en/rest/overview/endpoints-available-for-github-apps + operationsEnabledForGitHubApps[schemaName] = {} + for (const category of categories) { + const categoryOperations = operations.filter((operation) => operation.category === category) + + // This is a collection of operations that have `enabledForGitHubApps = true` + // It's grouped by resource title to make rendering easier + operationsEnabledForGitHubApps[schemaName][category] = categoryOperations + .filter((operation) => operation.enabledForGitHubApps) + .map((operation) => ({ + slug: slug(operation.title), + subcategory: operation.subcategory, + verb: operation.verb, + requestPath: operation.requestPath, + })) + } + } + + await writeFile(ENABLED_APPS, JSON.stringify(operationsEnabledForGitHubApps, null, 2)) + console.log('Wrote', ENABLED_APPS) + await writeFile(STATIC_REDIRECTS, JSON.stringify(clientSideRedirects, null, 2), 'utf8') + console.log('Wrote', STATIC_REDIRECTS) +} + +async function getDereferencedFiles(schemas) { + const schemaData = {} + for (const filename of schemas) { + const file = path.join(REST_DEREFERENCED_DIR, `${filename}.deref.json`) + const schema = JSON.parse(await readFile(file)) + schemaData[filename] = schema + } + return schemaData +} + +async function createStaticWebhookFiles(webhookSchemas) { + if (!Object.keys(webhookSchemas).length) { + console.log( + 'šŸŸ” No webhooks exist in the dereferenced files. No static webhook files will be generated.\n' + ) + return + } + rimraf.sync(`${WEBHOOK_DECORATED_DIR}/*`) + // Create a map of webhooks (e.g. check_run, issues, release) to the + // webhook's actions (e.g. created, deleted, etc.). + // + // Some webhooks like the ping webhook have no action types -- in cases + // like this we set a default action of 'default'. + // + // Example: + /* + { + 'branch-protection-rule': { + created: Webhook { + descriptionHtml: '

A branch protection rule was created.

', + summaryHtml: '

This event occurs when there is activity relating to branch protection rules. For more information, see "About protected branches." For information about the Branch protection APIs, see the GraphQL documentation and the REST API documentation.

\n' + + '

In order to install this event on a GitHub App, the app must have read-only access on repositories administration.

', + bodyParameters: [Array], + availability: [Array], + action: 'created', + category: 'branch-protection-rule' + }, + deleted: Webhook { + descriptionHtml: '

A branch protection rule was deleted.

', + summaryHtml: '

This event occurs when there is activity relating to branch protection rules. For more information, see "About protected branches." For information about the Branch protection APIs, see the GraphQL documentation and the REST API documentation.

\n' + + '

In order to install this event on a GitHub App, the app must have read-only access on repositories administration.

', + bodyParameters: [Array], + availability: [Array], + action: 'deleted', + category: 'branch-protection-rule' + }, + ... + } + */ + const categorizedWebhooks = {} + for (const [schemaName, webhooks] of Object.entries(webhookSchemas)) { + webhooks.forEach((webhook) => { + if (!webhook.action) webhook.action = 'default' + + if (categorizedWebhooks[webhook.category]) { + categorizedWebhooks[webhook.category][webhook.action] = webhook + } else { + categorizedWebhooks[webhook.category] = {} + categorizedWebhooks[webhook.category][webhook.action] = webhook + } + }) + const webhooksFilename = path + .join(WEBHOOK_DECORATED_DIR, `${schemaName}.json`) + .replace('.deref', '') + if (Object.keys(categorizedWebhooks).length > 0) { + if (!existsSync(WEBHOOK_DECORATED_DIR)) { + mkdirSync(WEBHOOK_DECORATED_DIR) + } + await writeFile(webhooksFilename, JSON.stringify(categorizedWebhooks, null, 2)) + console.log('Wrote', path.relative(process.cwd(), webhooksFilename)) + } + } } async function getCategoryOverrideRedirects() { @@ -250,3 +253,84 @@ async function getCategoryOverrideRedirects() { } return redirects } + +async function addRestClientSideRedirects(operations, clientSideRedirects) { + // For each rest operation that doesn't have an override defined + // in script/rest/utils/rest-api-overrides.json, + // add a client-side redirect + operations.forEach((operation) => { + // A handful of operations don't have external docs properties + const externalDocs = operation.getExternalDocs() + if (!externalDocs) { + return + } + const oldUrl = `/rest${externalDocs.url.replace('/rest/reference', '/rest').split('/rest')[1]}` + + if (!(oldUrl in clientSideRedirects)) { + // There are some operations that aren't nested in the sidebar + // For these, don't need to add a client-side redirect, the + // frontmatter redirect will handle it for us. + if (categoriesWithoutSubcategories.includes(operation.category)) { + return + } + const anchor = oldUrl.split('#')[1] + const subcategory = operation.subcategory + + // If there is no subcategory, a new page with the same name as the + // category was created. That page name may change going forward. + const redirectTo = subcategory + ? `/rest/${operation.category}/${subcategory}#${anchor}` + : `/rest/${operation.category}/${operation.category}#${anchor}` + clientSideRedirects[oldUrl] = redirectTo + } + + // There are a lot of section headings that we'll want to redirect too, + // now that subcategories are on their own page. For example, + // /rest/reference/actions#artifacts should redirect to + // /rest/actions/artifacts + if (operation.subcategory) { + const sectionRedirectFrom = `/rest/${operation.category}#${operation.subcategory}` + const sectionRedirectTo = `/rest/${operation.category}/${operation.subcategory}` + if (!(sectionRedirectFrom in clientSideRedirects)) { + clientSideRedirects[sectionRedirectFrom] = sectionRedirectTo + } + } + }) +} + +export async function getOpenApiSchemaFiles(schemas) { + const webhookSchemas = [] + const restSchemas = [] + + // All of the schema releases that we store in allVersions + // Ex: 'api.github.com', 'ghec', 'ghes-3.6', 'ghes-3.5', + // 'ghes-3.4', 'ghes-3.3', 'ghes-3.2', 'github.ae' + const openApiVersions = Object.keys(allVersions).map( + (elem) => allVersions[elem].openApiVersionName + ) + // The full list of dereferened OpenAPI schemas received from + // bundling the OpenAPI in github/github + const schemaBaseNames = schemas.map((schema) => path.basename(schema, '.deref.json')) + for (const schema of schemaBaseNames) { + // catches all of the schemas that are not + // calendar date versioned. Ex: ghec, ghes-3.7, and api.github.com + if (openApiVersions.includes(schema)) { + webhookSchemas.push(schema) + // Non-calendar date schemas could also match the calendar date versioned + // counterpart. + // Ex: api.github.com would match api.github.com and + // api.github.com.2022-09-09 + const filteredMatches = schemaBaseNames.filter((elem) => elem.includes(schema)) + // If there is only one match then there are no calendar date counterparts + // and this is the only schema for this plan and release. + if (filteredMatches.length === 1) { + restSchemas.push(schema) + } + // catches all of the calendar date versioned schemas in the + // format api.github.com.-- + } else { + restSchemas.push(schema) + } + } + return { restSchemas, webhookSchemas } +} diff --git a/script/rest/utils/get-openapi-schemas.js b/script/rest/utils/get-openapi-schemas.js new file mode 100644 index 0000000000..ef786b50ea --- /dev/null +++ b/script/rest/utils/get-openapi-schemas.js @@ -0,0 +1,80 @@ +import { readFile, readdir } from 'fs/promises' +import yaml from 'js-yaml' +import path from 'path' + +import { allVersions } from '../../../lib/all-versions.js' + +// Gets the full list of unpublished + active, deprecated + active, +// or active schemas from the github/github repo +// `openApiReleaseDir` is the path to the `app/api/description/config/releases` +// directory in `github/github` +// You can also specify getting specific versions of schemas. +export async function getSchemas( + openApiReleaseDir, + includeDeprecated = false, + includeUnpublished = false, + versions = [] +) { + const openAPIConfigs = await readdir(openApiReleaseDir) + const unpublished = [] + const deprecated = [] + const currentReleases = [] + + // The file content in the `github/github` repo is YAML before it is + // bundled into JSON. + for (const file of openAPIConfigs) { + const fileBaseName = path.basename(file, '.yaml') + const newFileName = `${fileBaseName}.deref.json` + const content = await readFile(path.join(openApiReleaseDir, file), 'utf8') + const yamlContent = yaml.load(content) + + const isDeprecatedInDocs = !Object.keys(allVersions).find( + (version) => allVersions[version].openApiVersionName === fileBaseName + ) + if (!yamlContent.published) { + unpublished.push(newFileName) + } + // If it's deprecated, it must have been published at some point in the past + // This checks if the schema is deprecated in github/github and + // github/docs-internal. Sometimes deprecating in github/github lags + // behind deprecating in github/docs-internal a few days + if ( + (yamlContent.deprecated && yamlContent.published) || + (isDeprecatedInDocs && yamlContent.published) + ) { + deprecated.push(newFileName) + } + if (!yamlContent.deprecated && !isDeprecatedInDocs && yamlContent.published) { + currentReleases.push(newFileName) + } + } + + const allSchemas = { currentReleases, unpublished, deprecated } + if (versions.length) { + await validateVersionsOptions(allSchemas, versions) + return versions.map((elem) => `${elem}.deref.json`) + } + const schemas = allSchemas.currentReleases + if (includeUnpublished) { + schemas.push(...allSchemas.unpublished) + } + if (includeDeprecated) { + schemas.push(...allSchemas.deprecated) + } + return schemas +} + +async function validateVersionsOptions(schemas, versions) { + // Validate individual versions provided + versions.forEach((version) => { + if ( + schemas.deprecated.includes(`${version}.deref.json`) || + schemas.unpublished.includes(`${version}.deref.json`) + ) { + const errorMsg = `šŸ›‘ This script doesn't support generating individual deprecated or unpublished schemas. Please reach out to #docs-engineering if this is a use case that you need.` + throw new Error(errorMsg) + } else if (!schemas.currentReleases.includes(`${version}.deref.json`)) { + throw new Error(`šŸ›‘ The version (${version}) you specified is not valid.`) + } + }) +} diff --git a/tests/fixtures/openapi-release-configs/api.github.com.yaml b/tests/fixtures/openapi-release-configs/api.github.com.yaml new file mode 100644 index 0000000000..8836967737 --- /dev/null +++ b/tests/fixtures/openapi-release-configs/api.github.com.yaml @@ -0,0 +1,19 @@ +--- +published: true +deprecated: false +variables: + externalDocsUrl: 'https://docs.github.com' + apiName: 'GitHub' +patch: + - op: add + path: /servers + value: + - url: https://api.github.com + - op: add + path: /externalDocs + value: + description: GitHub v3 REST API + url: https://docs.github.com/rest/ + - op: add + path: /info/x-github-plan + value: api.github.com diff --git a/tests/fixtures/openapi-release-configs/ghec.yaml b/tests/fixtures/openapi-release-configs/ghec.yaml new file mode 100644 index 0000000000..17933dc841 --- /dev/null +++ b/tests/fixtures/openapi-release-configs/ghec.yaml @@ -0,0 +1,31 @@ +--- +published: true +deprecated: false +variables: + externalDocsUrl: 'https://docs.github.com/enterprise-cloud@latest/' + apiName: 'GitHub Enterprise Cloud' +patch: + - op: add + path: /servers + value: + - url: https://api.github.com + - op: add + path: /externalDocs + value: + description: GitHub Enterprise Cloud REST API + url: https://docs.github.com/enterprise-cloud@latest/rest/ + - op: add + path: /tags/- + value: + name: oidc + description: Endpoints to manage GitHub OIDC configuration using the REST API. + x-displayName: OIDC + - op: add + path: /tags/- + value: + name: scim + description: Provisioning of GitHub organization membership for SCIM-enabled providers. + x-displayName: SCIM + - op: add + path: /info/x-github-plan + value: ghec diff --git a/tests/fixtures/openapi-release-configs/ghes-3.2.yaml b/tests/fixtures/openapi-release-configs/ghes-3.2.yaml new file mode 100644 index 0000000000..b01773c9bb --- /dev/null +++ b/tests/fixtures/openapi-release-configs/ghes-3.2.yaml @@ -0,0 +1,30 @@ +--- +published: true +deprecated: false +variables: + externalDocsUrl: https://docs.github.com/enterprise-server@3.2 + ghesVersion: '3.2' + apiName: GitHub Enterprise Server +patch: + - op: add + path: /servers + value: + - url: '{protocol}://{hostname}/api/v3' + variables: + hostname: + description: Self-hosted Enterprise Server or Enterprise Cloud hostname + default: HOSTNAME + protocol: + description: Self-hosted Enterprise Server or Enterprise Cloud protocol + default: http + - op: add + path: /externalDocs + value: + description: GitHub Enterprise Developer Docs + url: 'https://docs.github.com/enterprise-server@3.2/rest/' + - op: add + path: /info/x-github-plan + value: ghes + - op: add + path: /info/x-github-release + value: 3.2 diff --git a/tests/fixtures/openapi-release-configs/ghes-3.7.yaml b/tests/fixtures/openapi-release-configs/ghes-3.7.yaml new file mode 100644 index 0000000000..4231891a76 --- /dev/null +++ b/tests/fixtures/openapi-release-configs/ghes-3.7.yaml @@ -0,0 +1,42 @@ +--- +published: true +deprecated: false +variables: + externalDocsUrl: https://docs.github.com/enterprise-server@3.7 + ghesVersion: '3.7' + apiName: GitHub Enterprise Server +patch: + - op: add + path: /servers + value: + - url: '{protocol}://{hostname}/api/v3' + variables: + hostname: + description: Self-hosted Enterprise Server hostname + default: HOSTNAME + protocol: + description: Self-hosted Enterprise Server protocol + default: http + - op: add + path: /externalDocs + value: + description: GitHub Enterprise Developer Docs + url: 'https://docs.github.com/enterprise-server@3.7/rest/' + - op: add + path: /tags/- + value: + name: oidc + description: Endpoints to manage GitHub OIDC configuration using the REST API. + x-displayName: OIDC + - op: add + path: /tags/- + value: + name: scim + description: Provisioning of GitHub organization membership for SCIM-enabled providers. + x-displayName: SCIM + - op: add + path: /info/x-github-plan + value: ghes + - op: add + path: /info/x-github-release + value: 3.7 diff --git a/tests/fixtures/openapi-release-configs/ghes-3.8.yaml b/tests/fixtures/openapi-release-configs/ghes-3.8.yaml new file mode 100644 index 0000000000..730fb9ef4c --- /dev/null +++ b/tests/fixtures/openapi-release-configs/ghes-3.8.yaml @@ -0,0 +1,42 @@ +--- +published: false +deprecated: false +variables: + externalDocsUrl: https://docs.github.com/enterprise-server@3.8 + ghesVersion: '3.8' + apiName: GitHub Enterprise Server +patch: + - op: add + path: /servers + value: + - url: '{protocol}://{hostname}/api/v3' + variables: + hostname: + description: Self-hosted Enterprise Server hostname + default: HOSTNAME + protocol: + description: Self-hosted Enterprise Server protocol + default: http + - op: add + path: /externalDocs + value: + description: GitHub Enterprise Developer Docs + url: 'https://docs.github.com/enterprise-server@3.8/rest/' + - op: add + path: /tags/- + value: + name: oidc + description: Endpoints to manage GitHub OIDC configuration using the REST API. + x-displayName: OIDC + - op: add + path: /tags/- + value: + name: scim + description: Provisioning of GitHub organization membership for SCIM-enabled providers. + x-displayName: SCIM + - op: add + path: /info/x-github-plan + value: ghes + - op: add + path: /info/x-github-release + value: 3.8 diff --git a/tests/fixtures/openapi-release-configs/github.ae.yaml b/tests/fixtures/openapi-release-configs/github.ae.yaml new file mode 100644 index 0000000000..3e9f25f396 --- /dev/null +++ b/tests/fixtures/openapi-release-configs/github.ae.yaml @@ -0,0 +1,23 @@ +--- +published: true +deprecated: false +variables: + externalDocsUrl: https://docs.github.com/github-ae@latest + apiName: GitHub AE +patch: + - op: add + path: /servers + value: + - url: 'https://{hostname}/api/v3' + variables: + hostname: + description: Self-hosted GitHub AE hostname + default: HOSTNAME + - op: add + path: /externalDocs + value: + description: GitHub AE Developer Docs + url: 'https://docs.github.com/github-ae@latest/rest/' + - op: add + path: /info/x-github-plan + value: github.ae diff --git a/tests/unit/openapi-decorator.js b/tests/unit/openapi-decorator.js new file mode 100644 index 0000000000..1981cafbec --- /dev/null +++ b/tests/unit/openapi-decorator.js @@ -0,0 +1,91 @@ +import { describe, expect } from '@jest/globals' +import path from 'path' + +import { getOpenApiSchemaFiles } from '../../script/rest/utils/decorator.js' +import { getSchemas } from '../../script/rest/utils/get-openapi-schemas.js' +import { allVersions } from '../../lib/all-versions.js' + +const supportedReleases = Object.keys(allVersions).map( + (version) => allVersions[version].openApiVersionName +) +describe('decorated static files are generated correctly from dereferenced openapi files', () => { + // If there is a request with no request body parameters and all of + // the responses have no content, then we can create a docs + // example for just status codes below 300. All other status codes will + // be listed in the status code table in the docs. + test('webhook schema list should not include calendar date versions', async () => { + const schemas = [ + 'api.github.com.2022-08-09.deref.json', + 'api.github.com.2022-10-09.deref.json', + 'api.github.com.2022-11-09.deref.json', + 'ghec.2022-09-09.deref.json', + ...supportedReleases, + ] + + const expectedRestSchemas = [ + 'api.github.com.2022-08-09', + 'api.github.com.2022-10-09', + 'api.github.com.2022-11-09', + 'ghec.2022-09-09', + ...supportedReleases.filter((release) => release !== 'ghec' && release !== 'api.github.com'), + ] + + const { restSchemas, webhookSchemas } = await getOpenApiSchemaFiles(schemas) + expect(restSchemas.sort()).toEqual(expectedRestSchemas.sort()) + expect(webhookSchemas.sort()).toEqual(supportedReleases.sort()) + }) + + test('deprecated schemas in docs-internal are not included', async () => { + const myPath = path.join(process.cwd(), 'tests/fixtures/openapi-release-configs') + const currentReleaseSchemas = await getSchemas(myPath) + const expectedSchemas = [ + 'api.github.com.deref.json', + 'ghec.deref.json', + 'ghes-3.7.deref.json', + 'github.ae.deref.json', + ] + expect(currentReleaseSchemas.sort()).toEqual(expectedSchemas.sort()) + }) + + test('deprecated schemas in docs-internal are included when deprecated option is enabled', async () => { + const myPath = path.join(process.cwd(), 'tests/fixtures/openapi-release-configs') + const deprecatedSchemas = await getSchemas(myPath, true) + const expectedSchemas = [ + 'api.github.com.deref.json', + 'ghec.deref.json', + 'ghes-3.7.deref.json', + 'github.ae.deref.json', + 'ghes-3.2.deref.json', + ] + expect(deprecatedSchemas.sort()).toEqual(expectedSchemas.sort()) + }) + + test('unpublished schemas in github/github are included when unpublished option is enabled', async () => { + const myPath = path.join(process.cwd(), 'tests/fixtures/openapi-release-configs') + const unpublishedSchemas = await getSchemas(myPath, false, true) + const expectedSchemas = [ + 'api.github.com.deref.json', + 'ghec.deref.json', + 'ghes-3.7.deref.json', + 'github.ae.deref.json', + 'ghes-3.8.deref.json', + ] + expect(unpublishedSchemas.sort()).toEqual(expectedSchemas.sort()) + }) + + test('specifying specific openapi versions is successful', async () => { + const myPath = path.join(process.cwd(), 'tests/fixtures/openapi-release-configs') + const versionSchemas = await getSchemas(myPath, false, false, ['ghec', 'ghes-3.7']) + const expectedSchemas = ['ghec.deref.json', 'ghes-3.7.deref.json'] + expect(versionSchemas.sort()).toEqual(expectedSchemas.sort()) + }) + + test('specifying deprecated or unpublished versions fails', async () => { + const myPath = path.join(process.cwd(), 'tests/fixtures/openapi-release-configs') + // const testError = async () => { + // await getSchemas(myPath, false, false, ['ghes-3.8', 'ghes-3.2']) + // } + // await expect(await testError()).toThrow(Error) + await expect(getSchemas(myPath, false, false, ['ghes-3.8', 'ghes-3.2'])).rejects.toThrow() + }) +})