a340162d84
Co-authored-by: Lucas Costi <lucascosti@users.noreply.github.com> Co-authored-by: Peter Bengtsson <peterbe@github.com>
274 lines
10 KiB
JavaScript
Executable File
274 lines
10 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
||
|
||
// [start-readme]
|
||
//
|
||
// Run this script to pull openAPI files from github/github, dereference them, and decorate them.
|
||
//
|
||
// [end-readme]
|
||
|
||
import { stat, readFile, writeFile, readdir } from 'fs/promises'
|
||
import path from 'path'
|
||
import program from 'commander'
|
||
import { execSync } from 'child_process'
|
||
import mkdirp from 'mkdirp'
|
||
import rimraf from 'rimraf'
|
||
import getOperations from './utils/get-operations.js'
|
||
import yaml from 'js-yaml'
|
||
|
||
const tempDocsDir = path.join(process.cwd(), 'openapiTmp')
|
||
const githubRepoDir = path.join(process.cwd(), '../github')
|
||
const dereferencedPath = path.join(process.cwd(), 'lib/rest/static/dereferenced')
|
||
const decoratedPath = path.join(process.cwd(), 'lib/rest/static/decorated')
|
||
const openApiReleasesDir = `${githubRepoDir}/app/api/description/config/releases`
|
||
|
||
program
|
||
.description('Generate dereferenced OpenAPI and decorated schema files.')
|
||
.option(
|
||
'--decorate-only',
|
||
'⚠️ Only used by a 🤖 to generate decorated schema files from existing dereferenced schema files.'
|
||
)
|
||
.option(
|
||
'-v --versions <VERSIONS...>',
|
||
'A list of undeprecated, published versions to build, separated by a space. Example "ghes-3.1" or "api.github.com github.ae"'
|
||
)
|
||
.option('-d --include-deprecated', 'Includes schemas that are marked as `deprecated: true`')
|
||
.option('-u --include-unpublished', 'Includes schemas that are marked as `published: false`')
|
||
.option('--redirects-only', 'Only generate the redirects file')
|
||
.parse(process.argv)
|
||
|
||
const { decorateOnly, versions, includeUnpublished, includeDeprecated, redirectsOnly } =
|
||
program.opts()
|
||
|
||
// Check that the github/github repo exists. If the files are only being
|
||
// decorated, the github/github repo isn't needed.
|
||
if (!decorateOnly && !redirectsOnly) {
|
||
try {
|
||
await stat(githubRepoDir)
|
||
} catch (error) {
|
||
console.log(
|
||
`🛑 The ${githubRepoDir} does not exist. Make sure you have a local, bootstrapped checkout of github/github at the same level as your github/docs-internal repo before running this script.`
|
||
)
|
||
process.exit(1)
|
||
}
|
||
}
|
||
|
||
// When the input parameter type is decorate-only, use the local
|
||
// `github/docs-internal` repo to generate a list of schema files.
|
||
// Otherwise, use the `github/github` list of config files
|
||
const referenceSchemaDirectory = decorateOnly ? dereferencedPath : openApiReleasesDir
|
||
// A full list of unpublished, deprecated, and active schemas
|
||
const allSchemas = await getOpenApiSchemas(referenceSchemaDirectory)
|
||
|
||
await validateInputParameters(allSchemas)
|
||
|
||
// Format the command supplied to the bundle script in `github/github`
|
||
const commandParameters = await getCommandParameters()
|
||
// Get the list of schemas for this bundle, depending on options
|
||
const schemas = await getSchemas(allSchemas)
|
||
|
||
main()
|
||
|
||
async function main() {
|
||
// Generate the dereferenced OpenAPI schema files
|
||
if (!decorateOnly && !redirectsOnly) {
|
||
await getDereferencedFiles()
|
||
}
|
||
// Decorate the dereferenced files in a format ingestible by docs.github.com
|
||
if (!redirectsOnly) {
|
||
await decorate()
|
||
}
|
||
|
||
await updateRedirectOverrides()
|
||
|
||
console.log(
|
||
'\n🏁 The static REST API files are now up-to-date with your local `github/github` checkout. To revert uncommitted changes, run `git checkout lib/rest/static/*.\n\n'
|
||
)
|
||
}
|
||
|
||
async function getDereferencedFiles() {
|
||
// Get the github/github repo branch name and pull latest
|
||
const githubBranch = execSync('git rev-parse --abbrev-ref HEAD', { cwd: githubRepoDir })
|
||
.toString()
|
||
.trim()
|
||
|
||
// Only pull master branch because development mode branches are assumed
|
||
// to be up-to-date during active work.
|
||
if (githubBranch === 'master') {
|
||
execSync('git pull', { cwd: githubRepoDir })
|
||
}
|
||
|
||
// Create a tmp directory to store schema files generated from github/github
|
||
rimraf.sync(tempDocsDir)
|
||
await mkdirp(tempDocsDir)
|
||
|
||
console.log(
|
||
`\n🏃♀️🏃🏃♀️Running \`bin/openapi bundle\` in branch '${githubBranch}' of your github/github checkout to generate the dereferenced OpenAPI schema files.\n`
|
||
)
|
||
try {
|
||
console.log(`bundle -o ${tempDocsDir} ${commandParameters}`)
|
||
execSync(
|
||
`${path.join(githubRepoDir, 'bin/openapi')} bundle -o ${tempDocsDir} ${commandParameters}`,
|
||
{ stdio: 'inherit' }
|
||
)
|
||
} catch (error) {
|
||
console.error(error)
|
||
console.log(
|
||
'🛑 Whoops! It looks like the `bin/openapi bundle` command failed to run in your `github/github` repository checkout. To troubleshoot, ensure that your OpenAPI schema YAML is formatted correctly. A CI test runs on your `github/github` PR that flags malformed YAML. You can check the PR diff view for comments left by the openapi CI test to find and fix any formatting errors.'
|
||
)
|
||
process.exit(1)
|
||
}
|
||
|
||
execSync(`find ${tempDocsDir} -type f -name "*deref.json" -exec mv '{}' ${dereferencedPath} ';'`)
|
||
|
||
rimraf.sync(tempDocsDir)
|
||
|
||
// When running in development mode (locally), the the info.version
|
||
// property in the dereferenced schema is replaced with the branch
|
||
// name of the `github/github` checkout. A CI test
|
||
// checks the version and fails if it's not a semantic version.
|
||
for (const filename of schemas) {
|
||
const schema = JSON.parse(await readFile(path.join(dereferencedPath, filename)))
|
||
|
||
schema.info.version = `${githubBranch} !!DEVELOPMENT MODE - DO NOT MERGE!!`
|
||
await writeFile(path.join(dereferencedPath, filename), JSON.stringify(schema, null, 2))
|
||
}
|
||
}
|
||
|
||
async function updateRedirectOverrides() {
|
||
const overrides = JSON.parse(await readFile('script/rest/utils/rest-api-overrides.json', 'utf8'))
|
||
|
||
const redirects = {}
|
||
console.log('\n➡️ Updating REST API redirect exception list.\n')
|
||
for (const value of Object.values(overrides)) {
|
||
const oldUrl = value.originalUrl
|
||
const anchor = oldUrl.replace('/rest/reference', '').split('#')[1]
|
||
redirects[oldUrl] = `/rest/reference/${value.category}#${anchor}`
|
||
}
|
||
await writeFile(
|
||
'lib/redirects/static/rest-api-redirect-exceptions.json',
|
||
JSON.stringify(redirects, null, 2),
|
||
'utf8'
|
||
)
|
||
}
|
||
|
||
async function decorate() {
|
||
console.log('\n🎄 Decorating the OpenAPI schema files in lib/rest/static/dereferenced.\n')
|
||
const dereferencedSchemas = {}
|
||
for (const filename of schemas) {
|
||
const schema = JSON.parse(await readFile(path.join(dereferencedPath, filename)))
|
||
const key = filename.replace('.deref.json', '')
|
||
dereferencedSchemas[key] = schema
|
||
}
|
||
|
||
for (const [schemaName, schema] of Object.entries(dereferencedSchemas)) {
|
||
try {
|
||
// munge OpenAPI definitions object in an array of operations objects
|
||
const operations = await getOperations(schema)
|
||
|
||
// process each operation, asynchronously rendering markdown and stuff
|
||
await Promise.all(operations.map((operation) => operation.process()))
|
||
|
||
const filename = path.join(decoratedPath, `${schemaName}.json`).replace('.deref', '')
|
||
// write processed operations to disk
|
||
await writeFile(filename, JSON.stringify(operations, null, 2))
|
||
|
||
console.log('Wrote', path.relative(process.cwd(), filename))
|
||
} catch (error) {
|
||
console.error(error)
|
||
console.log(
|
||
"🐛 Whoops! It looks like the decorator script wasn't able to parse the dereferenced schema. A recent change may not yet be supported by the decorator. Please reach out in the #docs-engineering slack channel for help."
|
||
)
|
||
process.exit(1)
|
||
}
|
||
}
|
||
}
|
||
|
||
async function validateInputParameters(schemas) {
|
||
// The `--versions` and `--decorate-only` options cannot be used
|
||
// with the `--include-deprecated` or `--include-unpublished` options
|
||
const numberOfOptions = Object.keys(program.opts()).length
|
||
|
||
if (numberOfOptions > 1 && (decorateOnly || versions)) {
|
||
console.log(
|
||
`🛑 You cannot use the versions and decorate-only options with any other options.\nThe decorate-only switch will decorate all dereferenced schemas files in the docs-internal repo.\nThis script doesn't support generating individual deprecated or unpublished schemas.\nPlease reach out to #docs-engineering if this is a use case that you need.`
|
||
)
|
||
process.exit(1)
|
||
}
|
||
|
||
// Validate individual versions provided
|
||
if (versions) {
|
||
versions.forEach((version) => {
|
||
if (
|
||
schemas.deprecated.includes(`${version}.deref.json`) ||
|
||
schemas.unpublished.includes(`${version}.deref.json`)
|
||
) {
|
||
console.log(
|
||
`🛑 This script doesn't support generating individual deprecated or unpublished schemas. Please reach out to #docs-engineering if this is a use case that you need.`
|
||
)
|
||
process.exit(1)
|
||
} else if (!schemas.currentReleases.includes(`${version}.deref.json`)) {
|
||
console.log(`🛑 The version (${version}) you specified is not valid.`)
|
||
process.exit(1)
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
async function getOpenApiSchemas(directory) {
|
||
const openAPIConfigs = await readdir(directory)
|
||
const unpublished = []
|
||
const deprecated = []
|
||
const currentReleases = []
|
||
|
||
for (const file of openAPIConfigs) {
|
||
const newFileName = `${path.basename(file, 'yaml')}deref.json`
|
||
const content = await readFile(path.join(directory, file), 'utf8')
|
||
const yamlContent = yaml.load(content)
|
||
if (!yamlContent.published) {
|
||
unpublished.push(newFileName)
|
||
}
|
||
if (yamlContent.deprecated) {
|
||
deprecated.push(newFileName)
|
||
}
|
||
if (!yamlContent.deprecated && yamlContent.published) {
|
||
currentReleases.push(newFileName)
|
||
}
|
||
}
|
||
|
||
return { currentReleases, unpublished, deprecated }
|
||
}
|
||
|
||
async function getCommandParameters() {
|
||
let includeParams = []
|
||
|
||
if (versions) {
|
||
includeParams = versions
|
||
}
|
||
if (includeUnpublished) {
|
||
includeParams.push('--include_unpublished')
|
||
}
|
||
if (includeDeprecated) {
|
||
includeParams.push('--include_deprecated')
|
||
}
|
||
|
||
return includeParams.join(' ')
|
||
}
|
||
|
||
async function getSchemas(allSchemas) {
|
||
if (decorateOnly) {
|
||
const files = await readdir(dereferencedPath)
|
||
return files
|
||
} else if (versions) {
|
||
return versions.map((elem) => `${elem}.deref.json`)
|
||
} else {
|
||
const schemas = allSchemas.currentReleases
|
||
if (includeUnpublished) {
|
||
schemas.push(...allSchemas.unpublished)
|
||
}
|
||
if (includeDeprecated) {
|
||
schemas.push(...allSchemas.deprecated)
|
||
}
|
||
return schemas
|
||
}
|
||
}
|