import { readFile, writeFile } from 'fs/promises' import { existsSync } from 'fs' import path from 'path' import { mkdirp } from 'mkdirp' import { updateRestFiles } from './update-markdown' import { allVersions } from '@/versions/lib/all-versions' import { createOperations, processOperations } from './get-operations' import { getProgAccessData } from '@/github-apps/scripts/sync' import { REST_DATA_DIR, REST_SCHEMA_FILENAME } from '../../lib/index' type Schema = Record type Operation = { category: string; subcategory: string; [key: string]: any } type OperationsByCategory = Record> // All of the schema releases that we store in allVersions // Ex: 'api.github.com', 'ghec', 'ghes-3.6', 'ghes-3.5', // 'ghes-3.4', 'ghes-3.3', 'ghes-3.2', 'github.ae' const OPENAPI_VERSION_NAMES = Object.keys(allVersions).map( (elem) => allVersions[elem].openApiVersionName, ) export async function syncRestData( sourceDirectory: string, restSchemas: string[], progAccessSource: string, injectIntoSchema?: (schema: Schema, schemaName: string) => Schema, ): Promise { await Promise.all( restSchemas.map(async (schemaName) => { const file = path.join(sourceDirectory, schemaName) let schema = JSON.parse(await readFile(file, 'utf-8')) as Schema if (injectIntoSchema) { const injectedSchema = await injectIntoSchema(schema, schemaName) schema = injectedSchema || schema // Fallback to original if injection returns null } const operations: Operation[] = [] console.log('Instantiating operation instances from schema ', schemaName) try { const newOperations = await createOperations(schema) operations.push(...newOperations) } catch (error) { throw new Error( `${error}\n\nšŸ› Whoops! It looks like the script wasn't able to parse the dereferenced schema. A recent change may not yet be supported by the decorator. Please reach out in the #docs-engineering slack channel for help.`, ) } try { const { progAccessData } = await getProgAccessData(progAccessSource, true) await processOperations(operations, progAccessData) } catch (error) { throw new Error( `${error}\n\nšŸ› Whoops! It looks like some Markdown in the dereferenced schema wasn't able to be rendered. Please reach out in the #docs-engineering slack channel for help.`, ) } const formattedOperations = await formatRestData(operations) const versionDirectoryName = schemaName.replace('.json', '') const targetDirectoryPath = path.join(REST_DATA_DIR, versionDirectoryName) if (Object.keys(formattedOperations).length === 0) { throw new Error( `Generating REST data failed for ${sourceDirectory}/${schemaName}. The generated data file was empty.`, ) } if (!existsSync(targetDirectoryPath)) { await mkdirp(targetDirectoryPath) } const targetPath = path.join(targetDirectoryPath, REST_SCHEMA_FILENAME) await writeFile(targetPath, JSON.stringify(formattedOperations, null, 2)) console.log(`āœ… Wrote ${targetPath}`) }), ) await updateRestFiles() await updateRestConfigData(restSchemas) } async function formatRestData(operations: Operation[]): Promise { const categories = [...new Set(operations.map((operation) => operation.category))].sort() const operationsByCategory: OperationsByCategory = {} for (const category of categories) { operationsByCategory[category] = {} const categoryOperations = operations.filter((operation) => operation.category === category) const subcategories = [ ...new Set(categoryOperations.map((operation) => operation.subcategory)), ].sort() // the first item should be the item that has no subcategory // e.g., when the subcategory = category const firstItemIndex = subcategories.indexOf(category) if (firstItemIndex > -1) { const firstItem = subcategories.splice(firstItemIndex, 1)[0] subcategories.unshift(firstItem) } for (const subcategory of subcategories) { operationsByCategory[category][subcategory] = [] const subcategoryOperations = categoryOperations.filter( (operation) => operation.subcategory === subcategory, ) operationsByCategory[category][subcategory] = subcategoryOperations } } return operationsByCategory } // Every time we update the REST data files, we'll want to make sure the // config.json file is updated with the latest api versions. async function updateRestConfigData(schemas: string[]): Promise { const restConfigFilename = 'src/rest/lib/config.json' const restConfigData = JSON.parse(await readFile(restConfigFilename, 'utf8')) as Record< string, any > const restApiVersionData = restConfigData['api-versions'] || {} // If the version isn't one of the OpenAPI version, // then it's an api-versioned schema for (const schema of schemas) { const schemaBaseName = path.basename(schema, '.json') if (!OPENAPI_VERSION_NAMES.includes(schemaBaseName)) { const openApiVer = OPENAPI_VERSION_NAMES.find((ver) => schemaBaseName.startsWith(ver)) if (!openApiVer) { throw new Error(`Could not find the OpenAPI version for schema ${schemaBaseName}`) } const date = schemaBaseName.split(`${openApiVer}-`)[1] if (!restApiVersionData[openApiVer]) { restApiVersionData[openApiVer] = [] } if (!restApiVersionData[openApiVer].includes(date)) { const dates = restApiVersionData[openApiVer] dates.push(date) restApiVersionData[openApiVer] = dates } } restConfigData['api-versions'] = restApiVersionData } await writeFile(restConfigFilename, JSON.stringify(restConfigData, null, 2)) } export async function getOpenApiSchemaFiles( schemas: string[], ): Promise<{ restSchemas: string[]; webhookSchemas: string[] }> { const restSchemas: string[] = [] const webhookSchemas: string[] = [] // The full list of dereferened OpenAPI schemas received from // bundling the OpenAPI in github/github const schemaNames = schemas.map((schema) => path.basename(schema, '.json')) const versionNames = Object.keys(allVersions).map((elem) => allVersions[elem].openApiVersionName) for (const schema of schemaNames) { const schemaBasename = `${schema}.json` // If the version doesn't have calendar date versioning // it should have an exact match with one of the versions defined // in the allVersions object. if (versionNames.includes(schema)) { webhookSchemas.push(schemaBasename) } // If the schema version has calendar date versioning, then one of // the versions defined in allVersions should be a substring of the // schema version. This means the schema version is a supported version if (versionNames.some((elem) => schema.startsWith(elem))) { // If the schema being evaluated is a calendar-date version, then // there would only be one exact match in the list of schema names. // If the schema being evaluated is a non-calendar-date version, then // there will be two matches. // Ex: api.github.com would match api.github.com and // api.github.com.2022-09-09 const filteredMatches = schemaNames.filter((elem) => elem.includes(schema)) // If there is only one match then it's either a calendar-date version // or the version doesn't support calendar dates yet. We favor calendar-date // versions but default to non calendar-date versions. if (filteredMatches.length === 1) { restSchemas.push(schemaBasename) } } } return { restSchemas, webhookSchemas } }