diff --git a/lib/all-versions.js b/lib/all-versions.js index 696a283a64..07c72f1d64 100755 --- a/lib/all-versions.js +++ b/lib/all-versions.js @@ -1,13 +1,12 @@ import fs from 'fs' -import path from 'path' import enterpriseServerReleases from './enterprise-server-releases.js' - // version = "plan"@"release" // example: enterprise-server@2.21 // where "enterprise-server" is the plan and "2.21" is the release const versionDelimiter = '@' const latestNonNumberedRelease = 'latest' +const REST_DATA_META_FILE = 'src/rest/data/meta.json' // !Explanation of versionless redirect fallbacks! // This array is **in order** of the versions the site should try to fall back to if @@ -94,26 +93,13 @@ plans.forEach((planObj) => { }) }) -const apiFilesPath = path.join(process.cwd(), 'src/rest/data') -// This is what determines which versions are calendar date versioned for the REST API docs -// This is the source of truth for which versions are calendar date versioned. -fs.readdirSync(apiFilesPath) - .filter((file) => file.endsWith('.json')) - .forEach((file) => { - const fileName = file.split('.json')[0] - const version = getDocsVersion(fileName) - const apiVersion = fileName.split(allVersions[version].openApiVersionName)[1].replace('.', '') - - if (apiVersion !== '') { - allVersions[version].apiVersions.push(apiVersion) - if ( - allVersions[version].latestApiVersion === '' || - apiVersion > allVersions[version].latestApiVersion - ) { - allVersions[version].latestApiVersion = apiVersion - } - } - }) +// Adds the calendar date (or api versions) to the allVersions object +const apiVersions = JSON.parse(fs.readFileSync(REST_DATA_META_FILE, 'utf8'))['api-versions'] +Object.keys(apiVersions).forEach((key) => { + const docsVersion = getDocsVersion(key) + allVersions[docsVersion].apiVersions.push(...apiVersions[key].sort()) + allVersions[docsVersion].latestApiVersion = apiVersions[key].pop() +}) export const allVersionKeys = Object.keys(allVersions) export const allVersionShortnames = Object.fromEntries( diff --git a/src/rest/data/api.github.com.2022-11-28.json b/src/rest/data/api.github.com.2022-11-28/schema.json similarity index 100% rename from src/rest/data/api.github.com.2022-11-28.json rename to src/rest/data/api.github.com.2022-11-28/schema.json diff --git a/src/rest/data/ghec.2022-11-28.json b/src/rest/data/ghec.2022-11-28/schema.json similarity index 100% rename from src/rest/data/ghec.2022-11-28.json rename to src/rest/data/ghec.2022-11-28/schema.json diff --git a/src/rest/data/ghes-3.4.json b/src/rest/data/ghes-3.4/schema.json similarity index 100% rename from src/rest/data/ghes-3.4.json rename to src/rest/data/ghes-3.4/schema.json diff --git a/src/rest/data/ghes-3.5.json b/src/rest/data/ghes-3.5/schema.json similarity index 100% rename from src/rest/data/ghes-3.5.json rename to src/rest/data/ghes-3.5/schema.json diff --git a/src/rest/data/ghes-3.6.json b/src/rest/data/ghes-3.6/schema.json similarity index 100% rename from src/rest/data/ghes-3.6.json rename to src/rest/data/ghes-3.6/schema.json diff --git a/src/rest/data/ghes-3.7.json b/src/rest/data/ghes-3.7/schema.json similarity index 100% rename from src/rest/data/ghes-3.7.json rename to src/rest/data/ghes-3.7/schema.json diff --git a/src/rest/data/ghes-3.8.json b/src/rest/data/ghes-3.8/schema.json similarity index 100% rename from src/rest/data/ghes-3.8.json rename to src/rest/data/ghes-3.8/schema.json diff --git a/src/rest/data/github.ae.json b/src/rest/data/github.ae/schema.json similarity index 100% rename from src/rest/data/github.ae.json rename to src/rest/data/github.ae/schema.json diff --git a/src/rest/data/meta.json b/src/rest/data/meta.json new file mode 100644 index 0000000000..49dffb3311 --- /dev/null +++ b/src/rest/data/meta.json @@ -0,0 +1,6 @@ +{ + "api-versions": { + "api.github.com": ["2022-11-28"], + "ghec": ["2022-11-28"] + } +} \ No newline at end of file diff --git a/src/rest/lib/index.js b/src/rest/lib/index.js index 053382a82c..cedd6de7de 100644 --- a/src/rest/lib/index.js +++ b/src/rest/lib/index.js @@ -6,9 +6,9 @@ import { getAutomatedPageMiniTocItems } from '../../../lib/get-mini-toc-items.js import { allVersions, getOpenApiVersion } from '../../../lib/all-versions.js' import languages from '../../../lib/languages.js' -const schemasPath = 'src/rest/data' - -const contentPath = 'content/rest' +export const REST_DATA_DIR = 'src/rest/data' +export const REST_SCHEMA_FILENAME = 'schema.json' +const REST_CONTENT_DIR = 'content/rest' /* Loads the schemas from the static/decorated folder into a single @@ -59,7 +59,7 @@ Object.keys(languages).forEach((language) => { }) export const categoriesWithoutSubcategories = fs - .readdirSync(contentPath) + .readdirSync(REST_CONTENT_DIR) .filter((file) => { return file.endsWith('.md') && !file.includes('index.md') && !file.includes('README.md') }) @@ -71,26 +71,21 @@ export const categoriesWithoutSubcategories = fs // and the OpenApi Version bc it's not the same export default async function getRest(version, apiVersion, category, subCategory) { const openApiVersion = getOpenApiVersion(version) - const filename = apiVersion ? `${openApiVersion}.${apiVersion}.json` : `${openApiVersion}.json` + const openapiSchemaName = apiVersion ? `${openApiVersion}.${apiVersion}` : `${openApiVersion}` const apiDate = apiVersion || NOT_API_VERSIONED - + const fileName = path.join(REST_DATA_DIR, openapiSchemaName, REST_SCHEMA_FILENAME) if (!restOperations.has(openApiVersion)) { restOperations.set(openApiVersion, new Map()) restOperations.get(openApiVersion).set(apiDate, new Map()) // The `readCompressedJsonFileFallback()` function // will check for both a .br and .json extension. - restOperations - .get(openApiVersion) - .set(apiDate, readCompressedJsonFileFallback(path.join(schemasPath, filename))) + restOperations.get(openApiVersion).set(apiDate, readCompressedJsonFileFallback(fileName)) } else if (!restOperations.get(openApiVersion).has(apiDate)) { restOperations.get(openApiVersion).set(apiDate, new Map()) // The `readCompressedJsonFileFallback()` function // will check for both a .br and .json extension. - restOperations - .get(openApiVersion) - .set(apiDate, readCompressedJsonFileFallback(path.join(schemasPath, filename))) + restOperations.get(openApiVersion).set(apiDate, readCompressedJsonFileFallback(fileName)) } - if (subCategory) { return restOperations.get(openApiVersion).get(apiDate)[category][subCategory] } else if (category) { diff --git a/src/rest/scripts/test-open-api-schema.js b/src/rest/scripts/test-open-api-schema.js index 1ee7783abb..3b0f4801ed 100755 --- a/src/rest/scripts/test-open-api-schema.js +++ b/src/rest/scripts/test-open-api-schema.js @@ -12,11 +12,12 @@ import _ from 'lodash' import frontmatter from '../../../lib/read-frontmatter.js' import getApplicableVersions from '../../../lib/get-applicable-versions.js' import { allVersions, getDocsVersion } from '../../../lib/all-versions.js' +import { REST_DATA_DIR, REST_SCHEMA_FILENAME } from '../lib/index.js' const contentFiles = [] export async function getDiffOpenAPIContentRest() { - const contentPath = path.join(process.cwd(), 'content/rest') + const contentPath = 'content/rest' // Recursively go through the content/rest directory and add all categories/subcategories to contentFiles throughDirectory(contentPath) @@ -50,15 +51,14 @@ export async function getDiffOpenAPIContentRest() { } async function createOpenAPISchemasCheck() { - const schemasPath = path.join(process.cwd(), 'src/rest/data') const openAPICheck = createCheckObj() - const schemas = fs.readdirSync(schemasPath) + const restDirectory = fs.readdirSync(REST_DATA_DIR).filter((dir) => dir !== 'meta.json') - schemas.forEach((file) => { - const fileData = fs.readFileSync(path.join(schemasPath, file)) - const fileSchema = JSON.parse(fileData.toString()) + restDirectory.forEach((dir) => { + const filename = path.join(REST_DATA_DIR, dir, REST_SCHEMA_FILENAME) + const fileSchema = JSON.parse(fs.readFileSync(filename)) const categories = Object.keys(fileSchema).sort() - const version = getDocsVersion(file.split(/.json/)[0]) + const version = getDocsVersion(dir) categories.forEach((category) => { const subcategories = Object.keys(fileSchema[category]) diff --git a/src/rest/scripts/utils/decorator.js b/src/rest/scripts/utils/decorator.js index 386e3b28a9..5ccd87110d 100644 --- a/src/rest/scripts/utils/decorator.js +++ b/src/rest/scripts/utils/decorator.js @@ -4,14 +4,22 @@ import path from 'path' import { slug } from 'github-slugger' import { allVersions } from '../../../../lib/all-versions.js' -import { categoriesWithoutSubcategories } from '../../lib/index.js' +import { + categoriesWithoutSubcategories, + REST_DATA_DIR, + REST_SCHEMA_FILENAME, +} from '../../lib/index.js' import getOperations, { getWebhooks } from './get-operations.js' import { ENABLED_APPS_DIR, ENABLED_APPS_FILENAME } from '../../../github-apps/lib/index.js' import { WEBHOOK_DATA_DIR, WEBHOOK_SCHEMA_FILENAME } from '../../../webhooks/lib/index.js' - const STATIC_REDIRECTS = 'lib/redirects/static/client-side-rest-api-redirects.json' -const REST_DECORATED_DIR = 'src/rest/data' const REST_DEREFERENCED_DIR = 'src/rest/data/dereferenced' +// All of the schema releases that we store in allVersions +// Ex: 'api.github.com', 'ghec', 'ghes-3.6', 'ghes-3.5', +// 'ghes-3.4', 'ghes-3.3', 'ghes-3.2', 'github.ae' +const OPENAPI_VERSION_NAMES = Object.keys(allVersions).map( + (elem) => allVersions[elem].openApiVersionName +) export async function decorate(schemas) { console.log('\nšŸŽ„ Decorating the OpenAPI schema files in src/rest/data/dereferenced.\n') @@ -20,6 +28,7 @@ export async function decorate(schemas) { await createStaticWebhookFiles(webhookOperations) const restOperations = await getRestOperations(restSchemas) await createStaticRestFiles(restOperations) + await updateRestMetaData(restSchemas) } async function getRestOperations(restSchemas) { @@ -123,7 +132,9 @@ async function createStaticRestFiles(restOperations) { }) }) - const restFilename = path.join(REST_DECORATED_DIR, `${schemaName}.json`).replace('.deref', '') + const restFilename = path + .join(REST_DATA_DIR, schemaName, REST_SCHEMA_FILENAME) + .replace('.deref', '') // write processed operations to disk await writeFile(restFilename, JSON.stringify(operationsByCategory, null, 2)) @@ -304,19 +315,13 @@ export async function getOpenApiSchemaFiles(schemas) { const webhookSchemas = [] const restSchemas = [] - // All of the schema releases that we store in allVersions - // Ex: 'api.github.com', 'ghec', 'ghes-3.6', 'ghes-3.5', - // 'ghes-3.4', 'ghes-3.3', 'ghes-3.2', 'github.ae' - const openApiVersions = Object.keys(allVersions).map( - (elem) => allVersions[elem].openApiVersionName - ) // The full list of dereferened OpenAPI schemas received from // bundling the OpenAPI in github/github const schemaBaseNames = schemas.map((schema) => path.basename(schema, '.deref.json')) for (const schema of schemaBaseNames) { // catches all of the schemas that are not // calendar date versioned. Ex: ghec, ghes-3.7, and api.github.com - if (openApiVersions.includes(schema)) { + if (OPENAPI_VERSION_NAMES.includes(schema)) { webhookSchemas.push(schema) // Non-calendar date schemas could also match the calendar date versioned // counterpart. @@ -336,3 +341,27 @@ export async function getOpenApiSchemaFiles(schemas) { } return { restSchemas, webhookSchemas } } + +// Every time we update the REST data files, we'll want to make sure the +// meta.json file is updated with the latest api versions. +async function updateRestMetaData(schemas) { + const restMetaFilename = `${REST_DATA_DIR}/meta.json` + const restMetaData = JSON.parse(await readFile(restMetaFilename, 'utf8')) + const restApiVersionData = restMetaData['api-versions'] + schemas.forEach((schema) => { + // If the version isn't one of the OpenAPI version, + // then it's an api-versioned schema + if (!OPENAPI_VERSION_NAMES.includes(schema)) { + const openApiVer = OPENAPI_VERSION_NAMES.find((ver) => schema.startsWith(ver)) + const date = schema.split(`${openApiVer}.`)[1] + + if (!restApiVersionData[openApiVer].includes(date)) { + const dates = restApiVersionData[openApiVer] + dates.push(date) + restApiVersionData[openApiVer] = dates + } + } + }) + restMetaData['api-versions'] = restApiVersionData + await writeFile(restMetaFilename, JSON.stringify(restMetaData, null, 2)) +} diff --git a/tests/rendering/rest.js b/tests/rendering/rest.js index 8b88be5167..7356415d0a 100644 --- a/tests/rendering/rest.js +++ b/tests/rendering/rest.js @@ -4,7 +4,11 @@ import { readdirSync, readFileSync } from 'fs' import { join } from 'path' import { get, getDOM } from '../helpers/e2etest.js' -import getRest, { categoriesWithoutSubcategories } from '../../src/rest/lib/index.js' +import getRest, { + categoriesWithoutSubcategories, + REST_DATA_DIR, + REST_SCHEMA_FILENAME, +} from '../../src/rest/lib/index.js' import { getEnabledForApps } from '../../src/github-apps/lib/index.js' import { isApiVersioned, allVersions } from '../../lib/all-versions.js' import { getDiffOpenAPIContentRest } from '../../src/rest/scripts/test-open-api-schema.js' @@ -13,13 +17,14 @@ describe('REST references docs', () => { jest.setTimeout(3 * 60 * 1000) test('all category and subcategory REST pages render for free-pro-team', async () => { - const DECORATED_DIR = 'src/rest/data' // This currently just grabs the 'free-pro-team' schema, but ideally, we'd // get a list of all categories across all versions. - const freeProTeamFile = readdirSync(DECORATED_DIR) + const freeProTeamVersion = readdirSync(REST_DATA_DIR) .filter((file) => file.startsWith('api.github.com')) .shift() - const freeProTeamSchema = JSON.parse(readFileSync(join(DECORATED_DIR, freeProTeamFile), 'utf8')) + const freeProTeamSchema = JSON.parse( + readFileSync(join(REST_DATA_DIR, freeProTeamVersion, REST_SCHEMA_FILENAME), 'utf8') + ) // One off edge case for secret-scanning Docs-content issue 6637 if ('secret-scanning' in freeProTeamSchema) delete freeProTeamSchema['secret-scanning'] const restCategories = Object.entries(freeProTeamSchema)