add version directories for rest data (#34826)

lib/all-versions.js

@@ -1,13 +1,12 @@
 import fs from 'fs'
-import path from 'path'
 
 import enterpriseServerReleases from './enterprise-server-releases.js'
-
 // version = "plan"@"release"
 // example: enterprise-server@2.21
 // where "enterprise-server" is the plan and "2.21" is the release
 const versionDelimiter = '@'
 const latestNonNumberedRelease = 'latest'
+const REST_DATA_META_FILE = 'src/rest/data/meta.json'
 
 // !Explanation of versionless redirect fallbacks!
 // This array is **in order** of the versions the site should try to fall back to if
@@ -94,26 +93,13 @@ plans.forEach((planObj) => {
   })
 })
 
-const apiFilesPath = path.join(process.cwd(), 'src/rest/data')
+// Adds the calendar date (or api versions) to the allVersions object
-// This is what determines which versions are calendar date versioned for the REST API docs
+const apiVersions = JSON.parse(fs.readFileSync(REST_DATA_META_FILE, 'utf8'))['api-versions']
-// This is the source of truth for which versions are calendar date versioned.
+Object.keys(apiVersions).forEach((key) => {
-fs.readdirSync(apiFilesPath)
+  const docsVersion = getDocsVersion(key)
-  .filter((file) => file.endsWith('.json'))
+  allVersions[docsVersion].apiVersions.push(...apiVersions[key].sort())
-  .forEach((file) => {
+  allVersions[docsVersion].latestApiVersion = apiVersions[key].pop()
-    const fileName = file.split('.json')[0]
+})
-    const version = getDocsVersion(fileName)
-    const apiVersion = fileName.split(allVersions[version].openApiVersionName)[1].replace('.', '')
-
-    if (apiVersion !== '') {
-      allVersions[version].apiVersions.push(apiVersion)
-      if (
-        allVersions[version].latestApiVersion === '' ||
-        apiVersion > allVersions[version].latestApiVersion
-      ) {
-        allVersions[version].latestApiVersion = apiVersion
-      }
-    }
-  })
 
 export const allVersionKeys = Object.keys(allVersions)
 export const allVersionShortnames = Object.fromEntries(

src/rest/data/meta.json

@@ -0,0 +1,6 @@
+{
+  "api-versions": {
+    "api.github.com": ["2022-11-28"],
+    "ghec": ["2022-11-28"]
+  }
+}

src/rest/lib/index.js

@@ -6,9 +6,9 @@ import { getAutomatedPageMiniTocItems } from '../../../lib/get-mini-toc-items.js
 import { allVersions, getOpenApiVersion } from '../../../lib/all-versions.js'
 import languages from '../../../lib/languages.js'
 
-const schemasPath = 'src/rest/data'
+export const REST_DATA_DIR = 'src/rest/data'
-
+export const REST_SCHEMA_FILENAME = 'schema.json'
-const contentPath = 'content/rest'
+const REST_CONTENT_DIR = 'content/rest'
 
 /*
   Loads the schemas from the static/decorated folder into a single
@@ -59,7 +59,7 @@ Object.keys(languages).forEach((language) => {
 })
 
 export const categoriesWithoutSubcategories = fs
-  .readdirSync(contentPath)
+  .readdirSync(REST_CONTENT_DIR)
   .filter((file) => {
     return file.endsWith('.md') && !file.includes('index.md') && !file.includes('README.md')
   })
@@ -71,26 +71,21 @@ export const categoriesWithoutSubcategories = fs
 // and the OpenApi Version bc it's not the same
 export default async function getRest(version, apiVersion, category, subCategory) {
   const openApiVersion = getOpenApiVersion(version)
-  const filename = apiVersion ? `${openApiVersion}.${apiVersion}.json` : `${openApiVersion}.json`
+  const openapiSchemaName = apiVersion ? `${openApiVersion}.${apiVersion}` : `${openApiVersion}`
   const apiDate = apiVersion || NOT_API_VERSIONED
-
+  const fileName = path.join(REST_DATA_DIR, openapiSchemaName, REST_SCHEMA_FILENAME)
   if (!restOperations.has(openApiVersion)) {
     restOperations.set(openApiVersion, new Map())
     restOperations.get(openApiVersion).set(apiDate, new Map())
     // The `readCompressedJsonFileFallback()` function
     // will check for both a .br and .json extension.
-    restOperations
+    restOperations.get(openApiVersion).set(apiDate, readCompressedJsonFileFallback(fileName))
-      .get(openApiVersion)
-      .set(apiDate, readCompressedJsonFileFallback(path.join(schemasPath, filename)))
   } else if (!restOperations.get(openApiVersion).has(apiDate)) {
     restOperations.get(openApiVersion).set(apiDate, new Map())
     // The `readCompressedJsonFileFallback()` function
     // will check for both a .br and .json extension.
-    restOperations
+    restOperations.get(openApiVersion).set(apiDate, readCompressedJsonFileFallback(fileName))
-      .get(openApiVersion)
-      .set(apiDate, readCompressedJsonFileFallback(path.join(schemasPath, filename)))
   }
-
   if (subCategory) {
     return restOperations.get(openApiVersion).get(apiDate)[category][subCategory]
   } else if (category) {

src/rest/scripts/test-open-api-schema.js

@@ -12,11 +12,12 @@ import _ from 'lodash'
 import frontmatter from '../../../lib/read-frontmatter.js'
 import getApplicableVersions from '../../../lib/get-applicable-versions.js'
 import { allVersions, getDocsVersion } from '../../../lib/all-versions.js'
+import { REST_DATA_DIR, REST_SCHEMA_FILENAME } from '../lib/index.js'
 
 const contentFiles = []
 
 export async function getDiffOpenAPIContentRest() {
-  const contentPath = path.join(process.cwd(), 'content/rest')
+  const contentPath = 'content/rest'
 
   // Recursively go through the content/rest directory and add all categories/subcategories to contentFiles
   throughDirectory(contentPath)
@@ -50,15 +51,14 @@ export async function getDiffOpenAPIContentRest() {
 }
 
 async function createOpenAPISchemasCheck() {
-  const schemasPath = path.join(process.cwd(), 'src/rest/data')
   const openAPICheck = createCheckObj()
-  const schemas = fs.readdirSync(schemasPath)
+  const restDirectory = fs.readdirSync(REST_DATA_DIR).filter((dir) => dir !== 'meta.json')
 
-  schemas.forEach((file) => {
+  restDirectory.forEach((dir) => {
-    const fileData = fs.readFileSync(path.join(schemasPath, file))
+    const filename = path.join(REST_DATA_DIR, dir, REST_SCHEMA_FILENAME)
-    const fileSchema = JSON.parse(fileData.toString())
+    const fileSchema = JSON.parse(fs.readFileSync(filename))
     const categories = Object.keys(fileSchema).sort()
-    const version = getDocsVersion(file.split(/.json/)[0])
+    const version = getDocsVersion(dir)
 
     categories.forEach((category) => {
       const subcategories = Object.keys(fileSchema[category])

src/rest/scripts/utils/decorator.js

@@ -4,14 +4,22 @@ import path from 'path'
 import { slug } from 'github-slugger'
 
 import { allVersions } from '../../../../lib/all-versions.js'
-import { categoriesWithoutSubcategories } from '../../lib/index.js'
+import {
+  categoriesWithoutSubcategories,
+  REST_DATA_DIR,
+  REST_SCHEMA_FILENAME,
+} from '../../lib/index.js'
 import getOperations, { getWebhooks } from './get-operations.js'
 import { ENABLED_APPS_DIR, ENABLED_APPS_FILENAME } from '../../../github-apps/lib/index.js'
 import { WEBHOOK_DATA_DIR, WEBHOOK_SCHEMA_FILENAME } from '../../../webhooks/lib/index.js'
-
 const STATIC_REDIRECTS = 'lib/redirects/static/client-side-rest-api-redirects.json'
-const REST_DECORATED_DIR = 'src/rest/data'
 const REST_DEREFERENCED_DIR = 'src/rest/data/dereferenced'
+// All of the schema releases that we store in allVersions
+//  Ex: 'api.github.com', 'ghec', 'ghes-3.6', 'ghes-3.5',
+// 'ghes-3.4', 'ghes-3.3', 'ghes-3.2', 'github.ae'
+const OPENAPI_VERSION_NAMES = Object.keys(allVersions).map(
+  (elem) => allVersions[elem].openApiVersionName
+)
 
 export async function decorate(schemas) {
   console.log('\n🎄 Decorating the OpenAPI schema files in src/rest/data/dereferenced.\n')
@@ -20,6 +28,7 @@ export async function decorate(schemas) {
   await createStaticWebhookFiles(webhookOperations)
   const restOperations = await getRestOperations(restSchemas)
   await createStaticRestFiles(restOperations)
+  await updateRestMetaData(restSchemas)
 }
 
 async function getRestOperations(restSchemas) {
@@ -123,7 +132,9 @@ async function createStaticRestFiles(restOperations) {
       })
     })
 
-    const restFilename = path.join(REST_DECORATED_DIR, `${schemaName}.json`).replace('.deref', '')
+    const restFilename = path
+      .join(REST_DATA_DIR, schemaName, REST_SCHEMA_FILENAME)
+      .replace('.deref', '')
 
     // write processed operations to disk
     await writeFile(restFilename, JSON.stringify(operationsByCategory, null, 2))
@@ -304,19 +315,13 @@ export async function getOpenApiSchemaFiles(schemas) {
   const webhookSchemas = []
   const restSchemas = []
 
-  // All of the schema releases that we store in allVersions
-  //  Ex: 'api.github.com', 'ghec', 'ghes-3.6', 'ghes-3.5',
-  // 'ghes-3.4', 'ghes-3.3', 'ghes-3.2', 'github.ae'
-  const openApiVersions = Object.keys(allVersions).map(
-    (elem) => allVersions[elem].openApiVersionName
-  )
   // The full list of dereferened OpenAPI schemas received from
   // bundling the OpenAPI in github/github
   const schemaBaseNames = schemas.map((schema) => path.basename(schema, '.deref.json'))
   for (const schema of schemaBaseNames) {
     // catches all of the schemas that are not
     // calendar date versioned. Ex: ghec, ghes-3.7, and api.github.com
-    if (openApiVersions.includes(schema)) {
+    if (OPENAPI_VERSION_NAMES.includes(schema)) {
       webhookSchemas.push(schema)
       // Non-calendar date schemas could also match the calendar date versioned
       // counterpart.
@@ -336,3 +341,27 @@ export async function getOpenApiSchemaFiles(schemas) {
   }
   return { restSchemas, webhookSchemas }
 }
+
+// Every time we update the REST data files, we'll want to make sure the
+// meta.json file is updated with the latest api versions.
+async function updateRestMetaData(schemas) {
+  const restMetaFilename = `${REST_DATA_DIR}/meta.json`
+  const restMetaData = JSON.parse(await readFile(restMetaFilename, 'utf8'))
+  const restApiVersionData = restMetaData['api-versions']
+  schemas.forEach((schema) => {
+    // If the version isn't one of the OpenAPI version,
+    // then it's an api-versioned schema
+    if (!OPENAPI_VERSION_NAMES.includes(schema)) {
+      const openApiVer = OPENAPI_VERSION_NAMES.find((ver) => schema.startsWith(ver))
+      const date = schema.split(`${openApiVer}.`)[1]
+
+      if (!restApiVersionData[openApiVer].includes(date)) {
+        const dates = restApiVersionData[openApiVer]
+        dates.push(date)
+        restApiVersionData[openApiVer] = dates
+      }
+    }
+  })
+  restMetaData['api-versions'] = restApiVersionData
+  await writeFile(restMetaFilename, JSON.stringify(restMetaData, null, 2))
+}

tests/rendering/rest.js

@@ -4,7 +4,11 @@ import { readdirSync, readFileSync } from 'fs'
 import { join } from 'path'
 
 import { get, getDOM } from '../helpers/e2etest.js'
-import getRest, { categoriesWithoutSubcategories } from '../../src/rest/lib/index.js'
+import getRest, {
+  categoriesWithoutSubcategories,
+  REST_DATA_DIR,
+  REST_SCHEMA_FILENAME,
+} from '../../src/rest/lib/index.js'
 import { getEnabledForApps } from '../../src/github-apps/lib/index.js'
 import { isApiVersioned, allVersions } from '../../lib/all-versions.js'
 import { getDiffOpenAPIContentRest } from '../../src/rest/scripts/test-open-api-schema.js'
@@ -13,13 +17,14 @@ describe('REST references docs', () => {
   jest.setTimeout(3 * 60 * 1000)
 
   test('all category and subcategory REST pages render for free-pro-team', async () => {
-    const DECORATED_DIR = 'src/rest/data'
     // This currently just grabs the 'free-pro-team' schema, but ideally, we'd
     // get a list of all categories across all versions.
-    const freeProTeamFile = readdirSync(DECORATED_DIR)
+    const freeProTeamVersion = readdirSync(REST_DATA_DIR)
       .filter((file) => file.startsWith('api.github.com'))
       .shift()
-    const freeProTeamSchema = JSON.parse(readFileSync(join(DECORATED_DIR, freeProTeamFile), 'utf8'))
+    const freeProTeamSchema = JSON.parse(
+      readFileSync(join(REST_DATA_DIR, freeProTeamVersion, REST_SCHEMA_FILENAME), 'utf8')
+    )
     // One off edge case for secret-scanning Docs-content issue 6637
     if ('secret-scanning' in freeProTeamSchema) delete freeProTeamSchema['secret-scanning']
     const restCategories = Object.entries(freeProTeamSchema)