docs/script/rest/test-open-api-schema.js

#!/usr/bin/env node

// [start-readme]
//
// Run this script to check if OpenAPI operations match versions in content/rest operations
//
// [end-readme]

import fs from 'fs'
import path from 'path'
import { readFile, readdir } from 'fs/promises'
import readFileAsync from '../../lib/readfile-async.js'
import getOperations from './utils/get-operations.js'
import frontmatter from '../../lib/read-frontmatter.js'
import _ from 'lodash'

let LOWEST_SUPPORTED_GHES_VERSION = Number.MAX_VALUE
let HIGHEST_SUPPORTED_GHES_VERSION = Number.MIN_VALUE

const dereferencedPath = path.join(process.cwd(), 'lib/rest/static/dereferenced')
const contentPath = path.join(process.cwd(), 'content/rest')
const schemas = await readdir(dereferencedPath)
const contentFiles = []
const contentCheck = {}
const openAPISchemaCheck = {}
const dereferencedSchemas = {}

export async function getDiffOpenAPIContentRest() {
  // Recursively go through the content/rest directory and add all categories/subcategories to contentFiles
  throughDirectory(contentPath)

  // Add version keys to contentCheck and dereferencedSchema objects
  await addVersionKeys()

  // Creating the categories/subcategories based on the current content directory
  await createCheckContentDirectory()

  // Create categories/subcategories from OpenAPI Schemas
  await createOpenAPISchemasCheck()

  // Get Differences between categories/subcategories from dereferenced schemas and the content/rest directory frontmatter versions
  const differences = getDifferences(openAPISchemaCheck, contentCheck)
  const errorMessages = {}

  if (Object.keys(differences).length > 0) {
    for (const schemaName in differences) {
      errorMessages[schemaName] = {}
      for (const category in differences[schemaName]) {
        if (!errorMessages[schemaName]) errorMessages[schemaName] = category
        errorMessages[schemaName][category] = {
          contentDir: contentCheck[schemaName][category],
          openAPI: openAPISchemaCheck[schemaName][category],
        }
      }
    }
  }
  return errorMessages
}

async function addVersionKeys() {
  for (const filename of schemas) {
    const schema = JSON.parse(await readFile(path.join(dereferencedPath, filename)))
    const key = filename.replace('.deref.json', '')
    contentCheck[key] = {}
    if (key.startsWith('ghes')) {
      const version = parseFloat(key.split('-')[1]).toFixed(1)
      LOWEST_SUPPORTED_GHES_VERSION = Math.min(LOWEST_SUPPORTED_GHES_VERSION, version)
      HIGHEST_SUPPORTED_GHES_VERSION = Math.max(HIGHEST_SUPPORTED_GHES_VERSION, version)
    }
    dereferencedSchemas[key] = schema
  }
  // GitHub Enterprise Cloud is just github.com bc it is not in the OpenAPI schema yet. Once it is, this should be updated
  contentCheck['ghec.github.com'] = {}
  dereferencedSchemas['ghec.github.com'] = dereferencedSchemas['api.github.com']
}

async function createOpenAPISchemasCheck() {
  for (const [schemaName, schema] of Object.entries(dereferencedSchemas)) {
    try {
      const operationsByCategory = {}
      // munge OpenAPI definitions object in an array of operations objects
      const operations = await getOperations(schema)
      // process each operation, asynchronously rendering markdown and stuff
      await Promise.all(operations.map((operation) => operation.process()))

      // Remove any keys not needed in the decorated files
      const decoratedOperations = operations.map(
        ({
          tags,
          description,
          serverUrl,
          operationId,
          categoryLabel,
          subcategoryLabel,
          contentType,
          externalDocs,
          ...props
        }) => props
      )

      const categories = [
        ...new Set(decoratedOperations.map((operation) => operation.category)),
      ].sort()

      categories.forEach((category) => {
        operationsByCategory[category] = {}
        const categoryOperations = decoratedOperations.filter(
          (operation) => operation.category === category
        )
        categoryOperations
          .filter((operation) => !operation.subcategory)
          .map((operation) => (operation.subcategory = operation.category))

        const subcategories = [
          ...new Set(categoryOperations.map((operation) => operation.subcategory)),
        ].sort()
        // the first item should be the item that has no subcategory
        // e.g., when the subcategory = category
        const firstItemIndex = subcategories.indexOf(category)
        if (firstItemIndex > -1) {
          const firstItem = subcategories.splice(firstItemIndex, 1)[0]
          subcategories.unshift(firstItem)
        }
        operationsByCategory[category] = subcategories.sort()
      })
      openAPISchemaCheck[schemaName] = operationsByCategory
      // One off edge case where secret-scanning should be removed from FPT. Docs Content #6637
      delete openAPISchemaCheck['api.github.com']['secret-scanning']
    } catch (error) {
      console.error(error)
      console.log('🐛 Whoops! Could not get operations by category!')
      process.exit(1)
    }
  }
}

async function createCheckContentDirectory() {
  for (const filename of contentFiles) {
    const { data } = frontmatter(await readFileAsync(filename, 'utf8'))
    const splitPath = filename.split('/')
    const subCategory = splitPath[splitPath.length - 1].replace('.md', '')
    const category =
      splitPath[splitPath.length - 2] === 'rest' ? subCategory : splitPath[splitPath.length - 2]
    const versions = data.versions

    for (const version in versions) {
      const schemaNames = getSchemaName(version, versions[version])

      for (const name of schemaNames) {
        if (!contentCheck[name][category]) {
          contentCheck[name][category] = [subCategory]
        } else {
          contentCheck[name][category].push(subCategory)
        }
        contentCheck[name][category].sort()
      }
    }
  }
}

function getDifferences(openAPISchemaCheck, contentCheck) {
  const differences = {}
  for (const version in openAPISchemaCheck) {
    const diffOpenApiContent = difference(openAPISchemaCheck[version], contentCheck[version])
    if (Object.keys(diffOpenApiContent).length > 0) differences[version] = diffOpenApiContent
  }

  return differences
}

function getSchemaName(version, versionValues) {
  const versions = []
  if (version === 'fpt') {
    if (versionValues === '*') versions.push('api.github.com')
  } else if (version === 'ghec') {
    if (versionValues === '*') versions.push('ghec.github.com')
  } else if (version === 'ghae') {
    if (versionValues === '*') versions.push('github.ae')
  } else if (version === 'ghes') {
    if (versionValues === '*') {
      versions.push('ghes-3.1', 'ghes-3.2', 'ghes-3.3', 'ghes-3.4')
    } else {
      let ver = ''
      let includeVersion = false
      let goUp
      for (const char of versionValues) {
        if ((char >= '0' && char <= '9') || char === '.') {
          ver += char
        } else if (char === '=') {
          includeVersion = true
        } else if (char === '>') {
          goUp = true
        } else if (char === '<') {
          goUp = false
        }
      }
      let numVersion = parseFloat(ver).toFixed(1)

      if (!includeVersion) {
        numVersion = goUp
          ? (parseFloat(numVersion) + 0.1).toFixed(1)
          : (parseFloat(numVersion) - 0.1).toFixed(1)
      }

      while (
        numVersion <= HIGHEST_SUPPORTED_GHES_VERSION &&
        numVersion >= LOWEST_SUPPORTED_GHES_VERSION
      ) {
        numVersion = parseFloat(numVersion).toFixed(1)
        versions.push('ghes-' + numVersion)
        numVersion = goUp
          ? (parseFloat(numVersion) + 0.1).toFixed(1)
          : (parseFloat(numVersion) - 0.1).toFixed(1)
      }
    }
  }
  return versions
}

function throughDirectory(directory) {
  fs.readdirSync(directory).forEach((file) => {
    const absolute = path.join(directory, file)
    if (fs.statSync(absolute).isDirectory()) {
      return throughDirectory(absolute)
    } else if (
      !directory.includes('rest/guides') &&
      !directory.includes('rest/overview') &&
      !file.includes('index.md') &&
      !file.includes('README.md')
    ) {
      return contentFiles.push(absolute)
    }
  })
}

function difference(obj1, obj2) {
  const diff = Object.keys(obj1).reduce((result, key) => {
    if (!Object.prototype.hasOwnProperty.call(obj2, key)) {
      result.push(key)
    } else if (_.isEqual(obj1[key], obj2[key])) {
      const resultKeyIndex = result.indexOf(key)
      result.splice(resultKeyIndex, 1)
    }
    return result
  }, Object.keys(obj2))

  return diff
}