automate github apps docs (#35530)

Co-authored-by: Sarah Edwards <skedwards88@github.com>

src/github-apps/scripts/enabled-list-schema.js

@@ -0,0 +1,28 @@
+#!/usr/bin/env node
+// This schema is used to validate
+// src/github-apps/data/server-to-server-rest.json
+// src/github-apps/data/user-to-server-rest.json
+// and src/github-apps/data/fine-grained-pat.json
+
+export default {
+  type: 'object',
+  required: ['slug', 'subcategory', 'verb', 'requestPath'],
+  properties: {
+    slug: {
+      description: 'The documentation slug for the REST API operation.',
+      type: 'string',
+    },
+    subcategory: {
+      description: 'The subcategory of the REST API operation.',
+      type: 'string',
+    },
+    verb: {
+      description: 'The API request verb.',
+      type: 'string',
+    },
+    requestPath: {
+      description: 'The API request path.',
+      type: 'string',
+    },
+  },
+}

src/github-apps/scripts/permission-list-schema.js

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+// This schema is used to validate
+// src/github-apps/data/fine-grained-pat-permissions.json
+// and src/github-apps/data/server-to-server-permissions.json
+
+const permissionObjects = {
+  type: 'object',
+  required: ['access', 'category', 'subcategory', 'slug', 'verb', 'requestPath'],
+  properties: {
+    access: {
+      type: 'string',
+      enum: ['read', 'write', 'admin'],
+    },
+    category: {
+      type: 'string',
+    },
+    subcategory: {
+      type: 'string',
+    },
+    slug: {
+      type: 'string',
+    },
+    verb: {
+      type: 'string',
+      enum: ['get', 'patch', 'post', 'put', 'delete'],
+    },
+    requestPath: {
+      type: 'string',
+    },
+    'additional-permissions': {
+      type: 'array',
+    },
+  },
+}
+
+export default {
+  type: 'object',
+  required: ['title', 'displayTitle', 'permissions'],
+  properties: {
+    // Properties from the source OpenAPI schema that this module depends on
+    title: {
+      description: 'The name of the permission.',
+      type: 'string',
+    },
+    displayTitle: {
+      description: 'The display title, which includes the resource group and permission name',
+      type: 'string',
+    },
+    permissions: {
+      description: 'An oject with keys for read and write, each with an array of objects.',
+      type: 'array',
+      items: permissionObjects,
+    },
+  },
+}

src/github-apps/scripts/sync.js

@@ -5,59 +5,121 @@ import { mkdirp } from 'mkdirp'
 import { readFile, writeFile } from 'fs/promises'
 import path from 'path'
 import { slug } from 'github-slugger'
+import yaml from 'js-yaml'
 
 import { getOverrideCategory } from '../../rest/scripts/utils/operation.js'
-import { ENABLED_APPS_DIR, ENABLED_APPS_FILENAME } from '../lib/index.js'
+import { getContents } from '../../../script/helpers/git-utils.js'
+import permissionSchema from './permission-list-schema.js'
+import enabledSchema from './enabled-list-schema.js'
+import { validateData } from '../../rest/scripts/utils/validate-data.js'
+
+const ENABLED_APPS_DIR = 'src/github-apps/data'
+const CONFIG_FILE = 'src/github-apps/lib/config.json'
+
+export async function syncGitHubAppsData(openApiSource, sourceSchemas, progAccessSource = false) {
+  const { progAccessData, progActorResources } = await getProgAccessData(progAccessSource)
 
-// Creates the src/github-apps/data files used for
-// https://docs.github.com/en/rest/overview/endpoints-available-for-github-apps
-export async function syncGitHubAppsData(sourceDirectory, sourceSchemas) {
   for (const schemaName of sourceSchemas) {
-    const data = JSON.parse(await readFile(path.join(sourceDirectory, schemaName), 'utf8'))
+    const data = JSON.parse(await readFile(path.join(openApiSource, schemaName), 'utf8'))
+    const appsDataConfig = JSON.parse(await readFile(CONFIG_FILE, 'utf8'))
 
+    // Initialize the data structure with keys for each page type
+    const githubAppsData = {}
+    for (const pageType of Object.keys(appsDataConfig.pages)) {
+      githubAppsData[pageType] = {}
+    }
     // Because the information used on the apps page doesn't require any
     // rendered content we can parse the dereferenced files directly
-    const enabledForApps = {}
     for (const [requestPath, operationsAtPath] of Object.entries(data.paths)) {
       for (const [verb, operation] of Object.entries(operationsAtPath)) {
-        // We only want to process operations that are
-        // server-to-server GitHub App enabled
-        if (!operation['x-github'].enabledForGitHubApps) continue
+        // We only want to process operations that have programmatic access data
+        if (!progAccessData[operation.operationId]) continue
 
-        const schemaCategory = operation['x-github'].category
-        const schemaSubcategory = operation['x-github'].subcategory
-        const { category, subcategory } = getOverrideCategory(
-          operation.operationId,
-          schemaCategory,
-          schemaSubcategory
-        )
-
-        if (!enabledForApps[category]) {
-          enabledForApps[category] = []
-        }
-        enabledForApps[category].push({
+        const isInstallationAccessToken = progAccessData[operation.operationId].serverToServer
+        const isUserAccessToken = progAccessData[operation.operationId].userToServerRest
+        const isFineGrainedPat =
+          isUserAccessToken && !progAccessData[operation.operationId].disabledForPathv2
+        const { category, subcategory } = getCategory(operation)
+        const appDataOperation = {
           slug: slug(operation.summary),
           subcategory,
           verb,
           requestPath,
-        })
+        }
+        const appDataOperationWithCategory = Object.assign({ category }, appDataOperation)
+        // server-to-server
+        if (isInstallationAccessToken) {
+          initAppData(githubAppsData['server-to-server-rest'], category, appDataOperation)
+        }
+
+        // user-to-server
+        if (isUserAccessToken) {
+          initAppData(githubAppsData['user-to-server-rest'], category, appDataOperation)
+        }
+
+        // fine-grained pat
+        if (isFineGrainedPat) {
+          initAppData(githubAppsData['fine-grained-pat'], category, appDataOperation)
+        }
+
+        // permissions
+        for (const [permissionName, readOrWrite] of Object.entries(
+          progAccessData[operation.operationId].permissions
+        )) {
+          const tempTitle = permissionName.replace(/_/g, ' ')
+          const permissionNameExists = progActorResources[permissionName]
+          if (!permissionNameExists) {
+            console.warn(
+              `The permission ${permissionName} is missing from config/locales/programmatic_actor_fine_grained_resources.en.yml. Creating a placeholder value of ${tempTitle} until it's added.`
+            )
+          }
+          const title = progActorResources[permissionName]?.title || tempTitle
+          const resourceGroup = progActorResources[permissionName]?.resource_group || ''
+          const displayTitle = getDisplayTitle(title, resourceGroup)
+          const relatedPermissionNames = Object.keys(
+            progAccessData[operation.operationId].permissions
+          ).filter((permission) => permission !== permissionName)
+
+          // github app permissions
+          const serverToServerPermissions = githubAppsData['server-to-server-permissions']
+          if (!serverToServerPermissions[permissionName]) {
+            serverToServerPermissions[permissionName] = {
+              title,
+              displayTitle,
+              permissions: [],
+            }
+          }
+          const worksWithData = {
+            'user-to-server': Boolean(isUserAccessToken),
+            'server-to-server': Boolean(isInstallationAccessToken),
+            'additional-permissions': relatedPermissionNames,
+          }
+          serverToServerPermissions[permissionName].permissions.push(
+            Object.assign({}, appDataOperationWithCategory, { access: readOrWrite }, worksWithData)
+          )
+
+          // fine-grained pats
+          if (isFineGrainedPat) {
+            const findGrainedPatPermissions = githubAppsData['fine-grained-pat-permissions']
+            if (!findGrainedPatPermissions[permissionName]) {
+              findGrainedPatPermissions[permissionName] = {
+                title,
+                displayTitle,
+                permissions: [],
+              }
+            }
+
+            findGrainedPatPermissions[permissionName].permissions.push(
+              Object.assign({}, appDataOperationWithCategory, {
+                'additional-permissions': relatedPermissionNames,
+                access: readOrWrite,
+              })
+            )
+          }
+        }
       }
     }
 
-    if (Object.keys(enabledForApps).length === 0) {
-      throw new Error(
-        `Generating GitHub Apps data failed for ${sourceDirectory}/${schemaName}. The generated data file was empty.`
-      )
-    }
-
-    // Sort the operations by category for readability
-    const sortedOperations = Object.keys(enabledForApps)
-      .sort()
-      .reduce((acc, key) => {
-        acc[key] = enabledForApps[key]
-        return acc
-      }, {})
-
     const versionName = path.basename(schemaName, '.json')
     const targetDirectory = path.join(ENABLED_APPS_DIR, versionName)
 
@@ -66,8 +128,140 @@ export async function syncGitHubAppsData(sourceDirectory, sourceSchemas) {
       await mkdirp(targetDirectory)
     }
 
-    const targetPath = path.join(targetDirectory, ENABLED_APPS_FILENAME)
-    await writeFile(targetPath, JSON.stringify(sortedOperations, null, 2))
-    console.log(`✅ Wrote ${targetPath}`)
+    for (const pageType of Object.keys(githubAppsData)) {
+      const data = githubAppsData[pageType]
+      await validateAppData(data, pageType)
+
+      const filename = `${pageType}.json`
+      if (Object.keys(data).length === 0) {
+        throw new Error(
+          `Generating GitHub Apps data failed for ${openApiSource}/${schemaName}. The generated data file was empty.`
+        )
+      }
+      const sortedOperations = pageType.includes('permissions')
+        ? sortObjectByTitle(data)
+        : sortObjectByKeys(data)
+      const targetPath = path.join(targetDirectory, filename)
+      await writeFile(targetPath, JSON.stringify(sortedOperations, null, 2))
+      console.log(`✅ Wrote ${targetPath}`)
+    }
+  }
+}
+
+// When progAccessSource is defined, it will contain the root directory
+// of the repo with the programmatic access file. If it is not defined,
+// the file will be retrieved from the remote repo via the REST API.
+async function getProgAccessData(progAccessSource) {
+  let progAccessDataRaw
+  // config/locales/programmatic_actor_fine_grained_resources.en.yml
+  let progActorResources
+  const progAccessFilepath = 'config/access_control/programmatic_access.yaml'
+  const progActorFilepath = 'config/locales/programmatic_actor_fine_grained_resources.en.yml'
+  if (progAccessSource) {
+    progAccessDataRaw = yaml.load(
+      await readFile(path.join(progAccessSource, progAccessFilepath), 'utf8')
+    )
+    progActorResources = yaml.load(
+      await readFile(path.join(progAccessSource, progActorFilepath), 'utf8')
+    ).en.programmatic_actor_fine_grained_resources
+  } else {
+    progAccessDataRaw = yaml.load(
+      await getContents('github', 'github', 'master', progAccessFilepath)
+    )
+    progActorResources = yaml.load(
+      await getContents('github', 'github', 'master', progActorFilepath)
+    ).en.programmatic_actor_fine_grained_resources
+  }
+
+  const progAccessData = {}
+  for (const operation of progAccessDataRaw) {
+    const permissions = {}
+    if (operation.permission_sets) {
+      operation.permission_sets.forEach((permissionSet) => {
+        Object.assign(permissions, permissionSet)
+      })
+    }
+
+    const userToServerRest = operation.user_to_server.enabled
+    const serverToServer = operation.server_to_server.enabled
+    const allowPermissionlessAccess = operation.allows_permissionless_access
+    const disabledForPathv2 = operation.disabled_for_pathv2
+
+    progAccessData[operation.operation_ids] = {
+      userToServerRest,
+      serverToServer,
+      permissions,
+      allowPermissionlessAccess,
+      disabledForPathv2,
+    }
+  }
+  return { progAccessData, progActorResources }
+}
+
+function sortObjectByKeys(obj) {
+  return Object.keys(obj)
+    .sort()
+    .reduce((acc, key) => {
+      acc[key] = obj[key]
+      return acc
+    }, {})
+}
+
+function sortObjectByTitle(obj) {
+  return Object.keys(obj)
+    .sort((a, b) => {
+      if (obj[a].displayTitle > obj[b].displayTitle) {
+        return 1
+      }
+      if (obj[a].displayTitle < obj[b].displayTitle) {
+        return -1
+      }
+      return 0
+    })
+    .reduce((acc, key) => {
+      acc[key] = obj[key]
+      return acc
+    }, {})
+}
+
+function getCategory(operation) {
+  const schemaCategory = operation['x-github'].category
+  const schemaSubcategory = operation['x-github'].subcategory
+  return getOverrideCategory(operation.operationId, schemaCategory, schemaSubcategory)
+}
+
+function getDisplayTitle(title, resourceGroup) {
+  if (!title) {
+    console.warn(`No title found for title ${title} resource group ${resourceGroup}`)
+    return ''
+  }
+
+  return !resourceGroup
+    ? sentenceCase(title) + ' permissions'
+    : sentenceCase(resourceGroup) + ' permissions for ' + `"${title}"`
+}
+
+function sentenceCase(str) {
+  return str.charAt(0).toUpperCase() + str.slice(1)
+}
+
+function initAppData(storage, category, data) {
+  if (!storage[category]) {
+    storage[category] = []
+  }
+  storage[category].push(data)
+}
+
+async function validateAppData(data, pageType) {
+  if (pageType.includes('permissions')) {
+    for (const value of Object.values(data)) {
+      validateData(value, permissionSchema)
+    }
+  } else {
+    for (const arrayItems of Object.values(data)) {
+      for (const item of arrayItems) {
+        validateData(item, enabledSchema)
+      }
+    }
   }
 }