Restructure GraphQL automated content scripts (#34308)

src/graphql/scripts/README.md

@@ -0,0 +1,12 @@
+# GraphQL scripts
+
+A [scheduled workflow](../.github/workflows/update-graphql-files.yml) runs the following
+scripts on a daily basis:
+```
+src/graphql/scripts/update-files.js
+```
+These scripts update the [JSON data files](src/graphql/data) used to
+render GraphQL docs. See the [`src/graphql/README`](src/graphql/README.md)
+for more info.
+
+**Note**: The changelog script pulls content from the internal-developer repo. It relies on graphql-docs automation running daily to update the changelog files in internal-developer.

src/graphql/scripts/build-changelog.js

@@ -0,0 +1,308 @@
+#!/usr/bin/env node
+import { diff, ChangeType } from '@graphql-inspector/core'
+import { loadSchema } from '@graphql-tools/load'
+import fs from 'fs'
+import renderContent from '../../../lib/render-content/index.js'
+
+/**
+ * Tag `changelogEntry` with `date: YYYY-mm-dd`, then prepend it to the JSON
+ * structure written to `targetPath`. (`changelogEntry` and that file are modified in place.)
+ * @param {object} changelogEntry
+ * @param {string} targetPath
+ * @return {void}
+ */
+export function prependDatedEntry(changelogEntry, targetPath) {
+  // Build a `yyyy-mm-dd`-formatted date string
+  // and tag the changelog entry with it
+  const todayString = new Date().toISOString().slice(0, 10)
+  changelogEntry.date = todayString
+
+  const previousChangelogString = fs.readFileSync(targetPath)
+  const previousChangelog = JSON.parse(previousChangelogString)
+  // add a new entry to the changelog data
+  previousChangelog.unshift(changelogEntry)
+  // rewrite the updated changelog
+  fs.writeFileSync(targetPath, JSON.stringify(previousChangelog, null, 2))
+}
+
+/**
+ * Compare `oldSchemaString` to `newSchemaString`, and if there are any
+ * changes that warrant a changelog entry, return a changelog entry.
+ * Based on the parsed `previews`, identify changes that are under a preview.
+ * Otherwise, return null.
+ * @param {string} [oldSchemaString]
+ * @param {string} [newSchemaString]
+ * @param {Array<object>} [previews]
+ * @param {Array<object>} [oldUpcomingChanges]
+ * @param {Array<object>} [newUpcomingChanges]
+ * @return {object?}
+ */
+export async function createChangelogEntry(
+  oldSchemaString,
+  newSchemaString,
+  previews,
+  oldUpcomingChanges,
+  newUpcomingChanges
+) {
+  // Create schema objects out of the strings
+  const oldSchema = await loadSchema(oldSchemaString, {})
+  const newSchema = await loadSchema(newSchemaString, {})
+
+  // Generate changes between the two schemas
+  const changes = await diff(oldSchema, newSchema)
+  const changesToReport = []
+  changes.forEach(function (change) {
+    if (CHANGES_TO_REPORT.includes(change.type)) {
+      changesToReport.push(change)
+    } else if (CHANGES_TO_IGNORE.includes(change.type)) {
+      // Do nothing
+    } else {
+      throw new Error(
+        'This change type should be added to CHANGES_TO_REPORT or CHANGES_TO_IGNORE: ' + change.type
+      )
+    }
+  })
+
+  const { schemaChangesToReport, previewChangesToReport } = segmentPreviewChanges(
+    changesToReport,
+    previews
+  )
+
+  const addedUpcomingChanges = newUpcomingChanges.filter(function (change) {
+    // Manually check each of `newUpcomingChanges` for an equivalent entry
+    // in `oldUpcomingChanges`.
+    return !oldUpcomingChanges.find(function (oldChange) {
+      return (
+        oldChange.location === change.location &&
+        oldChange.date === change.date &&
+        oldChange.description === change.description
+      )
+    })
+  })
+
+  // If there were any changes, create a changelog entry
+  if (
+    schemaChangesToReport.length > 0 ||
+    previewChangesToReport.length > 0 ||
+    addedUpcomingChanges.length > 0
+  ) {
+    const changelogEntry = {
+      schemaChanges: [],
+      previewChanges: [],
+      upcomingChanges: [],
+    }
+
+    const cleanedSchemaChanges = cleanMessagesFromChanges(schemaChangesToReport)
+    const renderedScheamChanges = await Promise.all(
+      cleanedSchemaChanges.map(async (change) => {
+        return await renderContent(change)
+      })
+    )
+    const schemaChange = {
+      title: 'The GraphQL schema includes these changes:',
+      // Replace single quotes which wrap field/argument/type names with backticks
+      changes: renderedScheamChanges,
+    }
+    changelogEntry.schemaChanges.push(schemaChange)
+
+    for (const previewTitle in previewChangesToReport) {
+      const previewChanges = previewChangesToReport[previewTitle]
+      const cleanedPreviewChanges = cleanMessagesFromChanges(previewChanges.changes)
+      const renderedPreviewChanges = await Promise.all(
+        cleanedPreviewChanges.map(async (change) => {
+          return renderContent(change)
+        })
+      )
+      const cleanTitle = cleanPreviewTitle(previewTitle)
+      const entryTitle =
+        'The [' +
+        cleanTitle +
+        '](/graphql/overview/schema-previews#' +
+        previewAnchor(cleanTitle) +
+        ') includes these changes:'
+      changelogEntry.previewChanges.push({
+        title: entryTitle,
+        changes: renderedPreviewChanges,
+      })
+    }
+
+    if (addedUpcomingChanges.length > 0) {
+      const cleanedUpcomingChanges = addedUpcomingChanges.map((change) => {
+        const location = change.location
+        const description = change.description
+        const date = change.date.split('T')[0]
+        return 'On member `' + location + '`:' + description + ' **Effective ' + date + '**.'
+      })
+      const renderedUpcomingChanges = await Promise.all(
+        cleanedUpcomingChanges.map(async (change) => {
+          return await renderContent(change)
+        })
+      )
+      changelogEntry.upcomingChanges.push({
+        title: 'The following changes will be made to the schema:',
+        changes: renderedUpcomingChanges,
+      })
+    }
+
+    return changelogEntry
+  } else {
+    return null
+  }
+}
+
+/**
+ * Prepare the preview title from github/github source for the docs.
+ * @param {string} title
+ * @return {string}
+ */
+export function cleanPreviewTitle(title) {
+  if (title === 'UpdateRefsPreview') {
+    title = 'Update refs preview'
+  } else if (title === 'MergeInfoPreview') {
+    title = 'Merge info preview'
+  } else if (!title.endsWith('preview')) {
+    title = title + ' preview'
+  }
+  return title
+}
+
+/**
+ * Turn the given title into an HTML-ready anchor.
+ * (ported from graphql-docs/lib/graphql_docs/update_internal_developer/change_log.rb#L281)
+ * @param {string} [previewTitle]
+ * @return {string}
+ */
+export function previewAnchor(previewTitle) {
+  return previewTitle
+    .toLowerCase()
+    .replace(/ /g, '-')
+    .replace(/[^\w-]/g, '')
+}
+
+/**
+ * Turn changes from graphql-inspector into messages for the HTML changelog.
+ * @param {Array<object>} changes
+ * @return {Array<string>}
+ */
+export function cleanMessagesFromChanges(changes) {
+  return changes.map(function (change) {
+    // replace single quotes around graphql names with backticks,
+    // to match previous behavior from graphql-schema-comparator
+    return change.message.replace(/'([a-zA-Z. :!]+)'/g, '`$1`')
+  })
+}
+
+/**
+ * Split `changesToReport` into two parts,
+ * one for changes in the main schema,
+ * and another for changes that are under preview.
+ * (Ported from /graphql-docs/lib/graphql_docs/update_internal_developer/change_log.rb#L230)
+ * @param {Array<object>} changesToReport
+ * @param {object} previews
+ * @return {object}
+ */
+export function segmentPreviewChanges(changesToReport, previews) {
+  // Build a map of `{ path => previewTitle` }
+  // for easier lookup of change to preview
+  const pathToPreview = {}
+  previews.forEach(function (preview) {
+    preview.toggled_on.forEach(function (path) {
+      pathToPreview[path] = preview.title
+    })
+  })
+  const schemaChanges = []
+  const changesByPreview = {}
+
+  changesToReport.forEach(function (change) {
+    // For each change, see if its path _or_ one of its ancestors
+    // is covered by a preview. If it is, mark this change as belonging to a preview
+    const pathParts = change.path.split('.')
+    let testPath = null
+    let previewTitle = null
+    let previewChanges = null
+    while (pathParts.length > 0 && !previewTitle) {
+      testPath = pathParts.join('.')
+      previewTitle = pathToPreview[testPath]
+      // If that path didn't find a match, then we'll
+      // check the next ancestor.
+      pathParts.pop()
+    }
+    if (previewTitle) {
+      previewChanges =
+        changesByPreview[previewTitle] ||
+        (changesByPreview[previewTitle] = {
+          title: previewTitle,
+          changes: [],
+        })
+      previewChanges.changes.push(change)
+    } else {
+      schemaChanges.push(change)
+    }
+  })
+  return { schemaChangesToReport: schemaChanges, previewChangesToReport: changesByPreview }
+}
+
+// We only want to report changes to schema structure.
+// Deprecations are covered by "upcoming changes."
+// By listing the changes explicitly here, we can make sure that,
+// if the library changes, we don't miss publishing anything that we mean to.
+// This was originally ported from graphql-docs/lib/graphql_docs/update_internal_developer/change_log.rb#L35-L103
+const CHANGES_TO_REPORT = [
+  ChangeType.FieldArgumentDefaultChanged,
+  ChangeType.FieldArgumentTypeChanged,
+  ChangeType.EnumValueRemoved,
+  ChangeType.EnumValueAdded,
+  ChangeType.FieldRemoved,
+  ChangeType.FieldAdded,
+  ChangeType.FieldTypeChanged,
+  ChangeType.FieldArgumentAdded,
+  ChangeType.FieldArgumentRemoved,
+  ChangeType.ObjectTypeInterfaceAdded,
+  ChangeType.ObjectTypeInterfaceRemoved,
+  ChangeType.InputFieldRemoved,
+  ChangeType.InputFieldAdded,
+  ChangeType.InputFieldDefaultValueChanged,
+  ChangeType.InputFieldTypeChanged,
+  ChangeType.TypeRemoved,
+  ChangeType.TypeAdded,
+  ChangeType.TypeKindChanged,
+  ChangeType.UnionMemberRemoved,
+  ChangeType.UnionMemberAdded,
+  ChangeType.SchemaQueryTypeChanged,
+  ChangeType.SchemaMutationTypeChanged,
+  ChangeType.SchemaSubscriptionTypeChanged,
+]
+
+const CHANGES_TO_IGNORE = [
+  ChangeType.FieldArgumentDescriptionChanged,
+  ChangeType.DirectiveRemoved,
+  ChangeType.DirectiveAdded,
+  ChangeType.DirectiveDescriptionChanged,
+  ChangeType.DirectiveLocationAdded,
+  ChangeType.DirectiveLocationRemoved,
+  ChangeType.DirectiveArgumentAdded,
+  ChangeType.DirectiveArgumentRemoved,
+  ChangeType.DirectiveArgumentDescriptionChanged,
+  ChangeType.DirectiveArgumentDefaultValueChanged,
+  ChangeType.DirectiveArgumentTypeChanged,
+  ChangeType.EnumValueDescriptionChanged,
+  ChangeType.EnumValueDeprecationReasonChanged,
+  ChangeType.EnumValueDeprecationReasonAdded,
+  ChangeType.EnumValueDeprecationReasonRemoved,
+  ChangeType.FieldDescriptionChanged,
+  ChangeType.FieldDescriptionAdded,
+  ChangeType.FieldDescriptionRemoved,
+  ChangeType.FieldDeprecationAdded,
+  ChangeType.FieldDeprecationRemoved,
+  ChangeType.FieldDeprecationReasonChanged,
+  ChangeType.FieldDeprecationReasonAdded,
+  ChangeType.FieldDeprecationReasonRemoved,
+  ChangeType.InputFieldDescriptionAdded,
+  ChangeType.InputFieldDescriptionRemoved,
+  ChangeType.InputFieldDescriptionChanged,
+  ChangeType.TypeDescriptionChanged,
+  ChangeType.TypeDescriptionRemoved,
+  ChangeType.TypeDescriptionAdded,
+]
+
+export default { createChangelogEntry, cleanPreviewTitle, previewAnchor, prependDatedEntry }

src/graphql/scripts/update-files.js

@@ -0,0 +1,164 @@
+#!/usr/bin/env node
+import fs from 'fs/promises'
+import path from 'path'
+import mkdirp from 'mkdirp'
+import yaml from 'js-yaml'
+import { execSync } from 'child_process'
+import { getContents, listMatchingRefs } from '../../../script/helpers/git-utils.js'
+import { allVersions } from '../../../lib/all-versions.js'
+import processPreviews from './utils/process-previews.js'
+import processUpcomingChanges from './utils/process-upcoming-changes.js'
+import processSchemas from './utils/process-schemas.js'
+import { prependDatedEntry, createChangelogEntry } from './build-changelog.js'
+
+const graphqlDataDir = path.join(process.cwd(), 'data/graphql')
+const graphqlStaticDir = path.join(process.cwd(), 'src/graphql/data')
+const dataFilenames = JSON.parse(
+  await fs.readFile(path.join(process.cwd(), './src/graphql/scripts/utils/data-filenames.json'))
+)
+
+// check for required PAT
+if (!process.env.GITHUB_TOKEN) {
+  throw new Error('Error! You must have a GITHUB_TOKEN set in an .env file to run this script.')
+}
+
+const versionsToBuild = Object.keys(allVersions)
+
+main()
+
+async function main() {
+  const previewsJson = {}
+  const upcomingChangesJson = {}
+
+  for (const version of versionsToBuild) {
+    // Get the relevant GraphQL name  for the current version
+    // For example, free-pro-team@latest corresponds to dotcom,
+    // enterprise-server@2.22 corresponds to ghes-2.22,
+    // and github-ae@latest corresponds to ghae
+    const graphqlVersion = allVersions[version].miscVersionName
+
+    // 1. UPDATE PREVIEWS
+    const previewsPath = getDataFilepath('previews', graphqlVersion)
+    const safeForPublicPreviews = yaml.load(await getRemoteRawContent(previewsPath, graphqlVersion))
+    await updateFile(previewsPath, yaml.dump(safeForPublicPreviews))
+    previewsJson[graphqlVersion] = processPreviews(safeForPublicPreviews)
+
+    // 2. UPDATE UPCOMING CHANGES
+    const upcomingChangesPath = getDataFilepath('upcomingChanges', graphqlVersion)
+    const previousUpcomingChanges = yaml.load(await fs.readFile(upcomingChangesPath, 'utf8'))
+    const safeForPublicChanges = await getRemoteRawContent(upcomingChangesPath, graphqlVersion)
+    await updateFile(upcomingChangesPath, safeForPublicChanges)
+    upcomingChangesJson[graphqlVersion] = await processUpcomingChanges(safeForPublicChanges)
+
+    // 3. UPDATE SCHEMAS
+    // note: schemas live in separate files per version
+    const schemaPath = getDataFilepath('schemas', graphqlVersion)
+    const previousSchemaString = await fs.readFile(schemaPath, 'utf8')
+    const latestSchema = await getRemoteRawContent(schemaPath, graphqlVersion)
+    await updateFile(schemaPath, latestSchema)
+    const schemaJsonPerVersion = await processSchemas(latestSchema, safeForPublicPreviews)
+    await updateStaticFile(
+      schemaJsonPerVersion,
+      path.join(graphqlStaticDir, `schema-${graphqlVersion}.json`)
+    )
+
+    // 4. UPDATE CHANGELOG
+    if (allVersions[version].nonEnterpriseDefault) {
+      // The Changelog is only build for free-pro-team@latest
+      const changelogEntry = await createChangelogEntry(
+        previousSchemaString,
+        latestSchema,
+        safeForPublicPreviews,
+        previousUpcomingChanges.upcoming_changes,
+        yaml.load(safeForPublicChanges).upcoming_changes
+      )
+      if (changelogEntry) {
+        prependDatedEntry(
+          changelogEntry,
+          path.join(process.cwd(), 'src/graphql/data/changelog.json')
+        )
+      }
+    }
+  }
+
+  await updateStaticFile(previewsJson, path.join(graphqlStaticDir, 'previews.json'))
+  await updateStaticFile(upcomingChangesJson, path.join(graphqlStaticDir, 'upcoming-changes.json'))
+
+  // Ensure the YAML linter runs before checkinging in files
+  execSync('npx prettier -w "**/*.{yml,yaml}"')
+}
+
+// get latest from github/github
+async function getRemoteRawContent(filepath, graphqlVersion) {
+  const options = {
+    owner: 'github',
+    repo: 'github',
+  }
+
+  // find the relevant branch in github/github and set it as options.ref
+  await setBranchAsRef(options, graphqlVersion)
+
+  // add the filepath to the options so we can get the contents of the file
+  options.path = `config/${path.basename(filepath)}`
+
+  return getContents(...Object.values(options))
+}
+
+// find the relevant filepath in src/graphql/scripts/util/data-filenames.json
+function getDataFilepath(id, graphqlVersion) {
+  const versionType = getVersionType(graphqlVersion)
+
+  // for example, dataFilenames['schema']['ghes'] = schema.docs-enterprise.graphql
+  const filename = dataFilenames[id][versionType]
+
+  // dotcom files live at the root of data/graphql
+  // non-dotcom files live in data/graphql/<version_subdir>
+  const dataSubdir = graphqlVersion === 'dotcom' ? '' : graphqlVersion
+
+  return path.join(graphqlDataDir, dataSubdir, filename)
+}
+
+async function setBranchAsRef(options, graphqlVersion, branch = false) {
+  const versionType = getVersionType(graphqlVersion)
+  const defaultBranch = 'master'
+
+  const branches = {
+    dotcom: defaultBranch,
+    ghec: defaultBranch,
+    ghes: `enterprise-${graphqlVersion.replace('ghes-', '')}-release`,
+    // TODO confirm the below is accurate after the release branch is created
+    ghae: 'github-ae-release',
+  }
+
+  // the first time this runs, it uses the branch found for the version above
+  if (!branch) branch = branches[versionType]
+
+  // set the branch as the ref
+  options.ref = `heads/${branch}`
+
+  // check whether the branch can be found in github/github
+  const foundRefs = await listMatchingRefs(...Object.values(options))
+
+  // if foundRefs array is empty, the branch cannot be found, so try a fallback
+  if (!foundRefs.length) {
+    const fallbackBranch = defaultBranch
+    await setBranchAsRef(options, graphqlVersion, fallbackBranch)
+  }
+}
+
+// given a GraphQL version like `ghes-2.22`, return `ghes`;
+// given a GraphQL version like `ghae` or `dotcom`, return as is
+function getVersionType(graphqlVersion) {
+  return graphqlVersion.split('-')[0]
+}
+
+async function updateFile(filepath, content) {
+  console.log(`fetching latest data to ${filepath}`)
+  await mkdirp(path.dirname(filepath))
+  return fs.writeFile(filepath, content, 'utf8')
+}
+
+async function updateStaticFile(json, filepath) {
+  const jsonString = JSON.stringify(json, null, 2)
+  return updateFile(filepath, jsonString)
+}

src/graphql/scripts/utils/data-filenames.json

@@ -0,0 +1,20 @@
+{
+  "schemas": {
+    "dotcom": "schema.docs.graphql",
+    "ghec": "schema.docs.graphql",
+    "ghes": "schema.docs-enterprise.graphql",
+    "ghae": "schema.docs-ghae.graphql"
+  },
+  "previews": {
+    "dotcom": "graphql_previews.yml",
+    "ghec": "graphql_previews.yml",
+    "ghes": "graphql_previews.enterprise.yml",
+    "ghae": "graphql_previews.ghae.yml"
+  },
+  "upcomingChanges": {
+    "dotcom": "graphql_upcoming_changes.public.yml",
+    "ghec": "graphql_upcoming_changes.public.yml",
+    "ghes": "graphql_upcoming_changes.public-enterprise.yml",
+    "ghae": "graphql_upcoming_changes.public-ghae.yml"
+  }
+}

src/graphql/scripts/utils/process-previews.js

@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+import { sentenceCase } from 'change-case'
+import GithubSlugger from 'github-slugger'
+const slugger = new GithubSlugger()
+const inputOrPayload = /(Input|Payload)$/m
+
+export default function processPreviews(previews) {
+  // clean up raw yml data
+  previews.forEach((preview) => {
+    // remove any extra info that follows a hyphen
+    preview.title = sentenceCase(preview.title.replace(/ -.+/, '')).replace('it hub', 'itHub') // fix overcorrected `git hub` from sentenceCasing
+
+    // Add `preview` to the end of titles if needed
+    preview.title = preview.title.endsWith('preview') ? preview.title : `${preview.title} preview`
+
+    // filter out schema members that end in `Input` or `Payload`
+    preview.toggled_on = preview.toggled_on.filter(
+      (schemaMember) => !inputOrPayload.test(schemaMember)
+    )
+
+    // remove unnecessary leading colon
+    preview.toggled_by = preview.toggled_by.replace(':', '')
+
+    // add convenience properties
+    preview.accept_header = `application/vnd.github.${preview.toggled_by}+json`
+
+    delete preview.announcement
+    delete preview.updates
+
+    slugger.reset()
+    preview.href = `/graphql/overview/schema-previews#${slugger.slug(preview.title)}`
+  })
+
+  return previews
+}

src/graphql/scripts/utils/process-schemas.js

@@ -0,0 +1,454 @@
+#!/usr/bin/env node
+import { sortBy } from 'lodash-es'
+import { parse, buildASTSchema } from 'graphql'
+import helpers from './schema-helpers.js'
+import fs from 'fs/promises'
+import path from 'path'
+
+const externalScalarsJSON = JSON.parse(
+  await fs.readFile(path.join(process.cwd(), './src/graphql/lib/non-schema-scalars.json'))
+)
+const externalScalars = await Promise.all(
+  externalScalarsJSON.map(async (scalar) => {
+    scalar.description = await helpers.getDescription(scalar.description)
+    scalar.id = helpers.getId(scalar.name)
+    scalar.href = helpers.getFullLink('scalars', scalar.id)
+    return scalar
+  })
+)
+
+// select and format all the data from the schema that we need for the docs
+// used in the build step
+export default async function processSchemas(idl, previewsPerVersion) {
+  const schemaAST = parse(idl.toString())
+  const schema = buildASTSchema(schemaAST)
+
+  // list of objects is used when processing mutations
+  const objectsInSchema = schemaAST.definitions.filter((def) => def.kind === 'ObjectTypeDefinition')
+
+  const data = {}
+
+  data.queries = []
+  data.mutations = []
+  data.objects = []
+  data.interfaces = []
+  data.enums = []
+  data.unions = []
+  data.inputObjects = []
+  data.scalars = []
+
+  await Promise.all(
+    schemaAST.definitions.map(async (def) => {
+      // QUERIES
+      if (def.name.value === 'Query') {
+        await Promise.all(
+          def.fields.map(async (field) => {
+            const query = {}
+            const queryArgs = []
+
+            query.name = field.name.value
+            query.type = helpers.getType(field)
+            query.kind = helpers.getTypeKind(query.type, schema)
+            query.id = helpers.getId(query.type)
+            query.href = helpers.getFullLink(query.kind, query.id)
+            query.description = await helpers.getDescription(field.description.value)
+            query.isDeprecated = helpers.getDeprecationStatus(field.directives, query.name)
+            query.deprecationReason = await helpers.getDeprecationReason(field.directives, query)
+            query.preview = await helpers.getPreview(field.directives, query, previewsPerVersion)
+
+            await Promise.all(
+              field.arguments.map(async (arg) => {
+                const queryArg = {}
+                queryArg.name = arg.name.value
+                queryArg.defaultValue = arg.defaultValue ? arg.defaultValue.value : undefined
+                queryArg.type = helpers.getType(arg)
+                queryArg.id = helpers.getId(queryArg.type)
+                queryArg.kind = helpers.getTypeKind(queryArg.type, schema)
+                queryArg.href = helpers.getFullLink(queryArg.kind, queryArg.id)
+                queryArg.description = await helpers.getDescription(arg.description.value)
+                queryArg.isDeprecated = helpers.getDeprecationStatus(arg.directives, queryArg.name)
+                queryArg.deprecationReason = await helpers.getDeprecationReason(
+                  arg.directives,
+                  queryArg
+                )
+                queryArg.preview = await helpers.getPreview(
+                  arg.directives,
+                  queryArg,
+                  previewsPerVersion
+                )
+                queryArgs.push(queryArg)
+              })
+            )
+
+            query.args = sortBy(queryArgs, 'name')
+            data.queries.push(query)
+          })
+        )
+
+        return
+      }
+
+      // MUTATIONS
+      if (def.name.value === 'Mutation') {
+        await Promise.all(
+          def.fields.map(async (field) => {
+            const mutation = {}
+            const inputFields = []
+            const returnFields = []
+
+            mutation.name = field.name.value
+            mutation.kind = helpers.getKind(def.name.value)
+            mutation.id = helpers.getId(mutation.name)
+            mutation.href = helpers.getFullLink('mutations', mutation.id)
+            mutation.description = await helpers.getDescription(field.description.value)
+            mutation.isDeprecated = helpers.getDeprecationStatus(field.directives, mutation.name)
+            mutation.deprecationReason = await helpers.getDeprecationReason(
+              field.directives,
+              mutation
+            )
+            mutation.preview = await helpers.getPreview(
+              field.directives,
+              mutation,
+              previewsPerVersion
+            )
+
+            // there is only ever one input field argument, but loop anyway
+            await Promise.all(
+              field.arguments.map(async (field) => {
+                const inputField = {}
+                inputField.name = field.name.value
+                inputField.type = helpers.getType(field)
+                inputField.id = helpers.getId(inputField.type)
+                inputField.kind = helpers.getTypeKind(inputField.type, schema)
+                inputField.href = helpers.getFullLink(inputField.kind, inputField.id)
+                inputFields.push(inputField)
+              })
+            )
+
+            mutation.inputFields = sortBy(inputFields, 'name')
+
+            // get return fields
+            // first get the payload, then find payload object's fields. these are the mutation's return fields.
+            const returnType = helpers.getType(field)
+            const mutationReturnFields = objectsInSchema.find(
+              (obj) => obj.name.value === returnType
+            )
+
+            if (!mutationReturnFields) console.log(`no return fields found for ${returnType}`)
+
+            await Promise.all(
+              mutationReturnFields.fields.map(async (field) => {
+                const returnField = {}
+                returnField.name = field.name.value
+                returnField.type = helpers.getType(field)
+                returnField.id = helpers.getId(returnField.type)
+                returnField.kind = helpers.getTypeKind(returnField.type, schema)
+                returnField.href = helpers.getFullLink(returnField.kind, returnField.id)
+                returnField.description = await helpers.getDescription(field.description.value)
+                returnField.isDeprecated = helpers.getDeprecationStatus(
+                  field.directives,
+                  returnField.name
+                )
+                returnField.deprecationReason = await helpers.getDeprecationReason(
+                  field.directives,
+                  returnField
+                )
+                returnField.preview = await helpers.getPreview(
+                  field.directives,
+                  returnField,
+                  previewsPerVersion
+                )
+                returnFields.push(returnField)
+              })
+            )
+
+            mutation.returnFields = sortBy(returnFields, 'name')
+
+            data.mutations.push(mutation)
+          })
+        )
+        return
+      }
+
+      // OBJECTS
+      if (def.kind === 'ObjectTypeDefinition') {
+        // objects ending with 'Payload' are only used to derive mutation values
+        // they are not included in the objects docs
+        if (def.name.value.endsWith('Payload')) return
+
+        const object = {}
+        const objectImplements = []
+        const objectFields = []
+
+        object.name = def.name.value
+        object.kind = helpers.getKind(def.kind)
+        object.id = helpers.getId(object.name)
+        object.href = helpers.getFullLink('objects', object.id)
+        object.description = await helpers.getDescription(def.description.value)
+        object.isDeprecated = helpers.getDeprecationStatus(def.directives, object.name)
+        object.deprecationReason = await helpers.getDeprecationReason(def.directives, object)
+        object.preview = await helpers.getPreview(def.directives, object, previewsPerVersion)
+
+        // an object's interfaces render in the `Implements` section
+        // interfaces do not have directives so they cannot be under preview/deprecated
+        if (def.interfaces.length) {
+          await Promise.all(
+            def.interfaces.map(async (graphqlInterface) => {
+              const objectInterface = {}
+              objectInterface.name = graphqlInterface.name.value
+              objectInterface.id = helpers.getId(objectInterface.name)
+              objectInterface.href = helpers.getFullLink('interfaces', objectInterface.id)
+              objectImplements.push(objectInterface)
+            })
+          )
+        }
+
+        // an object's fields render in the `Fields` section
+        if (def.fields.length) {
+          await Promise.all(
+            def.fields.map(async (field) => {
+              if (!field.description) return
+              const objectField = {}
+
+              objectField.name = field.name.value
+              objectField.description = await helpers.getDescription(field.description.value)
+              objectField.type = helpers.getType(field)
+              objectField.id = helpers.getId(objectField.type)
+              objectField.kind = helpers.getTypeKind(objectField.type, schema)
+              objectField.href = helpers.getFullLink(objectField.kind, objectField.id)
+              objectField.arguments = await helpers.getArguments(field.arguments, schema)
+              objectField.isDeprecated = helpers.getDeprecationStatus(field.directives)
+              objectField.deprecationReason = await helpers.getDeprecationReason(
+                field.directives,
+                objectField
+              )
+              objectField.preview = await helpers.getPreview(
+                field.directives,
+                objectField,
+                previewsPerVersion
+              )
+
+              objectFields.push(objectField)
+            })
+          )
+        }
+
+        if (objectImplements.length) object.implements = sortBy(objectImplements, 'name')
+        if (objectFields.length) object.fields = sortBy(objectFields, 'name')
+
+        data.objects.push(object)
+        return
+      }
+
+      // INTERFACES
+      if (def.kind === 'InterfaceTypeDefinition') {
+        const graphqlInterface = {}
+        const interfaceFields = []
+
+        graphqlInterface.name = def.name.value
+        graphqlInterface.kind = helpers.getKind(def.kind)
+        graphqlInterface.id = helpers.getId(graphqlInterface.name)
+        graphqlInterface.href = helpers.getFullLink('interfaces', graphqlInterface.id)
+        graphqlInterface.description = await helpers.getDescription(def.description.value)
+        graphqlInterface.isDeprecated = helpers.getDeprecationStatus(def.directives)
+        graphqlInterface.deprecationReason = await helpers.getDeprecationReason(
+          def.directives,
+          graphqlInterface
+        )
+        graphqlInterface.preview = await helpers.getPreview(
+          def.directives,
+          graphqlInterface,
+          previewsPerVersion
+        )
+
+        // an interface's fields render in the "Fields" section
+        if (def.fields.length) {
+          await Promise.all(
+            def.fields.map(async (field) => {
+              if (!field.description) return
+              const interfaceField = {}
+
+              interfaceField.name = field.name.value
+              interfaceField.description = await helpers.getDescription(field.description.value)
+              interfaceField.type = helpers.getType(field)
+              interfaceField.id = helpers.getId(interfaceField.type)
+              interfaceField.kind = helpers.getTypeKind(interfaceField.type, schema)
+              interfaceField.href = helpers.getFullLink(interfaceField.kind, interfaceField.id)
+              interfaceField.arguments = await helpers.getArguments(field.arguments, schema)
+              interfaceField.isDeprecated = helpers.getDeprecationStatus(field.directives)
+              interfaceField.deprecationReason = await helpers.getDeprecationReason(
+                field.directives,
+                interfaceField
+              )
+              interfaceField.preview = await helpers.getPreview(
+                field.directives,
+                interfaceField,
+                previewsPerVersion
+              )
+
+              interfaceFields.push(interfaceField)
+            })
+          )
+        }
+
+        graphqlInterface.fields = sortBy(interfaceFields, 'name')
+
+        data.interfaces.push(graphqlInterface)
+        return
+      }
+
+      // ENUMS
+      if (def.kind === 'EnumTypeDefinition') {
+        const graphqlEnum = {}
+        const enumValues = []
+
+        graphqlEnum.name = def.name.value
+        graphqlEnum.kind = helpers.getKind(def.kind)
+        graphqlEnum.id = helpers.getId(graphqlEnum.name)
+        graphqlEnum.href = helpers.getFullLink('enums', graphqlEnum.id)
+        graphqlEnum.description = await helpers.getDescription(def.description.value)
+        graphqlEnum.isDeprecated = helpers.getDeprecationStatus(def.directives)
+        graphqlEnum.deprecationReason = await helpers.getDeprecationReason(
+          def.directives,
+          graphqlEnum
+        )
+        graphqlEnum.preview = await helpers.getPreview(
+          def.directives,
+          graphqlEnum,
+          previewsPerVersion
+        )
+
+        await Promise.all(
+          def.values.map(async (value) => {
+            const enumValue = {}
+            enumValue.name = value.name.value
+            enumValue.description = await helpers.getDescription(value.description.value)
+            enumValues.push(enumValue)
+          })
+        )
+
+        graphqlEnum.values = sortBy(enumValues, 'name')
+
+        data.enums.push(graphqlEnum)
+        return
+      }
+
+      // UNIONS
+      if (def.kind === 'UnionTypeDefinition') {
+        const union = {}
+        const possibleTypes = []
+
+        union.name = def.name.value
+        union.kind = helpers.getKind(def.kind)
+        union.id = helpers.getId(union.name)
+        union.href = helpers.getFullLink('unions', union.id)
+        union.description = await helpers.getDescription(def.description.value)
+        union.isDeprecated = helpers.getDeprecationStatus(def.directives)
+        union.deprecationReason = await helpers.getDeprecationReason(def.directives, union)
+        union.preview = await helpers.getPreview(def.directives, union, previewsPerVersion)
+
+        // union types do not have directives so cannot be under preview/deprecated
+        await Promise.all(
+          def.types.map(async (type) => {
+            const possibleType = {}
+            possibleType.name = type.name.value
+            possibleType.id = helpers.getId(possibleType.name)
+            possibleType.href = helpers.getFullLink('objects', possibleType.id)
+            possibleTypes.push(possibleType)
+          })
+        )
+
+        union.possibleTypes = sortBy(possibleTypes, 'name')
+
+        data.unions.push(union)
+        return
+      }
+
+      // INPUT OBJECTS
+      // NOTE: input objects ending with `Input` are NOT included in the v4 input objects sidebar
+      // but they are still present in the docs (e.g., https://developer.github.com/v4/input_object/acceptenterpriseadministratorinvitationinput/)
+      // so we will include them here
+      if (def.kind === 'InputObjectTypeDefinition') {
+        const inputObject = {}
+        const inputFields = []
+
+        inputObject.name = def.name.value
+        inputObject.kind = helpers.getKind(def.kind)
+        inputObject.id = helpers.getId(inputObject.name)
+        inputObject.href = helpers.getFullLink('input-objects', inputObject.id)
+        inputObject.description = await helpers.getDescription(def.description.value)
+        inputObject.isDeprecated = helpers.getDeprecationStatus(def.directives)
+        inputObject.deprecationReason = await helpers.getDeprecationReason(
+          def.directives,
+          inputObject
+        )
+        inputObject.preview = await helpers.getPreview(
+          def.directives,
+          inputObject,
+          previewsPerVersion
+        )
+
+        if (def.fields.length) {
+          await Promise.all(
+            def.fields.map(async (field) => {
+              const inputField = {}
+
+              inputField.name = field.name.value
+              inputField.description = await helpers.getDescription(field.description.value)
+              inputField.type = helpers.getType(field)
+              inputField.id = helpers.getId(inputField.type)
+              inputField.kind = helpers.getTypeKind(inputField.type, schema)
+              inputField.href = helpers.getFullLink(inputField.kind, inputField.id)
+              inputField.isDeprecated = helpers.getDeprecationStatus(field.directives)
+              inputField.deprecationReason = await helpers.getDeprecationReason(
+                field.directives,
+                inputField
+              )
+              inputField.preview = await helpers.getPreview(
+                field.directives,
+                inputField,
+                previewsPerVersion
+              )
+
+              inputFields.push(inputField)
+            })
+          )
+        }
+
+        inputObject.inputFields = sortBy(inputFields, 'name')
+
+        data.inputObjects.push(inputObject)
+        return
+      }
+
+      // SCALARS
+      if (def.kind === 'ScalarTypeDefinition') {
+        const scalar = {}
+        scalar.name = def.name.value
+        scalar.kind = helpers.getKind(def.kind)
+        scalar.id = helpers.getId(scalar.name)
+        scalar.href = helpers.getFullLink('scalars', scalar.id)
+        scalar.description = await helpers.getDescription(def.description.value)
+        scalar.isDeprecated = helpers.getDeprecationStatus(def.directives)
+        scalar.deprecationReason = await helpers.getDeprecationReason(def.directives, scalar)
+        scalar.preview = await helpers.getPreview(def.directives, scalar, previewsPerVersion)
+        data.scalars.push(scalar)
+      }
+    })
+  )
+
+  // add non-schema scalars and sort all scalars alphabetically
+  data.scalars = sortBy(data.scalars.concat(externalScalars), 'name')
+
+  // sort all the types alphabetically
+  data.queries = sortBy(data.queries, 'name')
+  data.mutations = sortBy(data.mutations, 'name')
+  data.objects = sortBy(data.objects, 'name')
+  data.interfaces = sortBy(data.interfaces, 'name')
+  data.enums = sortBy(data.enums, 'name')
+  data.unions = sortBy(data.unions, 'name')
+  data.inputObjects = sortBy(data.inputObjects, 'name')
+  data.scalars = sortBy(data.scalars, 'name')
+
+  return data
+}

src/graphql/scripts/utils/process-upcoming-changes.js

@@ -0,0 +1,16 @@
+#!/usr/bin/env node
+import yaml from 'js-yaml'
+import { groupBy } from 'lodash-es'
+import renderContent from '../../../../lib/render-content/index.js'
+
+export default async function processUpcomingChanges(upcomingChangesYml) {
+  const upcomingChanges = yaml.load(upcomingChangesYml).upcoming_changes
+
+  for (const change of upcomingChanges) {
+    change.date = change.date.slice(0, 10)
+    change.reason = await renderContent(change.reason)
+    change.description = await renderContent(change.description)
+  }
+
+  return groupBy(upcomingChanges.reverse(), 'date')
+}

src/graphql/scripts/utils/schema-helpers.js

@@ -0,0 +1,196 @@
+#!/usr/bin/env node
+import renderContent from '../../../../lib/render-content/index.js'
+import fs from 'fs/promises'
+import graphql from 'graphql'
+import path from 'path'
+
+const graphqlTypes = JSON.parse(
+  await fs.readFile(path.join(process.cwd(), './src/graphql/lib/types.json'))
+)
+const { isScalarType, isObjectType, isInterfaceType, isUnionType, isEnumType, isInputObjectType } =
+  graphql
+
+const singleQuotesInsteadOfBackticks = / '(\S+?)' /
+
+function addPeriod(string) {
+  return string.endsWith('.') ? string : string + '.'
+}
+
+async function getArguments(args, schema) {
+  if (!args.length) return
+
+  const newArgs = []
+
+  for (const arg of args) {
+    const newArg = {}
+    const type = {}
+    newArg.name = arg.name.value
+    newArg.defaultValue = arg.defaultValue ? arg.defaultValue.value : undefined
+    newArg.description = await getDescription(arg.description.value)
+    type.name = getType(arg)
+    type.id = getId(type.name)
+    type.kind = getTypeKind(type.name, schema)
+    type.href = getFullLink(type.kind, type.id)
+    newArg.type = type
+    newArgs.push(newArg)
+  }
+
+  return newArgs
+}
+
+async function getDeprecationReason(directives, schemaMember) {
+  if (!schemaMember.isDeprecated) return
+
+  // it's possible for a schema member to be deprecated and under preview
+  const deprecationDirective = directives.filter((dir) => dir.name.value === 'deprecated')
+
+  // catch any schema members that have more than one deprecation (none currently)
+  if (deprecationDirective.length > 1)
+    console.log(`more than one deprecation found for ${schemaMember.name}`)
+
+  return renderContent(deprecationDirective[0].arguments[0].value.value)
+}
+
+function getDeprecationStatus(directives) {
+  if (!directives.length) return
+
+  return directives[0].name.value === 'deprecated'
+}
+
+async function getDescription(rawDescription) {
+  rawDescription = rawDescription.replace(singleQuotesInsteadOfBackticks, '`$1`')
+
+  return renderContent(addPeriod(rawDescription))
+}
+
+function getFullLink(baseType, id) {
+  return `/graphql/reference/${baseType}#${id}`
+}
+
+function getId(path) {
+  return removeMarkers(path).toLowerCase()
+}
+
+// e.g., given `ObjectTypeDefinition`, get `objects`
+function getKind(type) {
+  return graphqlTypes.find((graphqlType) => graphqlType.type === type).kind
+}
+
+async function getPreview(directives, schemaMember, previewsPerVersion) {
+  if (!directives.length) return
+
+  // it's possible for a schema member to be deprecated and under preview
+  const previewDirective = directives.filter((dir) => dir.name.value === 'preview')
+  if (!previewDirective.length) return
+
+  // catch any schema members that are under more than one preview (none currently)
+  if (previewDirective.length > 1)
+    console.log(`more than one preview found for ${schemaMember.name}`)
+
+  // an input object's input field may have a ListValue directive that is not relevant to previews
+  if (previewDirective[0].arguments[0].value.kind !== 'StringValue') return
+
+  const previewName = previewDirective[0].arguments[0].value.value
+
+  const preview = previewsPerVersion.find((p) => p.toggled_by.includes(previewName))
+  if (!preview) console.error(`cannot find "${previewName}" in graphql_previews.yml`)
+
+  return preview
+}
+
+// the docs use brackets to denote list types: `[foo]`
+// and an exclamation mark to denote non-nullable types: `foo!`
+// both single items and lists can be non-nullable
+// so the permutations are:
+// 1. single items: `foo`, `foo!`
+// 2. nullable lists: `[foo]`, `[foo!]`
+// 3. non-null lists: `[foo]!`, `[foo!]!`
+// see https://github.com/rmosolgo/graphql-ruby/blob/master/guides/type_definitions/lists.md#lists-nullable-lists-and-lists-of-nulls
+function getType(field) {
+  // 1. single items
+  if (field.type.kind !== 'ListType') {
+    // nullable item, e.g. `license` query has `License` type
+    if (field.type.kind === 'NamedType') {
+      return field.type.name.value
+    }
+
+    // non-null item, e.g. `meta` query has `GitHubMetadata!` type
+    if (field.type.kind === 'NonNullType' && field.type.type.kind === 'NamedType') {
+      return `${field.type.type.name.value}!`
+    }
+  }
+  // 2. nullable lists
+  if (field.type.kind === 'ListType') {
+    // nullable items, e.g. `codesOfConduct` query has `[CodeOfConduct]` type
+    if (field.type.type.kind === 'NamedType') {
+      return `[${field.type.type.name.value}]`
+    }
+
+    // non-null items, e.g. `severities` arg has `[SecurityAdvisorySeverity!]` type
+    if (field.type.type.kind === 'NonNullType' && field.type.type.type.kind === 'NamedType') {
+      return `[${field.type.type.type.name.value}!]`
+    }
+  }
+
+  // 3. non-null lists
+  if (field.type.kind === 'NonNullType' && field.type.type.kind === 'ListType') {
+    // nullable items, e.g. `licenses` query has `[License]!` type
+    if (field.type.type.type.kind === 'NamedType') {
+      return `[${field.type.type.type.name.value}]!`
+    }
+
+    // non-null items, e.g. `marketplaceCategories` query has `[MarketplaceCategory!]!` type
+    if (
+      field.type.type.type.kind === 'NonNullType' &&
+      field.type.type.type.type.kind === 'NamedType'
+    ) {
+      return `[${field.type.type.type.type.name.value}!]!`
+    }
+  }
+
+  console.error(`cannot get type of ${field.name.value}`)
+}
+
+function getTypeKind(type, schema) {
+  type = removeMarkers(type)
+
+  const typeFromSchema = schema.getType(type)
+
+  if (isScalarType(typeFromSchema)) {
+    return 'scalars'
+  }
+  if (isObjectType(typeFromSchema)) {
+    return 'objects'
+  }
+  if (isInterfaceType(typeFromSchema)) {
+    return 'interfaces'
+  }
+  if (isUnionType(typeFromSchema)) {
+    return 'unions'
+  }
+  if (isEnumType(typeFromSchema)) {
+    return 'enums'
+  }
+  if (isInputObjectType(typeFromSchema)) {
+    return 'input-objects'
+  }
+
+  console.error(`cannot find type kind of ${type}`)
+}
+
+function removeMarkers(str) {
+  return str.replace('[', '').replace(']', '').replace(/!/g, '')
+}
+
+export default {
+  getArguments,
+  getDeprecationReason,
+  getDeprecationStatus,
+  getDescription,
+  getFullLink,
+  getId,
+  getKind,
+  getPreview,
+  getType,
+  getTypeKind,
+}