Convert more files to TypeScript sync.js & find-page.js (#51775)
This commit is contained in:
@@ -95,7 +95,7 @@ jobs:
|
||||
changes=$(git diff --name-only | wc -l)
|
||||
untracked=$(git status --untracked-files --short | wc -l)
|
||||
if [[ $changes -eq 0 ]] && [[ $untracked -eq 0 ]]; then
|
||||
echo "There are no changes to commit after running src/rest/scripts/update-files.js. Exiting..."
|
||||
echo "There are no changes to commit or untracked files. Exiting."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
2
.github/workflows/sync-audit-logs.yml
vendored
2
.github/workflows/sync-audit-logs.yml
vendored
@@ -55,7 +55,7 @@ jobs:
|
||||
changes=$(git diff --name-only | wc -l)
|
||||
untracked=$(git status --untracked-files --short | wc -l)
|
||||
if [[ $changes -eq 0 ]] && [[ $untracked -eq 0 ]]; then
|
||||
echo "There are no changes to commit after running src/rest/scripts/update-files.js. Exiting..."
|
||||
echo "There are no changes to commit or untracked files. Exiting..."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
4
.github/workflows/sync-openapi.yml
vendored
4
.github/workflows/sync-openapi.yml
vendored
@@ -49,7 +49,7 @@ jobs:
|
||||
# Needed for gh
|
||||
GITHUB_TOKEN: ${{ secrets.DOCS_BOT_PAT_WRITEORG_PROJECT }}
|
||||
run: |
|
||||
src/rest/scripts/update-files.js --source-repo rest-api-description --output rest github-apps webhooks rest-redirects
|
||||
npm run sync-rest -- --source-repo rest-api-description --output rest github-apps webhooks rest-redirects
|
||||
git status
|
||||
echo "Deleting the cloned github/rest-api-description repo..."
|
||||
rm -rf rest-api-description
|
||||
@@ -73,7 +73,7 @@ jobs:
|
||||
# If nothing to commit, exit now. It's fine. No orphans.
|
||||
changes=$(git diff --name-only | wc -l)
|
||||
if [[ $changes -eq 0 ]]; then
|
||||
echo "There are no changes to commit after running src/rest/scripts/update-files.js. Exiting..."
|
||||
echo "There are no changes to commit after running `npm run sync-rest` Exiting..."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
@@ -32,9 +32,9 @@ Repository admins can add any topics they'd like to a repository. Helpful topics
|
||||
You can search for repositories that are associated with a particular topic. For more information, see "[AUTOTITLE](/search-github/searching-on-github/searching-for-repositories#search-by-topic)." You can also search for a list of topics on {% data variables.product.product_name %}. For more information, see "[AUTOTITLE](/search-github/searching-on-github/searching-topics)."
|
||||
|
||||
When creating a topic:
|
||||
* use lowercase letters, numbers, and hyphens.
|
||||
* use 50 characters or less.
|
||||
* add no more than 20 topics.
|
||||
* Use lowercase letters, numbers, and hyphens.
|
||||
* Use 50 characters or less.
|
||||
* Add no more than 20 topics.
|
||||
|
||||
## Adding topics to your repository
|
||||
|
||||
|
||||
@@ -67,7 +67,7 @@
|
||||
"start-all-languages": "cross-env NODE_ENV=development tsx src/frame/server.ts",
|
||||
"start-for-playwright": "cross-env ROOT=src/fixtures/fixtures TRANSLATIONS_FIXTURE_ROOT=src/fixtures/fixtures/translations ENABLED_LANGUAGES=en,ja NODE_ENV=test tsx src/frame/server.ts",
|
||||
"symlink-from-local-repo": "node src/early-access/scripts/symlink-from-local-repo.js",
|
||||
"sync-rest": "node src/rest/scripts/update-files.js",
|
||||
"sync-rest": "tsx src/rest/scripts/update-files.ts",
|
||||
"sync-search": "cross-env NODE_OPTIONS='--max_old_space_size=8192' start-server-and-test sync-search-server 4002 sync-search-indices",
|
||||
"sync-search-ghes-release": "cross-env GHES_RELEASE=1 start-server-and-test sync-search-server 4002 sync-search-indices",
|
||||
"sync-search-indices": "node src/search/scripts/sync-search-indices.js",
|
||||
|
||||
@@ -310,6 +310,10 @@ async function getIndexFileVersions(directory, files) {
|
||||
`File ${filepath} does not exist while assembling directory index.md files to create parent version.`,
|
||||
)
|
||||
}
|
||||
// If not a markdown(x) file, skip it
|
||||
if (!file.endsWith('.md') && !file.endsWith('.mdx')) {
|
||||
return
|
||||
}
|
||||
const { data } = matter(await readFile(filepath, 'utf-8'))
|
||||
if (!data || !data.versions) {
|
||||
throw new Error(`Frontmatter in ${filepath} does not contain versions.`)
|
||||
|
||||
@@ -24,9 +24,9 @@ To run the CodeQL CLI pipeline locally:
|
||||
|
||||
## About this directory
|
||||
|
||||
- `src/rest/lib/config.json` - A configuration file used to specify metadata about the REST pipeline.
|
||||
- `src/rest/scripts` - The scripts and source code used run the CodeQL CLI pipeline.
|
||||
- `src/rest/scripts/sync.js` - The entrypoint script that runs the CodeQL CLI pipeline.
|
||||
- `src/codeql-cli/lib/config.json` - A configuration file used to specify metadata about the CodeQL CLI pipeline.
|
||||
- `src/codeql-cli/scripts` - The scripts and source code used run the CodeQL CLI pipeline.
|
||||
- `src/codeql-cli/scripts/sync.js` - The entrypoint script that runs the CodeQL CLI pipeline.
|
||||
|
||||
## Content team
|
||||
|
||||
|
||||
@@ -133,7 +133,7 @@ The benefit of the first method is that you don't need to deal with merging two
|
||||
- [ ] To update the OpenAPI data, run the following command.
|
||||
|
||||
```shell
|
||||
src/rest/scripts/update-files.js --source-repo rest-api-description --output rest github-apps webhooks rest-redirects
|
||||
npm run sync-rest -- --source-repo rest-api-description --output rest github-apps webhooks rest-redirects
|
||||
```
|
||||
|
||||
You may see an error that indicates that "...you must have the GITHUB_TOKEN environment variable set to access the programmatic access and resource files via the GitHub REST API." You can ignore this error.
|
||||
|
||||
@@ -34,7 +34,8 @@ If the OpenAPI has changed, you will need to first wait for the OpenAPI to be me
|
||||
To run the GitHub Apps pipeline locally:
|
||||
|
||||
1. Clone the [`github/rest-api-description`](https://github.com/github/rest-api-description) repository inside your local `docs-internal` repository.
|
||||
1. Run `src/rest/scripts/update-files.js -s rest-api-description -o github-apps`.
|
||||
1. Set a `GITHUB_TOKEN` in your `.env` with (classic) `repo` scopes & enable SSO for the github org.
|
||||
1. Run `npm run sync-rest -- -s rest-api-description -o github-apps`.
|
||||
|
||||
## About this directory
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ A [workflow](.github/workflows/sync-openapi.yml) is used to trigger the automati
|
||||
- REST
|
||||
- Webhooks
|
||||
|
||||
The workflow automatically creates a pull request with the changes (for all three pipelines) and the label `github-openapi-bot`. The workflow runs the `src/rest/scripts/update-files.js` script, which creates, deletes, or updates Markdown files in the `content/rest` directory.
|
||||
The workflow automatically creates a pull request with the changes (for all three pipelines) and the label `github-openapi-bot`. The workflow runs the `npm run sync-rest` script, which creates, deletes, or updates Markdown files in the `content/rest` directory.
|
||||
|
||||
### Triggering the workflow sooner than the scheduled time
|
||||
|
||||
@@ -35,7 +35,8 @@ Then, you can manually sync the data used by the REST, Webhooks, and GitHub App
|
||||
To run the REST pipeline locally:
|
||||
|
||||
1. Clone the [`github/rest-api-description`](https://github.com/github/rest-api-description) repository inside your local `docs-internal` repository.
|
||||
1. Run `src/rest/scripts/update-files.js -s rest-api-description -o rest`. Note, by default `-o rest` is specified, so you can omit it.
|
||||
1. Set a `GITHUB_TOKEN` in your `.env` with (classic) `repo` scopes & enable SSO for the github org.
|
||||
1. Run `npm run sync-rest -- -s rest-api-description -o rest`. Note, by default `-o rest` is specified, so you can omit it.
|
||||
|
||||
## About this directory
|
||||
|
||||
@@ -45,7 +46,7 @@ To run the REST pipeline locally:
|
||||
- `src/rest/lib` - The source code used in production for the automated documentation generated by the REST pipeline and configuration files edited by content and engineering team members.
|
||||
- `src/rest/lib/config.json` - A configuration file used to specify metadata about the REST pipeline.
|
||||
- `src/rest/scripts` - The scripts and source code used run the REST pipeline, which updates the `src/rest/data` directory.
|
||||
- `src/rest/scripts/update-files.js` - The entrypoint script that runs the REST pipeline.
|
||||
- `src/rest/scripts/update-files.ts` - The entrypoint script that runs the REST pipeline.
|
||||
- `src/rest/tests` - The tests used to verify the REST pipeline.
|
||||
|
||||
## Configuring the pipeline
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# REST scripts
|
||||
|
||||
Writers run the [update-files.js](./update-files.js) script to get the latest dereferenced OpenAPI schema files.
|
||||
Writers run the [update-files.ts](./update-files.ts) script to get the latest dereferenced OpenAPI schema files.
|
||||
```
|
||||
src/rest/scripts/update-files.js
|
||||
npm run sync-rest
|
||||
```
|
||||
These scripts update the dereferenced OpenAPI files to create [the decorated files](../../src/rest/data) used to
|
||||
render REST docs. See the [`src/rest/README`](../../src/rest/README.md)
|
||||
@@ -20,7 +20,7 @@ Writers and developers depend on this script to preview OpenAPI changes in the d
|
||||
|
||||
## Production `--decorate-only` option
|
||||
|
||||
When changes to the OpenAPI are merged to the default branch of the `github/github` repository, a pull request is automatically opened with the updated dereferenced OpenAPI files. When pull requests are authored by `github-openapi-bot`, a CI test runs the `src/rest/scripts/update-files.js` script with the `--decorate-only` option. The `--decorate-only` option only decorates the dereferenced OpenAPI files, using the existing dereferenced OpenAPI schema files, and checks those changes in to the existing branch. The `--decorate-only` option is only used by a 🤖 and is only used on production dereferenced OpenAPI schema files.
|
||||
When changes to the OpenAPI are merged to the default branch of the `github/github` repository, a pull request is automatically opened with the updated dereferenced OpenAPI files. When pull requests are authored by `github-openapi-bot`, a CI test runs the `npm run sync-rest` script with the `--decorate-only` option. The `--decorate-only` option only decorates the dereferenced OpenAPI files, using the existing dereferenced OpenAPI schema files, and checks those changes in to the existing branch. The `--decorate-only` option is only used by a 🤖 and is only used on production dereferenced OpenAPI schema files.
|
||||
|
||||
The `.github/workflows/openapi-schema-check.yml` CI test checks that the dereferenced and decorated schema files match. If the files don't match, potential causes could be:
|
||||
- something went wrong when the schema changes (created by `github-openapi-bot`) were merged into another branch
|
||||
@@ -28,4 +28,4 @@ The `.github/workflows/openapi-schema-check.yml` CI test checks that the derefer
|
||||
|
||||
⚠️ Only do this if you know exactly what the `--decorate-only` option does. ⚠️
|
||||
|
||||
If you know that the dereferenced schema files are correct, you can run the `src/rest/scripts/update-files.js --decorate-only` command on the branch locally to update the decorated files in your branch.
|
||||
If you know that the dereferenced schema files are correct, you can run the `npm run sync-rest -- --decorate-only` command on the branch locally to update the decorated files in your branch.
|
||||
|
||||
@@ -17,12 +17,12 @@ import { fileURLToPath } from 'url'
|
||||
import walk from 'walk-sync'
|
||||
import { existsSync } from 'fs'
|
||||
|
||||
import { syncRestData, getOpenApiSchemaFiles } from './utils/sync.js'
|
||||
import { validateVersionsOptions } from './utils/get-openapi-schemas.js'
|
||||
import { allVersions } from '#src/versions/lib/all-versions.js'
|
||||
import { syncWebhookData } from '../../webhooks/scripts/sync.js'
|
||||
import { syncGitHubAppsData } from '../../github-apps/scripts/sync.js'
|
||||
import { syncRestRedirects } from './utils/get-redirects.js'
|
||||
import { syncRestData, getOpenApiSchemaFiles } from './utils/sync'
|
||||
import { validateVersionsOptions } from './utils/get-openapi-schemas'
|
||||
import { allVersions } from '@/versions/lib/all-versions'
|
||||
import { syncWebhookData } from '../../webhooks/scripts/sync'
|
||||
import { syncGitHubAppsData } from '../../github-apps/scripts/sync'
|
||||
import { syncRestRedirects } from './utils/get-redirects'
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
const TEMP_OPENAPI_DIR = path.join(__dirname, '../../../rest-api-description/openApiTemp')
|
||||
@@ -30,7 +30,9 @@ const TEMP_BUNDLED_OPENAPI_DIR = path.join(TEMP_OPENAPI_DIR, 'bundled')
|
||||
const GITHUB_REP_DIR = '../github'
|
||||
const REST_API_DESCRIPTION_ROOT = 'rest-api-description'
|
||||
const REST_DESCRIPTION_DIR = path.join(REST_API_DESCRIPTION_ROOT, 'descriptions-next')
|
||||
const VERSION_NAMES = JSON.parse(await readFile('src/rest/lib/config.json', 'utf8')).versionMapping
|
||||
const VERSION_NAMES: Record<string, string> = JSON.parse(
|
||||
await readFile('src/rest/lib/config.json', 'utf8'),
|
||||
).versionMapping
|
||||
const noConfig = ['rest-redirects']
|
||||
|
||||
program
|
||||
@@ -105,7 +107,10 @@ async function main() {
|
||||
// so that we don't spend time generating data files for them.
|
||||
if (sourceRepo === REST_API_DESCRIPTION_ROOT) {
|
||||
const derefDir = await readdir(TEMP_OPENAPI_DIR)
|
||||
const currentOpenApiVersions = Object.values(allVersions).map((elem) => elem.openApiVersionName)
|
||||
// TODO: After migrating all-version.js to TypeScript, we can remove the type assertion
|
||||
const currentOpenApiVersions = Object.values(allVersions).map(
|
||||
(elem) => (elem as any).openApiVersionName,
|
||||
)
|
||||
|
||||
for (const schema of derefDir) {
|
||||
// if the schema does not start with a current version name, delete it
|
||||
@@ -162,7 +167,7 @@ async function main() {
|
||||
)
|
||||
}
|
||||
|
||||
async function getBundledFiles() {
|
||||
async function getBundledFiles(): Promise<void> {
|
||||
// Get the github/github repo branch name and pull latest
|
||||
const githubBranch = execSync('git rev-parse --abbrev-ref HEAD', { cwd: GITHUB_REP_DIR })
|
||||
.toString()
|
||||
@@ -197,7 +202,7 @@ async function getBundledFiles() {
|
||||
}
|
||||
}
|
||||
|
||||
async function getBundlerOptions() {
|
||||
async function getBundlerOptions(): Promise<string> {
|
||||
let includeParams = ['--generate_dref_json_only']
|
||||
|
||||
if (versions) {
|
||||
@@ -213,7 +218,7 @@ async function getBundlerOptions() {
|
||||
return includeParams.join(' ')
|
||||
}
|
||||
|
||||
async function validateInputParameters() {
|
||||
async function validateInputParameters(): Promise<void> {
|
||||
// The `--versions` option cannot be used
|
||||
// with the `--include-deprecated` option
|
||||
if (includeDeprecated && versions) {
|
||||
@@ -248,7 +253,7 @@ async function validateInputParameters() {
|
||||
// the short name of the version defined in lib/allVersions.js.
|
||||
// This function also translates calendar-date format from .2022-11-28 to
|
||||
// -2022-11-28
|
||||
export async function normalizeDataVersionNames(sourceDirectory) {
|
||||
export async function normalizeDataVersionNames(sourceDirectory: string): Promise<void> {
|
||||
const schemas = await readdir(sourceDirectory)
|
||||
|
||||
for (const schema of schemas) {
|
||||
@@ -259,8 +264,8 @@ export async function normalizeDataVersionNames(sourceDirectory) {
|
||||
// Update the version name to use docs convention, e.g.,
|
||||
// api.github.com.2022-11-28 -> fpt.2022-11-28
|
||||
const docsBaseName = baseName.replace(
|
||||
matchingSourceVersion,
|
||||
VERSION_NAMES[matchingSourceVersion],
|
||||
matchingSourceVersion!,
|
||||
VERSION_NAMES[matchingSourceVersion!],
|
||||
)
|
||||
// Match a calendar version if it exists, e.g., .2022-11-28
|
||||
const regex = /.\d{4}-\d{2}-\d{2}/
|
||||
@@ -1,5 +1,39 @@
|
||||
#!/usr/bin/env node
|
||||
import { renderContent } from '#src/content-render/index.js'
|
||||
import { renderContent } from '@/content-render/index'
|
||||
|
||||
interface Schema {
|
||||
oneOf?: any[]
|
||||
type?: string
|
||||
items?: any
|
||||
properties?: Record<string, any>
|
||||
required?: string[]
|
||||
additionalProperties?: any
|
||||
description?: string
|
||||
enum?: string[]
|
||||
nullable?: boolean
|
||||
allOf?: any[]
|
||||
anyOf?: any[]
|
||||
[key: string]: any
|
||||
}
|
||||
|
||||
export interface TransformedParam {
|
||||
type: string
|
||||
name: string
|
||||
description: string
|
||||
isRequired?: boolean
|
||||
in?: string
|
||||
childParamsGroups?: TransformedParam[]
|
||||
enum?: string[]
|
||||
oneOfObject?: boolean
|
||||
default?: any
|
||||
}
|
||||
|
||||
interface BodyParamProps {
|
||||
paramKey?: string
|
||||
required?: string[]
|
||||
childParamsGroups?: TransformedParam[]
|
||||
topLevel?: boolean
|
||||
}
|
||||
|
||||
// If there is a oneOf at the top level, then we have to present just one
|
||||
// in the docs. We don't currently have a convention for showing more than one
|
||||
@@ -8,14 +42,16 @@ import { renderContent } from '#src/content-render/index.js'
|
||||
// Currently there aren't very many operations that require this treatment.
|
||||
// As an example, the 'Add status check contexts' and 'Set status check contexts'
|
||||
// operations have a top-level oneOf.
|
||||
|
||||
async function getTopLevelOneOfProperty(schema) {
|
||||
async function getTopLevelOneOfProperty(
|
||||
schema: Schema,
|
||||
): Promise<{ properties: Record<string, any>; required: string[] }> {
|
||||
if (!schema.oneOf) {
|
||||
throw new Error('Schema does not have a requestBody oneOf property defined')
|
||||
}
|
||||
if (!(Array.isArray(schema.oneOf) && schema.oneOf.length > 0)) {
|
||||
throw new Error('Schema requestBody oneOf property is not an array')
|
||||
}
|
||||
|
||||
// When a oneOf exists but the `type` differs, the case has historically
|
||||
// been that the alternate option is an array, where the first option
|
||||
// is the array as a property of the object. We need to ensure that the
|
||||
@@ -39,8 +75,8 @@ async function getTopLevelOneOfProperty(schema) {
|
||||
}
|
||||
|
||||
// Gets the body parameters for a given schema recursively.
|
||||
export async function getBodyParams(schema, topLevel = false) {
|
||||
const bodyParametersParsed = []
|
||||
export async function getBodyParams(schema: Schema, topLevel = false): Promise<TransformedParam[]> {
|
||||
const bodyParametersParsed: TransformedParam[] = []
|
||||
const schemaObject = schema.oneOf && topLevel ? await getTopLevelOneOfProperty(schema) : schema
|
||||
const properties = schemaObject.properties || {}
|
||||
const required = schemaObject.required || []
|
||||
@@ -48,7 +84,7 @@ export async function getBodyParams(schema, topLevel = false) {
|
||||
// Most operation requestBody schemas are objects. When the type is an array,
|
||||
// there will not be properties on the `schema` object.
|
||||
if (topLevel && schema.type === 'array') {
|
||||
const childParamsGroups = []
|
||||
const childParamsGroups: TransformedParam[] = []
|
||||
const arrayType = schema.items.type
|
||||
const paramType = [schema.type]
|
||||
if (arrayType === 'object') {
|
||||
@@ -76,7 +112,7 @@ export async function getBodyParams(schema, topLevel = false) {
|
||||
? param.additionalProperties.type
|
||||
: [param.additionalProperties.type]
|
||||
: []
|
||||
const childParamsGroups = []
|
||||
const childParamsGroups: TransformedParam[] = []
|
||||
|
||||
// If the parameter is an array or object there may be child params
|
||||
// If the parameter has oneOf or additionalProperties, they need to be
|
||||
@@ -88,7 +124,7 @@ export async function getBodyParams(schema, topLevel = false) {
|
||||
// Create a snapshot of dependencies for a repository
|
||||
// Update a gist
|
||||
if (param.additionalProperties && additionalPropertiesType.includes('object')) {
|
||||
const keyParam = {
|
||||
const keyParam: TransformedParam = {
|
||||
type: 'object',
|
||||
name: 'key',
|
||||
description: await renderContent(
|
||||
@@ -99,11 +135,13 @@ export async function getBodyParams(schema, topLevel = false) {
|
||||
default: param.default,
|
||||
childParamsGroups: [],
|
||||
}
|
||||
if (keyParam.childParamsGroups) {
|
||||
keyParam.childParamsGroups.push(...(await getBodyParams(param.additionalProperties, false)))
|
||||
}
|
||||
childParamsGroups.push(keyParam)
|
||||
} else if (paramType && paramType.includes('array') && param.items) {
|
||||
if (param.items && param.items.oneOf) {
|
||||
if (param.items.oneOf.every((object) => object.type === 'object')) {
|
||||
} else if (paramType.includes('array') && param.items) {
|
||||
if (param.items.oneOf) {
|
||||
if (param.items.oneOf.every((object: TransformedParam) => object.type === 'object')) {
|
||||
paramType.splice(paramType.indexOf('array'), 1, `array of objects`)
|
||||
param.oneOfObject = true
|
||||
childParamsGroups.push(...(await getOneOfChildParams(param.items)))
|
||||
@@ -116,31 +154,25 @@ export async function getBodyParams(schema, topLevel = false) {
|
||||
if (arrayType === 'object') {
|
||||
childParamsGroups.push(...(await getBodyParams(param.items, false)))
|
||||
}
|
||||
// If the type is an enumerated list of strings
|
||||
if (arrayType === 'string' && param.items.enum) {
|
||||
param.description += `${
|
||||
param.description ? '\n' : ''
|
||||
}Supported values are: ${param.items.enum
|
||||
.map((lang) => `<code>${lang}</code>`)
|
||||
.join(', ')}`
|
||||
}Supported values are: ${param.items.enum.map((lang: string) => `<code>${lang}</code>`).join(', ')}`
|
||||
}
|
||||
}
|
||||
} else if (paramType && paramType.includes('object')) {
|
||||
if (param && param.oneOf) {
|
||||
if (param.oneOf.every((object) => object.type === 'object')) {
|
||||
} else if (paramType.includes('object')) {
|
||||
if (param.oneOf) {
|
||||
if (param.oneOf.every((object: TransformedParam) => object.type === 'object')) {
|
||||
param.oneOfObject = true
|
||||
childParamsGroups.push(...(await getOneOfChildParams(param)))
|
||||
}
|
||||
} else {
|
||||
childParamsGroups.push(...(await getBodyParams(param, false)))
|
||||
}
|
||||
} else if (param && param.oneOf) {
|
||||
// get concatenated description and type
|
||||
const descriptions = []
|
||||
} else if (param.oneOf) {
|
||||
const descriptions: { type: string; description: string }[] = []
|
||||
for (const childParam of param.oneOf) {
|
||||
paramType.push(childParam.type)
|
||||
// If there is no parent description, create a description from
|
||||
// each type
|
||||
if (!param.description) {
|
||||
if (childParam.type === 'array') {
|
||||
if (childParam.items.description) {
|
||||
@@ -164,13 +196,14 @@ export async function getBodyParams(schema, topLevel = false) {
|
||||
if (!param.description) param.description = oneOfDescriptions
|
||||
|
||||
// This is a workaround for an operation that incorrectly defines anyOf
|
||||
// for a body parameter. As a workaround, we will use the first object
|
||||
// in the list of the anyOf array. Otherwise, fallback to the first item
|
||||
// in the array. There is currently only one occurrence for the operation
|
||||
// id repos/update-information-about-pages-site. See Ecosystem API issue
|
||||
// for a body parameter. We use the first object in the list of the anyOf array.
|
||||
// There is currently only one occurrence for the operation id
|
||||
// repos/update-information-about-pages-site. See Ecosystem API issue
|
||||
// number #3332 for future plans to fix this in the OpenAPI
|
||||
} else if (param && param.anyOf && Object.keys(param).length === 1) {
|
||||
const firstObject = Object.values(param.anyOf).find((item) => item.type === 'object')
|
||||
} else if (param.anyOf && Object.keys(param).length === 1) {
|
||||
const firstObject = Object.values(param.anyOf).find(
|
||||
(item) => (item as Schema).type === 'object',
|
||||
) as Schema
|
||||
if (firstObject) {
|
||||
paramType.push('object')
|
||||
param.description = firstObject.description
|
||||
@@ -181,8 +214,8 @@ export async function getBodyParams(schema, topLevel = false) {
|
||||
param.description = param.anyOf[0].description
|
||||
param.isRequired = param.anyOf[0].required
|
||||
}
|
||||
} else if (param && param.allOf) {
|
||||
// this else is only used for webhooks handling of allOf
|
||||
// Used only for webhooks handling allOf
|
||||
} else if (param.allOf) {
|
||||
for (const prop of param.allOf) {
|
||||
paramType.push('object')
|
||||
childParamsGroups.push(...(await getBodyParams(prop, false)))
|
||||
@@ -200,20 +233,24 @@ export async function getBodyParams(schema, topLevel = false) {
|
||||
return bodyParametersParsed
|
||||
}
|
||||
|
||||
async function getTransformedParam(param, paramType, props) {
|
||||
async function getTransformedParam(
|
||||
param: Schema,
|
||||
paramType: string[],
|
||||
props: BodyParamProps,
|
||||
): Promise<TransformedParam> {
|
||||
const { paramKey, required, childParamsGroups, topLevel } = props
|
||||
const paramDecorated = {}
|
||||
const paramDecorated: TransformedParam = {} as TransformedParam
|
||||
// Supports backwards compatibility for OpenAPI 3.0
|
||||
// In 3.1 a nullable type is part of the param.type array and
|
||||
// the property param.nullable does not exist.
|
||||
if (param.nullable) paramType.push('null')
|
||||
paramDecorated.type = Array.from(new Set(paramType.filter(Boolean))).join(' or ')
|
||||
paramDecorated.name = paramKey
|
||||
paramDecorated.name = paramKey || ''
|
||||
if (topLevel) {
|
||||
paramDecorated.in = 'body'
|
||||
}
|
||||
paramDecorated.description = await renderContent(param.description)
|
||||
if (required && required.includes(paramKey)) {
|
||||
paramDecorated.description = await renderContent(param.description || '')
|
||||
if (required && required.includes(paramKey || '')) {
|
||||
paramDecorated.isRequired = true
|
||||
}
|
||||
if (childParamsGroups && childParamsGroups.length > 0 && !param.oneOfObject) {
|
||||
@@ -227,12 +264,12 @@ async function getTransformedParam(param, paramType, props) {
|
||||
obj.name,
|
||||
curr ? (!Object.hasOwn(curr, 'isRequired') ? obj : curr) : obj,
|
||||
)
|
||||
}, new Map())
|
||||
}, new Map<string, TransformedParam>())
|
||||
.values(),
|
||||
)
|
||||
|
||||
paramDecorated.childParamsGroups = mergedChildParamsGroups
|
||||
} else if (childParamsGroups.length > 0) {
|
||||
} else if (childParamsGroups && childParamsGroups.length > 0) {
|
||||
paramDecorated.childParamsGroups = childParamsGroups
|
||||
}
|
||||
if (param.enum) {
|
||||
@@ -243,24 +280,28 @@ async function getTransformedParam(param, paramType, props) {
|
||||
paramDecorated.oneOfObject = true
|
||||
}
|
||||
|
||||
// we also want to catch default values of `false` for booleans
|
||||
if (param.default !== undefined) {
|
||||
paramDecorated.default = param.default
|
||||
}
|
||||
return paramDecorated
|
||||
}
|
||||
|
||||
async function getOneOfChildParams(param) {
|
||||
const childParamsGroups = []
|
||||
async function getOneOfChildParams(param: Schema): Promise<TransformedParam[]> {
|
||||
const childParamsGroups: TransformedParam[] = []
|
||||
if (!param.oneOf) {
|
||||
return childParamsGroups
|
||||
}
|
||||
for (const oneOfParam of param.oneOf) {
|
||||
const objParam = {
|
||||
const objParam: TransformedParam = {
|
||||
type: 'object',
|
||||
name: oneOfParam.title,
|
||||
description: await renderContent(oneOfParam.description),
|
||||
isRequired: oneOfParam.required,
|
||||
childParamsGroups: [],
|
||||
}
|
||||
if (objParam.childParamsGroups) {
|
||||
objParam.childParamsGroups.push(...(await getBodyParams(oneOfParam, false)))
|
||||
}
|
||||
childParamsGroups.push(objParam)
|
||||
}
|
||||
return childParamsGroups
|
||||
@@ -8,7 +8,7 @@ import { renderContent } from '#src/content-render/index.js'
|
||||
import getCodeSamples from './create-rest-examples.js'
|
||||
import operationSchema from './operation-schema.js'
|
||||
import { validateJson } from '#src/tests/lib/validate-json-schema.js'
|
||||
import { getBodyParams } from './get-body-params.js'
|
||||
import { getBodyParams } from './get-body-params'
|
||||
|
||||
export default class Operation {
|
||||
#operation
|
||||
|
||||
@@ -3,11 +3,15 @@ import { existsSync } from 'fs'
|
||||
import path from 'path'
|
||||
import { mkdirp } from 'mkdirp'
|
||||
|
||||
import { updateRestFiles } from './update-markdown.js'
|
||||
import { allVersions } from '#src/versions/lib/all-versions.js'
|
||||
import { createOperations, processOperations } from './get-operations.js'
|
||||
import { getProgAccessData } from '#src/github-apps/scripts/sync.js'
|
||||
import { REST_DATA_DIR, REST_SCHEMA_FILENAME } from '../../lib/index.js'
|
||||
import { updateRestFiles } from './update-markdown'
|
||||
import { allVersions } from '@/versions/lib/all-versions'
|
||||
import { createOperations, processOperations } from './get-operations'
|
||||
import { getProgAccessData } from '@/github-apps/scripts/sync'
|
||||
import { REST_DATA_DIR, REST_SCHEMA_FILENAME } from '../../lib/index'
|
||||
|
||||
type Schema = Record<string, any>
|
||||
type Operation = { category: string; subcategory: string; [key: string]: any }
|
||||
type OperationsByCategory = Record<string, Record<string, Operation[]>>
|
||||
|
||||
// All of the schema releases that we store in allVersions
|
||||
// Ex: 'api.github.com', 'ghec', 'ghes-3.6', 'ghes-3.5',
|
||||
@@ -16,13 +20,17 @@ const OPENAPI_VERSION_NAMES = Object.keys(allVersions).map(
|
||||
(elem) => allVersions[elem].openApiVersionName,
|
||||
)
|
||||
|
||||
export async function syncRestData(sourceDirectory, restSchemas, progAccessSource) {
|
||||
export async function syncRestData(
|
||||
sourceDirectory: string,
|
||||
restSchemas: string[],
|
||||
progAccessSource: string,
|
||||
): Promise<void> {
|
||||
await Promise.all(
|
||||
restSchemas.map(async (schemaName) => {
|
||||
const file = path.join(sourceDirectory, schemaName)
|
||||
const schema = JSON.parse(await readFile(file, 'utf-8'))
|
||||
const schema = JSON.parse(await readFile(file, 'utf-8')) as Schema
|
||||
|
||||
const operations = []
|
||||
const operations: Operation[] = []
|
||||
console.log('Instantiating operation instances from schema ', schemaName)
|
||||
try {
|
||||
const newOperations = await createOperations(schema)
|
||||
@@ -62,30 +70,10 @@ export async function syncRestData(sourceDirectory, restSchemas, progAccessSourc
|
||||
await updateRestConfigData(restSchemas)
|
||||
}
|
||||
|
||||
/*
|
||||
Orders the operations by their category and subcategories.
|
||||
All operations must have a category, but operations don't need
|
||||
a subcategory. When no subcategory is present, the subcategory
|
||||
property is an empty string ('').
|
||||
|
||||
Example:
|
||||
{
|
||||
[category]: {
|
||||
'': {
|
||||
"description": "",
|
||||
"operations": []
|
||||
},
|
||||
[subcategory sorted alphabetically]: {
|
||||
"description": "",
|
||||
"operations": []
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
async function formatRestData(operations) {
|
||||
async function formatRestData(operations: Operation[]): Promise<OperationsByCategory> {
|
||||
const categories = [...new Set(operations.map((operation) => operation.category))].sort()
|
||||
|
||||
const operationsByCategory = {}
|
||||
const operationsByCategory: OperationsByCategory = {}
|
||||
categories.forEach((category) => {
|
||||
operationsByCategory[category] = {}
|
||||
const categoryOperations = operations.filter((operation) => operation.category === category)
|
||||
@@ -102,7 +90,7 @@ async function formatRestData(operations) {
|
||||
}
|
||||
|
||||
subcategories.forEach((subcategory) => {
|
||||
operationsByCategory[category][subcategory] = {}
|
||||
operationsByCategory[category][subcategory] = []
|
||||
|
||||
const subcategoryOperations = categoryOperations.filter(
|
||||
(operation) => operation.subcategory === subcategory,
|
||||
@@ -116,9 +104,12 @@ async function formatRestData(operations) {
|
||||
|
||||
// Every time we update the REST data files, we'll want to make sure the
|
||||
// config.json file is updated with the latest api versions.
|
||||
async function updateRestConfigData(schemas) {
|
||||
async function updateRestConfigData(schemas: string[]): Promise<void> {
|
||||
const restConfigFilename = 'src/rest/lib/config.json'
|
||||
const restConfigData = JSON.parse(await readFile(restConfigFilename, 'utf8'))
|
||||
const restConfigData = JSON.parse(await readFile(restConfigFilename, 'utf8')) as Record<
|
||||
string,
|
||||
any
|
||||
>
|
||||
const restApiVersionData = restConfigData['api-versions'] || {}
|
||||
// If the version isn't one of the OpenAPI version,
|
||||
// then it's an api-versioned schema
|
||||
@@ -126,6 +117,9 @@ async function updateRestConfigData(schemas) {
|
||||
const schemaBaseName = path.basename(schema, '.json')
|
||||
if (!OPENAPI_VERSION_NAMES.includes(schemaBaseName)) {
|
||||
const openApiVer = OPENAPI_VERSION_NAMES.find((ver) => schemaBaseName.startsWith(ver))
|
||||
if (!openApiVer) {
|
||||
throw new Error(`Could not find the OpenAPI version for schema ${schemaBaseName}`)
|
||||
}
|
||||
const date = schemaBaseName.split(`${openApiVer}-`)[1]
|
||||
|
||||
if (!restApiVersionData[openApiVer]) {
|
||||
@@ -142,9 +136,11 @@ async function updateRestConfigData(schemas) {
|
||||
await writeFile(restConfigFilename, JSON.stringify(restConfigData, null, 2))
|
||||
}
|
||||
|
||||
export async function getOpenApiSchemaFiles(schemas) {
|
||||
const restSchemas = []
|
||||
const webhookSchemas = []
|
||||
export async function getOpenApiSchemaFiles(
|
||||
schemas: string[],
|
||||
): Promise<{ restSchemas: string[]; webhookSchemas: string[] }> {
|
||||
const restSchemas: string[] = []
|
||||
const webhookSchemas: string[] = []
|
||||
// The full list of dereferened OpenAPI schemas received from
|
||||
// bundling the OpenAPI in github/github
|
||||
const schemaNames = schemas.map((schema) => path.basename(schema, '.json'))
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, test } from 'vitest'
|
||||
|
||||
import { getOpenApiSchemaFiles } from '../scripts/utils/sync.js'
|
||||
import { getOpenApiSchemaFiles } from '../scripts/utils/sync'
|
||||
import { allVersions } from '#src/versions/lib/all-versions.js'
|
||||
|
||||
const supportedReleases = Object.keys(allVersions).map(
|
||||
@@ -16,7 +16,7 @@ A [workflow](.github/workflows/sync-openapi.yml) is used to trigger the automati
|
||||
|
||||
The workflow automatically creates a pull request with the changes (for all three pipelines) and the label `github-openapi-bot`.
|
||||
|
||||
The workflow runs the `src/rest/scripts/update-files.js` script, which then calls the `src/webhooks/scripts/sync.js` script.
|
||||
The workflow runs the `npm run sync-rest` script, which then calls the `src/webhooks/scripts/sync.ts` script.
|
||||
|
||||
## Manually running the pipeline
|
||||
|
||||
@@ -29,7 +29,8 @@ Then, you can manually sync the data used by the REST, Webhooks, and GitHub App
|
||||
To run the webhooks pipeline locally:
|
||||
|
||||
1. Clone the [`github/rest-api-description`](https://github.com/github/rest-api-description) repository inside your local `docs-internal` repository.
|
||||
1. Run `src/rest/scripts/update-files.js -s rest-api-description -o webhooks`.
|
||||
1. Set a `GITHUB_TOKEN` in your `.env` with (classic) `repo` scopes & enable SSO for the github org.
|
||||
1. Run `npm run sync-rest -- -s rest-api-description -o webhooks`.
|
||||
|
||||
## About this directory
|
||||
|
||||
@@ -37,7 +38,7 @@ To run the webhooks pipeline locally:
|
||||
- `src/webhooks/lib` - The source code used in production to display the webhook docs and configuration files edited by content and engineering team members.
|
||||
- `src/webhooks/lib/config.json` - A configuration file used to specify metadata about the webhooks pipeline.
|
||||
- `src/webhooks/scripts` - The scripts and source code used run the webhooks pipeline, which updates the `src/webhooks/data` directory.
|
||||
- `src/webhooks/scripts/sync.js` - The entrypoint script that runs the webhooks pipeline.
|
||||
- `src/webhooks/scripts/sync.ts` - The entrypoint script that runs the webhooks pipeline.
|
||||
- `src/webhooks/tests` - The tests used to verify the webhooks pipeline.
|
||||
|
||||
## Configuring the pipeline
|
||||
|
||||
@@ -3,14 +3,26 @@ import { existsSync } from 'fs'
|
||||
import path from 'path'
|
||||
import { mkdirp } from 'mkdirp'
|
||||
|
||||
import { WEBHOOK_DATA_DIR, WEBHOOK_SCHEMA_FILENAME } from '../lib/index.js'
|
||||
import Webhook from './webhook.js'
|
||||
import { WEBHOOK_DATA_DIR, WEBHOOK_SCHEMA_FILENAME } from '../lib/index'
|
||||
import Webhook, { WebhookSchema } from '@/webhooks/scripts/webhook'
|
||||
|
||||
export async function syncWebhookData(sourceDirectory, webhookSchemas) {
|
||||
interface WebhookFile {
|
||||
webhooks?: {
|
||||
post: WebhookSchema
|
||||
}[]
|
||||
'x-webhooks'?: {
|
||||
post: WebhookSchema
|
||||
}[]
|
||||
}
|
||||
|
||||
export async function syncWebhookData(
|
||||
sourceDirectory: string,
|
||||
webhookSchemas: string[],
|
||||
): Promise<void> {
|
||||
await Promise.all(
|
||||
webhookSchemas.map(async (schemaName) => {
|
||||
const file = path.join(sourceDirectory, schemaName)
|
||||
const schema = JSON.parse(await readFile(file, 'utf-8'))
|
||||
const schema: WebhookFile = JSON.parse(await readFile(file, 'utf-8'))
|
||||
// In OpenAPI version 3.1, the schema data is under the `webhooks`
|
||||
// key, but in 3.0 the schema data was in `x-webhooks`.
|
||||
// We just fallback to `x-webhooks` for now since there's
|
||||
@@ -47,7 +59,7 @@ export async function syncWebhookData(sourceDirectory, webhookSchemas) {
|
||||
)
|
||||
}
|
||||
|
||||
async function processWebhookSchema(webhooks) {
|
||||
async function processWebhookSchema(webhooks: Webhook[]): Promise<void> {
|
||||
try {
|
||||
if (webhooks.length) {
|
||||
await Promise.all(webhooks.map((webhook) => webhook.process()))
|
||||
@@ -62,9 +74,11 @@ async function processWebhookSchema(webhooks) {
|
||||
// Create an object with all webhooks where the key is the webhook name.
|
||||
// Webhooks typically have a property called `action` that describes the
|
||||
// events that trigger the webhook. Some webhooks (like `ping`) don't have
|
||||
// action types -- in that case we set a the value of action to 'default'.
|
||||
async function formatWebhookData(webhooks) {
|
||||
const categorizedWebhooks = {}
|
||||
// action types -- in that case we set the value of action to 'default'.
|
||||
async function formatWebhookData(
|
||||
webhooks: Webhook[],
|
||||
): Promise<Record<string, Record<string, Webhook>>> {
|
||||
const categorizedWebhooks: Record<string, Record<string, Webhook>> = {}
|
||||
for (const webhook of Object.values(webhooks)) {
|
||||
if (!webhook.action) webhook.action = 'default'
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
#!/usr/bin/env node
|
||||
import { get, isPlainObject } from 'lodash-es'
|
||||
|
||||
import { getJsonValidator } from '#src/tests/lib/validate-json-schema.js'
|
||||
import { renderContent } from '#src/content-render/index.js'
|
||||
import webhookSchema from './webhook-schema.js'
|
||||
import { getBodyParams } from '../../rest/scripts/utils/get-body-params.js'
|
||||
import { getJsonValidator } from '@/tests/lib/validate-json-schema'
|
||||
import { renderContent } from '@/content-render/index'
|
||||
import webhookSchema from './webhook-schema'
|
||||
import { getBodyParams, TransformedParam } from '../../rest/scripts/utils/get-body-params'
|
||||
|
||||
const NO_CHILD_PROPERTIES = [
|
||||
'action',
|
||||
@@ -17,13 +16,45 @@ const NO_CHILD_PROPERTIES = [
|
||||
|
||||
const validate = getJsonValidator(webhookSchema)
|
||||
|
||||
export default class Webhook {
|
||||
#webhook
|
||||
constructor(webhook) {
|
||||
export interface WebhookSchema {
|
||||
description: string
|
||||
summary: string
|
||||
requestBody?: {
|
||||
content: {
|
||||
'application/json': {
|
||||
schema: Record<string, any>
|
||||
}
|
||||
}
|
||||
}
|
||||
'x-github': {
|
||||
'supported-webhook-types': string[]
|
||||
subcategory: string
|
||||
}
|
||||
}
|
||||
|
||||
interface WebhookInterface {
|
||||
descriptionHtml: string
|
||||
summaryHtml: string
|
||||
bodyParameters: TransformedParam[]
|
||||
availability: string[]
|
||||
action: string | null
|
||||
category: string
|
||||
process(): Promise<void>
|
||||
renderDescription(): Promise<this>
|
||||
renderBodyParameterDescriptions(): Promise<void>
|
||||
}
|
||||
|
||||
export default class Webhook implements WebhookInterface {
|
||||
#webhook: WebhookSchema
|
||||
descriptionHtml: string = ''
|
||||
summaryHtml: string = ''
|
||||
bodyParameters: TransformedParam[] = []
|
||||
availability: string[]
|
||||
action: string | null
|
||||
category: string
|
||||
|
||||
constructor(webhook: WebhookSchema) {
|
||||
this.#webhook = webhook
|
||||
this.descriptionHtml = ''
|
||||
this.summaryHtml = ''
|
||||
this.bodyParameters = []
|
||||
this.availability = webhook['x-github']['supported-webhook-types']
|
||||
this.action = get(
|
||||
webhook,
|
||||
@@ -45,30 +76,29 @@ export default class Webhook {
|
||||
// The OpenAPI uses hyphens for the webhook names, but the webhooks
|
||||
// are sent using underscores (e.g. `branch_protection_rule` instead
|
||||
// of `branch-protection-rule`)
|
||||
this.category = webhook['x-github'].subcategory.replaceAll('-', '_')
|
||||
return this
|
||||
this.category = webhook['x-github'].subcategory.replace(/-/g, '_')
|
||||
}
|
||||
|
||||
async process() {
|
||||
async process(): Promise<void> {
|
||||
await Promise.all([this.renderDescription(), this.renderBodyParameterDescriptions()])
|
||||
|
||||
const isValid = validate(this)
|
||||
const isValid = validate(this as WebhookInterface) // Add type assertion here
|
||||
if (!isValid) {
|
||||
console.error(JSON.stringify(validate.errors, null, 2))
|
||||
throw new Error(`Invalid OpenAPI webhook found: ${this.category}`)
|
||||
}
|
||||
}
|
||||
|
||||
async renderDescription() {
|
||||
async renderDescription(): Promise<this> {
|
||||
this.descriptionHtml = await renderContent(this.#webhook.description)
|
||||
this.summaryHtml = await renderContent(this.#webhook.summary)
|
||||
return this
|
||||
}
|
||||
|
||||
async renderBodyParameterDescriptions() {
|
||||
if (!this.#webhook.requestBody) return []
|
||||
const schema = get(this.#webhook, `requestBody.content.['application/json'].schema`, {})
|
||||
this.bodyParameters = isPlainObject(schema) ? await getBodyParams(schema, true, this.title) : []
|
||||
async renderBodyParameterDescriptions(): Promise<void> {
|
||||
if (!this.#webhook.requestBody) return
|
||||
const schema = get(this.#webhook, `requestBody.content['application/json'].schema`, {})
|
||||
this.bodyParameters = isPlainObject(schema) ? await getBodyParams(schema, true) : []
|
||||
|
||||
// Removes the children of the common properties
|
||||
this.bodyParameters.forEach((param) => {
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, test } from 'vitest'
|
||||
|
||||
import { getOpenApiSchemaFiles } from '../../rest/scripts/utils/sync.js'
|
||||
import { getOpenApiSchemaFiles } from '../../rest/scripts/utils/sync'
|
||||
import { allVersions } from '#src/versions/lib/all-versions.js'
|
||||
|
||||
describe('webhook data files are generated correctly from dereferenced openapi files', () => {
|
||||
Reference in New Issue
Block a user