diff --git a/.github/workflows/openapi-decorate.yml b/.github/workflows/openapi-decorate.yml index d0e3d9a29f..749a24f18c 100644 --- a/.github/workflows/openapi-decorate.yml +++ b/.github/workflows/openapi-decorate.yml @@ -27,7 +27,7 @@ concurrency: jobs: generate-decorated-files: if: github.repository == 'github/docs-internal' - runs-on: ubuntu-latest + runs-on: ubuntu-20.04-xl steps: - if: ${{ env.FREEZE == 'true' }} run: | @@ -49,14 +49,6 @@ jobs: - uses: ./.github/actions/node-npm-setup - - name: Get the rest-api-description SHA being synced - id: rest-api-description - run: | - cd rest-api-description - OPENAPI_COMMIT_SHA=$(git rev-parse HEAD) - echo "OPENAPI_COMMIT_SHA=$OPENAPI_COMMIT_SHA" >> $GITHUB_OUTPUT - echo "Copied files from github/rest-api-description repo. Commit SHA: $OPENAPI_COMMIT_SHA" - - name: Sync the REST, Webhooks, and GitHub Apps schemas run: | src/rest/scripts/update-files.js --source-repo rest-api-description --output rest github-apps webhooks rest-redirects @@ -64,6 +56,17 @@ jobs: echo "Deleting the cloned github/rest-api-description repo..." rm -rf rest-api-description + - name: Get the rest-api-description SHA being synced + id: rest-api-description + run: | + OPENAPI_COMMIT_SHA=$(cat src/rest/lib/config.json | jq -r '.sha') + echo "OPENAPI_COMMIT_SHA=$OPENAPI_COMMIT_SHA" >> $GITHUB_OUTPUT + echo "Copied files from github/rest-api-description repo. Commit SHA: $OPENAPI_COMMIT_SHA" + if [ -z $OPENAPI_COMMIT_SHA ]; then + echo "OpenAPI commit SHA is empty!" + exit 1 + fi + - name: Create pull request env: # Needed for gh diff --git a/lib/all-versions.js b/lib/all-versions.js index 532fc8f701..7709fd7076 100755 --- a/lib/all-versions.js +++ b/lib/all-versions.js @@ -6,7 +6,7 @@ import enterpriseServerReleases from './enterprise-server-releases.js' // where "enterprise-server" is the plan and "2.21" is the release const versionDelimiter = '@' const latestNonNumberedRelease = 'latest' -const REST_DATA_META_FILE = 'src/rest/data/meta.json' +const REST_DATA_META_FILE = 'src/rest/lib/config.json' // !Explanation of versionless redirect fallbacks! // This array is **in order** of the versions the site should try to fall back to if diff --git a/src/github-apps/lib/config.json b/src/github-apps/lib/config.json new file mode 100644 index 0000000000..031e6115f6 --- /dev/null +++ b/src/github-apps/lib/config.json @@ -0,0 +1,3 @@ +{ + "sha": "4c7e9b68a3aeb05689b7240c48ea5c4ad44e4d7c" +} diff --git a/src/rest/data/meta.json b/src/rest/lib/config.json similarity index 85% rename from src/rest/data/meta.json rename to src/rest/lib/config.json index 94f311bd78..fe5572311d 100644 --- a/src/rest/data/meta.json +++ b/src/rest/lib/config.json @@ -19,5 +19,6 @@ ], "autogenerated": "rest", "allowTitleToDifferFromFilename": true - } -} \ No newline at end of file + }, + "sha": "4c7e9b68a3aeb05689b7240c48ea5c4ad44e4d7c" +} diff --git a/src/rest/scripts/update-files.js b/src/rest/scripts/update-files.js index 46f28798c2..2acbbe0280 100755 --- a/src/rest/scripts/update-files.js +++ b/src/rest/scripts/update-files.js @@ -7,7 +7,7 @@ // // [end-readme] -import { stat, readdir, copyFile, readFile, rename } from 'fs/promises' +import { readdir, copyFile, readFile, writeFile, rename } from 'fs/promises' import path from 'path' import { program, Option } from 'commander' import { execSync } from 'child_process' @@ -15,6 +15,7 @@ import mkdirp from 'mkdirp' import rimraf from 'rimraf' import { fileURLToPath } from 'url' import walk from 'walk-sync' +import { existsSync } from 'fs' import { syncRestData, getOpenApiSchemaFiles } from './utils/sync.js' import { validateVersionsOptions } from './utils/get-openapi-schemas.js' @@ -27,8 +28,10 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)) const TEMP_OPENAPI_DIR = path.join(__dirname, '../../../rest-api-description/openApiTemp') const TEMP_BUNDLED_OPENAPI_DIR = path.join(TEMP_OPENAPI_DIR, 'bundled') const GITHUB_REP_DIR = '../github' -const REST_DESCRIPTION_DIR = 'rest-api-description/descriptions-next' -const VERSION_NAMES = JSON.parse(await readFile('src/rest/data/meta.json', 'utf8')).versionMapping +const REST_API_DESCRIPTION_ROOT = 'rest-api-description' +const REST_DESCRIPTION_DIR = path.join(REST_API_DESCRIPTION_ROOT, 'descriptions-next') +const VERSION_NAMES = JSON.parse(await readFile('src/rest/lib/config.json', 'utf8')).versionMapping +const noConfig = ['rest-redirects'] program .description('Update rest, webhooks, and github-apps automated pipeline data files.') @@ -43,9 +46,9 @@ program .addOption( new Option( '-s, --source-repo ', - 'The source repository to get the dereferenced files from. When the source repo is rest-api-description, the bundler is not run to generate the source dereferenced OpenAPI files because the rest-api-description repo already contains them.' + `The source repository to get the dereferenced files from. When the source repo is ${REST_API_DESCRIPTION_ROOT}, the bundler is not run to generate the source dereferenced OpenAPI files because the ${REST_API_DESCRIPTION_ROOT} repo already contains them.` ) - .choices(['github', 'rest-api-description']) + .choices(['github', REST_API_DESCRIPTION_ROOT]) .default('github', 'github') ) .option( @@ -73,8 +76,9 @@ async function main() { } // When we get the dereferenced OpenAPI files from the open-source - // github/rest-api-description repo, we need to remove any versions - // that are deprecated because that repo contains all past versions. + // rest description repo (REST_API_DESCRIPTION_ROOT), we need to + // remove any versions that are deprecated because that repo contains + // all past versions. const sourceDirectory = sourceRepo === 'github' ? TEMP_BUNDLED_OPENAPI_DIR : REST_DESCRIPTION_DIR const dereferencedFiles = walk(sourceDirectory, { @@ -90,10 +94,10 @@ async function main() { rimraf.sync(TEMP_BUNDLED_OPENAPI_DIR) await normalizeDataVersionNames(TEMP_OPENAPI_DIR) - // The rest-api-description repo contains all current and deprecated versions - // We need to remove the deprecated versions so that we don't spend time - // generating data files for them. - if (sourceRepo === 'rest-api-description') { + // The REST_API_DESCRIPTION_ROOT repo contains all current and + // deprecated versions. We need to remove the deprecated versions + // so that we don't spend time generating data files for them. + if (sourceRepo === REST_API_DESCRIPTION_ROOT) { const derefDir = await readdir(TEMP_OPENAPI_DIR) const currentOpenApiVersions = Object.values(allVersions).map((elem) => elem.openApiVersionName) derefDir.forEach((schema) => { @@ -126,6 +130,26 @@ async function main() { await syncRestRedirects(TEMP_OPENAPI_DIR, restSchemas) } + // If the source repo is REST_API_DESCRIPTION_ROOT, we want to update + // the pipeline config files with the SHA of the synced commit. + if (sourceRepo === REST_API_DESCRIPTION_ROOT) { + const syncedSha = execSync('git rev-parse HEAD', { + cwd: REST_API_DESCRIPTION_ROOT, + encoding: 'utf8', + }).trim() + if (!syncedSha) { + throw new Error(`Could not get the SHA of the synced ${REST_API_DESCRIPTION_ROOT} repo.`) + } + + const pipelinesWithConfigs = output.filter((pipeline) => !noConfig.includes(pipeline)) + for (const pipeline of pipelinesWithConfigs) { + const configFilepath = `src/${pipeline}/lib/config.json` + const configData = JSON.parse(await readFile(configFilepath, 'utf8')) + configData.sha = syncedSha + await writeFile(configFilepath, JSON.stringify(configData, null, 2)) + } + } + console.log( `\nšŸ The static REST API files are now up-to-date with \`github/${sourceRepo}\`. To revert uncommitted data changes, run \`git checkout src/**/data/*\`\n` ) @@ -190,15 +214,14 @@ async function validateInputParameters() { throw new Error(errorMsg) } - // Check that the github/github repo exists. If the files are only being - // decorated, the github/github repo isn't needed. - if (sourceRepo === 'github') { - try { - await stat(GITHUB_REP_DIR) - } catch (error) { - const errorMsg = `šŸ›‘ The ${GITHUB_REP_DIR} does not exist. Make sure you have a codespace with a checkout of \`github/github\` at the same level as your \`github/docs-internal \`repo before running this script. See this documentation for details: https://thehub.github.com/epd/engineering/products-and-services/public-apis/rest/openapi/openapi-in-the-docs/#previewing-changes-in-the-docs.` - throw new Error(errorMsg) - } + // Check that the source repo exists. + const sourceRepoDirectory = sourceRepo === 'github' ? GITHUB_REP_DIR : REST_API_DESCRIPTION_ROOT + if (!existsSync(sourceRepoDirectory)) { + const errorMsg = + sourceRepo === 'github' + ? `šŸ›‘ The ${GITHUB_REP_DIR} does not exist. Make sure you have a codespace with a checkout of \`github/github\` at the same level as your \`github/docs-internal \`repo before running this script. See this documentation for details: https://thehub.github.com/epd/engineering/products-and-services/public-apis/rest/openapi/openapi-in-the-docs/#previewing-changes-in-the-docs.` + : `šŸ›‘ You must have a clone of the ${REST_API_DESCRIPTION_ROOT} repo in the root of this repo.` + throw new Error(errorMsg) } if (versions && versions.length) { await validateVersionsOptions(versions) @@ -207,8 +230,8 @@ async function validateInputParameters() { // Version names in the data consumed by the docs site varies depending on the // team that owns the data we consume. This function translates the version -// names to use the names in the src//data/meta.json file. -// The names in the meta.json file maps the incoming version name to +// names to use the names in the src//lib/config.json file. +// The names in the config.json file maps the incoming version name to // the short name of the version defined in lib/allVersions.js. export async function normalizeDataVersionNames(sourceDirectory) { const schemas = await readdir(sourceDirectory) diff --git a/src/rest/scripts/utils/get-openapi-schemas.js b/src/rest/scripts/utils/get-openapi-schemas.js index 64fd6066aa..34ed65d7ef 100644 --- a/src/rest/scripts/utils/get-openapi-schemas.js +++ b/src/rest/scripts/utils/get-openapi-schemas.js @@ -5,7 +5,7 @@ import path from 'path' import { allVersions } from '../../../../lib/all-versions.js' const OPEN_API_RELEASES_DIR = '../github/app/api/description/config/releases' -const metaData = JSON.parse(await readFile('src/rest/data/meta.json', 'utf8')) +const configData = JSON.parse(await readFile('src/rest/lib/config.json', 'utf8')) // Gets the full list of unpublished + active, deprecated + active, // or active schemas from the github/github repo // `openApiReleaseDir` is the path to the `app/api/description/config/releases` @@ -25,19 +25,19 @@ export async function getSchemas(directory = OPEN_API_RELEASES_DIR) { const content = await readFile(path.join(directory, file), 'utf8') const yamlContent = yaml.load(content) - const releaseMatch = Object.keys(metaData.versionMapping).find((name) => + const releaseMatch = Object.keys(configData.versionMapping).find((name) => fileBaseName.startsWith(name) ) if (!releaseMatch) { throw new Error( `šŸ›‘ The file ${fileBaseName} does not match any known docs version name. (not one of ${Object.keys( - metaData.versionMapping + configData.versionMapping )})` ) } const docsName = - metaData.versionMapping[fileBaseName] || - fileBaseName.replace(releaseMatch, metaData.versionMapping[releaseMatch]) + configData.versionMapping[fileBaseName] || + fileBaseName.replace(releaseMatch, configData.versionMapping[releaseMatch]) const isDeprecatedInDocs = !Object.keys(allVersions).find( (version) => allVersions[version].openApiVersionName === docsName ) diff --git a/src/rest/scripts/utils/sync.js b/src/rest/scripts/utils/sync.js index cf1c0345a3..3a3dfd9a70 100644 --- a/src/rest/scripts/utils/sync.js +++ b/src/rest/scripts/utils/sync.js @@ -49,7 +49,7 @@ export async function syncRestData(sourceDirectory, restSchemas) { }) ) await updateMarkdownFiles() - await updateRestMetaData(restSchemas) + await updateRestConfigData(restSchemas) } /* @@ -105,11 +105,11 @@ async function formatRestData(operations) { } // Every time we update the REST data files, we'll want to make sure the -// meta.json file is updated with the latest api versions. -async function updateRestMetaData(schemas) { - const restMetaFilename = `${REST_DATA_DIR}/meta.json` - const restMetaData = JSON.parse(await readFile(restMetaFilename, 'utf8')) - const restApiVersionData = restMetaData['api-versions'] +// config.json file is updated with the latest api versions. +async function updateRestConfigData(schemas) { + const restConfigFilename = 'src/rest/lib/config.json' + const restConfigData = JSON.parse(await readFile(restConfigFilename, 'utf8')) + const restApiVersionData = restConfigData['api-versions'] // If the version isn't one of the OpenAPI version, // then it's an api-versioned schema for (const schema of schemas) { @@ -124,9 +124,9 @@ async function updateRestMetaData(schemas) { restApiVersionData[openApiVer] = dates } } - restMetaData['api-versions'] = restApiVersionData - await writeFile(restMetaFilename, JSON.stringify(restMetaData, null, 2)) + restConfigData['api-versions'] = restApiVersionData } + await writeFile(restConfigFilename, JSON.stringify(restConfigData, null, 2)) } export async function getOpenApiSchemaFiles(schemas) { diff --git a/src/rest/scripts/utils/update-markdown.js b/src/rest/scripts/utils/update-markdown.js index 39a7310cf9..938d177659 100644 --- a/src/rest/scripts/utils/update-markdown.js +++ b/src/rest/scripts/utils/update-markdown.js @@ -11,7 +11,7 @@ import { allVersions, getDocsVersion } from '../../../../lib/all-versions.js' import { REST_DATA_DIR, REST_SCHEMA_FILENAME } from '../../lib/index.js' const frontmatterDefaults = JSON.parse( - await readFile(path.join(REST_DATA_DIR, 'meta.json'), 'utf-8') + await readFile('src/rest/lib/config.json', 'utf-8') ).frontmatterDefaults export async function updateMarkdownFiles() { diff --git a/src/webhooks/lib/config.json b/src/webhooks/lib/config.json new file mode 100644 index 0000000000..031e6115f6 --- /dev/null +++ b/src/webhooks/lib/config.json @@ -0,0 +1,3 @@ +{ + "sha": "4c7e9b68a3aeb05689b7240c48ea5c4ad44e4d7c" +}