Automation pipelines update markdown (#35194)
This commit is contained in:
Rachael Sewell
@@ -12,5 +12,12 @@
|
||||
"github.ae": "ghae",
|
||||
"ghes": "ghes",
|
||||
"ghec": "ghec"
|
||||
},
|
||||
"frontmatterDefaults": {
|
||||
"topics": [
|
||||
"API"
|
||||
],
|
||||
"autogenerated": "rest",
|
||||
"allowTitleToDifferFromFilename": true
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { existsSync } from 'fs'
|
||||
import path from 'path'
|
||||
import mkdirp from 'mkdirp'
|
||||
|
||||
import { updateMarkdownFiles } from './update-markdown.js'
|
||||
import { allVersions } from '../../../../lib/all-versions.js'
|
||||
import { createOperations, processOperations } from './get-operations.js'
|
||||
import { REST_DATA_DIR, REST_SCHEMA_FILENAME } from '../../lib/index.js'
|
||||
@@ -47,6 +48,7 @@ export async function syncRestData(sourceDirectory, restSchemas) {
|
||||
console.log(`✅ Wrote ${targetPath}`)
|
||||
})
|
||||
)
|
||||
await updateMarkdownFiles()
|
||||
await updateRestMetaData(restSchemas)
|
||||
}
|
||||
|
||||
|
||||
273
src/rest/scripts/utils/update-markdown.js
Normal file
273
src/rest/scripts/utils/update-markdown.js
Normal file
@@ -0,0 +1,273 @@
|
||||
import path from 'path'
|
||||
import { existsSync } from 'fs'
|
||||
import walk from 'walk-sync'
|
||||
import matter from 'gray-matter'
|
||||
import { difference } from 'lodash-es'
|
||||
import { readFile, writeFile, unlink } from 'fs/promises'
|
||||
|
||||
import { allVersions, getDocsVersion } from '../../../../lib/all-versions.js'
|
||||
import { REST_DATA_DIR, REST_SCHEMA_FILENAME } from '../../lib/index.js'
|
||||
|
||||
const frontmatterDefaults = JSON.parse(
|
||||
await readFile(path.join(REST_DATA_DIR, 'meta.json'), 'utf-8')
|
||||
).frontmatterDefaults
|
||||
|
||||
export async function updateMarkdownFiles() {
|
||||
const restVersions = await getDataFrontmatter(REST_DATA_DIR, REST_SCHEMA_FILENAME)
|
||||
const restContentFrontmatter = await getMarkdownFrontmatter(restVersions)
|
||||
const restContentFiles = Object.keys(restContentFrontmatter)
|
||||
// Get a list of existing markdown files so we can make deletions
|
||||
const restMarkdownFiles = walk('content/rest', {
|
||||
includeBasePath: true,
|
||||
directories: false,
|
||||
}).filter((file) => !file.includes('index.md') && !file.includes('README.md'))
|
||||
const existingAutogeneratedFiles = (
|
||||
await Promise.all(
|
||||
restMarkdownFiles.map(async (file) => {
|
||||
const data = await readFile(file, 'utf-8')
|
||||
const frontmatter = matter(data)
|
||||
if (frontmatter.data.autogenerated === 'rest') {
|
||||
return file
|
||||
}
|
||||
})
|
||||
)
|
||||
).filter(Boolean)
|
||||
|
||||
// If the first array contains items that the second array does not,
|
||||
// it means that a Markdown page was deleted from the OpenAPI schema
|
||||
const filesToRemove = difference(existingAutogeneratedFiles, restContentFiles)
|
||||
|
||||
// Markdown files that need to be deleted
|
||||
for (const file of filesToRemove) {
|
||||
await unlink(file)
|
||||
await updateIndexFile(file, 'remove')
|
||||
}
|
||||
|
||||
// Markdown files that need to be added or updated
|
||||
for (const [file, newFrontmatter] of Object.entries(restContentFrontmatter)) {
|
||||
if (existsSync(file)) {
|
||||
// update only the versions property of the file, assuming
|
||||
// the other properties have already been added and edited
|
||||
const { data, content } = matter(await readFile(file, 'utf-8'))
|
||||
data.versions = newFrontmatter.versions
|
||||
await writeFile(file, matter.stringify(content, data))
|
||||
} else {
|
||||
// create a new file placeholder metadata
|
||||
await writeFile(file, matter.stringify('', newFrontmatter))
|
||||
updateIndexFile(file, 'add')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Adds or removes children properties from index.md pages
|
||||
async function updateIndexFile(file, changeType) {
|
||||
const filename = path.basename(file, '.md')
|
||||
const indexDirectory = path.basename(path.dirname(file))
|
||||
const rootDir = file.split(indexDirectory)[0]
|
||||
const indexFilePath = path.join(rootDir, indexDirectory, 'index.md')
|
||||
|
||||
const { data, content } = matter(await readFile(indexFilePath, 'utf-8'))
|
||||
|
||||
if (changeType === 'remove') {
|
||||
const index = data.children.indexOf(`/${filename}`)
|
||||
data.children.splice(index, 1)
|
||||
}
|
||||
if (changeType === 'add') {
|
||||
data.children.push(`/${filename}`)
|
||||
}
|
||||
await writeFile(indexFilePath, matter.stringify(content, data))
|
||||
}
|
||||
|
||||
/* Takes a list of versions in the format:
|
||||
[
|
||||
'free-pro-team@latest',
|
||||
'enterprise-cloud@latest',
|
||||
'enterprise-server@3.3',
|
||||
'enterprise-server@3.4',
|
||||
'enterprise-server@3.5',
|
||||
'enterprise-server@3.6',
|
||||
'enterprise-server@3.7',
|
||||
'github-ae@latest'
|
||||
]
|
||||
and returns the frontmatter equivalent JSON:
|
||||
{
|
||||
fpt: '*',
|
||||
ghae: '*',
|
||||
ghec: '*',
|
||||
ghes: '*'
|
||||
}
|
||||
*/
|
||||
export async function convertVersionsToFrontmatter(versions) {
|
||||
const frontmatterVersions = {}
|
||||
const numberedReleases = {}
|
||||
|
||||
// Currently, only GHES is numbered. Number releases have to be
|
||||
// handled differently because they use semantic versioning.
|
||||
versions.forEach((version) => {
|
||||
const docsVersion = allVersions[version]
|
||||
if (!docsVersion.hasNumberedReleases) {
|
||||
frontmatterVersions[docsVersion.shortName] = '*'
|
||||
} else {
|
||||
// Each version that has numbered releases in allVersions
|
||||
// has a string for the number (currentRelease) and an array
|
||||
// of all of the available releases (e.g. ['3.3', '3.4', '3.5'])
|
||||
// This creates an array of the applicable releases in the same
|
||||
// order as the available releases array. This is used to track when
|
||||
// a release is no longer supported.
|
||||
const i = docsVersion.releases.sort().indexOf(docsVersion.currentRelease)
|
||||
if (!numberedReleases[docsVersion.shortName]) {
|
||||
const availableReleases = Array(docsVersion.releases.length).fill(undefined)
|
||||
availableReleases[i] = docsVersion.currentRelease
|
||||
numberedReleases[docsVersion.shortName] = {
|
||||
availableReleases,
|
||||
}
|
||||
} else {
|
||||
numberedReleases[docsVersion.shortName].availableReleases[i] = docsVersion.currentRelease
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// Create semantic versions for numbered releases
|
||||
Object.keys(numberedReleases).forEach((key) => {
|
||||
const availableReleases = numberedReleases[key].availableReleases
|
||||
const versionContinuity = checkVersionContinuity(availableReleases)
|
||||
if (availableReleases.every(Boolean)) {
|
||||
frontmatterVersions[key] = '*'
|
||||
} else if (!versionContinuity) {
|
||||
// If there happens to be version gaps, just enumerate each version
|
||||
// using syntax like =3.x || =3.x
|
||||
const semVer = availableReleases
|
||||
.filter(Boolean)
|
||||
.map((release) => `=${release}`)
|
||||
.join(' || ')
|
||||
frontmatterVersions[key] = semVer
|
||||
} else {
|
||||
const semVer = []
|
||||
if (!availableReleases[availableReleases.length - 1]) {
|
||||
const endVersion = availableReleases.filter(Boolean).pop()
|
||||
semVer.push(`<=${endVersion}`)
|
||||
}
|
||||
if (!availableReleases[0]) {
|
||||
const startVersion = availableReleases.filter(Boolean).shift()
|
||||
semVer.push(`>=${startVersion}`)
|
||||
}
|
||||
frontmatterVersions[key] = semVer.join(' ')
|
||||
}
|
||||
})
|
||||
const sortedFrontmatterVersions = Object.keys(frontmatterVersions)
|
||||
.sort()
|
||||
.reduce((acc, key) => {
|
||||
acc[key] = frontmatterVersions[key]
|
||||
return acc
|
||||
}, {})
|
||||
return sortedFrontmatterVersions
|
||||
}
|
||||
|
||||
// This is uncommon, but we potentially could have the case where an
|
||||
// article was versioned for say 3.2, not for 3.3, and then again
|
||||
// versioned for 3.4. This will result in a custom semantic version range
|
||||
function checkVersionContinuity(versions) {
|
||||
const availableVersions = [...versions]
|
||||
|
||||
// values at the beginning or end of the array are not gaps but normal
|
||||
// starts and ends of version ranges
|
||||
while (!availableVersions[0]) {
|
||||
availableVersions.shift()
|
||||
}
|
||||
while (!availableVersions[availableVersions.length - 1]) {
|
||||
availableVersions.pop()
|
||||
}
|
||||
return availableVersions.every(Boolean)
|
||||
}
|
||||
|
||||
// Reads data files from the directory provided and returns a
|
||||
// JSON object that lists the versions for each category/subcategory
|
||||
// The data files are split up by version, so all files must be
|
||||
// read to get a complete list of versions.
|
||||
async function getDataFrontmatter(dataDirectory, schemaFilename) {
|
||||
const fileList = walk(dataDirectory, { includeBasePath: true }).filter(
|
||||
(file) => path.basename(file) === schemaFilename
|
||||
)
|
||||
|
||||
const restVersions = {}
|
||||
|
||||
for (const file of fileList) {
|
||||
const data = JSON.parse(await readFile(file, 'utf-8'))
|
||||
const docsVersionName = getDocsVersion(path.basename(path.dirname(file)))
|
||||
Object.keys(data).forEach((category) => {
|
||||
// Used to automatically update Markdown files
|
||||
const subcategories = Object.keys(data[category])
|
||||
subcategories.forEach((subcategory) => {
|
||||
if (!restVersions[category]) {
|
||||
restVersions[category] = {}
|
||||
}
|
||||
if (!restVersions[category][subcategory]) {
|
||||
restVersions[category][subcategory] = {
|
||||
versions: [docsVersionName],
|
||||
}
|
||||
} else if (!restVersions[category][subcategory].versions.includes(docsVersionName)) {
|
||||
restVersions[category][subcategory].versions.push(docsVersionName)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
return restVersions
|
||||
}
|
||||
|
||||
/*
|
||||
Take an object that includes the version frontmatter
|
||||
that should be applied to the Markdown page that corresponds
|
||||
to the category and subcategory. The format looks like this:
|
||||
{
|
||||
"actions": {
|
||||
"artifacts": {
|
||||
"versions": {
|
||||
"free-pro-team@latest",
|
||||
"github-ae@latest",
|
||||
"enterprise-cloud@latest",
|
||||
"enterprise-server@3.4",
|
||||
"enterprise-server@3.5",
|
||||
"enterprise-server@3.6",
|
||||
"enterprise-server@3.7",
|
||||
"enterprise-server@3.8"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
async function getMarkdownFrontmatter(versions) {
|
||||
const markdownUpdates = {}
|
||||
|
||||
for (const category of Object.keys(versions)) {
|
||||
const subcategories = Object.keys(versions[category])
|
||||
// When there is only a single subcategory, the Markdown file
|
||||
// will be at the root of the content/rest directory. The
|
||||
// filen path will be content/rest/<category>.md
|
||||
if (subcategories.length === 1) {
|
||||
// this will be a file in the root of the rest directory
|
||||
const filepath = path.join('content/rest', `${category}.md`)
|
||||
markdownUpdates[filepath] = {
|
||||
title: category,
|
||||
shortTitle: category,
|
||||
intro: '',
|
||||
versions: await convertVersionsToFrontmatter(versions[category][subcategories[0]].versions),
|
||||
...frontmatterDefaults,
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// The file path will be content/rest/<category>/<subcategory>.md
|
||||
for (const subcategory of subcategories) {
|
||||
const filepath = path.join('content/rest', category, `${subcategory}.md`)
|
||||
markdownUpdates[filepath] = {
|
||||
title: subcategory,
|
||||
shortTitle: subcategory,
|
||||
intro: '',
|
||||
versions: await convertVersionsToFrontmatter(versions[category][subcategory].versions),
|
||||
...frontmatterDefaults,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return markdownUpdates
|
||||
}
|
||||
83
tests/unit/automated-pipelines.js
Normal file
83
tests/unit/automated-pipelines.js
Normal file
@@ -0,0 +1,83 @@
|
||||
import { describe, expect } from '@jest/globals'
|
||||
|
||||
import { supported } from '../../lib/enterprise-server-releases.js'
|
||||
import { allVersionKeys, allVersions } from '../../lib/all-versions.js'
|
||||
import { convertVersionsToFrontmatter } from '../../src/rest/scripts/utils/update-markdown.js'
|
||||
|
||||
describe('frontmatter versions are generated correctly from REST data', () => {
|
||||
test('non-continuous enterprise server versions', async () => {
|
||||
const fromVersions = allVersionKeys.filter(
|
||||
(version) =>
|
||||
version !== `enterprise-server@${supported[0]}` &&
|
||||
version !== `enterprise-server@${supported[2]}`
|
||||
)
|
||||
|
||||
const expectedEneterpriseServerVersions = fromVersions
|
||||
.map(
|
||||
(version) =>
|
||||
version.includes('enterprise-server@') && version.replace('enterprise-server@', '')
|
||||
)
|
||||
.filter(Boolean)
|
||||
|
||||
const expectedEnterpriseServerVersions = expectedEneterpriseServerVersions
|
||||
.sort()
|
||||
.map((version) => `=${version}`)
|
||||
.join(' || ')
|
||||
|
||||
const expectedVersions = {
|
||||
fpt: '*',
|
||||
ghae: '*',
|
||||
ghec: '*',
|
||||
ghes: expectedEnterpriseServerVersions,
|
||||
}
|
||||
const toFrontmatter = await convertVersionsToFrontmatter(fromVersions)
|
||||
expect(toFrontmatter).toEqual(expectedVersions)
|
||||
})
|
||||
|
||||
test('less than the latest enterprise server version', async () => {
|
||||
const fromVersions = Object.values(allVersions)
|
||||
.filter(
|
||||
(version) =>
|
||||
!(version.currentRelease === version.latestRelease && version.hasNumberedReleases)
|
||||
)
|
||||
.map((version) => version.version)
|
||||
const nextLatestRelease = [...supported].sort()[supported.length - 2]
|
||||
const expectedVersions = {
|
||||
fpt: '*',
|
||||
ghae: '*',
|
||||
ghec: '*',
|
||||
ghes: `<=${nextLatestRelease}`,
|
||||
}
|
||||
const toFrontmatter = await convertVersionsToFrontmatter(fromVersions)
|
||||
expect(toFrontmatter).toEqual(expectedVersions)
|
||||
})
|
||||
|
||||
test('greater than the oldest enterprise server version', async () => {
|
||||
const oldestRelease = [...supported].sort()[0]
|
||||
const fromVersions = Object.values(allVersions)
|
||||
.filter((version) => !(version.version === `enterprise-server@${oldestRelease}`))
|
||||
.map((version) => version.version)
|
||||
|
||||
const secondOldestRelease = [...supported].sort()[1]
|
||||
const expectedVersions = {
|
||||
fpt: '*',
|
||||
ghae: '*',
|
||||
ghec: '*',
|
||||
ghes: `>=${secondOldestRelease}`,
|
||||
}
|
||||
const toFrontmatter = await convertVersionsToFrontmatter(fromVersions)
|
||||
expect(toFrontmatter).toEqual(expectedVersions)
|
||||
})
|
||||
|
||||
test('no non-numbered release versions', async () => {
|
||||
const fromVersions = Object.values(allVersions)
|
||||
.filter((version) => version.hasNumberedReleases)
|
||||
.map((version) => version.version)
|
||||
|
||||
const expectedVersions = {
|
||||
ghes: `*`,
|
||||
}
|
||||
const toFrontmatter = await convertVersionsToFrontmatter(fromVersions)
|
||||
expect(toFrontmatter).toEqual(expectedVersions)
|
||||
})
|
||||
})
|
||||
Reference in New Issue
Block a user