Refactor Secret Scanning pattern docs (#58633)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

.github/workflows/sync-secret-scanning.yml

@@ -53,7 +53,7 @@ jobs:
           git config --global user.name "docs-bot"
           git config --global user.email "77750099+docs-bot@users.noreply.github.com"
 
-          branchname=sync-secret-scanning-${{ steps.secret-scanning-sync.outputs.sha }}
+          branchname=sync-secret-scanning-`date +%Y%m%d%H%M%S`
 
           remotesha=$(git ls-remote --heads origin $branchname)
           if [ -n "$remotesha" ]; then

src/github-apps/scripts/sync.ts

@@ -579,7 +579,9 @@ async function getProgActorResourceContent({
   if (gitHubSourceDirectory) {
     files = await getProgActorContentFromDisk(gitHubSourceDirectory)
   } else {
-    files = await getDirectoryContents(owner!, repo!, branch!, resourcePath!)
+    files = (await getDirectoryContents(owner!, repo!, branch!, resourcePath!)).map(
+      (file) => file.content,
+    )
   }
 
   // We need to format the file content into a single object. Each file

src/secret-scanning/data/public-docs-schema.ts

@@ -50,8 +50,6 @@ export default {
       'isPrivateWithGhas',
       'hasPushProtection',
       'hasValidityCheck',
-      'base64Supported',
-      'isduplicate',
     ],
     properties: {
       provider: {

src/secret-scanning/middleware/secret-scanning.ts

@@ -6,8 +6,10 @@ import type { NextFunction, Response } from 'express'
 import getApplicableVersions from '@/versions/lib/get-applicable-versions'
 import { liquid } from '@/content-render/index'
 import { ExtendedRequest, SecretScanningData } from '@/types'
+import { allVersions } from '@/versions/lib/all-versions'
+import { getVersionInfo } from '@/app/lib/constants'
 
-const secretScanningPath = 'src/secret-scanning/data/public-docs.yml'
+const secretScanningDir = 'src/secret-scanning/data/pattern-docs'
 
 // This is the path to the file that contains the secret scanning data.
 // Currently it's:
@@ -23,12 +25,20 @@ export default async function secretScanning(
 ) {
   if (!req.pagePath!.endsWith(targetFilename)) return next()
 
-  const secretScanningData = yaml.load(
-    fs.readFileSync(secretScanningPath, 'utf-8'),
-  ) as SecretScanningData[]
-
   if (!req.context) throw new Error('request not contextualized')
   const { currentVersion } = req.context
+  if (!currentVersion) throw new Error('currentVersion not set in context')
+
+  const { isEnterpriseCloud, isEnterpriseServer } = getVersionInfo(currentVersion)
+
+  const versionPath = isEnterpriseCloud
+    ? 'ghec'
+    : isEnterpriseServer
+      ? `ghes-${allVersions[currentVersion].currentRelease}`
+      : 'fpt'
+  const filepath = `${secretScanningDir}/${versionPath}/public-docs.yml`
+
+  const secretScanningData = yaml.load(fs.readFileSync(filepath, 'utf-8')) as SecretScanningData[]
 
   req.context.secretScanningData = secretScanningData.filter((entry) =>
     currentVersion ? getApplicableVersions(entry.versions).includes(currentVersion) : false,

src/secret-scanning/scripts/sync.ts

@@ -4,22 +4,20 @@
  * GITHUB_TOKEN
  *
  * Syncs the
- * https://github.com/github/token-scanning-service/blob/main/docs/public-docs.yml
- * file to src/secret-scanning/data/public-docs.yml
+ * https://github.com/github/token-scanning-service/blob/main/docs/public-docs
+ * directory to src/secret-scanning/data/pattern-docs
  */
-import { readFile, writeFile } from 'fs/promises'
-import core from '@actions/core'
+import { writeFile } from 'fs/promises'
 import yaml from 'js-yaml'
 
-import { getContentAndData, getCommitSha } from '@/workflows/git-utils'
+import { getDirectoryContents } from '@/workflows/git-utils'
 import schema from '@/secret-scanning/data/public-docs-schema'
 // This is temporarily being imported until the subsequent modules
 // have been converted to TypeScript.
 import { validateJson } from '@/tests/lib/validate-json-schema'
 import { formatAjvErrors } from '@/tests/helpers/schemas'
 
-const SECRET_SCANNING_FILEPATH = 'src/secret-scanning/data/public-docs.yml'
-type PipelineConfig = { sha: string; 'blob-sha': string }
+const SECRET_SCANNING_DIR = 'src/secret-scanning/data/pattern-docs'
 
 async function main() {
   if (!process.env.GITHUB_TOKEN) {
@@ -29,44 +27,33 @@ async function main() {
   const owner = 'github'
   const repo = 'token-scanning-service'
   const ref = 'main'
-  const filepath = 'docs/public-docs.yml'
+  const directory = 'docs/public-docs'
 
-  const { content, blobSha } = await getContentAndData(owner, repo, ref, filepath)
+  const files = await getDirectoryContents(owner, repo, ref, directory)
 
-  const configFilepath = 'src/secret-scanning/lib/config.json'
-  const pipelineConfig: PipelineConfig = JSON.parse(await readFile(configFilepath, 'utf8'))
-  if (pipelineConfig['blob-sha'] === blobSha) {
-    console.log('No changes detected in the public-docs.yml file')
-    return
+  for (const file of files) {
+    // ensure yaml can be parsed
+    let yamlData
+    try {
+      yamlData = yaml.load(file.content)
+    } catch (error) {
+      console.error('The public-docs.yml file being synced is not valid yaml')
+      throw error
+    }
+
+    // ensure yaml is valid against the schema
+    const { isValid, errors } = validateJson(schema, yamlData)
+
+    if (!isValid && errors) {
+      console.error(formatAjvErrors(errors))
+      throw new Error('The public-docs.yml file being synced does not have a valid schema')
+    }
+
+    const filePath = file.path.replace(`${directory}/`, '')
+    const localFilePath = `${SECRET_SCANNING_DIR}/${filePath}`
+
+    await writeFile(localFilePath, yaml.dump(yamlData))
   }
-
-  // ensure yaml can be parsed
-  let yamlData
-  try {
-    yamlData = yaml.load(content)
-  } catch (error) {
-    console.error('The public-docs.yml file being synced is not valid yaml')
-    throw error
-  }
-
-  // ensure yaml is valid against the schema
-  const { isValid, errors } = validateJson(schema, yamlData)
-
-  if (!isValid && errors) {
-    console.error(formatAjvErrors(errors))
-    throw new Error('The public-docs.yml file being synced does not have a valid schema')
-  }
-
-  await writeFile(SECRET_SCANNING_FILEPATH, yaml.dump(yamlData))
-
-  // update the config file with the latest sha
-  pipelineConfig.sha = await getCommitSha(owner, repo, `heads/${ref}`)
-  pipelineConfig['blob-sha'] = blobSha
-  await writeFile(configFilepath, JSON.stringify(pipelineConfig, null, 2))
-
-  // the workflow that runs this script needs the synced sha to use
-  // when creating the PR.
-  core.setOutput('sha', pipelineConfig.sha)
 }
 
 main()

src/workflows/git-utils.ts

@@ -295,11 +295,11 @@ export async function getDirectoryContents(
     } else if (blob.type === 'file') {
       if (!data.content) {
         const blobContents = await getContentsForBlob(owner, repo, blob.sha)
-        files.push(blobContents)
+        files.push({ path: blob.path, content: blobContents })
       } else {
         // decode Base64 encoded contents
         const decodedContent = Buffer.from(blob.content, 'base64').toString()
-        files.push(decodedContent)
+        files.push({ path: blob.path, content: decodedContent })
       }
     }
   }