Add secret scanning pipeline (#51188)

Co-authored-by: mc <42146119+mchammer01@users.noreply.github.com>
Co-authored-by: Peter Bengtsson <peterbe@github.com>

.github/workflows/secret-scanning-pattern-table-updates.yml

@@ -1,50 +0,0 @@
-name: Secret Scanning Pattern Table Updates
-
-# **What it does**: When a PR that updates `data/secret-scanning.yml` is opened in docs-internal, it adds the `ready-for-docs-review` label, as well as a comment explaining what this PR is for and that it needs to be reviewed quickly. It also provides reviewing instructions, and gives details of who can help.
-# **Why we have it**: To help Docs Content team members know what to do with this sort of PRs, or to direct them to who can help if they don't feel comfortable reviewing the PR themselves.
-# **Who does it impact**: docs-internal maintainers and docs content first responders.
-
-on:
-  pull_request:
-    types: [opened]
-    paths:
-      - data/secret-scanning.yml
-
-permissions:
-  contents: read
-  pull-requests: write
-  repository-projects: write
-
-jobs:
-  Process-secret-scanning-PR:
-    runs-on: ubuntu-latest
-    if: github.repository == 'github/docs-internal'
-    steps:
-      - name: Check out repo
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-
-      - name: Label pull requests updating the secret-scanning.yml file with ready-for-doc-review
-        run: gh pr edit $PR --add-label "ready-for-doc-review"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PR: ${{ github.event.pull_request.html_url }}
-      - name: Comment on the secret scanning partners PR
-        run: >
-          gh pr comment $PR --body "This PR updates data for secret scanning patterns
-          in the _/data/secret-scanning.yml_ file. The data in this file is used
-          to populate the tables in the '[Secret scanning
-          patterns](https://docs.github.com/en/code-security/secret-scanning/secret-scanning-patterns#supported-secrets)' article at build time.
-
-          - The secret scanning team updates this file fairly regularly, and raises PRs in the `docs-internal` repository to update our docs accordingly. We've agreed to review these PRs **quickly** as the changes are already effective when these PRs reach us.
-
-          - Anyone in the Docs Content team can review and merge this PR. A few guidelines:
-              - You can only merge this PR if it's had a technical review (see who's approved it in the 'Reviewers' section in the top right corner).
-              - To test that the changes appear on Staging, look at the preview of the 'Secret scanning patterns' file. You may need to use the product picker to look at the table for different GitHub products, and test the versioning.
-              - If you don't feel comfortable reviewing this PR, please post a link to it in the #code-security-docs Slack channel so someone from the Dependencies & Secrets focus team can take a look.
-
-          - For more information about this automation, and the reasons why we have decided to implement it, see [About automations for Dependencies & Secrets](https://github.com/github/docs-content/blob/main/focus-areas/code-security/about-automations-for-dependencies-and-secrets.md#secret-scanning-prs-adding-new-supported-patterns) in the 'docs-content' repository.
-
-          - Thank you :fishsticks: :sparkling_heart:"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PR: ${{ github.event.pull_request.html_url }}

.github/workflows/sync-secret-scanning.yml

@@ -0,0 +1,83 @@
+name: Sync Secret Scanning data
+
+# **What it does**: This updates the data used by the secret scanning patterns page.
+# **Why we have it**: To automate updates to the secret scanning pattern data in our public-facing documentation.
+# **Who does it impact**: Docs engineering, content writers.
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '20 16 * * *' # Run every day at 16:20 UTC / 8:22 PST
+
+permissions:
+  contents: write
+  pull-requests: write
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
+
+jobs:
+  update-secret-scanning-file:
+    if: github.repository == 'github/docs-internal'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - uses: ./.github/actions/node-npm-setup
+
+      - name: Sync secret scanning data
+        id: secret-scanning-sync
+        env:
+          # need to use a token from a user with access to
+          # github/token-scanning-service for this step
+          GITHUB_TOKEN: ${{ secrets.DOCS_BOT_PAT_WRITEORG_PROJECT }}
+        run: |
+          npm run sync-secret-scanning
+
+      - name: Create and merge pull request
+        env:
+          # Needed for gh
+          GITHUB_TOKEN: ${{ secrets.DOCS_BOT_PAT_READPUBLICKEY }}
+        run: |
+          # If nothing to commit, exit now. It's fine.
+          changes=$(git diff --name-only | wc -l)
+          untracked=$(git status --untracked-files --short | wc -l)
+          if [[ $changes -eq 0 ]] && [[ $untracked -eq 0 ]]; then
+            echo "There are no changes to commit. Exiting..."
+            exit 0
+          fi
+
+          git config --global user.name "docs-bot"
+          git config --global user.email "77750099+docs-bot@users.noreply.github.com"
+
+          branchname=sync-secret-scanning-${{ steps.secret-scanning-sync.outputs.sha }}
+
+          remotesha=$(git ls-remote --heads origin $branchname)
+          if [ -n "$remotesha" ]; then
+            # output is not empty, it means the remote branch exists
+            echo "Branch $branchname already exists in 'github/docs-internal'. Exiting..."
+            exit 0
+          fi
+
+          git checkout -b $branchname
+          git add .
+          git commit -m "Add updated secret scanning data"
+          git push origin $branchname
+
+          echo "Creating pull request..."
+          gh pr create \
+            --title "Sync secret scanning data" \
+            --body '👋 humans. This PR updates the secret scanning data with the latest changes from github/token-scanning-service.\n\n/cc @github/docs-content-security-products
+
+            If CI does not pass or other problems arise, contact #docs-engineering on slack.' \
+            --repo github/docs-internal \
+            --label secret-scanning-pipeline,ready-for-docs-review
+
+      - uses: ./.github/actions/slack-alert
+        if: ${{ failure() && github.event_name != 'workflow_dispatch' }}
+        with:
+          slack_channel_id: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
+          slack_token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}

package.json

@@ -71,6 +71,7 @@
     "sync-search-ghes-release": "cross-env GHES_RELEASE=1 start-server-and-test sync-search-server 4002 sync-search-indices",
     "sync-search-indices": "node src/search/scripts/sync-search-indices.js",
     "sync-search-server": "cross-env NODE_ENV=production PORT=4002 MINIMAL_RENDER=true CHANGELOG_DISABLED=true tsx src/frame/server.ts",
+    "sync-secret-scanning": "tsx src/secret-scanning/scripts/sync.ts",
     "sync-webhooks": "src/rest/scripts/update-files.js -o webhooks",
     "test": "vitest",
     "test-local-dev": "node src/workflows/test-local-dev.js",

src/data-directory/lib/data-schemas/index.js

@@ -3,7 +3,6 @@ export default {
   'data/variables': '#src/data-directory/lib/data-schemas/variables.js',
   'data/learning-tracks': '#src/data-directory/lib/data-schemas/learning-tracks.js',
   'data/release-notes': '#src/data-directory/lib/data-schemas/release-notes.js',
-  'data/secret-scanning.yml': '#src/data-directory/lib/data-schemas/secret-scanning.js',
   'data/code-languages.yml': '#src/data-directory/lib/data-schemas/code-languages.js',
   'data/glossaries/candidates.yml': '#src/data-directory/lib/data-schemas/glossaries-candidates.js',
   'data/glossaries/external.yml': '#src/data-directory/lib/data-schemas/glossaries-external.js',

src/secret-scanning/lib/config.json

@@ -0,0 +1,3 @@
+{
+  "sha": "b0ba22279023b8ee79e209fb3de1f2c8b1a8a246"
+}

src/secret-scanning/middleware/secret-scanning.ts

@@ -7,7 +7,7 @@ import getApplicableVersions from '@/versions/lib/get-applicable-versions.js'
 import { liquid } from '@/content-render/index.js'
 import { ExtendedRequest, SecretScanningData } from '@/types'
 
-const secretScanningPath = 'data/secret-scanning.yml'
+const secretScanningPath = 'src/secret-scanning/data/public-docs.yml'
 
 export default async function secretScanning(
   req: ExtendedRequest,

src/secret-scanning/scripts/sync.ts

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+
+/**
+ * Required env variables:
+ *
+ * GITHUB_TOKEN
+ *
+ * Syncs the
+ * https://github.com/github/token-scanning-service/blob/main/docs/public-docs.yml
+ * file to src/secret-scanning/data/public-docs.yml
+ */
+import { readFile, writeFile } from 'fs/promises'
+import core from '@actions/core'
+import yaml from 'js-yaml'
+
+import { getContents, getCommitSha } from '@/workflows/git-utils.js'
+import schema from '@/secret-scanning/data/public-docs-schema.js'
+// This is temporarily being imported until the subsequent modules
+// have beeen converted to TypeScript.
+import { validateJson } from '@/tests/lib/validate-json-schema.js'
+import { formatAjvErrors } from '@/tests/helpers/schemas.js'
+
+const SECRET_SCANNING_FILEPATH = 'src/secret-scanning/data/public-docs.yml'
+type PipelineConfig = { sha: string }
+
+async function main() {
+  if (!process.env.GITHUB_TOKEN) {
+    throw new Error('GITHUB_TOKEN environment variable must be set to run this script')
+  }
+
+  const owner = 'github'
+  const repo = 'token-scanning-service'
+  const ref = 'main'
+  const filepath = 'docs/public-docs.yml'
+
+  const data = await getContents(owner, repo, ref, filepath)
+
+  // ensure yaml can be parsed
+  let yamlData
+  try {
+    yamlData = yaml.load(data)
+  } catch (error) {
+    console.error('The public-docs.yml file being synced is not valid yaml')
+    throw error
+  }
+
+  // ensure yaml is valid against the schema
+  const { isValid, errors } = validateJson(schema, yamlData)
+
+  if (!isValid && errors) {
+    console.error(formatAjvErrors(errors))
+    throw new Error('The public-docs.yml file being synced does not have a valid schema')
+  }
+
+  await writeFile(SECRET_SCANNING_FILEPATH, data)
+
+  // update the config file with the latest sha
+  const configFilepath = 'src/secret-scanning/lib/config.json'
+  const mainSha = await getCommitSha(owner, repo, `heads/${ref}`)
+  const pipelineConfig: PipelineConfig = JSON.parse(await readFile(configFilepath, 'utf8'))
+  pipelineConfig.sha = mainSha
+  await writeFile(configFilepath, JSON.stringify(pipelineConfig, null, 2))
+
+  // the workflow that runs this script needs the synced sha to use
+  // when creating the PR.
+  core.setOutput('sha', mainSha)
+}
+
+main()