1
0
mirror of synced 2025-12-19 18:10:59 -05:00

Rewrite code scanning query list in JS (#47287)

Co-authored-by: Shati Patel <42641846+shati-patel@users.noreply.github.com>
Co-authored-by: Felicity Chapman <felicitymay@github.com>
This commit is contained in:
Peter Bengtsson
2023-12-07 15:55:02 -05:00
committed by GitHub
parent c2888b942b
commit 9472f235e1
5 changed files with 694 additions and 257 deletions

View File

@@ -22,11 +22,13 @@ permissions:
jobs:
generate-query-lists:
if: github.repository == 'github/docs-internal'
runs-on: ubuntu-latest
runs-on: ubuntu-20.04-xl
steps:
- name: Checkout repository code
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
- uses: ./.github/actions/node-npm-setup
- name: Checkout codeql repo
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
with:
@@ -42,22 +44,38 @@ jobs:
echo "OPENAPI_COMMIT_SHA=$OPENAPI_COMMIT_SHA" >> $GITHUB_OUTPUT
echo "Copied files from github/codeql repo. Commit SHA: $OPENAPI_COMMIT_SHA"
- name: Set up Python 3.8
uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
with:
python-version: 3.8
- name: Download CodeQL CLI
# Look under the `codeql` directory, as this is where we checked out the `github/codeql` repo
# Look under the `codeql` directory, as this is where we checked out the `github/codeql` repo
uses: ./codeql/.github/actions/fetch-codeql
- name: Test CodeQL CLI Download
run: codeql --version
# "Server for running multiple commands while avoiding repeated JVM initialization."
# Having started this should speed up the execution of the various
# CLI calls of the executable.
- name: Start CodeQL CLI server in the background
run: |
codeql execute cli-server &
sleep 3
codeql --version
- name: Build code scanning query list
run: |
for lang in "cpp" "csharp" "go" "java" "javascript" "python" "ruby" "swift"; do
echo "Generating code scanning query list for $lang"
python src/code-scanning/generate-code-scanning-query-list.py $lang > data/reusables/code-scanning/codeql-query-tables/$lang.md
npm run generate-code-scanning-query-list -- \
--verbose \
--codeql-path codeql \
--codeql-dir codeql \
-o data/reusables/code-scanning/codeql-query-tables/$lang.md \
$lang
done
- name: Debug
run: |
git diff
- name: Create pull request
env:
GITHUB_TOKEN: ${{ secrets.DOCS_BOT_PAT_READPUBLICKEY }}
@@ -75,7 +93,7 @@ jobs:
branchname=codeql-query-tables-${{ steps.codeql.outputs.OPENAPI_COMMIT_SHA }}
# Exit if the branch already exists. Since the actions/checkout fetch-depth is 1,
# Exit if the branch already exists. Since the actions/checkout fetch-depth is 1,
# it doesn't "know" about branches locally, so we need to manually list them.
branchExists=$(git ls-remote --heads origin refs/heads/$branchname | wc -l)
@@ -93,7 +111,6 @@ jobs:
echo "Creating pull request..."
gh pr create \
--title "Update CodeQL query tables" \
--draft \
--repo github/docs-internal \
--label "codeql-query-tables,skip FR board" \
--body '👋 humans. This PR updates the **CodeQL query table reusables** with the latest changes in preparation for the next **CodeQL CLI** release.

429
package-lock.json generated
View File

@@ -152,6 +152,7 @@
"sass": "^1.52.3",
"start-server-and-test": "^2.0.3",
"ts-jest": "29.1.1",
"tsx": "4.6.2",
"typescript": "^5.2.2",
"unist-util-remove": "^4.0.0",
"unist-util-visit-parents": "6.0.1",
@@ -930,6 +931,358 @@
"version": "0.7.5",
"license": "MIT"
},
"node_modules/@esbuild/android-arm": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.18.20.tgz",
"integrity": "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==",
"cpu": [
"arm"
],
"dev": true,
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/android-arm64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.18.20.tgz",
"integrity": "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/android-x64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.18.20.tgz",
"integrity": "sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"android"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/darwin-arm64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.18.20.tgz",
"integrity": "sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/darwin-x64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.18.20.tgz",
"integrity": "sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/freebsd-arm64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.18.20.tgz",
"integrity": "sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/freebsd-x64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.18.20.tgz",
"integrity": "sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"freebsd"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/linux-arm": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.18.20.tgz",
"integrity": "sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==",
"cpu": [
"arm"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/linux-arm64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.18.20.tgz",
"integrity": "sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/linux-ia32": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.18.20.tgz",
"integrity": "sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==",
"cpu": [
"ia32"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/linux-loong64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.18.20.tgz",
"integrity": "sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==",
"cpu": [
"loong64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/linux-mips64el": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.18.20.tgz",
"integrity": "sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==",
"cpu": [
"mips64el"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/linux-ppc64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.18.20.tgz",
"integrity": "sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==",
"cpu": [
"ppc64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/linux-riscv64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.18.20.tgz",
"integrity": "sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==",
"cpu": [
"riscv64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/linux-s390x": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.18.20.tgz",
"integrity": "sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==",
"cpu": [
"s390x"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/linux-x64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.18.20.tgz",
"integrity": "sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"linux"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/netbsd-x64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.18.20.tgz",
"integrity": "sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"netbsd"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/openbsd-x64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.18.20.tgz",
"integrity": "sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"openbsd"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/sunos-x64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.18.20.tgz",
"integrity": "sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"sunos"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/win32-arm64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.18.20.tgz",
"integrity": "sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==",
"cpu": [
"arm64"
],
"dev": true,
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/win32-ia32": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.18.20.tgz",
"integrity": "sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==",
"cpu": [
"ia32"
],
"dev": true,
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@esbuild/win32-x64": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.18.20.tgz",
"integrity": "sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==",
"cpu": [
"x64"
],
"dev": true,
"optional": true,
"os": [
"win32"
],
"engines": {
"node": ">=12"
}
},
"node_modules/@eslint-community/eslint-utils": {
"version": "4.4.0",
"dev": true,
@@ -5252,6 +5605,43 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/esbuild": {
"version": "0.18.20",
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.18.20.tgz",
"integrity": "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA==",
"dev": true,
"hasInstallScript": true,
"bin": {
"esbuild": "bin/esbuild"
},
"engines": {
"node": ">=12"
},
"optionalDependencies": {
"@esbuild/android-arm": "0.18.20",
"@esbuild/android-arm64": "0.18.20",
"@esbuild/android-x64": "0.18.20",
"@esbuild/darwin-arm64": "0.18.20",
"@esbuild/darwin-x64": "0.18.20",
"@esbuild/freebsd-arm64": "0.18.20",
"@esbuild/freebsd-x64": "0.18.20",
"@esbuild/linux-arm": "0.18.20",
"@esbuild/linux-arm64": "0.18.20",
"@esbuild/linux-ia32": "0.18.20",
"@esbuild/linux-loong64": "0.18.20",
"@esbuild/linux-mips64el": "0.18.20",
"@esbuild/linux-ppc64": "0.18.20",
"@esbuild/linux-riscv64": "0.18.20",
"@esbuild/linux-s390x": "0.18.20",
"@esbuild/linux-x64": "0.18.20",
"@esbuild/netbsd-x64": "0.18.20",
"@esbuild/openbsd-x64": "0.18.20",
"@esbuild/sunos-x64": "0.18.20",
"@esbuild/win32-arm64": "0.18.20",
"@esbuild/win32-ia32": "0.18.20",
"@esbuild/win32-x64": "0.18.20"
}
},
"node_modules/escalade": {
"version": "3.1.1",
"license": "MIT",
@@ -6764,9 +7154,9 @@
"license": "MIT"
},
"node_modules/get-tsconfig": {
"version": "4.7.0",
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.0.tgz",
"integrity": "sha512-pmjiZ7xtB8URYm74PlGJozDNyhvsVLUcpBa8DZBG3bWHwaHa9bPiRpiSfovw+fjhwONSCWKRyk+JQHEGZmMrzw==",
"version": "4.7.2",
"resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.7.2.tgz",
"integrity": "sha512-wuMsz4leaj5hbGgg4IvDU0bqJagpftG5l5cXIAvo8uZrqn0NJqwtfupTN00VnkQJPcIRrxYrm1Ue24btpCha2A==",
"dev": true,
"dependencies": {
"resolve-pkg-maps": "^1.0.0"
@@ -14913,6 +15303,39 @@
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.3.tgz",
"integrity": "sha512-mSxlJJwl3BMEQCUNnxXBU9jP4JBktcEGhURcPR6VQVlnP0FdDEsIaz0C35dXNGLyRfrATNofF0F5p2KPxQgB+w=="
},
"node_modules/tsx": {
"version": "4.6.2",
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.6.2.tgz",
"integrity": "sha512-QPpBdJo+ZDtqZgAnq86iY/PD2KYCUPSUGIunHdGwyII99GKH+f3z3FZ8XNFLSGQIA4I365ui8wnQpl8OKLqcsg==",
"dev": true,
"dependencies": {
"esbuild": "~0.18.20",
"get-tsconfig": "^4.7.2"
},
"bin": {
"tsx": "dist/cli.mjs"
},
"engines": {
"node": ">=18.0.0"
},
"optionalDependencies": {
"fsevents": "~2.3.3"
}
},
"node_modules/tsx/node_modules/fsevents": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
"integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
"dev": true,
"hasInstallScript": true,
"optional": true,
"os": [
"darwin"
],
"engines": {
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
}
},
"node_modules/tunnel": {
"version": "0.0.6",
"dev": true,

View File

@@ -31,6 +31,7 @@
"lint": "eslint '**/*.{js,mjs,ts,tsx}'",
"lint-content": "node src/content-linter/scripts/lint-content.js",
"lint-translation": "cross-env NODE_OPTIONS=--experimental-vm-modules jest src/content-linter/tests/lint-files.js",
"generate-code-scanning-query-list": "tsx src/code-scanning/scripts/generate-code-scanning-query-list.ts",
"move-content": "node src/content-render/scripts/move-content.js",
"openapi-docs": "node src/rest/docs.js",
"playwright-test": "playwright test --config src/fixtures/playwright.config.ts --project=\"Google Chrome\"",
@@ -367,6 +368,7 @@
"sass": "^1.52.3",
"start-server-and-test": "^2.0.3",
"ts-jest": "29.1.1",
"tsx": "4.6.2",
"typescript": "^5.2.2",
"unist-util-remove": "^4.0.0",
"unist-util-visit-parents": "6.0.1",

View File

@@ -1,244 +0,0 @@
import subprocess
import json
import shutil
import sys
import os
import argparse
"""
This script collects CodeQL queries that are part of code scanning query packs
and prints a markdown table to stdout that describes which packs contain which queries.
Errors are printed to stderr. This script requires that 'git' and 'codeql' commands
are on the PATH. It'll try to automatically set the CodeQL search path correctly,
as long as you run the script from one of the following locations:
- anywhere from within a clone of the CodeQL Git repo
- from the parent directory of a clone of the CodeQL Git repo (assuming 'codeql'
directory exists)
"""
parser = argparse.ArgumentParser(__name__)
parser.add_argument(
"--ignore-missing-query-packs",
action="store_true",
help="Don't fail if a query pack can't be found",
)
parser.add_argument(
"language",
help="The language to generate the query list for",
)
arguments = parser.parse_args()
assert hasattr(arguments, "ignore_missing_query_packs")
assert hasattr(arguments, "language")
# Define which languages and query packs to consider
language = arguments.language
packs = [ "code-scanning", "security-extended" ]
class CodeQL:
def __init__(self):
pass
def __enter__(self):
self.proc = subprocess.Popen(['codeql', 'execute','cli-server'],
executable=shutil.which('codeql'),
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=sys.stderr,
env=os.environ.copy(),
)
return self
def __exit__(self, type, value, tb):
self.proc.stdin.write(b'["shutdown"]\0')
self.proc.stdin.close()
try:
self.proc.wait(5)
except:
self.proc.kill()
def command(self, args):
data = json.dumps(args)
data_bytes = data.encode('utf-8')
self.proc.stdin.write(data_bytes)
self.proc.stdin.write(b'\0')
self.proc.stdin.flush()
res = b''
while True:
b = self.proc.stdout.read(1)
if b == b'\0':
return res.decode('utf-8')
res += b
def get_docs_link(language, query_id):
"""
Gets the documentation link for a query, given the query ID.
The documentation link contains the query language, followed by the ID, with slashes
replaced by dashes.
For example:
cpp/external-entity-expansion
becomes:
https://codeql.github.com/codeql-query-help/cpp/cpp-external-entity-expansion/
"""
return "https://codeql.github.com/codeql-query-help/%s/%s/" % (language, query_id.replace("/","-"))
def single_spaces(input):
"""
Workaround for https://github.com/github/codeql-coreql-team/issues/470 which causes
some metadata strings to contain newlines and spaces without a good reason.
"""
return " ".join(input.split())
def get_query_metadata(key, metadata, queryfile):
"""Returns query metadata or prints a warning to stderr if a particular piece of metadata is not available."""
if key in metadata: return single_spaces(metadata[key])
query_id = metadata['id'] if 'id' in metadata else 'unknown'
return ""
def get_query_cwes(tags):
"""
Returns a list of CWEs that are associated with a query, given its tags.
For example, if the list of tags is:
maintainability readability external/cwe/cwe-1078 external/cwe/cwe-670 security
We return:
["1078", "670"]
"""
cwes = []
tags = tags.split()
for tag in tags:
if tag.startswith("external/cwe/cwe-"):
cwe = tag.split("-")[-1]
cwes.append(cwe)
return cwes
def subprocess_run(cmd):
"""Runs a command through subprocess.run, with a few tweaks. Raises an Exception if exit code != 0."""
return subprocess.run(cmd, capture_output=True, text=True, env=os.environ.copy(), check=True)
try: # Check for `git` on path
subprocess_run(["git","--version"])
except Exception as e:
print("Error: couldn't invoke 'git'. Is it on the path? Aborting.", file=sys.stderr)
raise e
with CodeQL() as codeql:
try: # Check for `codeql` on path
codeql.command(["--version"])
except Exception as e:
print("Error: couldn't invoke CodeQL CLI 'codeql'. Is it on the path? Aborting.", file=sys.stderr)
raise e
# Define CodeQL search path so it'll find the CodeQL repositories:
# - anywhere in the current Git clone (including current working directory)
# - the 'codeql' subdirectory of the cwd
codeql_search_path = "./codeql:." # will be extended further down
# Extend CodeQL search path by detecting root of the current Git repo (if any). This means that you
# can run this script from any location within the CodeQL git repository.
try:
git_toplevel_dir = subprocess_run(["git","rev-parse","--show-toplevel"])
# Current working directory is in a Git repo. Add it to the search path, just in case it's the CodeQL repo
git_toplevel_dir = git_toplevel_dir.stdout.strip()
codeql_search_path += ":" + git_toplevel_dir
except:
# git rev-parse --show-toplevel exited with non-zero exit code. We're not in a Git repo
pass
# Write a markdown table to stdout
table_data = []
# Define the header row
table_data.append(["Query name", "Related CWEs", "Default", "Extended"])
# Iterate over all packs, and resolve which queries are part of those packs
queries_dict = {}
for pack in packs:
# Get absolute paths to queries in this pack by using 'codeql resolve queries'
try:
queries_subp = codeql.command(["resolve","queries","--search-path", codeql_search_path, "%s-%s.qls" % (language, pack)])
except Exception as e:
# Resolving queries might go wrong if the github/codeql repository is not
# on the search path.
level = "Warning" if arguments.ignore_missing_query_packs else "Error"
print(
"%s: couldn't find query pack '%s' for language '%s'. Do you have the right repositories in the right places (search path: '%s')?" % (level, pack, language, codeql_search_path),
file=sys.stderr
)
if arguments.ignore_missing_query_packs:
continue
else:
sys.exit("You can use '--ignore-missing-query-packs' to ignore this error")
# Investigate metadata for every query by using 'codeql resolve metadata'
for queryfile in queries_subp.strip().split("\n"):
query_metadata_json = codeql.command(["resolve","metadata",queryfile]).strip()
# Turn an absolute path to a query file into an nwo-prefixed path (e.g. github/codeql/java/ql/src/....)
meta = json.loads(query_metadata_json)
# Format a markdown link from the query name and the docs link
query_id = get_query_metadata('id', meta, queryfile)
query_docs_link = get_docs_link(language, query_id)
query_name = get_query_metadata('name', meta, queryfile)
query_name = query_name.replace('"', '&quot;') # Replace quotation marks with HTML entities
query_markdown_link = "[%s](%s)" % (query_name, query_docs_link)
# Get query CWEs
query_tags = get_query_metadata('tags', meta, queryfile)
query_cwes = get_query_cwes(query_tags)
# Only include queries that have CWEs, since the other queries deal with code scanning
# metadata and metrics (e.g. counting lines of code or number of files) and have no docs link
if query_cwes:
if query_id not in queries_dict:
queries_dict[query_id] = {
'query_markdown_link': query_markdown_link,
'packs': [pack],
'cwes': query_cwes
}
else:
queries_dict[query_id]['packs'].append(pack)
# Sort the queries alphabetically
sorted_queries = sorted(queries_dict.items(), key=lambda item: item[1]['query_markdown_link'])
# Write the sorted rows to the markdown table
for query_id, query_dict in sorted_queries:
default = '{% octicon "x" aria-label="Not included" %}'
extended = '{% octicon "x" aria-label="Not included" %}'
if "code-scanning" in query_dict['packs']:
default = '{% octicon "check" aria-label="Included" %}'
if "security-extended" in query_dict['packs']:
extended = '{% octicon "check" aria-label="Included" %}'
table_data.append([
query_dict['query_markdown_link'],
', '.join(query_dict['cwes']),
default,
extended
])
# Write "row headers" to make rows accessible
print("{% rowheaders %}")
# Write 'table_data' as markdown
md_table = '\n| ' + ' | '.join(table_data[0]) + ' |\n'
md_table += '| ' + ' | '.join(['---'] * len(table_data[0])) + ' |\n'
for row in table_data[1:]:
md_table += '| ' + ' | '.join(row) + ' |\n'
md_table = md_table.rstrip()
print(md_table)
# Write end of "row headers"
print("\n{% endrowheaders %}")

View File

@@ -0,0 +1,239 @@
#!/usr/bin/env node
/**
* This script generates a block of Markdown that can be saved as a reusable.
* The reusable lists all the queries for one programming language, with CWEs, as a Markdown table.
*
* To be able to execute this script, you need to have the CodeQL CLI installed.
* To do that, you need two things:
*
* 1. The directory where the github/codeql repo is clone
* 2. The path to the executable `codeql` file.
*
* The directory where the github/codeql repo is cloned is needed because
* that's how it looks up files. You can set it up like this:
*
* cd /tmp
* git clone git@github.com:github/codeql.git
* cd codeql
* pwd
*
* To install the codeql executable, use `gh` like this:
*
* gh extension install github/gh-codeql
* gh codeql set-channel nightly
* gh codeql version
*
* Note that when you run the `gh codeql version` command, it will tell you
* where the executable is installed. For example:
*
* /Users/peterbe/.local/share/gh/extensions/gh-codeql/dist/nightly/codeql-bundle-20231204/codeql
*
* If you've git cloned github/codeql in /tmp/ now you can execute this script.
* For example, to generate the Markdown
* for Python:
*
* npm run generate-code-scanning-query-list -- \
* --codeql-path ~/.local/share/gh/extensions/gh-codeql/dist/nightly/codeql-bundle-20231204/codeql \
* --codeql-dir /tmp/codeql python | tee /tmp/python.md
* less /tmp/python.md
*/
import fs from 'fs'
import { execFileSync } from 'child_process'
import chalk from 'chalk'
import { program } from 'commander'
program
.description('Generate a reusable Markdown for for a code scanning query language')
.option('--verbose', 'Verbose outputs')
.option('--codeql-path <path>', 'path to the codeql executable', 'codeql')
.option('--codeql-dir <path>', 'path to the codeql executable', '.codeql/')
.option('-o, --output-file <path>', 'path to the codeql executable', 'stdout')
.option('-p, --pack <pack>', 'which packs to search for', ['code-scanning', 'security-extended'])
.argument('<language>', 'for example java')
.parse(process.argv)
type Options = {
codeqlPath: string
codeqlDir: string
outputFile: string
packs: string[]
verbose: boolean
}
type QueryMetadata = {
id?: string
name?: string
tags?: string
}
type Query = {
name: string
url: string
packs: string[]
cwes: string[]
}
const opts = program.opts()
main(
{
codeqlPath: opts.codeqlPath,
codeqlDir: opts.codeqlDir,
outputFile: opts.outputFile,
packs: opts.pack,
verbose: Boolean(opts.verbose),
},
program.args[0],
)
async function main(options: Options, language: string) {
if (!options.packs.length) {
throw new Error("no packs specified, use '--pack code-scanning' or '--pack security-extended'")
}
if (options.verbose && options.outputFile === 'stdout') {
console.warn(chalk.yellow('Verbose mode is on but output is going to stdout'))
}
if (!testCodeQLPath(options)) {
process.exit(1)
}
const queries: {
[id: string]: Query
} = {}
for (const pack of options.packs) {
const languagePack = `${language}-${pack}.qls`
if (options.verbose) console.log(chalk.dim(`Searching for queries in ${languagePack}`))
const res = execFileSync(
options.codeqlPath,
['resolve', 'queries', `--search-path=${options.codeqlDir}`, languagePack],
{
encoding: 'utf-8',
},
)
for (const line of res.split('\n')) {
if (line.trim()) {
if (options.verbose) console.log('found', line)
const metadata = getMetadata(options, line)
const { id, name, tags } = metadata
if (id && name) {
const cwes = getCWEs(tags || '')
const url = getDocsLink(language, id)
// Only include queries that have CWEs, since the other queries deal with code scanning
// metadata and metrics (e.g. counting lines of code or number of files) and have no docs link
if (cwes.length) {
if (!(id in queries)) {
queries[id] = { url, name, packs: [], cwes }
}
queries[id].packs.push(pack)
} else {
if (options.verbose) {
console.log(chalk.dim(`Skipping ${id} because it has no CWEs`))
}
}
}
}
}
}
const entries = Object.values(queries)
entries.sort((a, b) => a.name.localeCompare(b.name))
printQueries(options, entries)
}
function printQueries(options: Options, queries: Query[]) {
const markdown = []
markdown.push('{% rowheaders %}')
markdown.push('') // blank line
const header = ['Query name', 'Related CWEs', 'Default', 'Extended']
markdown.push(`| ${header.join(' | ')} |`)
markdown.push(`| ${header.map(() => '---').join(' | ')} |`)
const notIncludedOcticon = '{% octicon "x" aria-label="Not included" %}'
const includedOcticon = '{% octicon "check" aria-label="Included" %}'
for (const query of queries) {
const markdownLink = `[${query.name}](${query.url})`
let defaultIcon = notIncludedOcticon
let extendedIcon = notIncludedOcticon
if (query.packs.includes('code-scanning')) {
defaultIcon = includedOcticon
}
if (query.packs.includes('security-extended')) {
extendedIcon = includedOcticon
}
markdown.push(
`| ${markdownLink} | ${query.cwes.join(', ')} | ${defaultIcon} | ${extendedIcon} |`,
)
}
markdown.push('') // blank line
markdown.push('{% endrowheaders %}')
markdown.push('') // always end with a blank line
if (options.outputFile === 'stdout') {
console.log(markdown.join('\n'))
} else {
fs.writeFileSync(options.outputFile, markdown.join('\n'), 'utf-8')
}
}
function getMetadata(options: Options, queryFile: string): QueryMetadata {
const metadataJson = execFileSync(options.codeqlPath, ['resolve', 'metadata', queryFile], {
encoding: 'utf-8',
})
const parsed = JSON.parse(metadataJson)
return parsed
}
/**
*
* @param language 'cpp'
* @param queryId 'external-entity-expansion'
* @returns https://codeql.github.com/codeql-query-help/cpp/cpp-external-entity-expansion/
*/
function getDocsLink(language: string, queryId: string) {
return `https://codeql.github.com/codeql-query-help/${language}/${queryId.replaceAll('/', '-')}/`
}
/**
*
* @param tags 'maintainability readability external/cwe/cwe-1078 external/cwe/cwe-670 security'
* @returns ['1078', '670']
*/
function getCWEs(tags: string) {
const cwes: string[] = []
for (const tag of tags.split(/\s+/g)) {
if (tag.startsWith('external/cwe/cwe-')) {
const cwe = tag.split('-').pop() || ''
if (cwe) cwes.push(cwe)
}
}
return cwes
}
function testCodeQLPath(options: Options) {
try {
const output = execFileSync(options.codeqlPath, ['--version'], { encoding: 'utf-8' })
if (options.verbose) {
const matched = output.match(/CodeQL command-line toolchain release ([\d.+]+)/)
if (matched) {
console.log('codeql version', chalk.green(matched[0]))
return true
}
}
return true
} catch (error) {
console.error('Could not find codeql executable at', options.codeqlPath)
if (options.verbose) {
throw error
} else {
console.log(chalk.yellow(`${options.codeqlPath} --version`), 'failed')
return false
}
}
}