Merge pull request #25761 from github/repo-sync

repo sync

content/code-security/codeql-cli/codeql-cli-manual/database-bundle.md

@@ -50,6 +50,13 @@ that results, logs, TRAP, or similar should be included.
 
 Include any precomputed query results in the bundle.
 
+#### `--include-temp`
+
+Includes the "temp" directory, where any generated packs, queries, and
+suites are located.
+
+Available since `v2.13.3`.
+
 #### `--include-uncompressed-source`
 
 Include an uncompressed version of the source archive directory. This is

content/code-security/codeql-cli/codeql-cli-manual/index.md

@@ -74,6 +74,7 @@ children:
   - /query-run
   - /resolve-database
   - /resolve-extensions
+  - /resolve-extensions-by-pack
   - /resolve-extractor
   - /resolve-files
   - /resolve-languages

content/code-security/codeql-cli/codeql-cli-manual/pack-bundle.md

@@ -74,6 +74,13 @@ directory. This will reduce the size of the pack and the time it takes
 to create it, but will require compilation before the pack can be run.
 Only meaningful for query packs.
 
+#### `--no-validate-extensions`
+
+\[Advanced] Avoid validating data extensions as part of the compile
+step.
+
+Available since `v2.13.3`.
+
 #### `--no-overwrite`
 
 \[Advanced] Avoid recompiling and overwriting any existing compiled

content/code-security/codeql-cli/codeql-cli-manual/pack-ci.md

@@ -32,8 +32,8 @@ codeql pack ci [--force] <options>... -- <dir>
 
 ## Description
 
-\[Experimental] Install dependencies for this pack, verifying that the
-existing lock file is up to date.
+\[Experimental] Clean install dependencies for this pack, verifying
+that the existing lock file is up to date.
 
 This command installs the dependencies of the pack, using the versions
 specified in the codeql-pack.lock.yml file. If any of the versions
@@ -41,6 +41,11 @@ specified in the lock file are incompatible with the version constraints
 specified in the qlpack.yml file, or if no lock file is present, this
 command fails.
 
+This command is similar to `codeql pack install`, except it's meant to
+be used in automated environments such as test platforms, continuous
+integration, and deployment -- or any situation where you want to make
+sure you're doing a clean install of your dependencies.
+
 Available since `v2.12.4`.
 
 ## Primary options

content/code-security/codeql-cli/codeql-cli-manual/pack-create.md

@@ -76,6 +76,13 @@ directory. This will reduce the size of the pack and the time it takes
 to create it, but will require compilation before the pack can be run.
 Only meaningful for query packs.
 
+#### `--no-validate-extensions`
+
+\[Advanced] Avoid validating data extensions as part of the compile
+step.
+
+Available since `v2.13.3`.
+
 #### `--no-overwrite`
 
 \[Advanced] Avoid recompiling and overwriting any existing compiled

content/code-security/codeql-cli/codeql-cli-manual/pack-publish.md

@@ -75,6 +75,13 @@ directory. This will reduce the size of the pack and the time it takes
 to create it, but will require compilation before the pack can be run.
 Only meaningful for query packs.
 
+#### `--no-validate-extensions`
+
+\[Advanced] Avoid validating data extensions as part of the compile
+step.
+
+Available since `v2.13.3`.
+
 #### `--no-overwrite`
 
 \[Advanced] Avoid recompiling and overwriting any existing compiled

content/code-security/codeql-cli/codeql-cli-manual/resolve-extensions-by-pack.md

@@ -0,0 +1,149 @@
+---
+title: resolve extensions-by-pack
+intro: |-
+  [Experimental] [Deep plumbing] Determine accessible extensions for
+  the given paths to pack roots. This includes machine learning models and
+  data extensions.
+versions:
+  fpt: '*'
+  ghae: '*'
+  ghec: '*'
+  ghes: '*'
+topics:
+  - Advanced Security
+  - Code scanning
+  - CodeQL
+type: reference
+product: '{% data reusables.gated-features.codeql %}'
+autogenerated: codeql-cli
+---
+
+<!-- Content after this section is automatically generated -->
+
+{% data reusables.codeql-cli.man-pages-version-note %}
+
+## Synopsis
+
+```shell{:copy}
+codeql resolve extensions-by-pack <options>... -- <pack>...
+```
+
+## Description
+
+\[Experimental] \[Deep plumbing] Determine accessible extensions for
+the given paths to pack roots. This includes machine learning models and
+data extensions.
+
+This plumbing command resolves the set of data extensions and
+GitHub-created machine learning models that are available to the paths
+passed in as command line arguments.
+
+Available since `v2.13.3`.
+
+## Primary options
+
+#### `<pack>...`
+
+The path to the root of the packs to resolve extensions for.
+
+#### `--search-path=<dir>[:<dir>...]`
+
+A list of directories under which QL packs may be found. Each directory
+can either be a QL pack (or bundle of packs containing a
+`.codeqlmanifest.json` file at the root) or the immediate parent of one
+or more such directories.
+
+If the path contains more than one directory, their order defines
+precedence between them: when a pack name that must be resolved is
+matched in more than one of the directory trees, the one given first
+wins.
+
+Pointing this at a checkout of the open-source CodeQL repository ought
+to work when querying one of the languages that live there.
+
+If you have checked out the CodeQL repository as a sibling of the
+unpacked CodeQL toolchain, you don't need to give this option; such
+sibling directories will always be searched for QL packs that cannot be
+found otherwise. (If this default does not work, it is strongly
+recommended to set up `--search-path` once and for all in a per-user
+configuration file).
+
+(Note: On Windows the path separator is `;`).
+
+#### `--additional-packs=<dir>[:<dir>...]`
+
+If this list of directories is given, they will be searched for packs
+before the ones in `--search-path`. The order between these doesn't
+matter; it is an error if a pack name is found in two different places
+through this list.
+
+This is useful if you're temporarily developing a new version of a pack
+that also appears in the default path. On the other hand, it is *not
+recommended* to override this option in a config file; some internal
+actions will add this option on the fly, overriding any configured
+value.
+
+(Note: On Windows the path separator is `;`).
+
+### Options for configuring the CodeQL package manager
+
+#### `--registries-auth-stdin`
+
+Authenticate to GitHub Enterprise Server Container registries by passing
+a comma-separated list of \<registry\_url>=\<token> pairs.
+
+For example, you can pass
+`https://containers.GHEHOSTNAME1/v2/=TOKEN1,https://containers.GHEHOSTNAME2/v2/=TOKEN2`
+to authenticate to two GitHub Enterprise Server instances.
+
+This overrides the CODEQL\_REGISTRIES\_AUTH and GITHUB\_TOKEN environment
+variables. If you only need to authenticate to the github.com Container
+registry, you can instead authenticate using the simpler
+`--github-auth-stdin` option.
+
+#### `--github-auth-stdin`
+
+Authenticate to the github.com Container registry by passing a
+github.com GitHub Apps token or personal access token via standard
+input.
+
+To authenticate to GitHub Enterprise Server Container registries, pass
+`--registries-auth-stdin` or use the CODEQL\_REGISTRIES\_AUTH environment
+variable.
+
+This overrides the GITHUB\_TOKEN environment variable.
+
+### Common options
+
+#### `-h, --help`
+
+Show this help text.
+
+#### `-J=<opt>`
+
+\[Advanced] Give option to the JVM running the command.
+
+(Beware that options containing spaces will not be handled correctly.)
+
+#### `-v, --verbose`
+
+Incrementally increase the number of progress messages printed.
+
+#### `-q, --quiet`
+
+Incrementally decrease the number of progress messages printed.
+
+#### `--verbosity=<level>`
+
+\[Advanced] Explicitly set the verbosity level to one of errors,
+warnings, progress, progress+, progress++, progress+++. Overrides `-v`
+and `-q`.
+
+#### `--logdir=<dir>`
+
+\[Advanced] Write detailed logs to one or more files in the given
+directory, with generated names that include timestamps and the name of
+the running subcommand.
+
+(To write a log file with a name you have full control over, instead
+give `--log-to-stderr` and redirect stderr as desired.)

middleware/helmet.js

@@ -48,6 +48,7 @@ const DEFAULT_OPTIONS = {
       frameAncestors: isDev ? ['*'] : [...GITHUB_DOMAINS],
       styleSrc: ["'self'", "'unsafe-inline'", 'data:', AZURE_STORAGE_URL],
       childSrc: ["'self'"], // exception for search in deprecated GHE versions
+      manifestSrc: ["'self'"],
       upgradeInsecureRequests: isDev ? null : [],
     },
   },

middleware/index.js

@@ -30,6 +30,7 @@ import archivedEnterpriseVersionsAssets from './archived-enterprise-versions-ass
 import api from './api/index.js'
 import healthz from './healthz.js'
 import productIcons from './product-icons.js'
+import manifestJson from './manifest-json.js'
 import remoteIP from './remote-ip.js'
 import buildInfo from './build-info.js'
 import archivedEnterpriseVersions from './archived-enterprise-versions.js'
@@ -248,6 +249,7 @@ export default function (app) {
   app.get('/_ip', instrument(remoteIP, './remoteIP'))
   app.get('/_build', instrument(buildInfo, './buildInfo'))
   app.use('/producticons', instrument(productIcons, './product-icons'))
+  app.use('/manifest.json', asyncMiddleware(instrument(manifestJson, './manifest')))
 
   // Things like `/api` sets their own Fastly surrogate keys.
   // Now that the `req.language` is known, set it for the remaining endpoints

middleware/manifest-json.js

@@ -0,0 +1,52 @@
+import fs from 'fs'
+import path from 'path'
+
+import { defaultCacheControl } from './cache-control.js'
+
+const ICONS = [
+  './assets/images/site/apple-touch-icon-57x57.png',
+  './assets/images/site/apple-touch-icon-60x60.png',
+  './assets/images/site/apple-touch-icon-72x72.png',
+  './assets/images/site/apple-touch-icon-76x76.png',
+  './assets/images/site/apple-touch-icon-114x114.png',
+  './assets/images/site/apple-touch-icon-120x120.png',
+  './assets/images/site/apple-touch-icon-144x144.png',
+  './assets/images/site/apple-touch-icon-152x152.png',
+  './assets/images/site/apple-touch-icon-180x180.png',
+  './assets/images/site/apple-touch-icon-192x192.png',
+  './assets/images/site/apple-touch-icon-512x512.png',
+]
+
+export default async function manifestJson(req, res) {
+  // This is modelled after https://github.com/manifest.json
+  const manifest = {
+    // In the future we might want to have a different manifest for each
+    // language. Particularly, the `name` property.
+    // But as of May 2023, this is overkill because all translations's
+    // home page refer to the name of the site as "GitHub Docs".
+    // For example, on https://docs.github.com/ja the `<title>`
+    // is "GitHub Docs".
+    name: 'GitHub Docs',
+    short_name: 'GitHub Docs',
+    start_url: '/',
+    display: 'standalone',
+    icons: [],
+  }
+  for (const icon of ICONS) {
+    for (const sizes of path.basename(icon).match(/\d+x\d+/g)) {
+      const stats = fs.statSync(icon)
+      const split = icon.slice(1).split(path.sep)
+      const hash = `${stats.size}`
+      split.splice(2, 0, `cb-${hash}`)
+      const src = split.join('/')
+      const type = path.extname(icon) === '.png' ? 'image/png' : undefined
+      manifest.icons.push({
+        sizes,
+        src,
+        type,
+      })
+    }
+  }
+  defaultCacheControl(res)
+  res.json(manifest)
+}

pages/_app.tsx

@@ -75,6 +75,8 @@ const MyApp = ({ Component, pageProps, languagesContext }: MyAppProps) => {
         <link rel="alternate icon" type="image/png" href="/assets/cb-600/images/site/favicon.png" />
         <link rel="icon" type="image/svg+xml" href="/assets/cb-803/images/site/favicon.svg" />
 
+        <link href="/manifest.json" rel="manifest" />
+
         <meta
           name="google-site-verification"
           content="OgdQc0GZfjDI52wDv1bkMT-SLpBUo_h5nn9mI9L22xQ"

script/find-orphaned-assets.js

@@ -18,6 +18,17 @@ import languages from '../lib/languages.js'
 const EXCEPTIONS = new Set([
   'assets/images/site/favicon.ico',
   'assets/images/site/apple-touch-icon.png',
+  'assets/images/site/apple-touch-icon-114x114.png',
+  'assets/images/site/apple-touch-icon-120x120.png',
+  'assets/images/site/apple-touch-icon-144x144.png',
+  'assets/images/site/apple-touch-icon-152x152.png',
+  'assets/images/site/apple-touch-icon-180x180.png',
+  'assets/images/site/apple-touch-icon-192x192.png',
+  'assets/images/site/apple-touch-icon-512x512.png',
+  'assets/images/site/apple-touch-icon-57x57.png',
+  'assets/images/site/apple-touch-icon-60x60.png',
+  'assets/images/site/apple-touch-icon-72x72.png',
+  'assets/images/site/apple-touch-icon-76x76.png',
 ])
 
 function isExceptionPath(imagePath) {

src/pageinfo/tests/pageinfo.js

@@ -16,6 +16,12 @@ describe('pageinfo api', () => {
         'WARNING: The pageinfo tests require the ROOT environment variable to be set to the fixture root'
       )
     }
+    // Ditto for fixture-based translations to work
+    if (!process.env.TRANSLATIONS_FIXTURE_ROOT) {
+      console.warn(
+        'WARNING: The pageinfo tests require the TRANSLATIONS_FIXTURE_ROOT environment variable to be set'
+      )
+    }
   })
 
   test('redirects without version suffix', async () => {
@@ -195,4 +201,29 @@ describe('pageinfo api', () => {
     const { error } = JSON.parse(res.body)
     expect(error).toBe("'pathname' can not contain whitespace")
   })
+
+  describe('translations', () => {
+    test('Japanese page', async () => {
+      const res = await get(makeURL('/ja/get-started/quickstart/hello-world'))
+      expect(res.statusCode).toBe(200)
+      const { info } = JSON.parse(res.body)
+      expect(info.product).toBe('Get started')
+      expect(info.title).toBe('こんにちは World')
+      expect(info.intro).toBe('この Hello World 演習に従って、GitHub の使用を開始します。')
+    })
+
+    test('falls back to English if translation is not present', async () => {
+      const enRes = await get(makeURL('/en/get-started/quickstart'))
+      expect(enRes.statusCode).toBe(200)
+      // This page doesn't have a Japanese translation. I.e. it doesn't
+      // even exist on disk. So it'll fall back to English.
+      const translationRes = await get(makeURL('/ja/get-started/quickstart'))
+      expect(translationRes.statusCode).toBe(200)
+      const en = JSON.parse(enRes.body)
+      const translation = JSON.parse(translationRes.body)
+      expect(en.title).toBe(translation.title)
+      expect(en.intro).toBe(translation.intro)
+      expect(en.product).toBe(translation.product)
+    })
+  })
 })

tests/rendering/manifest.js

@@ -0,0 +1,41 @@
+import sharp from 'sharp'
+
+import { SURROGATE_ENUMS } from '../../middleware/set-fastly-surrogate-key.js'
+import { get, getDOM } from '../helpers/e2etest.js'
+
+describe('manifest', () => {
+  test('download manifest from HTML and check content', async () => {
+    const $ = await getDOM('/')
+    const url = $('link[rel="manifest"]').attr('href')
+    const res = await get(url)
+    expect(res.statusCode).toBe(200)
+
+    // Check that it can be cached at the CDN
+    expect(res.headers['set-cookie']).toBeUndefined()
+    expect(res.headers['cache-control']).toContain('public')
+    expect(res.headers['cache-control']).toMatch(/max-age=[1-9]/)
+    expect(res.headers['surrogate-control']).toContain('public')
+    expect(res.headers['surrogate-control']).toMatch(/max-age=[1-9]/)
+    expect(res.headers['surrogate-key']).toBe(`${SURROGATE_ENUMS.DEFAULT} no-language`)
+
+    const manifest = JSON.parse(res.body)
+    expect(manifest.name).toBe('GitHub Docs')
+    expect(manifest.short_name).toBe('GitHub Docs')
+    expect(manifest.start_url).toBe('/')
+    expect(manifest.display).toBe('standalone')
+    expect(manifest.icons.length).toBeGreaterThan(0)
+    await Promise.all(
+      manifest.icons.map(async (icon) => {
+        const res = await get(icon.src, { responseType: 'buffer' })
+        expect(res.statusCode).toBe(200)
+        expect(res.headers['content-type']).toBe(icon.type)
+        // The `sizes` should match the payload
+        const image = sharp(res.body)
+        const [width, height] = icon.sizes.split('x').map((s) => parseInt(s))
+        const dimensions = await image.metadata()
+        expect(dimensions.width).toBe(width)
+        expect(dimensions.height).toBe(height)
+      })
+    )
+  })
+})

tests/rendering/server.js

@@ -135,6 +135,8 @@ describe('server', () => {
 
     expect(csp.get('style-src').includes("'self'")).toBe(true)
     expect(csp.get('style-src').includes("'unsafe-inline'")).toBe(true)
+
+    expect(csp.get('manifest-src').includes("'self'")).toBe(true)
   })
 
   test('sets Fastly cache control headers', async () => {