Code-headers plugin rewrite (#37654)

Co-authored-by: Peter Bengtsson <peterbe@github.com>

components/ui/MarkdownContent/stylesheets/code.scss

@@ -10,7 +10,7 @@
     overflow: auto;
   }
 
-  [class~="code-extra"] {
+  [class~="code-example"] {
     margin-top: 1.5rem;
 
     pre,

lib/render-content/create-processor.js

@@ -14,7 +14,6 @@ import erb from 'highlight.js/lib/languages/erb'
 import powershell from 'highlight.js/lib/languages/powershell'
 import graphql from 'highlight.js/lib/languages/graphql'
 import html from 'rehype-stringify'
-import remarkCodeExtra from 'remark-code-extra'
 import codeHeader from './plugins/code-header.js'
 import rewriteLocalLinks from './plugins/rewrite-local-links.js'
 import rewriteImgSources from './plugins/rewrite-asset-urls.js'
@@ -34,7 +33,6 @@ export default function createProcessor(context) {
       .use(process.env.COMMONMARK ? gfm : null)
       // Markdown AST below vvv
       .use(parseInfoString)
-      .use(remarkCodeExtra, { transform: codeHeader })
       .use(emoji)
       // Markdown AST above ^^^
       .use(remark2rehype, { allowDangerousHtml: true })
@@ -42,6 +40,7 @@ export default function createProcessor(context) {
       .use(slug)
       .use(useEnglishHeadings, context)
       .use(headingLinks)
+      .use(codeHeader)
       .use(annotate)
       .use(highlight, {
         languages: { graphql, dockerfile, http, groovy, erb, powershell },

lib/render-content/plugins/annotate.js

@@ -35,6 +35,7 @@ import { visit } from 'unist-util-visit'
 import { h } from 'hastscript'
 import { fromMarkdown } from 'mdast-util-from-markdown'
 import { toHast } from 'mdast-util-to-hast'
+// import { header } from './code-header.js'
 
 const languages = yaml.load(fs.readFileSync('./data/variables/code-languages.yml', 'utf8'))
 
@@ -71,7 +72,7 @@ function createAnnotatedNode(node) {
   const rows = chunk(groups, 2)
 
   // Render the HTML
-  return template({ lang, rows })
+  return template({ lang, code, rows })
 }
 
 function validate(lang, code) {
@@ -116,10 +117,11 @@ function matchComment(lang) {
   return (line) => regex.test(line)
 }
 
-function template({ lang, rows }) {
+function template({ lang, code, rows }) {
   return h(
     'div',
     { class: 'annotate' },
+    // header(lang, code),
     rows.map(([note, code]) =>
       h('div', { className: 'annotate-row' }, [
         h(

lib/render-content/plugins/code-header.js

@@ -1,105 +1,42 @@
+/**
+ * Adds a bar above code blocks that shows the language and a copy button
+ */
+
+import yaml from 'js-yaml'
+import fs from 'fs'
+import { visit } from 'unist-util-visit'
 import { h } from 'hastscript'
 import octicons from '@primer/octicons'
 import { parse } from 'parse5'
 import { fromParse5 } from 'hast-util-from-parse5'
 
-const LANGUAGE_MAP = {
+const languages = yaml.load(fs.readFileSync('./data/variables/code-languages.yml', 'utf8'))
-  asp: 'ASP',
-  aspx: 'ASP',
-  'aspx-vb': 'ASP',
-  as3: 'ActionScript',
-  apache: 'ApacheConf',
-  nasm: 'Assembly',
-  bat: 'Batchfile',
-  'c#': 'C#',
-  csharp: 'C#',
-  c: 'C',
-  'c++': 'C++',
-  cpp: 'C++',
-  chpl: 'Chapel',
-  coffee: 'CoffeeScript',
-  'coffee-script': 'CoffeeScript',
-  cfm: 'ColdFusion',
-  'common-lisp': 'Common Lisp',
-  lisp: 'Common Lisp',
-  dpatch: 'Darcs Patch',
-  dart: 'Dart',
-  elisp: 'Emacs Lisp',
-  emacs: 'Emacs Lisp',
-  'emacs-lisp': 'Emacs Lisp',
-  pot: 'Gettext Catalog',
-  html: 'HTML',
-  xhtml: 'HTML',
-  'html+erb': 'HTML+ERB',
-  erb: 'HTML+ERB',
-  irc: 'IRC log',
-  json: 'JSON',
-  jsp: 'Java Server Pages',
-  java: 'Java',
-  javascript: 'JavaScript',
-  js: 'JavaScript',
-  lhs: 'Literate Haskell',
-  'literate-haskell': 'Literate Haskell',
-  objc: 'Objective-C',
-  openedge: 'OpenEdge ABL',
-  progress: 'OpenEdge ABL',
-  abl: 'OpenEdge ABL',
-  pir: 'Parrot Internal Representation',
-  posh: 'PowerShell',
-  puppet: 'Puppet',
-  'pure-data': 'Pure Data',
-  raw: 'Raw token data',
-  rb: 'Ruby',
-  ruby: 'Ruby',
-  r: 'R',
-  scheme: 'Scheme',
-  bash: 'Shell',
-  sh: 'Shell',
-  shell: 'Shell',
-  zsh: 'Shell',
-  supercollider: 'SuperCollider',
-  tex: 'TeX',
-  ts: 'TypeScript',
-  vim: 'Vim script',
-  viml: 'Vim script',
-  rst: 'reStructuredText',
-  xbm: 'X BitMap',
-  xpm: 'X PixMap',
-  yaml: 'YAML',
-  yml: 'YAML',
 
-  // Unofficial languages
+const matcher = (node) =>
-  shellsession: 'Shell',
+  node.type === 'element' &&
-  jsx: 'JSX',
+  node.tagName === 'pre' &&
+  // For now, limit to ones with the copy meta,
+  // but we may enable for all examples later.
+  getPreMeta(node).copy &&
+  // Don't add this header for annotated examples.
+  !getPreMeta(node).annotate
+
+export default function codeHeader() {
+  return (tree) => {
+    visit(tree, matcher, (node, index, parent) => {
+      parent.children[index] = wrapCodeExample(node)
+    })
+  }
 }
 
-const COPY_REGEX = /\{:copy\}$/
+function wrapCodeExample(node) {
-
+  const lang = node.children[0].properties.className?.[0].replace('language-', '')
-/**
+  const code = node.children[0].children[0].value
- * Adds a bar above code blocks that shows the language and a copy button
+  return h('div', { className: 'code-example' }, [header(lang, code), node])
- */
-export default function addCodeHeader(node) {
-  // Check if the language matches `lang{:copy}`
-  const hasCopy = node.lang && COPY_REGEX.test(node.lang)
-
-  if (hasCopy) {
-    // js{:copy} => js
-    node.lang = node.lang.replace(COPY_REGEX, '')
-  } else {
-    // It doesn't have the copy annotation, so don't add the header
-    return
 }
 
-  // Display the language using the above map of `{ [shortCode]: language }`
+export function header(lang, code) {
-  const language = LANGUAGE_MAP[node.lang] || node.lang || 'Code'
+  return h(
-
-  const btnIconHtml = octicons.copy.toSVG()
-  const btnIconAst = parse(String(btnIconHtml), { sourceCodeLocationInfo: true })
-  const btnIcon = fromParse5(btnIconAst, { file: btnIconHtml })
-
-  // Need to create the header using Markdown AST utilities, to fit
-  // into the Unified processor ecosystem.
-  const header = h(
     'header',
     {
       class: [
@@ -109,24 +46,35 @@ export default function addCodeHeader(node) {
         'p-2',
         'text-small',
         'rounded-top-1',
-        'border',
+        'border-top',
+        'border-left',
+        'border-right',
       ],
     },
     [
-      h('span', language),
+      h('span', languages[lang]?.name),
       h(
         'button',
         {
           class: ['js-btn-copy', 'btn', 'btn-sm', 'tooltipped', 'tooltipped-nw'],
-          'data-clipboard-text': node.value,
+          'data-clipboard-text': code,
           'aria-label': 'Copy code to clipboard',
         },
-        btnIcon
+        btnIcon()
       ),
     ]
   )
+}
 
-  return {
+function btnIcon() {
-    before: [header],
+  const btnIconHtml = octicons.copy.toSVG()
+  const btnIconAst = parse(String(btnIconHtml), { sourceCodeLocationInfo: true })
+  const btnIcon = fromParse5(btnIconAst, { file: btnIconHtml })
+  return btnIcon
 }
+
+function getPreMeta(node) {
+  // Here's why this monstrosity works:
+  // https://github.com/syntax-tree/mdast-util-to-hast/blob/c87cd606731c88a27dbce4bfeaab913a9589bf83/lib/handlers/code.js#L40-L42
+  return node.children[0]?.data?.meta || {}
 }

lib/render-content/plugins/parse-info-string.js

@@ -7,17 +7,26 @@
 
 import { visit } from 'unist-util-visit'
 
-const matcher = (node) => node.type === 'code' && node.lang && node.meta
+const matcher = (node) => node.type === 'code' && node.lang
 
 export default function parseInfoString() {
   return (tree) => {
     visit(tree, matcher, (node) => {
       node.meta = strToObj(node.meta)
+
+      // Temporary, change {:copy} to ` copy` to avoid conflict in styles.
+      // We may end up enabling copy on all code examples later.
+      const copyTag = '{:copy}'
+      if (node.lang.includes(copyTag)) {
+        node.meta.copy = true
+        node.lang = node.lang.replace(copyTag, '')
+      }
     })
   }
 }
 
 function strToObj(str) {
+  if (!str) return {}
   return Object.fromEntries(
     str
       .split(/\s+/g)

package-lock.json

@@ -73,7 +73,6 @@
         "rehype-raw": "^6.1.1",
         "rehype-slug": "^5.0.1",
         "rehype-stringify": "^9.0.3",
-        "remark-code-extra": "^1.0.1",
         "remark-gemoji-to-emoji": "^1.1.0",
         "remark-gfm": "^3.0.1",
         "remark-parse": "^10.0.1",
@@ -16424,31 +16423,6 @@
         "url": "https://opencollective.com/unified"
       }
     },
-    "node_modules/remark-code-extra": {
-      "version": "1.0.1",
-      "license": "MIT",
-      "dependencies": {
-        "unist-util-visit": "^1.4.1"
-      }
-    },
-    "node_modules/remark-code-extra/node_modules/unist-util-is": {
-      "version": "3.0.0",
-      "license": "MIT"
-    },
-    "node_modules/remark-code-extra/node_modules/unist-util-visit": {
-      "version": "1.4.1",
-      "license": "MIT",
-      "dependencies": {
-        "unist-util-visit-parents": "^2.0.0"
-      }
-    },
-    "node_modules/remark-code-extra/node_modules/unist-util-visit-parents": {
-      "version": "2.1.2",
-      "license": "MIT",
-      "dependencies": {
-        "unist-util-is": "^3.0.0"
-      }
-    },
     "node_modules/remark-gemoji-to-emoji": {
       "version": "1.1.0",
       "license": "MIT",
@@ -30242,29 +30216,6 @@
         "unified": "^10.0.0"
       }
     },
-    "remark-code-extra": {
-      "version": "1.0.1",
-      "requires": {
-        "unist-util-visit": "^1.4.1"
-      },
-      "dependencies": {
-        "unist-util-is": {
-          "version": "3.0.0"
-        },
-        "unist-util-visit": {
-          "version": "1.4.1",
-          "requires": {
-            "unist-util-visit-parents": "^2.0.0"
-          }
-        },
-        "unist-util-visit-parents": {
-          "version": "2.1.2",
-          "requires": {
-            "unist-util-is": "^3.0.0"
-          }
-        }
-      }
-    },
     "remark-gemoji-to-emoji": {
       "version": "1.1.0",
       "requires": {

package.json

@@ -120,7 +120,6 @@
     "rehype-raw": "^6.1.1",
     "rehype-slug": "^5.0.1",
     "rehype-stringify": "^9.0.3",
-    "remark-code-extra": "^1.0.1",
     "remark-gemoji-to-emoji": "^1.1.0",
     "remark-gfm": "^3.0.1",
     "remark-parse": "^10.0.1",

src/codeql-cli/scripts/convert-markdown-for-docs.js

@@ -86,7 +86,8 @@ export async function convertContentToDocs(content, frontmatterDefaults = {}) {
   visitParents(ast, matcher, (node, ancestors) => {
     // Add the copy button to the example command
     if (node.type === 'code' && node.value.startsWith(`codeql ${frontmatter.title}`)) {
-      node.lang = 'shell{:copy}'
+      node.lang = 'shell'
+      node.meta = 'copy'
     }
 
     // This is the beginning of a secondary options section. For example,

tests/unit/render-content.js

@@ -248,12 +248,12 @@ $resourceGroupName = "octocat-testgroup"
   test('does not autoguess code block language', async () => {
     const template = nl(`
 \`\`\`
-some code
+var a = 1
-\`\`\`\
+\`\`\`
     `)
     const html = await renderContent(template)
     const $ = cheerio.load(html, { xmlMode: true })
-    expect($.html().includes('<pre><code>some code\n</code></pre>')).toBeTruthy()
+    expect($.html().includes('var a = 1')).toBeTruthy()
   })
 
   test('renders a line break in a table', async () => {
@@ -267,15 +267,15 @@ some code
     )
   })
 
-  test('renders a copy button for code blocks with {:copy} annotation', async () => {
+  test('renders a copy button for code blocks with language specified', async () => {
     const template = nl(`
-\`\`\`js{:copy}
+\`\`\`javascript copy
-some code
+var a = 1
-\`\`\`\
+\`\`\`
     `)
     const html = await renderContent(template)
     const $ = cheerio.load(html)
     const el = $('button.js-btn-copy')
-    expect(el.data('clipboard-text')).toBe('some code')
+    expect(el.data('clipboard-text')).toBe('var a = 1\n')
   })
 })