Fix code syntax highlighting

website/package.json

@@ -11,6 +11,7 @@
         "prettier": "prettier . --write"
     },
     "dependencies": {
+        "@mapbox/rehype-prism": "^0.8.0",
         "@mdx-js/loader": "^2.1.5",
         "@mdx-js/react": "^2.1.5",
         "@next/mdx": "^13.0.2",
@@ -18,6 +19,7 @@
         "@types/node": "18.11.9",
         "@types/react": "18.0.25",
         "@types/react-dom": "18.0.8",
+        "acorn": "^8.8.1",
         "browser-monads": "^1.0.0",
         "classnames": "^2.3.2",
         "eslint": "8.27.0",
@@ -27,7 +29,9 @@
         "md-attr-parser": "^1.3.0",
         "next": "13.0.2",
         "next-mdx-remote": "^4.2.0",
+        "parse-numeric-range": "^1.3.0",
         "prettier": "^2.7.1",
+        "prismjs": "^1.29.0",
         "prop-types": "^15.8.1",
         "react": "18.2.0",
         "react-dom": "18.2.0",

website/pages/[...listPathPage].tsx

@@ -92,9 +92,7 @@ export const getStaticProps: GetStaticProps<PropsPage, ParsedUrlQuery> = async (
 
     const mdx = await serialize(fs.readFileSync(pathFileWithIndexAndExtension, 'utf-8'), {
         parseFrontmatter: true,
-        mdxOptions: {
-            remarkPlugins,
-        },
+        mdxOptions: { remarkPlugins },
     })
 
     if (!mdx.frontmatter) {

website/plugins/getProps.mjs

@@ -17,7 +17,8 @@ const getProps = (estree) => {
         return {}
     }
 
-    const estreeBodyFirstNode = estree.body[0]
+    const estreeBodyFirstNode =
+        estree.body[0].type === 'BlockStatement' ? estree.body[0].body[0] : estree.body[0]
 
     if (estreeBodyFirstNode.type !== 'ExpressionStatement' || !estreeBodyFirstNode.expression) {
         return {}

website/plugins/remarkCodeBlocks.mjs

@@ -1,10 +1,12 @@
 /**
  * Support titles, line highlights and more for code blocks
  */
-
+import { Parser } from 'acorn'
 import { visit } from 'unist-util-visit'
 import parseAttr from 'md-attr-parser'
 
+import getProps from './getProps.mjs'
+
 const defaultOptions = {
     defaultPrefix: '###',
     prefixMap: {
@@ -43,12 +45,15 @@ function remarkCodeBlocks(userOptions = {}) {
                     firstLine.startsWith('%%GITHUB_')
                 ) {
                     // GitHub URL
-                    attrs.github = 'true'
+                    attrs.github = node.value
                 }
 
                 const data = node.data || (node.data = {})
                 const hProps = data.hProperties || (data.hProperties = {})
-                node.data.hProperties = Object.assign({}, hProps, attrs)
+
+                const meta = getProps(Parser.parse(node.meta, { ecmaVersion: 'latest' }))
+
+                node.data.hProperties = Object.assign({}, hProps, attrs, meta)
             }
         })
 

website/src/components/code.js

@@ -1,10 +1,18 @@
-import React, { Fragment } from 'react'
+import React, { Fragment, useEffect, useState } from 'react'
 import PropTypes from 'prop-types'
 import classNames from 'classnames'
-import highlightCode from 'gatsby-remark-prismjs/highlight-code.js'
-import 'prismjs-bibtex'
 import rangeParser from 'parse-numeric-range'
-import { window } from 'browser-monads'
+import Prism from 'prismjs'
+
+// We manually load all the languages that are needed, which are currently only those:
+import 'prismjs/components/prism-diff.min.js'
+import 'prismjs/components/prism-bash.min.js'
+import 'prismjs/components/prism-ini.min.js'
+import 'prismjs/components/prism-jsx.min.js'
+import 'prismjs/components/prism-json.min.js'
+import 'prismjs/components/prism-markdown.min.js'
+import 'prismjs/components/prism-python.min.js'
+import 'prismjs/components/prism-yaml.min.js'
 
 import CUSTOM_TYPES from '../../meta/type-annotations.json'
 import { isString, htmlToReact } from './util'
@@ -71,7 +79,8 @@ export const TypeAnnotation = ({ lang = 'python', link = true, children }) => {
     const code = Array.isArray(children) ? children.join('') : children || ''
     const [rawText, meta] = code.split(/(?= \(.+\)$)/)
     const rawStr = rawText.replace(/\./g, TMP_DOT)
-    const rawHtml = lang === 'none' || !code ? code : highlightCode(lang, rawStr)
+    const rawHtml =
+        lang === 'none' || !code ? code : Prism.highlight(rawStr, Prism.languages[lang], lang)
     const html = rawHtml.replace(new RegExp(TMP_DOT, 'g'), '.').replace(/\n/g, ' ')
     const result = htmlToReact(html)
     const elements = Array.isArray(result) ? result : [result]
@@ -94,12 +103,48 @@ export const TypeAnnotation = ({ lang = 'python', link = true, children }) => {
     )
 }
 
-function replacePrompt(line, prompt, isFirst = false) {
-    let result = line
-    const hasPrompt = result.startsWith(`${prompt} `)
-    const showPrompt = hasPrompt || isFirst
-    if (hasPrompt) result = result.slice(2)
-    return result && showPrompt ? `<span data-prompt="${prompt}">${result}</span>` : result
+const splitLines = (children) => {
+    const listChildrenPerLine = []
+
+    if (typeof children === 'string') {
+        listChildrenPerLine.push(...children.split('\n'))
+    } else {
+        listChildrenPerLine.push([])
+        let indexLine = 0
+        if (Array.isArray(children)) {
+            children.forEach((child) => {
+                if (typeof child === 'string' && child.includes('\n')) {
+                    const listString = child.split('\n')
+                    listString.forEach((string, index) => {
+                        listChildrenPerLine[indexLine].push(string)
+
+                        if (index !== listString.length - 1) {
+                            indexLine += 1
+                            listChildrenPerLine[indexLine] = []
+                        }
+                    })
+                } else {
+                    listChildrenPerLine[indexLine].push(child)
+                }
+            })
+        } else {
+            listChildrenPerLine[indexLine].push(children)
+            indexLine += 1
+            listChildrenPerLine[indexLine] = []
+        }
+    }
+
+    const listLine = listChildrenPerLine[listChildrenPerLine.length - 1]
+    if (listLine === '' || (listLine.length === 1 && listLine[0] === '')) {
+        listChildrenPerLine.pop()
+    }
+
+    return listChildrenPerLine.map((childrenPerLine, index) => (
+        <>
+            {childrenPerLine}
+            {index !== listChildrenPerLine.length - 1 && '\n'}
+        </>
+    ))
 }
 
 function parseArgs(raw) {
@@ -123,82 +168,140 @@ function parseArgs(raw) {
     return result
 }
 
-function convertLine(line, i) {
-    const cliRegex = /^(\$ )?python -m spacy/
-    if (cliRegex.test(line)) {
-        const text = line.replace(cliRegex, '')
-        const args = parseArgs(text)
-        const cmd = Object.keys(args).map((key, i) => {
-            const value = args[key]
-            return value === null || value === true || i === 0 ? key : `${key} ${value}`
-        })
-        return (
-            <Fragment key={line}>
-                <span data-prompt={i === 0 ? '$' : null} className={classes['cli-arg-subtle']}>
-                    python -m
-                </span>{' '}
-                <span>spacy</span>{' '}
-                {cmd.map((item, j) => {
-                    const isCmd = j === 0
-                    const url = isCmd ? `/api/cli#${item.replace(' ', '-')}` : null
-                    const isAbstract = isString(item) && /^\[(.+)\]$/.test(item)
-                    const itemClassNames = classNames(classes['cli-arg'], {
-                        [classes['cli-arg-highlight']]: isCmd,
-                        [classes['cli-arg-emphasis']]: isAbstract,
-                    })
-                    const text = isAbstract ? item.slice(1, -1) : item
-                    return (
-                        <Fragment key={j}>
-                            {j !== 0 && ' '}
-                            <span className={itemClassNames}>
-                                <OptionalLink hidden hideIcon to={url}>
-                                    {text}
-                                </OptionalLink>
-                            </span>
-                        </Fragment>
-                    )
-                })}
-            </Fragment>
-        )
+const flattenReact = (children) => {
+    if (children === null || children === undefined || children === false) {
+        return []
     }
-    const htmlLine = replacePrompt(highlightCode('bash', line), '$')
-    return htmlToReact(htmlLine)
+
+    if (typeof children === 'string') {
+        return [children]
+    }
+
+    if (children.props) {
+        return flattenReact(children.props.children)
+    }
+
+    return children.flatMap(flattenReact)
 }
 
-function formatCode(html, lang, prompt) {
-    if (lang === 'cli') {
-        const lines = html
-            .trim()
-            .split('\n')
-            .map((line) =>
-                line
-                    .split(' | ')
-                    .map((l, i) => convertLine(l, i))
-                    .map((l, j) => (
-                        <Fragment key={j}>
-                            {j !== 0 && <span> | </span>}
-                            {l}
-                        </Fragment>
-                    ))
-            )
-        return lines.map((line, i) => (
-            <Fragment key={i}>
-                {i !== 0 && <br />}
-                {line}
-            </Fragment>
-        ))
+const checkoutForComment = (line) => {
+    const lineParts = line.split(' # ')
+
+    if (lineParts.length !== 2) {
+        return line
     }
-    const result = html
-        .split('\n')
-        .map((line, i) => {
-            let newLine = prompt ? replacePrompt(line, prompt, i === 0) : line
-            if (lang === 'diff' && !line.startsWith('<')) {
-                newLine = highlightCode('python', line)
-            }
-            return newLine
-        })
-        .join('\n')
-    return htmlToReact(result)
+
+    return (
+        <>
+            {lineParts[0]}
+            {` `}
+            <span class="token comment">
+                {`# `}
+                {lineParts[1]}
+            </span>
+        </>
+    )
+}
+
+const handlePromot = ({ lineFlat, prompt }) => {
+    const lineWithoutPrompt = lineFlat.slice(prompt.length + 1)
+
+    const cliRegex = /^python -m spacy/
+
+    if (!cliRegex.test(lineWithoutPrompt)) {
+        return <span data-prompt={prompt}>{checkoutForComment(lineWithoutPrompt)}</span>
+    }
+
+    const text = lineWithoutPrompt.replace(cliRegex, '')
+    const args = parseArgs(text)
+    const cmd = Object.keys(args).map((key, i) => {
+        const value = args[key]
+        return value === null || value === true || i === 0 ? key : `${key} ${value}`
+    })
+    return (
+        <span data-prompt={prompt}>
+            <span className={classes['cli-arg-subtle']}>python -m</span> <span>spacy</span>{' '}
+            {cmd.map((item, j) => {
+                const isCmd = j === 0
+                const url = isCmd ? `/api/cli#${item.replace(' ', '-')}` : null
+                const isAbstract = isString(item) && /^\[(.+)\]$/.test(item)
+                const itemClassNames = classNames(classes['cli-arg'], {
+                    [classes['cli-arg-highlight']]: isCmd,
+                    [classes['cli-arg-emphasis']]: isAbstract,
+                })
+                const text = isAbstract ? item.slice(1, -1) : item
+                return (
+                    <Fragment key={j}>
+                        {j !== 0 && ' '}
+                        <span className={itemClassNames}>
+                            <OptionalLink hidden hideIcon to={url}>
+                                {text}
+                            </OptionalLink>
+                        </span>
+                    </Fragment>
+                )
+            })}
+        </span>
+    )
+}
+
+const convertLine = ({ line, prompt, lang }) => {
+    const lineFlat = flattenReact(line).join('')
+    if (lineFlat.startsWith(`${prompt} `)) {
+        return handlePromot({ lineFlat, prompt })
+    }
+
+    return lang === 'none' || !lineFlat ? (
+        lineFlat
+    ) : (
+        <span
+            dangerouslySetInnerHTML={{
+                __html: Prism.highlight(lineFlat, Prism.languages[lang], lang),
+            }}
+        />
+    )
+}
+
+const addLineHighlight = (children, highlight) => {
+    if (!highlight) {
+        return children
+    }
+    const listHighlight = rangeParser(highlight)
+
+    if (listHighlight.length === 0) {
+        return children
+    }
+
+    return children.map((child, index) => {
+        const isHighlight = listHighlight.includes(index + 1)
+        return (
+            <span
+                className={classNames({
+                    'gatsby-highlight-code-line': isHighlight,
+                })}
+                key={index}
+            >
+                {child}
+            </span>
+        )
+    })
+}
+
+export const CodeHighlighted = ({ children, highlight, lang }) => {
+    const [html, setHtml] = useState()
+
+    useEffect(
+        () =>
+            setHtml(
+                addLineHighlight(
+                    splitLines(children).map((line) => convertLine({ line, prompt: '$', lang })),
+                    highlight
+                )
+            ),
+        [children, highlight, lang]
+    )
+
+    return <>{html}</>
 }
 
 export class Code extends React.Component {
@@ -207,14 +310,13 @@ export class Code extends React.Component {
     static defaultProps = {
         lang: 'none',
         executable: null,
-        github: false,
     }
 
     static propTypes = {
         lang: PropTypes.string,
         title: PropTypes.string,
         executable: PropTypes.oneOf(['true', 'false', true, false, null]),
-        github: PropTypes.oneOf(['true', 'false', true, false, null]),
+        github: PropTypes.string,
         prompt: PropTypes.string,
         highlight: PropTypes.string,
         className: PropTypes.string,
@@ -236,8 +338,7 @@ export class Code extends React.Component {
     }
 
     render() {
-        const { lang, title, executable, github, prompt, wrap, highlight, className, children } =
-            this.props
+        const { lang, title, executable, github, wrap, highlight, className, children } = this.props
         const codeClassNames = classNames(classes['code'], className, `language-${lang}`, {
             [classes['wrap']]: !!highlight || !!wrap || lang === 'cli',
             [classes['cli']]: lang === 'cli',
@@ -246,7 +347,7 @@ export class Code extends React.Component {
         const { Juniper } = this.state
 
         if (github) {
-            return <GitHubCode url={children} className={ghClassNames} lang={lang} />
+            return <GitHubCode url={github} className={ghClassNames} lang={lang} />
         }
         if (!!executable && Juniper) {
             return (
@@ -256,16 +357,14 @@ export class Code extends React.Component {
             )
         }
 
-        const codeText = Array.isArray(children) ? children.join('') : children || ''
-        const highlightRange = highlight ? rangeParser.parse(highlight).filter((n) => n > 0) : []
-        const rawHtml = ['none', 'cli'].includes(lang)
-            ? codeText
-            : highlightCode(lang, codeText, highlightRange)
-        const html = formatCode(rawHtml, lang, prompt)
         return (
             <>
                 {title && <h4 className={classes['title']}>{title}</h4>}
-                <code className={codeClassNames}>{html}</code>
+                <code className={codeClassNames}>
+                    <CodeHighlighted highlight={highlight} lang={lang}>
+                        {children}
+                    </CodeHighlighted>
+                </code>
             </>
         )
     }

website/src/components/github.js

@@ -1,11 +1,11 @@
 import React, { useState, useEffect } from 'react'
 import PropTypes from 'prop-types'
 import classNames from 'classnames'
-import highlightCode from 'gatsby-remark-prismjs/highlight-code.js'
 
 import Icon from './icon'
 import Link from './link'
 import classes from '../styles/code.module.sass'
+import { Code } from './code'
 
 const defaultErrorMsg = `Can't fetch code example from GitHub :(
 
@@ -39,8 +39,6 @@ const GitHubCode = ({ url, lang, errorMsg = defaultErrorMsg, className }) => {
         }
     }, [initialized, rawUrl, errorMsg])
 
-    const highlighted = lang === 'none' || !code ? code : highlightCode(lang, code)
-
     return (
         <>
             <header className={classes.header}>
@@ -53,7 +51,11 @@ const GitHubCode = ({ url, lang, errorMsg = defaultErrorMsg, className }) => {
                     </code>
                 </Link>
             </header>
-            <code className={codeClassNames} dangerouslySetInnerHTML={{ __html: highlighted }} />
+            {code && (
+                <Code className={codeClassNames} lang={lang}>
+                    {code}
+                </Code>
+            )}
         </>
     )
 }

website/src/templates/models.js

@@ -298,8 +298,8 @@ const Model = ({
                 {name}
             </H2>
             <Aside title="Installation">
-                <CodeBlock lang="cli" prompt="$">
-                    python -m spacy download {name}
+                <CodeBlock lang="bash" prompt="$">
+                    $ python -m spacy download {name}
                 </CodeBlock>
             </Aside>
             {meta.description && <MarkdownToReact markdown={meta.description} />}