Add code block handling

website/package-lock.json

@@ -16,6 +16,7 @@
                 "@types/node": "18.11.9",
                 "@types/react": "18.0.25",
                 "@types/react-dom": "18.0.8",
+                "acorn": "^8.8.1",
                 "browser-monads": "^1.0.0",
                 "classnames": "^2.3.2",
                 "eslint": "8.27.0",
@@ -25,6 +26,7 @@
                 "md-attr-parser": "^1.3.0",
                 "next": "13.0.2",
                 "next-mdx-remote": "^4.2.0",
+                "parse-numeric-range": "^1.3.0",
                 "prettier": "^2.7.1",
                 "prop-types": "^15.8.1",
                 "react": "18.2.0",
@@ -4303,6 +4305,11 @@
                 "url": "https://github.com/sponsors/wooorm"
             }
         },
+        "node_modules/parse-numeric-range": {
+            "version": "1.3.0",
+            "resolved": "https://registry.npmjs.org/parse-numeric-range/-/parse-numeric-range-1.3.0.tgz",
+            "integrity": "sha512-twN+njEipszzlMJd4ONUYgSfZPDxgHhT9Ahed5uTigpQn90FggW4SA/AIPq/6a149fTbE9qBEcSwE3FAEp6wQQ=="
+        },
         "node_modules/path-exists": {
             "version": "4.0.0",
             "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
@@ -8765,6 +8772,11 @@
                 "is-hexadecimal": "^2.0.0"
             }
         },
+        "parse-numeric-range": {
+            "version": "1.3.0",
+            "resolved": "https://registry.npmjs.org/parse-numeric-range/-/parse-numeric-range-1.3.0.tgz",
+            "integrity": "sha512-twN+njEipszzlMJd4ONUYgSfZPDxgHhT9Ahed5uTigpQn90FggW4SA/AIPq/6a149fTbE9qBEcSwE3FAEp6wQQ=="
+        },
         "path-exists": {
             "version": "4.0.0",
             "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",

website/package.json

@@ -21,6 +21,7 @@
         "@types/node": "18.11.9",
         "@types/react": "18.0.25",
         "@types/react-dom": "18.0.8",
+        "acorn": "^8.8.1",
         "browser-monads": "^1.0.0",
         "classnames": "^2.3.2",
         "eslint": "8.27.0",
@@ -30,6 +31,7 @@
         "md-attr-parser": "^1.3.0",
         "next": "13.0.2",
         "next-mdx-remote": "^4.2.0",
+        "parse-numeric-range": "^1.3.0",
         "prettier": "^2.7.1",
         "prop-types": "^15.8.1",
         "react": "18.2.0",

website/plugins/getProps.mjs

@@ -17,7 +17,8 @@ const getProps = (estree) => {
         return {}
     }
 
-    const estreeBodyFirstNode = estree.body[0]
+    const estreeBodyFirstNode =
+        estree.body[0].type === 'BlockStatement' ? estree.body[0].body[0] : estree.body[0]
 
     if (estreeBodyFirstNode.type !== 'ExpressionStatement' || !estreeBodyFirstNode.expression) {
         return {}

website/plugins/remarkCodeBlocks.mjs

@@ -1,10 +1,12 @@
 /**
  * Support titles, line highlights and more for code blocks
  */
-
+import { Parser } from 'acorn'
 import { visit } from 'unist-util-visit'
 import parseAttr from 'md-attr-parser'
 
+import getProps from './getProps.mjs'
+
 const defaultOptions = {
     defaultPrefix: '###',
     prefixMap: {
@@ -48,7 +50,13 @@ function remarkCodeBlocks(userOptions = {}) {
 
                 const data = node.data || (node.data = {})
                 const hProps = data.hProperties || (data.hProperties = {})
-                node.data.hProperties = Object.assign({}, hProps, attrs)
+
+                node.data.hProperties = Object.assign(
+                    {},
+                    hProps,
+                    attrs,
+                    getProps(Parser.parse(node.meta, { ecmaVersion: 'latest' }))
+                )
             }
         })
 

website/src/components/code.js

@@ -1,4 +1,4 @@
-import React, { Fragment } from 'react'
+import React, { Fragment, useEffect, useState } from 'react'
 import PropTypes from 'prop-types'
 import classNames from 'classnames'
 import highlightCode from 'gatsby-remark-prismjs/highlight-code.js'
@@ -94,12 +94,48 @@ export const TypeAnnotation = ({ lang = 'python', link = true, children }) => {
     )
 }
 
-function replacePrompt(line, prompt, isFirst = false) {
+const splitLines = (children) => {
-    let result = line
+    const listChildrenPerLine = []
-    const hasPrompt = result.startsWith(`${prompt} `)
+
-    const showPrompt = hasPrompt || isFirst
+    if (typeof children === 'string') {
-    if (hasPrompt) result = result.slice(2)
+        listChildrenPerLine.push(...children.split('\n'))
-    return result && showPrompt ? `<span data-prompt="${prompt}">${result}</span>` : result
+    } else {
+        listChildrenPerLine.push([])
+        let indexLine = 0
+        if (Array.isArray(children)) {
+            children.forEach((child) => {
+                if (typeof child === 'string' && child.includes('\n')) {
+                    const listString = child.split('\n')
+                    listString.forEach((string, index) => {
+                        listChildrenPerLine[indexLine].push(string)
+
+                        if (index !== listString.length - 1) {
+                            indexLine += 1
+                            listChildrenPerLine[indexLine] = []
+                        }
+                    })
+                } else {
+                    listChildrenPerLine[indexLine].push(child)
+                }
+            })
+        } else {
+            listChildrenPerLine[indexLine].push(children)
+            indexLine += 1
+            listChildrenPerLine[indexLine] = []
+        }
+    }
+
+    const listLine = listChildrenPerLine[listChildrenPerLine.length - 1]
+    if (listLine === '' || (listLine.length === 1 && listLine[0] === '')) {
+        listChildrenPerLine.pop()
+    }
+
+    return listChildrenPerLine.map((childrenPerLine, index) => (
+        <>
+            {childrenPerLine}
+            {index !== listChildrenPerLine.length - 1 && '\n'}
+        </>
+    ))
 }
 
 function parseArgs(raw) {
@@ -123,82 +159,128 @@ function parseArgs(raw) {
     return result
 }
 
-function convertLine(line, i) {
+const flattenReact = (children) => {
-    const cliRegex = /^(\$ )?python -m spacy/
+    if (children === null || children === undefined || children === false) {
-    if (cliRegex.test(line)) {
+        return []
-        const text = line.replace(cliRegex, '')
-        const args = parseArgs(text)
-        const cmd = Object.keys(args).map((key, i) => {
-            const value = args[key]
-            return value === null || value === true || i === 0 ? key : `${key} ${value}`
-        })
-        return (
-            <Fragment key={line}>
-                <span data-prompt={i === 0 ? '$' : null} className={classes['cli-arg-subtle']}>
-                    python -m
-                </span>{' '}
-                <span>spacy</span>{' '}
-                {cmd.map((item, j) => {
-                    const isCmd = j === 0
-                    const url = isCmd ? `/api/cli#${item.replace(' ', '-')}` : null
-                    const isAbstract = isString(item) && /^\[(.+)\]$/.test(item)
-                    const itemClassNames = classNames(classes['cli-arg'], {
-                        [classes['cli-arg-highlight']]: isCmd,
-                        [classes['cli-arg-emphasis']]: isAbstract,
-                    })
-                    const text = isAbstract ? item.slice(1, -1) : item
-                    return (
-                        <Fragment key={j}>
-                            {j !== 0 && ' '}
-                            <span className={itemClassNames}>
-                                <OptionalLink hidden hideIcon to={url}>
-                                    {text}
-                                </OptionalLink>
-                            </span>
-                        </Fragment>
-                    )
-                })}
-            </Fragment>
-        )
     }
-    const htmlLine = replacePrompt(highlightCode('bash', line), '$')
+
-    return htmlToReact(htmlLine)
+    if (typeof children === 'string') {
+        return [children]
+    }
+
+    if (children.props) {
+        return flattenReact(children.props.children)
+    }
+
+    return children.flatMap(flattenReact)
 }
 
-function formatCode(html, lang, prompt) {
+const checkoutForComment = (line) => {
-    if (lang === 'cli') {
+    const lineParts = line.split(' # ')
-        const lines = html
+
-            .trim()
+    if (lineParts.length !== 2) {
-            .split('\n')
+        return line
-            .map((line) =>
-                line
-                    .split(' | ')
-                    .map((l, i) => convertLine(l, i))
-                    .map((l, j) => (
-                        <Fragment key={j}>
-                            {j !== 0 && <span> | </span>}
-                            {l}
-                        </Fragment>
-                    ))
-            )
-        return lines.map((line, i) => (
-            <Fragment key={i}>
-                {i !== 0 && <br />}
-                {line}
-            </Fragment>
-        ))
     }
-    const result = html
+
-        .split('\n')
+    return (
-        .map((line, i) => {
+        <>
-            let newLine = prompt ? replacePrompt(line, prompt, i === 0) : line
+            {lineParts[0]}
-            if (lang === 'diff' && !line.startsWith('<')) {
+            {` `}
-                newLine = highlightCode('python', line)
+            <span class="token comment">
-            }
+                {`# `}
-            return newLine
+                {lineParts[1]}
-        })
+            </span>
-        .join('\n')
+        </>
-    return htmlToReact(result)
+    )
+}
+
+const convertLine = (line, prompt) => {
+    const lineFlat = flattenReact(line).join('')
+    if (!lineFlat.startsWith(`${prompt} `)) {
+        return line
+    }
+
+    const lineWithoutPrompt = lineFlat.slice(prompt.length + 1)
+
+    const cliRegex = /^python -m spacy/
+
+    if (!cliRegex.test(lineWithoutPrompt)) {
+        return <span data-prompt={prompt}>{checkoutForComment(lineWithoutPrompt)}</span>
+    }
+
+    const text = lineWithoutPrompt.replace(cliRegex, '')
+    const args = parseArgs(text)
+    const cmd = Object.keys(args).map((key, i) => {
+        const value = args[key]
+        return value === null || value === true || i === 0 ? key : `${key} ${value}`
+    })
+    return (
+        <span data-prompt={prompt}>
+            <span className={classes['cli-arg-subtle']}>python -m</span> <span>spacy</span>{' '}
+            {cmd.map((item, j) => {
+                const isCmd = j === 0
+                const url = isCmd ? `/api/cli#${item.replace(' ', '-')}` : null
+                const isAbstract = isString(item) && /^\[(.+)\]$/.test(item)
+                const itemClassNames = classNames(classes['cli-arg'], {
+                    [classes['cli-arg-highlight']]: isCmd,
+                    [classes['cli-arg-emphasis']]: isAbstract,
+                })
+                const text = isAbstract ? item.slice(1, -1) : item
+                return (
+                    <Fragment key={j}>
+                        {j !== 0 && ' '}
+                        <span className={itemClassNames}>
+                            <OptionalLink hidden hideIcon to={url}>
+                                {text}
+                            </OptionalLink>
+                        </span>
+                    </Fragment>
+                )
+            })}
+        </span>
+    )
+}
+
+const addLineHighlight = (children, highlight) => {
+    if (!highlight) {
+        return children
+    }
+    const listHighlight = rangeParser(highlight)
+
+    if (listHighlight.length === 0) {
+        return children
+    }
+
+    return children.map((child, index) => {
+        const isHighlight = listHighlight.includes(index + 1)
+        return (
+            <span
+                className={classNames({
+                    'gatsby-highlight-code-line': isHighlight,
+                })}
+                key={index}
+            >
+                {child}
+            </span>
+        )
+    })
+}
+
+const CodeHighlighted = ({ children, highlight }) => {
+    const [html, setHtml] = useState()
+
+    useEffect(
+        () =>
+            setHtml(
+                addLineHighlight(
+                    splitLines(children).map((line) => convertLine(line, '$')),
+                    highlight
+                )
+            ),
+        [children, highlight]
+    )
+
+    return <>{html}</>
 }
 
 export class Code extends React.Component {
@@ -255,16 +337,12 @@ export class Code extends React.Component {
             )
         }
 
-        const codeText = Array.isArray(children) ? children.join('') : children || ''
-        const highlightRange = highlight ? rangeParser.parse(highlight).filter((n) => n > 0) : []
-        const rawHtml = ['none', 'cli'].includes(lang)
-            ? codeText
-            : highlightCode(lang, codeText, highlightRange)
-        const html = formatCode(rawHtml, lang, prompt)
         return (
             <>
                 {title && <h4 className={classes['title']}>{title}</h4>}
-                <code className={codeClassNames}>{html}</code>
+                <code className={codeClassNames}>
+                    <CodeHighlighted highlight={highlight}>{children}</CodeHighlighted>
+                </code>
             </>
         )
     }