Merge branch 'develop' into spacy.io

bin/push-tag.sh

@@ -7,6 +7,7 @@ git diff-index --quiet HEAD
 
 git checkout $1
 git pull origin $1
+git push origin $1
 
 version=$(grep "__version__ = " spacy/about.py)
 version=${version/__version__ = }
@@ -15,4 +16,4 @@ version=${version/\'/}
 version=${version/\"/}
 version=${version/\"/}
 git tag "v$version"
-git push origin --tags
+git push origin "v$version" --tags

bin/train_word_vectors.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+from __future__ import print_function, unicode_literals, division
+
+import logging
+from pathlib import Path
+from collections import defaultdict
+from gensim.models import Word2Vec
+from preshed.counter import PreshCounter
+import plac
+import spacy
+
+logger = logging.getLogger(__name__)
+
+
+class Corpus(object):
+    def __init__(self, directory, min_freq=10):
+        self.directory = directory
+        self.counts = PreshCounter()
+        self.strings = {}
+        self.min_freq = min_freq
+
+    def count_doc(self, doc):
+        # Get counts for this document
+        for word in doc:
+            self.counts.inc(word.orth, 1)
+        return len(doc)
+
+    def __iter__(self):
+        for text_loc in iter_dir(self.directory):
+            with text_loc.open("r", encoding="utf-8") as file_:
+                text = file_.read()
+            yield text
+
+
+def iter_dir(loc):
+    dir_path = Path(loc)
+    for fn_path in dir_path.iterdir():
+        if fn_path.is_dir():
+            for sub_path in fn_path.iterdir():
+                yield sub_path
+        else:
+            yield fn_path
+
+
+@plac.annotations(
+    lang=("ISO language code"),
+    in_dir=("Location of input directory"),
+    out_loc=("Location of output file"),
+    n_workers=("Number of workers", "option", "n", int),
+    size=("Dimension of the word vectors", "option", "d", int),
+    window=("Context window size", "option", "w", int),
+    min_count=("Min count", "option", "m", int),
+    negative=("Number of negative samples", "option", "g", int),
+    nr_iter=("Number of iterations", "option", "i", int),
+)
+def main(
+    lang,
+    in_dir,
+    out_loc,
+    negative=5,
+    n_workers=4,
+    window=5,
+    size=128,
+    min_count=10,
+    nr_iter=2,
+):
+    logging.basicConfig(
+        format="%(asctime)s : %(levelname)s : %(message)s", level=logging.INFO
+    )
+    model = Word2Vec(
+        size=size,
+        window=window,
+        min_count=min_count,
+        workers=n_workers,
+        sample=1e-5,
+        negative=negative,
+    )
+    nlp = spacy.blank(lang)
+    corpus = Corpus(in_dir)
+    total_words = 0
+    total_sents = 0
+    for text_no, text_loc in enumerate(iter_dir(corpus.directory)):
+        with text_loc.open("r", encoding="utf-8") as file_:
+            text = file_.read()
+        total_sents += text.count("\n")
+        doc = nlp(text)
+        total_words += corpus.count_doc(doc)
+        logger.info(
+            "PROGRESS: at batch #%i, processed %i words, keeping %i word types",
+            text_no,
+            total_words,
+            len(corpus.strings),
+        )
+    model.corpus_count = total_sents
+    model.raw_vocab = defaultdict(int)
+    for orth, freq in corpus.counts:
+        if freq >= min_count:
+            model.raw_vocab[nlp.vocab.strings[orth]] = freq
+    model.scale_vocab()
+    model.finalize_vocab()
+    model.iter = nr_iter
+    model.train(corpus)
+    model.save(out_loc)
+
+
+if __name__ == "__main__":
+    plac.call(main)

spacy/about.py

@@ -4,7 +4,7 @@
 # fmt: off
 
 __title__ = "spacy-nightly"
-__version__ = "2.1.0a10"
+__version__ = "2.1.0a13"
 __summary__ = "Industrial-strength Natural Language Processing (NLP) with Python and Cython"
 __uri__ = "https://spacy.io"
 __author__ = "Explosion AI"

spacy/displacy/__init__.py

@@ -161,7 +161,7 @@ def parse_deps(orig_doc, options={}):
                     "dir": "right",
                 }
             )
-    return {"words": words, "arcs": arcs}
+    return {"words": words, "arcs": arcs, "settings": get_doc_settings(orig_doc)}
 
 
 def parse_ents(doc, options={}):
@@ -177,7 +177,8 @@ def parse_ents(doc, options={}):
     if not ents:
         user_warning(Warnings.W006)
     title = doc.user_data.get("title", None) if hasattr(doc, "user_data") else None
-    return {"text": doc.text, "ents": ents, "title": title}
+    settings = get_doc_settings(doc)
+    return {"text": doc.text, "ents": ents, "title": title, "settings": settings}
 
 
 def set_render_wrapper(func):
@@ -195,3 +196,10 @@ def set_render_wrapper(func):
     if not hasattr(func, "__call__"):
         raise ValueError(Errors.E110.format(obj=type(func)))
     RENDER_WRAPPER = func
+
+
+def get_doc_settings(doc):
+    return {
+        "lang": doc.lang_,
+        "direction": doc.vocab.writing_system.get("direction", "ltr"),
+    }

spacy/displacy/render.py

@@ -3,10 +3,13 @@ from __future__ import unicode_literals
 
 import uuid
 
-from .templates import TPL_DEP_SVG, TPL_DEP_WORDS, TPL_DEP_ARCS
+from .templates import TPL_DEP_SVG, TPL_DEP_WORDS, TPL_DEP_ARCS, TPL_ENTS
-from .templates import TPL_ENT, TPL_ENTS, TPL_FIGURE, TPL_TITLE, TPL_PAGE
+from .templates import TPL_ENT, TPL_ENT_RTL, TPL_FIGURE, TPL_TITLE, TPL_PAGE
 from ..util import minify_html, escape_html
 
+DEFAULT_LANG = "en"
+DEFAULT_DIR = "ltr"
+
 
 class DependencyRenderer(object):
     """Render dependency parses as SVGs."""
@@ -30,6 +33,8 @@ class DependencyRenderer(object):
         self.color = options.get("color", "#000000")
         self.bg = options.get("bg", "#ffffff")
         self.font = options.get("font", "Arial")
+        self.direction = DEFAULT_DIR
+        self.lang = DEFAULT_LANG
 
     def render(self, parsed, page=False, minify=False):
         """Render complete markup.
@@ -42,13 +47,19 @@ class DependencyRenderer(object):
         # Create a random ID prefix to make sure parses don't receive the
         # same ID, even if they're identical
         id_prefix = uuid.uuid4().hex
-        rendered = [
+        rendered = []
-            self.render_svg("{}-{}".format(id_prefix, i), p["words"], p["arcs"])
+        for i, p in enumerate(parsed):
-            for i, p in enumerate(parsed)
+            if i == 0:
-        ]
+                self.direction = p["settings"].get("direction", DEFAULT_DIR)
+                self.lang = p["settings"].get("lang", DEFAULT_LANG)
+            render_id = "{}-{}".format(id_prefix, i)
+            svg = self.render_svg(render_id, p["words"], p["arcs"])
+            rendered.append(svg)
         if page:
             content = "".join([TPL_FIGURE.format(content=svg) for svg in rendered])
-            markup = TPL_PAGE.format(content=content)
+            markup = TPL_PAGE.format(
+                content=content, lang=self.lang, dir=self.direction
+            )
         else:
             markup = "".join(rendered)
         if minify:
@@ -83,6 +94,8 @@ class DependencyRenderer(object):
             bg=self.bg,
             font=self.font,
             content=content,
+            dir=self.direction,
+            lang=self.lang,
         )
 
     def render_word(self, text, tag, i):
@@ -95,11 +108,13 @@ class DependencyRenderer(object):
         """
         y = self.offset_y + self.word_spacing
         x = self.offset_x + i * self.distance
+        if self.direction == "rtl":
+            x = self.width - x
         html_text = escape_html(text)
         return TPL_DEP_WORDS.format(text=html_text, tag=tag, x=x, y=y)
 
     def render_arrow(self, label, start, end, direction, i):
-        """Render indivicual arrow.
+        """Render individual arrow.
 
         label (unicode): Dependency label.
         start (int): Index of start word.
@@ -110,6 +125,8 @@ class DependencyRenderer(object):
         """
         level = self.levels.index(end - start) + 1
         x_start = self.offset_x + start * self.distance + self.arrow_spacing
+        if self.direction == "rtl":
+            x_start = self.width - x_start
         y = self.offset_y
         x_end = (
             self.offset_x
@@ -117,6 +134,8 @@ class DependencyRenderer(object):
             + start * self.distance
             - self.arrow_spacing * (self.highest_level - level) / 4
         )
+        if self.direction == "rtl":
+            x_end = self.width - x_end
         y_curve = self.offset_y - level * self.distance / 2
         if self.compact:
             y_curve = self.offset_y - level * self.distance / 6
@@ -124,12 +143,14 @@ class DependencyRenderer(object):
             y_curve = -self.distance
         arrowhead = self.get_arrowhead(direction, x_start, y, x_end)
         arc = self.get_arc(x_start, y, y_curve, x_end)
+        label_side = "right" if self.direction == "rtl" else "left"
         return TPL_DEP_ARCS.format(
             id=self.id,
             i=i,
             stroke=self.arrow_stroke,
             head=arrowhead,
             label=label,
+            label_side=label_side,
             arc=arc,
         )
 
@@ -219,6 +240,8 @@ class EntityRenderer(object):
         self.default_color = "#ddd"
         self.colors = colors
         self.ents = options.get("ents", None)
+        self.direction = DEFAULT_DIR
+        self.lang = DEFAULT_LANG
 
     def render(self, parsed, page=False, minify=False):
         """Render complete markup.
@@ -228,12 +251,15 @@ class EntityRenderer(object):
         minify (bool): Minify HTML markup.
         RETURNS (unicode): Rendered HTML markup.
         """
-        rendered = [
+        rendered = []
-            self.render_ents(p["text"], p["ents"], p.get("title", None)) for p in parsed
+        for i, p in enumerate(parsed):
-        ]
+            if i == 0:
+                self.direction = p["settings"].get("direction", DEFAULT_DIR)
+                self.lang = p["settings"].get("lang", DEFAULT_LANG)
+            rendered.append(self.render_ents(p["text"], p["ents"], p["title"]))
         if page:
             docs = "".join([TPL_FIGURE.format(content=doc) for doc in rendered])
-            markup = TPL_PAGE.format(content=docs)
+            markup = TPL_PAGE.format(content=docs, lang=self.lang, dir=self.direction)
         else:
             markup = "".join(rendered)
         if minify:
@@ -261,12 +287,16 @@ class EntityRenderer(object):
                     markup += "</br>"
             if self.ents is None or label.upper() in self.ents:
                 color = self.colors.get(label.upper(), self.default_color)
-                markup += TPL_ENT.format(label=label, text=entity, bg=color)
+                ent_settings = {"label": label, "text": entity, "bg": color}
+                if self.direction == "rtl":
+                    markup += TPL_ENT_RTL.format(**ent_settings)
+                else:
+                    markup += TPL_ENT.format(**ent_settings)
             else:
                 markup += entity
             offset = end
         markup += escape_html(text[offset:])
-        markup = TPL_ENTS.format(content=markup, colors=self.colors)
+        markup = TPL_ENTS.format(content=markup, dir=self.direction)
         if title:
             markup = TPL_TITLE.format(title=title) + markup
         return markup

spacy/displacy/templates.py

@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 # Jupyter to render it properly in a cell
 
 TPL_DEP_SVG = """
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" id="{id}" class="displacy" width="{width}" height="{height}" style="max-width: none; height: {height}px; color: {color}; background: {bg}; font-family: {font}">{content}</svg>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:lang="{lang}" id="{id}" class="displacy" width="{width}" height="{height}" direction="{dir}" style="max-width: none; height: {height}px; color: {color}; background: {bg}; font-family: {font}; direction: {dir}">{content}</svg>
 """
 
 
@@ -22,7 +22,7 @@ TPL_DEP_ARCS = """
 <g class="displacy-arrow">
     <path class="displacy-arc" id="arrow-{id}-{i}" stroke-width="{stroke}px" d="{arc}" fill="none" stroke="currentColor"/>
     <text dy="1.25em" style="font-size: 0.8em; letter-spacing: 1px">
-        <textPath xlink:href="#arrow-{id}-{i}" class="displacy-label" startOffset="50%" fill="currentColor" text-anchor="middle">{label}</textPath>
+        <textPath xlink:href="#arrow-{id}-{i}" class="displacy-label" startOffset="50%" side="{label_side}" fill="currentColor" text-anchor="middle">{label}</textPath>
     </text>
     <path class="displacy-arrowhead" d="{head}" fill="currentColor"/>
 </g>
@@ -39,7 +39,7 @@ TPL_TITLE = """
 
 
 TPL_ENTS = """
-<div class="entities" style="line-height: 2.5">{content}</div>
+<div class="entities" style="line-height: 2.5; direction: {dir}">{content}</div>
 """
 
 
@@ -50,14 +50,21 @@ TPL_ENT = """
 </mark>
 """
 
+TPL_ENT_RTL = """
+<mark class="entity" style="background: {bg}; padding: 0.45em 0.6em; margin: 0 0.25em; line-height: 1; border-radius: 0.35em;">
+    {text}
+    <span style="font-size: 0.8em; font-weight: bold; line-height: 1; border-radius: 0.35em; text-transform: uppercase; vertical-align: middle; margin-right: 0.5rem">{label}</span>
+</mark>
+"""
+
 
 TPL_PAGE = """
 <!DOCTYPE html>
-<html>
+<html lang="{lang}">
     <head>
         <title>displaCy</title>
     </head>
 
-    <body style="font-size: 16px; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'; padding: 4rem 2rem;">{content}</body>
+    <body style="font-size: 16px; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif, 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'; padding: 4rem 2rem; direction: {dir}">{content}</body>
 </html>
 """

spacy/lang/ar/__init__.py

@@ -23,6 +23,7 @@ class ArabicDefaults(Language.Defaults):
     tokenizer_exceptions = update_exc(BASE_EXCEPTIONS, TOKENIZER_EXCEPTIONS)
     stop_words = STOP_WORDS
     suffixes = TOKENIZER_SUFFIXES
+    writing_system = {"direction": "rtl", "has_case": False, "has_letters": True}
 
 
 class Arabic(Language):

spacy/lang/fa/__init__.py

@@ -27,6 +27,7 @@ class PersianDefaults(Language.Defaults):
     stop_words = STOP_WORDS
     tag_map = TAG_MAP
     suffixes = TOKENIZER_SUFFIXES
+    writing_system = {"direction": "rtl", "has_case": False, "has_letters": True}
 
 
 class Persian(Language):

spacy/lang/he/__init__.py

@@ -14,6 +14,7 @@ class HebrewDefaults(Language.Defaults):
     lex_attr_getters[LANG] = lambda text: "he"
     tokenizer_exceptions = update_exc(BASE_EXCEPTIONS)
     stop_words = STOP_WORDS
+    writing_system = {"direction": "rtl", "has_case": False, "has_letters": True}
 
 
 class Hebrew(Language):

spacy/lang/ja/__init__.py

@@ -8,15 +8,13 @@ from .stop_words import STOP_WORDS
 from .tag_map import TAG_MAP
 from ...attrs import LANG
 from ...language import Language
-from ...tokens import Doc, Token
+from ...tokens import Doc
+from ...compat import copy_reg
 from ...util import DummyTokenizer
 
 
 ShortUnitWord = namedtuple("ShortUnitWord", ["surface", "lemma", "pos"])
 
-# TODO: Is this the right place for this?
-Token.set_extension("mecab_tag", default=None)
-
 
 def try_mecab_import():
     """Mecab is required for Japanese support, so check for it.
@@ -81,10 +79,12 @@ class JapaneseTokenizer(DummyTokenizer):
         words = [x.surface for x in dtokens]
         spaces = [False] * len(words)
         doc = Doc(self.vocab, words=words, spaces=spaces)
+        mecab_tags = []
         for token, dtoken in zip(doc, dtokens):
-            token._.mecab_tag = dtoken.pos
+            mecab_tags.append(dtoken.pos)
             token.tag_ = resolve_pos(dtoken)
             token.lemma_ = dtoken.lemma
+        doc.user_data["mecab_tags"] = mecab_tags
         return doc
 
 
@@ -93,6 +93,7 @@ class JapaneseDefaults(Language.Defaults):
     lex_attr_getters[LANG] = lambda _text: "ja"
     stop_words = STOP_WORDS
     tag_map = TAG_MAP
+    writing_system = {"direction": "ltr", "has_case": False, "has_letters": False}
 
     @classmethod
     def create_tokenizer(cls, nlp=None):
@@ -107,4 +108,11 @@ class Japanese(Language):
         return self.tokenizer(text)
 
 
+def pickle_japanese(instance):
+    return Japanese, tuple()
+
+
+copy_reg.pickle(Japanese, pickle_japanese)
+
+
 __all__ = ["Japanese"]

spacy/lang/zh/__init__.py

@@ -14,6 +14,7 @@ class ChineseDefaults(Language.Defaults):
     use_jieba = True
     tokenizer_exceptions = BASE_EXCEPTIONS
     stop_words = STOP_WORDS
+    writing_system = {"direction": "ltr", "has_case": False, "has_letters": False}
 
 
 class Chinese(Language):

spacy/language.py

@@ -94,6 +94,7 @@ class BaseDefaults(object):
     morph_rules = {}
     lex_attr_getters = LEX_ATTRS
     syntax_iterators = {}
+    writing_system = {"direction": "ltr", "has_case": True, "has_letters": True}
 
 
 class Language(object):
@@ -899,6 +900,11 @@ class DisabledPipes(list):
 
 
 def _pipe(func, docs, kwargs):
+    # We added some args for pipe that __call__ doesn't expect.
+    kwargs = dict(kwargs)
+    for arg in ["n_threads", "batch_size"]:
+        if arg in kwargs:
+            kwargs.pop(arg)
     for doc in docs:
         doc = func(doc, **kwargs)
         yield doc

spacy/lexeme.pyx

@@ -161,15 +161,15 @@ cdef class Lexeme:
         Lexeme.c_from_bytes(self.c, lex_data)
         self.orth = self.c.orth
 
-    property has_vector:
+    @property
+    def has_vector(self):
         """RETURNS (bool): Whether a word vector is associated with the object.
         """
-        def __get__(self):
         return self.vocab.has_vector(self.c.orth)
 
-    property vector_norm:
+    @property
+    def vector_norm(self):
         """RETURNS (float): The L2 norm of the vector representation."""
-        def __get__(self):
         vector = self.vector
         return numpy.sqrt((vector**2).sum())
 
@@ -209,16 +209,16 @@ cdef class Lexeme:
         def __set__(self, float sentiment):
             self.c.sentiment = sentiment
 
-    property orth_:
+    @property
+    def orth_(self):
         """RETURNS (unicode): The original verbatim text of the lexeme
             (identical to `Lexeme.text`). Exists mostly for consistency with
             the other attributes."""
-        def __get__(self):
         return self.vocab.strings[self.c.orth]
 
-    property text:
+    @property
+    def text(self):
         """RETURNS (unicode): The original verbatim text of the lexeme."""
-        def __get__(self):
         return self.orth_
 
     property lower:

spacy/syntax/arc_eager.pyx

@@ -369,8 +369,8 @@ cdef class ArcEager(TransitionSystem):
         actions[LEFT].setdefault('dep', 0)
         return actions
 
-    property action_types:
+    @property
-        def __get__(self):
+    def action_types(self):
         return (SHIFT, REDUCE, LEFT, RIGHT, BREAK)
 
     def get_cost(self, StateClass state, GoldParse gold, action):

spacy/syntax/ner.pyx

@@ -80,8 +80,8 @@ cdef class BiluoPushDown(TransitionSystem):
                             actions[action][label] += 1
         return actions
 
-    property action_types:
+    @property
-        def __get__(self):
+    def action_types(self):
         return (BEGIN, IN, LAST, UNIT, OUT)
 
     def move_name(self, int move, attr_t label):

spacy/tests/doc/test_doc_api.py

@@ -272,3 +272,9 @@ def test_doc_is_nered(en_vocab):
     # Test serialization
     new_doc = Doc(en_vocab).from_bytes(doc.to_bytes())
     assert new_doc.is_nered
+
+
+def test_doc_lang(en_vocab):
+    doc = Doc(en_vocab, words=["Hello", "world"])
+    assert doc.lang_ == "en"
+    assert doc.lang == en_vocab.strings["en"]

spacy/tests/doc/test_token_api.py

@@ -199,3 +199,31 @@ def test_token0_has_sent_start_true():
     assert doc[0].is_sent_start is True
     assert doc[1].is_sent_start is None
     assert not doc.is_sentenced
+
+
+def test_token_api_conjuncts_chain(en_vocab):
+    words = "The boy and the girl and the man went .".split()
+    heads = [1, 7, -1, 1, -3, -1, 1, -3, 0, -1]
+    deps = ["det", "nsubj", "cc", "det", "conj", "cc", "det", "conj", "ROOT", "punct"]
+    doc = get_doc(en_vocab, words=words, heads=heads, deps=deps)
+    assert [w.text for w in doc[1].conjuncts] == ["girl", "man"]
+    assert [w.text for w in doc[4].conjuncts] == ["boy", "man"]
+    assert [w.text for w in doc[7].conjuncts] == ["boy", "girl"]
+
+
+def test_token_api_conjuncts_simple(en_vocab):
+    words = "They came and went .".split()
+    heads = [1, 0, -1, -2, -1]
+    deps = ["nsubj", "ROOT", "cc", "conj"]
+    doc = get_doc(en_vocab, words=words, heads=heads, deps=deps)
+    assert [w.text for w in doc[1].conjuncts] == ["went"]
+    assert [w.text for w in doc[3].conjuncts] == ["came"]
+
+
+def test_token_api_non_conjuncts(en_vocab):
+    words = "They came .".split()
+    heads = [1, 0, -1]
+    deps = ["nsubj", "ROOT", "punct"]
+    doc = get_doc(en_vocab, words=words, heads=heads, deps=deps)
+    assert [w.text for w in doc[0].conjuncts] == []
+    assert [w.text for w in doc[1].conjuncts] == []

spacy/tests/regression/test_issue2001-2500.py

@@ -7,7 +7,6 @@ from spacy.tokens import Doc
 from spacy.displacy import render
 from spacy.gold import iob_to_biluo
 from spacy.lang.it import Italian
-import numpy
 from spacy.lang.en import English
 
 from ..util import add_vecs_to_vocab, get_doc

spacy/tests/test_displacy.py

@@ -0,0 +1,90 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import pytest
+from spacy import displacy
+from spacy.tokens import Span
+from spacy.lang.fa import Persian
+
+from .util import get_doc
+
+
+def test_displacy_parse_ents(en_vocab):
+    """Test that named entities on a Doc are converted into displaCy's format."""
+    doc = get_doc(en_vocab, words=["But", "Google", "is", "starting", "from", "behind"])
+    doc.ents = [Span(doc, 1, 2, label=doc.vocab.strings["ORG"])]
+    ents = displacy.parse_ents(doc)
+    assert isinstance(ents, dict)
+    assert ents["text"] == "But Google is starting from behind "
+    assert ents["ents"] == [{"start": 4, "end": 10, "label": "ORG"}]
+
+
+def test_displacy_parse_deps(en_vocab):
+    """Test that deps and tags on a Doc are converted into displaCy's format."""
+    words = ["This", "is", "a", "sentence"]
+    heads = [1, 0, 1, -2]
+    pos = ["DET", "VERB", "DET", "NOUN"]
+    tags = ["DT", "VBZ", "DT", "NN"]
+    deps = ["nsubj", "ROOT", "det", "attr"]
+    doc = get_doc(en_vocab, words=words, heads=heads, pos=pos, tags=tags, deps=deps)
+    deps = displacy.parse_deps(doc)
+    assert isinstance(deps, dict)
+    assert deps["words"] == [
+        {"text": "This", "tag": "DET"},
+        {"text": "is", "tag": "VERB"},
+        {"text": "a", "tag": "DET"},
+        {"text": "sentence", "tag": "NOUN"},
+    ]
+    assert deps["arcs"] == [
+        {"start": 0, "end": 1, "label": "nsubj", "dir": "left"},
+        {"start": 2, "end": 3, "label": "det", "dir": "left"},
+        {"start": 1, "end": 3, "label": "attr", "dir": "right"},
+    ]
+
+
+def test_displacy_spans(en_vocab):
+    """Test that displaCy can render Spans."""
+    doc = get_doc(en_vocab, words=["But", "Google", "is", "starting", "from", "behind"])
+    doc.ents = [Span(doc, 1, 2, label=doc.vocab.strings["ORG"])]
+    html = displacy.render(doc[1:4], style="ent")
+    assert html.startswith("<div")
+
+
+def test_displacy_raises_for_wrong_type(en_vocab):
+    with pytest.raises(ValueError):
+        displacy.render("hello world")
+
+
+def test_displacy_rtl():
+    # Source: http://www.sobhe.ir/hazm/ – is this correct?
+    words = ["ما", "بسیار", "کتاب", "می\u200cخوانیم"]
+    # These are (likely) wrong, but it's just for testing
+    pos = ["PRO", "ADV", "N_PL", "V_SUB"]  # needs to match lang.fa.tag_map
+    deps = ["foo", "bar", "foo", "baz"]
+    heads = [1, 0, 1, -2]
+    nlp = Persian()
+    doc = get_doc(nlp.vocab, words=words, pos=pos, tags=pos, heads=heads, deps=deps)
+    doc.ents = [Span(doc, 1, 3, label="TEST")]
+    html = displacy.render(doc, page=True, style="dep")
+    assert "direction: rtl" in html
+    assert 'direction="rtl"' in html
+    assert 'lang="{}"'.format(nlp.lang) in html
+    html = displacy.render(doc, page=True, style="ent")
+    assert "direction: rtl" in html
+    assert 'lang="{}"'.format(nlp.lang) in html
+
+
+def test_displacy_render_wrapper(en_vocab):
+    """Test that displaCy accepts custom rendering wrapper."""
+
+    def wrapper(html):
+        return "TEST" + html + "TEST"
+
+    displacy.set_render_wrapper(wrapper)
+    doc = get_doc(en_vocab, words=["But", "Google", "is", "starting", "from", "behind"])
+    doc.ents = [Span(doc, 1, 2, label=doc.vocab.strings["ORG"])]
+    html = displacy.render(doc, style="ent")
+    assert html.startswith("TEST<div")
+    assert html.endswith("/div>TEST")
+    # Restore
+    displacy.set_render_wrapper(lambda html: html)

spacy/tests/test_misc.py

@@ -4,13 +4,9 @@ from __future__ import unicode_literals
 import pytest
 from pathlib import Path
 from spacy import util
-from spacy import displacy
 from spacy import prefer_gpu, require_gpu
-from spacy.tokens import Span
 from spacy._ml import PrecomputableAffine
 
-from .util import get_doc
-
 
 @pytest.mark.parametrize("text", ["hello/world", "hello world"])
 def test_util_ensure_path_succeeds(text):
@@ -31,66 +27,6 @@ def test_util_get_package_path(package):
     assert isinstance(path, Path)
 
 
-def test_displacy_parse_ents(en_vocab):
-    """Test that named entities on a Doc are converted into displaCy's format."""
-    doc = get_doc(en_vocab, words=["But", "Google", "is", "starting", "from", "behind"])
-    doc.ents = [Span(doc, 1, 2, label=doc.vocab.strings["ORG"])]
-    ents = displacy.parse_ents(doc)
-    assert isinstance(ents, dict)
-    assert ents["text"] == "But Google is starting from behind "
-    assert ents["ents"] == [{"start": 4, "end": 10, "label": "ORG"}]
-
-
-def test_displacy_parse_deps(en_vocab):
-    """Test that deps and tags on a Doc are converted into displaCy's format."""
-    words = ["This", "is", "a", "sentence"]
-    heads = [1, 0, 1, -2]
-    pos = ["DET", "VERB", "DET", "NOUN"]
-    tags = ["DT", "VBZ", "DT", "NN"]
-    deps = ["nsubj", "ROOT", "det", "attr"]
-    doc = get_doc(en_vocab, words=words, heads=heads, pos=pos, tags=tags, deps=deps)
-    deps = displacy.parse_deps(doc)
-    assert isinstance(deps, dict)
-    assert deps["words"] == [
-        {"text": "This", "tag": "DET"},
-        {"text": "is", "tag": "VERB"},
-        {"text": "a", "tag": "DET"},
-        {"text": "sentence", "tag": "NOUN"},
-    ]
-    assert deps["arcs"] == [
-        {"start": 0, "end": 1, "label": "nsubj", "dir": "left"},
-        {"start": 2, "end": 3, "label": "det", "dir": "left"},
-        {"start": 1, "end": 3, "label": "attr", "dir": "right"},
-    ]
-
-
-def test_displacy_spans(en_vocab):
-    """Test that displaCy can render Spans."""
-    doc = get_doc(en_vocab, words=["But", "Google", "is", "starting", "from", "behind"])
-    doc.ents = [Span(doc, 1, 2, label=doc.vocab.strings["ORG"])]
-    html = displacy.render(doc[1:4], style="ent")
-    assert html.startswith("<div")
-
-
-def test_displacy_render_wrapper(en_vocab):
-    """Test that displaCy accepts custom rendering wrapper."""
-
-    def wrapper(html):
-        return "TEST" + html + "TEST"
-
-    displacy.set_render_wrapper(wrapper)
-    doc = get_doc(en_vocab, words=["But", "Google", "is", "starting", "from", "behind"])
-    doc.ents = [Span(doc, 1, 2, label=doc.vocab.strings["ORG"])]
-    html = displacy.render(doc, style="ent")
-    assert html.startswith("TEST<div")
-    assert html.endswith("/div>TEST")
-
-
-def test_displacy_raises_for_wrong_type(en_vocab):
-    with pytest.raises(ValueError):
-        displacy.render("hello world")
-
-
 def test_PrecomputableAffine(nO=4, nI=5, nF=3, nP=2):
     model = PrecomputableAffine(nO=nO, nI=nI, nF=nF, nP=nP)
     assert model.W.shape == (nF, nO, nP, nI)

spacy/tests/vocab_vectors/test_vocab_api.py

@@ -45,3 +45,8 @@ def test_vocab_api_contains(en_vocab, text):
     _ = en_vocab[text]  # noqa: F841
     assert text in en_vocab
     assert "LKsdjvlsakdvlaksdvlkasjdvljasdlkfvm" not in en_vocab
+
+
+def test_vocab_writing_system(en_vocab):
+    assert en_vocab.writing_system["direction"] == "ltr"
+    assert en_vocab.writing_system["has_case"] is True

spacy/tokens/doc.pyx

@@ -384,7 +384,8 @@ cdef class Doc:
         xp = get_array_module(vector)
         return xp.dot(vector, other.vector) / (self.vector_norm * other.vector_norm)
 
-    property has_vector:
+    @property
+    def has_vector(self):
         """A boolean value indicating whether a word vector is associated with
         the object.
 
@@ -392,7 +393,6 @@ cdef class Doc:
 
         DOCS: https://spacy.io/api/doc#has_vector
         """
-        def __get__(self):
         if "has_vector" in self.user_hooks:
             return self.user_hooks["has_vector"](self)
         elif self.vocab.vectors.data.size:
@@ -453,21 +453,21 @@ cdef class Doc:
         def __set__(self, value):
             self._vector_norm = value
 
-    property text:
+    @property
+    def text(self):
         """A unicode representation of the document text.
 
         RETURNS (unicode): The original verbatim text of the document.
         """
-        def __get__(self):
         return "".join(t.text_with_ws for t in self)
 
-    property text_with_ws:
+    @property
+    def text_with_ws(self):
         """An alias of `Doc.text`, provided for duck-type compatibility with
         `Span` and `Token`.
 
         RETURNS (unicode): The original verbatim text of the document.
         """
-        def __get__(self):
         return self.text
 
     property ents:
@@ -545,7 +545,8 @@ cdef class Doc:
                     # Set start as B
                     self.c[start].ent_iob = 3
 
-    property noun_chunks:
+    @property
+    def noun_chunks(self):
         """Iterate over the base noun phrases in the document. Yields base
         noun-phrase #[code Span] objects, if the document has been
         syntactically parsed. A base noun phrase, or "NP chunk", is a noun
@@ -557,7 +558,6 @@ cdef class Doc:
 
         DOCS: https://spacy.io/api/doc#noun_chunks
         """
-        def __get__(self):
         if not self.is_parsed:
             raise ValueError(Errors.E029)
         # Accumulate the result before beginning to iterate over it. This
@@ -572,7 +572,8 @@ cdef class Doc:
         for span in spans:
             yield span
 
-    property sents:
+    @property
+    def sents(self):
         """Iterate over the sentences in the document. Yields sentence `Span`
         objects. Sentence spans have no label. To improve accuracy on informal
         texts, spaCy calculates sentence boundaries from the syntactic
@@ -583,7 +584,6 @@ cdef class Doc:
 
         DOCS: https://spacy.io/api/doc#sents
         """
-        def __get__(self):
         if not self.is_sentenced:
             raise ValueError(Errors.E030)
         if "sents" in self.user_hooks:
@@ -597,6 +597,16 @@ cdef class Doc:
             if start != self.length:
                 yield Span(self, start, self.length)
 
+    @property
+    def lang(self):
+        """RETURNS (uint64): ID of the language of the doc's vocabulary."""
+        return self.vocab.strings[self.vocab.lang]
+
+    @property
+    def lang_(self):
+        """RETURNS (unicode): Language of the doc's vocabulary, e.g. 'en'."""
+        return self.vocab.lang
+
     cdef int push_back(self, LexemeOrToken lex_or_tok, bint has_space) except -1:
         if self.length == 0:
             # Flip these to false when we see the first token.

spacy/tokens/span.pyx

@@ -322,14 +322,14 @@ cdef class Span:
             self.start = start
             self.end = end + 1
 
-    property vocab:
+    @property
+    def vocab(self):
         """RETURNS (Vocab): The Span's Doc's vocab."""
-        def __get__(self):
         return self.doc.vocab
 
-    property sent:
+    @property
+    def sent(self):
         """RETURNS (Span): The sentence span that the span is a part of."""
-        def __get__(self):
         if "sent" in self.doc.user_span_hooks:
             return self.doc.user_span_hooks["sent"](self)
         # This should raise if not parsed / no custom sentence boundaries
@@ -361,7 +361,8 @@ cdef class Span:
                     break
             return self.doc[start:end]
 
-    property ents:
+    @property
+    def ents(self):
         """The named entities in the span. Returns a tuple of named entity
         `Span` objects, if the entity recognizer has been applied.
 
@@ -369,14 +370,14 @@ cdef class Span:
 
         DOCS: https://spacy.io/api/span#ents
         """
-        def __get__(self):
         ents = []
         for ent in self.doc.ents:
             if ent.start >= self.start and ent.end <= self.end:
                 ents.append(ent)
         return ents
 
-    property has_vector:
+    @property
+    def has_vector(self):
         """A boolean value indicating whether a word vector is associated with
         the object.
 
@@ -384,7 +385,6 @@ cdef class Span:
 
         DOCS: https://spacy.io/api/span#has_vector
         """
-        def __get__(self):
         if "has_vector" in self.doc.user_span_hooks:
             return self.doc.user_span_hooks["has_vector"](self)
         elif self.vocab.vectors.data.size > 0:
@@ -394,7 +394,8 @@ cdef class Span:
         else:
             return False
 
-    property vector:
+    @property
+    def vector(self):
         """A real-valued meaning representation. Defaults to an average of the
         token vectors.
 
@@ -403,21 +404,20 @@ cdef class Span:
 
         DOCS: https://spacy.io/api/span#vector
         """
-        def __get__(self):
         if "vector" in self.doc.user_span_hooks:
             return self.doc.user_span_hooks["vector"](self)
         if self._vector is None:
             self._vector = sum(t.vector for t in self) / len(self)
         return self._vector
 
-    property vector_norm:
+    @property
+    def vector_norm(self):
         """The L2 norm of the span's vector representation.
 
         RETURNS (float): The L2 norm of the vector representation.
 
         DOCS: https://spacy.io/api/span#vector_norm
         """
-        def __get__(self):
         if "vector_norm" in self.doc.user_span_hooks:
             return self.doc.user_span_hooks["vector"](self)
         cdef float value
@@ -429,35 +429,36 @@ cdef class Span:
             self._vector_norm = sqrt(norm) if norm != 0 else 0
         return self._vector_norm
 
-    property sentiment:
+    @property
+    def sentiment(self):
         """RETURNS (float): A scalar value indicating the positivity or
             negativity of the span.
         """
-        def __get__(self):
         if "sentiment" in self.doc.user_span_hooks:
             return self.doc.user_span_hooks["sentiment"](self)
         else:
             return sum([token.sentiment for token in self]) / len(self)
 
-    property text:
+    @property
+    def text(self):
         """RETURNS (unicode): The original verbatim text of the span."""
-        def __get__(self):
         text = self.text_with_ws
         if self[-1].whitespace_:
             text = text[:-1]
         return text
 
-    property text_with_ws:
+    @property
+    def text_with_ws(self):
         """The text content of the span with a trailing whitespace character if
         the last token has one.
 
         RETURNS (unicode): The text content of the span (with trailing
             whitespace).
         """
-        def __get__(self):
         return "".join([t.text_with_ws for t in self])
 
-    property noun_chunks:
+    @property
+    def noun_chunks(self):
         """Yields base noun-phrase `Span` objects, if the document has been
         syntactically parsed. A base noun phrase, or "NP chunk", is a noun
         phrase that does not permit other NPs to be nested within it – so no
@@ -468,7 +469,6 @@ cdef class Span:
 
         DOCS: https://spacy.io/api/span#noun_chunks
         """
-        def __get__(self):
         if not self.doc.is_parsed:
             raise ValueError(Errors.E029)
         # Accumulate the result before beginning to iterate over it. This
@@ -484,7 +484,8 @@ cdef class Span:
         for span in spans:
             yield span
 
-    property root:
+    @property
+    def root(self):
         """The token with the shortest path to the root of the
         sentence (or the root itself). If multiple tokens are equally
         high in the tree, the first token is taken.
@@ -493,7 +494,6 @@ cdef class Span:
 
         DOCS: https://spacy.io/api/span#root
         """
-        def __get__(self):
         self._recalculate_indices()
         if "root" in self.doc.user_span_hooks:
             return self.doc.user_span_hooks["root"](self)
@@ -527,7 +527,18 @@ cdef class Span:
         else:
             return self.doc[root]
 
-    property lefts:
+    @property
+    def conjuncts(self):
+        """Tokens that are conjoined to the span's root.
+
+        RETURNS (tuple): A tuple of Token objects.
+
+        DOCS: https://spacy.io/api/span#lefts
+        """
+        return self.root.conjuncts
+
+    @property
+    def lefts(self):
         """Tokens that are to the left of the span, whose head is within the
         `Span`.
 
@@ -535,13 +546,13 @@ cdef class Span:
 
         DOCS: https://spacy.io/api/span#lefts
         """
-        def __get__(self):
         for token in reversed(self):  # Reverse, so we get tokens in order
             for left in token.lefts:
                 if left.i < self.start:
                     yield left
 
-    property rights:
+    @property
+    def rights(self):
         """Tokens that are to the right of the Span, whose head is within the
         `Span`.
 
@@ -549,13 +560,13 @@ cdef class Span:
 
         DOCS: https://spacy.io/api/span#rights
         """
-        def __get__(self):
         for token in self:
             for right in token.rights:
                 if right.i >= self.end:
                     yield right
 
-    property n_lefts:
+    @property
+    def n_lefts(self):
         """The number of tokens that are to the left of the span, whose
         heads are within the span.
 
@@ -564,10 +575,10 @@ cdef class Span:
 
         DOCS: https://spacy.io/api/span#n_lefts
         """
-        def __get__(self):
         return len(list(self.lefts))
 
-    property n_rights:
+    @property
+    def n_rights(self):
         """The number of tokens that are to the right of the span, whose
         heads are within the span.
 
@@ -576,17 +587,16 @@ cdef class Span:
 
         DOCS: https://spacy.io/api/span#n_rights
         """
-        def __get__(self):
         return len(list(self.rights))
 
-    property subtree:
+    @property
+    def subtree(self):
         """Tokens within the span and tokens which descend from them.
 
         YIELDS (Token): A token within the span, or a descendant from it.
 
         DOCS: https://spacy.io/api/span#subtree
         """
-        def __get__(self):
         for word in self.lefts:
             yield from word.subtree
         yield from self
@@ -609,32 +619,32 @@ cdef class Span:
         def __set__(self, hash_t key):
             raise NotImplementedError(TempErrors.T007.format(attr="ent_id_"))
 
-    property orth_:
+    @property
+    def orth_(self):
         """Verbatim text content (identical to `Span.text`). Exists mostly for
         consistency with other attributes.
 
         RETURNS (unicode): The span's text."""
-        def __get__(self):
         return self.text
 
-    property lemma_:
+    @property
+    def lemma_(self):
         """RETURNS (unicode): The span's lemma."""
-        def __get__(self):
         return " ".join([t.lemma_ for t in self]).strip()
 
-    property upper_:
+    @property
+    def upper_(self):
         """Deprecated. Use `Span.text.upper()` instead."""
-        def __get__(self):
         return "".join([t.text_with_ws.upper() for t in self]).strip()
 
-    property lower_:
+    @property
+    def lower_(self):
         """Deprecated. Use `Span.text.lower()` instead."""
-        def __get__(self):
         return "".join([t.text_with_ws.lower() for t in self]).strip()
 
-    property string:
+    @property
+    def string(self):
         """Deprecated: Use `Span.text_with_ws` instead."""
-        def __get__(self):
         return "".join([t.text_with_ws for t in self])
 
     property label_:

spacy/tokens/token.pyx

@@ -218,110 +218,110 @@ cdef class Token:
         xp = get_array_module(vector)
         return (xp.dot(vector, other.vector) / (self.vector_norm * other.vector_norm))
 
-    property lex_id:
+    @property
+    def lex_id(self):
         """RETURNS (int): Sequential ID of the token's lexical type."""
-        def __get__(self):
         return self.c.lex.id
 
-    property rank:
+    @property
+    def rank(self):
         """RETURNS (int): Sequential ID of the token's lexical type, used to
         index into tables, e.g. for word vectors."""
-        def __get__(self):
         return self.c.lex.id
 
-    property string:
+    @property
+    def string(self):
         """Deprecated: Use Token.text_with_ws instead."""
-        def __get__(self):
         return self.text_with_ws
 
-    property text:
+    @property
+    def text(self):
         """RETURNS (unicode): The original verbatim text of the token."""
-        def __get__(self):
         return self.orth_
 
-    property text_with_ws:
+    @property
+    def text_with_ws(self):
         """RETURNS (unicode): The text content of the span (with trailing
             whitespace).
         """
-        def __get__(self):
         cdef unicode orth = self.vocab.strings[self.c.lex.orth]
         if self.c.spacy:
             return orth + " "
         else:
             return orth
 
-    property prob:
+    @property
+    def prob(self):
         """RETURNS (float): Smoothed log probability estimate of token type."""
-        def __get__(self):
         return self.c.lex.prob
 
-    property sentiment:
+    @property
+    def sentiment(self):
         """RETURNS (float): A scalar value indicating the positivity or
             negativity of the token."""
-        def __get__(self):
         if "sentiment" in self.doc.user_token_hooks:
             return self.doc.user_token_hooks["sentiment"](self)
         return self.c.lex.sentiment
 
-    property lang:
+    @property
+    def lang(self):
         """RETURNS (uint64): ID of the language of the parent document's
             vocabulary.
         """
-        def __get__(self):
         return self.c.lex.lang
 
-    property idx:
+    @property
+    def idx(self):
         """RETURNS (int): The character offset of the token within the parent
             document.
         """
-        def __get__(self):
         return self.c.idx
 
-    property cluster:
+    @property
+    def cluster(self):
         """RETURNS (int): Brown cluster ID."""
-        def __get__(self):
         return self.c.lex.cluster
 
-    property orth:
+    @property
+    def orth(self):
         """RETURNS (uint64): ID of the verbatim text content."""
-        def __get__(self):
         return self.c.lex.orth
 
-    property lower:
+    @property
+    def lower(self):
         """RETURNS (uint64): ID of the lowercase token text."""
-        def __get__(self):
         return self.c.lex.lower
 
-    property norm:
+    @property
+    def norm(self):
         """RETURNS (uint64): ID of the token's norm, i.e. a normalised form of
             the token text. Usually set in the language's tokenizer exceptions
             or norm exceptions.
         """
-        def __get__(self):
         if self.c.norm == 0:
             return self.c.lex.norm
         else:
             return self.c.norm
 
-    property shape:
+    @property
+    def shape(self):
         """RETURNS (uint64): ID of the token's shape, a transform of the
             tokens's string, to show orthographic features (e.g. "Xxxx", "dd").
         """
-        def __get__(self):
         return self.c.lex.shape
 
-    property prefix:
+    @property
+    def prefix(self):
         """RETURNS (uint64): ID of a length-N substring from the start of the
             token. Defaults to `N=1`.
         """
-        def __get__(self):
         return self.c.lex.prefix
 
-    property suffix:
+    @property
+    def suffix(self):
         """RETURNS (uint64): ID of a length-N substring from the end of the
             token. Defaults to `N=3`.
         """
-        def __get__(self):
         return self.c.lex.suffix
 
     property lemma:
@@ -362,7 +362,8 @@ cdef class Token:
         def __set__(self, attr_t label):
             self.c.dep = label
 
-    property has_vector:
+    @property
+    def has_vector(self):
         """A boolean value indicating whether a word vector is associated with
         the object.
 
@@ -370,14 +371,14 @@ cdef class Token:
 
         DOCS: https://spacy.io/api/token#has_vector
         """
-        def __get__(self):
+        if "has_vector" in self.doc.user_token_hooks:
-            if 'has_vector' in self.doc.user_token_hooks:
             return self.doc.user_token_hooks["has_vector"](self)
         if self.vocab.vectors.size == 0 and self.doc.tensor.size != 0:
             return True
         return self.vocab.has_vector(self.c.lex.orth)
 
-    property vector:
+    @property
+    def vector(self):
         """A real-valued meaning representation.
 
         RETURNS (numpy.ndarray[ndim=1, dtype='float32']): A 1D numpy array
@@ -385,28 +386,28 @@ cdef class Token:
 
         DOCS: https://spacy.io/api/token#vector
         """
-        def __get__(self):
+        if "vector" in self.doc.user_token_hooks:
-            if 'vector' in self.doc.user_token_hooks:
             return self.doc.user_token_hooks["vector"](self)
         if self.vocab.vectors.size == 0 and self.doc.tensor.size != 0:
             return self.doc.tensor[self.i]
         else:
             return self.vocab.get_vector(self.c.lex.orth)
 
-    property vector_norm:
+    @property
+    def vector_norm(self):
         """The L2 norm of the token's vector representation.
 
         RETURNS (float): The L2 norm of the vector representation.
 
         DOCS: https://spacy.io/api/token#vector_norm
         """
-        def __get__(self):
+        if "vector_norm" in self.doc.user_token_hooks:
-            if 'vector_norm' in self.doc.user_token_hooks:
             return self.doc.user_token_hooks["vector_norm"](self)
         vector = self.vector
         return numpy.sqrt((vector ** 2).sum())
 
-    property n_lefts:
+    @property
+    def n_lefts(self):
         """The number of leftward immediate children of the word, in the
         syntactic dependency parse.
 
@@ -415,10 +416,10 @@ cdef class Token:
 
         DOCS: https://spacy.io/api/token#n_lefts
         """
-        def __get__(self):
         return self.c.l_kids
 
-    property n_rights:
+    @property
+    def n_rights(self):
         """The number of rightward immediate children of the word, in the
         syntactic dependency parse.
 
@@ -427,12 +428,11 @@ cdef class Token:
 
         DOCS: https://spacy.io/api/token#n_rights
         """
-        def __get__(self):
         return self.c.r_kids
 
-    property sent:
+    @property
+    def sent(self):
         """RETURNS (Span): The sentence span that the token is a part of."""
-        def __get__(self):
         if 'sent' in self.doc.user_token_hooks:
             return self.doc.user_token_hooks["sent"](self)
         return self.doc[self.i : self.i+1].sent
@@ -479,7 +479,8 @@ cdef class Token:
             else:
                 raise ValueError(Errors.E044.format(value=value))
 
-    property lefts:
+    @property
+    def lefts(self):
         """The leftward immediate children of the word, in the syntactic
         dependency parse.
 
@@ -487,7 +488,6 @@ cdef class Token:
 
         DOCS: https://spacy.io/api/token#lefts
         """
-        def __get__(self):
         cdef int nr_iter = 0
         cdef const TokenC* ptr = self.c - (self.i - self.c.l_edge)
         while ptr < self.c:
@@ -499,7 +499,8 @@ cdef class Token:
             if nr_iter >= 10000000:
                 raise RuntimeError(Errors.E045.format(attr="token.lefts"))
 
-    property rights:
+    @property
+    def rights(self):
         """The rightward immediate children of the word, in the syntactic
         dependency parse.
 
@@ -507,7 +508,6 @@ cdef class Token:
 
         DOCS: https://spacy.io/api/token#rights
         """
-        def __get__(self):
         cdef const TokenC* ptr = self.c + (self.c.r_edge - self.i)
         tokens = []
         cdef int nr_iter = 0
@@ -522,18 +522,19 @@ cdef class Token:
         for t in tokens:
             yield t
 
-    property children:
+    @property
+    def children(self):
         """A sequence of the token's immediate syntactic children.
 
         YIELDS (Token): A child token such that `child.head==self`.
 
         DOCS: https://spacy.io/api/token#children
         """
-        def __get__(self):
         yield from self.lefts
         yield from self.rights
 
-    property subtree:
+    @property
+    def subtree(self):
         """A sequence containing the token and all the token's syntactic
         descendants.
 
@@ -542,30 +543,30 @@ cdef class Token:
 
         DOCS: https://spacy.io/api/token#subtree
         """
-        def __get__(self):
         for word in self.lefts:
             yield from word.subtree
         yield self
         for word in self.rights:
             yield from word.subtree
 
-    property left_edge:
+    @property
+    def left_edge(self):
         """The leftmost token of this token's syntactic descendents.
 
         RETURNS (Token): The first token such that `self.is_ancestor(token)`.
         """
-        def __get__(self):
         return self.doc[self.c.l_edge]
 
-    property right_edge:
+    @property
+    def right_edge(self):
         """The rightmost token of this token's syntactic descendents.
 
         RETURNS (Token): The last token such that `self.is_ancestor(token)`.
         """
-        def __get__(self):
         return self.doc[self.c.r_edge]
 
-    property ancestors:
+    @property
+    def ancestors(self):
         """A sequence of this token's syntactic ancestors.
 
         YIELDS (Token): A sequence of ancestor tokens such that
@@ -573,7 +574,6 @@ cdef class Token:
 
         DOCS: https://spacy.io/api/token#ancestors
         """
-        def __get__(self):
         cdef const TokenC* head_ptr = self.c
         # Guard against infinite loop, no token can have
         # more ancestors than tokens in the tree.
@@ -685,23 +685,31 @@ cdef class Token:
             # Set new head
             self.c.head = rel_newhead_i
 
-    property conjuncts:
+    @property
+    def conjuncts(self):
         """A sequence of coordinated tokens, including the token itself.
 
-        YIELDS (Token): A coordinated token.
+        RETURNS (tuple): The coordinated tokens.
 
         DOCS: https://spacy.io/api/token#conjuncts
         """
-        def __get__(self):
+        cdef Token word, child
-            cdef Token word
         if "conjuncts" in self.doc.user_token_hooks:
-                yield from self.doc.user_token_hooks["conjuncts"](self)
+            return tuple(self.doc.user_token_hooks["conjuncts"](self))
+        start = self
+        while start.i != start.head.i:
+            if start.dep == conj:
+                start = start.head
             else:
-                if self.dep != conj:
+                break
-                    for word in self.rights:
+        queue = [start]
-                        if word.dep == conj:
+        output = [start]
-                            yield word
+        for word in queue:
-                            yield from word.conjuncts
+            for child in word.rights:
+                if child.c.dep == conj:
+                    output.append(child)
+                    queue.append(child)
+        return tuple([w for w in output if w.i != self.i])
 
     property ent_type:
         """RETURNS (uint64): Named entity type."""
@@ -711,15 +719,6 @@ cdef class Token:
         def __set__(self, ent_type):
             self.c.ent_type = ent_type
 
-    property ent_iob:
-        """IOB code of named entity tag. `1="I", 2="O", 3="B"`. 0 means no tag
-        is assigned.
-
-        RETURNS (uint64): IOB code of named entity tag.
-        """
-        def __get__(self):
-            return self.c.ent_iob
-
     property ent_type_:
         """RETURNS (unicode): Named entity type."""
         def __get__(self):
@@ -728,14 +727,23 @@ cdef class Token:
         def __set__(self, ent_type):
             self.c.ent_type = self.vocab.strings.add(ent_type)
 
-    property ent_iob_:
+    @property
+    def ent_iob(self):
+        """IOB code of named entity tag. `1="I", 2="O", 3="B"`. 0 means no tag
+        is assigned.
+
+        RETURNS (uint64): IOB code of named entity tag.
+        """
+        return self.c.ent_iob
+
+    @property
+    def ent_iob_(self):
         """IOB code of named entity tag. "B" means the token begins an entity,
         "I" means it is inside an entity, "O" means it is outside an entity,
         and "" means no entity tag is set.
 
         RETURNS (unicode): IOB code of named entity tag.
         """
-        def __get__(self):
         iob_strings = ("", "I", "O", "B")
         return iob_strings[self.c.ent_iob]
 
@@ -759,25 +767,24 @@ cdef class Token:
         def __set__(self, name):
             self.c.ent_id = self.vocab.strings.add(name)
 
-    property whitespace_:
+    @property
-        """RETURNS (unicode): The trailing whitespace character, if present.
+    def whitespace_(self):
-        """
+        """RETURNS (unicode): The trailing whitespace character, if present."""
-        def __get__(self):
         return " " if self.c.spacy else ""
 
-    property orth_:
+    @property
+    def orth_(self):
         """RETURNS (unicode): Verbatim text content (identical to
             `Token.text`). Exists mostly for consistency with the other
             attributes.
         """
-        def __get__(self):
         return self.vocab.strings[self.c.lex.orth]
 
-    property lower_:
+    @property
+    def lower_(self):
         """RETURNS (unicode): The lowercase token text. Equivalent to
             `Token.text.lower()`.
         """
-        def __get__(self):
         return self.vocab.strings[self.c.lex.lower]
 
     property norm_:
@@ -791,32 +798,32 @@ cdef class Token:
         def __set__(self, unicode norm_):
             self.c.norm = self.vocab.strings.add(norm_)
 
-    property shape_:
+    @property
+    def shape_(self):
         """RETURNS (unicode): Transform of the tokens's string, to show
             orthographic features. For example, "Xxxx" or "dd".
         """
-        def __get__(self):
         return self.vocab.strings[self.c.lex.shape]
 
-    property prefix_:
+    @property
+    def prefix_(self):
         """RETURNS (unicode): A length-N substring from the start of the token.
             Defaults to `N=1`.
         """
-        def __get__(self):
         return self.vocab.strings[self.c.lex.prefix]
 
-    property suffix_:
+    @property
+    def suffix_(self):
         """RETURNS (unicode): A length-N substring from the end of the token.
             Defaults to `N=3`.
         """
-        def __get__(self):
         return self.vocab.strings[self.c.lex.suffix]
 
-    property lang_:
+    @property
+    def lang_(self):
         """RETURNS (unicode): Language of the parent document's vocabulary,
             e.g. 'en'.
         """
-        def __get__(self):
         return self.vocab.strings[self.c.lex.lang]
 
     property lemma_:
@@ -856,110 +863,110 @@ cdef class Token:
         def __set__(self, unicode label):
             self.c.dep = self.vocab.strings.add(label)
 
-    property is_oov:
+    @property
+    def is_oov(self):
         """RETURNS (bool): Whether the token is out-of-vocabulary."""
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_OOV)
 
-    property is_stop:
+    @property
+    def is_stop(self):
         """RETURNS (bool): Whether the token is a stop word, i.e. part of a
             "stop list" defined by the language data.
         """
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_STOP)
 
-    property is_alpha:
+    @property
+    def is_alpha(self):
         """RETURNS (bool): Whether the token consists of alpha characters.
             Equivalent to `token.text.isalpha()`.
         """
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_ALPHA)
 
-    property is_ascii:
+    @property
+    def is_ascii(self):
         """RETURNS (bool): Whether the token consists of ASCII characters.
             Equivalent to `[any(ord(c) >= 128 for c in token.text)]`.
         """
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_ASCII)
 
-    property is_digit:
+    @property
+    def is_digit(self):
         """RETURNS (bool): Whether the token consists of digits. Equivalent to
             `token.text.isdigit()`.
         """
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_DIGIT)
 
-    property is_lower:
+    @property
+    def is_lower(self):
         """RETURNS (bool): Whether the token is in lowercase. Equivalent to
             `token.text.islower()`.
         """
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_LOWER)
 
-    property is_upper:
+    @property
+    def is_upper(self):
         """RETURNS (bool): Whether the token is in uppercase. Equivalent to
             `token.text.isupper()`
         """
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_UPPER)
 
-    property is_title:
+    @property
+    def is_title(self):
         """RETURNS (bool): Whether the token is in titlecase. Equivalent to
             `token.text.istitle()`.
         """
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_TITLE)
 
-    property is_punct:
+    @property
+    def is_punct(self):
         """RETURNS (bool): Whether the token is punctuation."""
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_PUNCT)
 
-    property is_space:
+    @property
+    def is_space(self):
         """RETURNS (bool): Whether the token consists of whitespace characters.
             Equivalent to `token.text.isspace()`.
         """
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_SPACE)
 
-    property is_bracket:
+    @property
+    def is_bracket(self):
         """RETURNS (bool): Whether the token is a bracket."""
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_BRACKET)
 
-    property is_quote:
+    @property
+    def is_quote(self):
         """RETURNS (bool): Whether the token is a quotation mark."""
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_QUOTE)
 
-    property is_left_punct:
+    @property
+    def is_left_punct(self):
         """RETURNS (bool): Whether the token is a left punctuation mark."""
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_LEFT_PUNCT)
 
-    property is_right_punct:
+    @property
+    def is_right_punct(self):
         """RETURNS (bool): Whether the token is a right punctuation mark."""
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_RIGHT_PUNCT)
 
-    property is_currency:
+    @property
+    def is_currency(self):
         """RETURNS (bool): Whether the token is a currency symbol."""
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, IS_CURRENCY)
 
-    property like_url:
+    @property
+    def like_url(self):
         """RETURNS (bool): Whether the token resembles a URL."""
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, LIKE_URL)
 
-    property like_num:
+    @property
+    def like_num(self):
         """RETURNS (bool): Whether the token resembles a number, e.g. "10.9",
             "10", "ten", etc.
         """
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, LIKE_NUM)
 
-    property like_email:
+    @property
+    def like_email(self):
         """RETURNS (bool): Whether the token resembles an email address."""
-        def __get__(self):
         return Lexeme.c_check_flag(self.c.lex, LIKE_EMAIL)

spacy/util.py

@@ -38,6 +38,18 @@ def set_env_log(value):
     _PRINT_ENV = value
 
 
+def lang_class_is_loaded(lang):
+    """Check whether a Language class is already loaded. Language classes are
+    loaded lazily, to avoid expensive setup code associated with the language
+    data.
+
+    lang (unicode): Two-letter language code, e.g. 'en'.
+    RETURNS (bool): Whether a Language class has been loaded.
+    """
+    global LANGUAGES
+    return lang in LANGUAGES
+
+
 def get_lang_class(lang):
     """Import and load a Language class.
 

spacy/vocab.pyx

@@ -60,13 +60,24 @@ cdef class Vocab:
         self.morphology = Morphology(self.strings, tag_map, lemmatizer)
         self.vectors = Vectors()
 
-    property lang:
+    @property
-        def __get__(self):
+    def lang(self):
         langfunc = None
         if self.lex_attr_getters:
             langfunc = self.lex_attr_getters.get(LANG, None)
         return langfunc("_") if langfunc else ""
 
+    property writing_system:
+        """A dict with information about the language's writing system. To get
+        the data, we use the vocab.lang property to fetch the Language class.
+        If the Language class is not loaded, an empty dict is returned.
+        """
+        def __get__(self):
+            if not util.lang_class_is_loaded(self.lang):
+                return {}
+            lang_class = util.get_lang_class(self.lang)
+            return dict(lang_class.Defaults.writing_system)
+
     def __len__(self):
         """The current number of lexemes stored.
 

website/.eslintrc

@@ -0,0 +1,27 @@
+{
+    "extends": ["standard", "prettier"],
+    "plugins": ["standard", "react", "react-hooks"],
+    "rules": {
+        "no-var": "error",
+        "no-unused-vars": 1,
+        "arrow-spacing": ["error", { "before": true, "after": true }],
+        "indent": ["error", 4],
+        "semi": ["error", "never"],
+        "arrow-parens": ["error", "as-needed"],
+        "standard/object-curly-even-spacing": ["error", "either"],
+        "standard/array-bracket-even-spacing": ["error", "either"],
+        "standard/computed-property-even-spacing": ["error", "even"],
+        "standard/no-callback-literal": ["error", ["cb", "callback"]],
+        "react/jsx-uses-react": "error",
+        "react/jsx-uses-vars": "error",
+        "react-hooks/rules-of-hooks": "error",
+        "react-hooks/exhaustive-deps": "warn"
+    },
+    "parser": "babel-eslint",
+    "parserOptions": {
+        "ecmaVersion": 8
+    },
+    "env": {
+        "browser": true
+    }
+}

website/docs/api/doc.md

@@ -654,6 +654,8 @@ The L2 norm of the document's vector representation.
 | `tensor` <Tag variant="new">2</Tag>     | object       | Container for dense vector representations.                                                                                                                                                                                                                                                |
 | `cats` <Tag variant="new">2</Tag>       | dictionary   | Maps either a label to a score for categories applied to whole document, or `(start_char, end_char, label)` to score for categories applied to spans. `start_char` and `end_char` should be character offsets, label can be either a string or an integer ID, and score should be a float. |
 | `user_data`                             | -            | A generic storage area, for user custom data.                                                                                                                                                                                                                                              |
+| `lang` <Tag variant="new">2.1</Tag>     | int          | Language of the document's vocabulary.                                                                                                                                                                                                                                                     |
+| `lang_` <Tag variant="new">2.1</Tag>    | unicode      | Language of the document's vocabulary.                                                                                                                                                                                                                                                     |
 | `is_tagged`                             | bool         | A flag indicating that the document has been part-of-speech tagged.                                                                                                                                                                                                                        |
 | `is_parsed`                             | bool         | A flag indicating that the document has been syntactically parsed.                                                                                                                                                                                                                         |
 | `is_sentenced`                          | bool         | A flag indicating that sentence boundaries have been applied to the document.                                                                                                                                                                                                              |

website/docs/api/span.md

@@ -316,6 +316,22 @@ taken.
 | ----------- | ------- | --------------- |
 | **RETURNS** | `Token` | The root token. |
 
+## Span.conjuncts {#conjuncts tag="property" model="parser"}
+
+A tuple of tokens coordinated to `span.root`.
+
+> #### Example
+>
+> ```python
+> doc = nlp(u"I like apples and oranges")
+> apples_conjuncts = doc[2:3].conjuncts
+> assert [t.text for t in apples_conjuncts] == [u"oranges"]
+> ```
+
+| Name        | Type    | Description             |
+| ----------- | ------- | ----------------------- |
+| **RETURNS** | `tuple` | The coordinated tokens. |
+
 ## Span.lefts {#lefts tag="property" model="parser"}
 
 Tokens that are to the left of the span, whose heads are within the span.

website/docs/api/token.md

@@ -211,7 +211,7 @@ The rightmost token of this token's syntactic descendants.
 
 ## Token.conjuncts {#conjuncts tag="property" model="parser"}
 
-A sequence of coordinated tokens, including the token itself.
+A tuple of coordinated tokens, not including the token itself.
 
 > #### Example
 >
@@ -222,8 +222,8 @@ A sequence of coordinated tokens, including the token itself.
 > ```
 
 | Name        | Type    | Description             |
-| ---------- | ------- | -------------------- |
+| ----------- | ------- | ----------------------- |
-| **YIELDS** | `Token` | A coordinated token. |
+| **RETURNS** | `tuple` | The coordinated tokens. |
 
 ## Token.children {#children tag="property" model="parser"}
 

website/docs/api/top-level.md

@@ -351,6 +351,24 @@ the two-letter language code.
 | `name` | unicode    | Two-letter language code, e.g. `'en'`. |
 | `cls`  | `Language` | The language class, e.g. `English`.    |
 
+### util.lang_class_is_loaded (#util.lang_class_is_loaded tag="function" new="2.1")
+
+Check whether a `Language` class is already loaded. `Language` classes are
+loaded lazily, to avoid expensive setup code associated with the language data.
+
+> #### Example
+>
+> ```python
+> lang_cls = util.get_lang_class("en")
+> assert util.lang_class_is_loaded("en") is True
+> assert util.lang_class_is_loaded("de") is False
+> ```
+
+| Name        | Type    | Description                            |
+| ----------- | ------- | -------------------------------------- |
+| `name`      | unicode | Two-letter language code, e.g. `'en'`. |
+| **RETURNS** | bool    | Whether the class has been loaded.     |
+
 ### util.load_model {#util.load_model tag="function" new="2"}
 
 Load a model from a shortcut link, package or data path. If called with a

website/docs/api/vocab.md

@@ -289,10 +289,11 @@ Load state from a binary string.
 > ```
 
 | Name                                          | Type          | Description                                                  |
-| ------------------------------------ | ------------- | --------------------------------------------- |
+| --------------------------------------------- | ------------- | ------------------------------------------------------------ |
 | `strings`                                     | `StringStore` | A table managing the string-to-int mapping.                  |
 | `vectors` <Tag variant="new">2</Tag>          | `Vectors`     | A table associating word IDs to word vectors.                |
 | `vectors_length`                              | int           | Number of dimensions for each word vector.                   |
+| `writing_system` <Tag variant="new">2.1</Tag> | dict          | A dict with information about the language's writing system. |
 
 ## Serialization fields {#serialization-fields}
 

website/docs/usage/101/_language-data.md

@@ -39,9 +39,9 @@ together all components and creating the `Language` subclass – for example,
 | **Morph rules**<br />[`morph_rules.py`][morph_rules.py]                            | Exception rules for morphological analysis of irregular words like personal pronouns.                                                                    |
 
 [stop_words.py]:
-  https://github.com/explosion/spacy-dev-resources/tree/master/templates/new_language/stop_words.py
+  https://github.com/explosion/spaCy/tree/master/spacy/lang/en/stop_words.py
 [tokenizer_exceptions.py]:
-  https://github.com/explosion/spacy-dev-resources/tree/master/templates/new_language/tokenizer_exceptions.py
+  https://github.com/explosion/spaCy/tree/master/spacy/lang/de/tokenizer_exceptions.py
 [norm_exceptions.py]:
   https://github.com/explosion/spaCy/tree/master/spacy/lang/norm_exceptions.py
 [punctuation.py]:
@@ -49,12 +49,12 @@ together all components and creating the `Language` subclass – for example,
 [char_classes.py]:
   https://github.com/explosion/spaCy/tree/master/spacy/lang/char_classes.py
 [lex_attrs.py]:
-  https://github.com/explosion/spacy-dev-resources/tree/master/templates/new_language/lex_attrs.py
+  https://github.com/explosion/spaCy/tree/master/spacy/lang/en/lex_attrs.py
 [syntax_iterators.py]:
   https://github.com/explosion/spaCy/tree/master/spacy/lang/en/syntax_iterators.py
 [lemmatizer.py]:
-  https://github.com/explosion/spacy-dev-resources/tree/master/templates/new_language/lemmatizer.py
+  https://github.com/explosion/spaCy/tree/master/spacy/lang/de/lemmatizer.py
 [tag_map.py]:
-  https://github.com/explosion/spacy-dev-resources/tree/master/templates/new_language/tag_map.py
+  https://github.com/explosion/spaCy/tree/master/spacy/lang/en/tag_map.py
 [morph_rules.py]:
   https://github.com/explosion/spaCy/tree/master/spacy/lang/en/morph_rules.py

website/docs/usage/adding-languages.md

@@ -105,11 +105,11 @@ to know the language's character set. If the language you're adding uses
 non-latin characters, you might need to define the required character classes in
 the global
 [`char_classes.py`](https://github.com/explosion/spaCy/tree/master/spacy/lang/char_classes.py).
-For efficiency, spaCy uses hard-coded unicode ranges to define character classes,
+For efficiency, spaCy uses hard-coded unicode ranges to define character
-the definitions of which can be found on [Wikipedia](https://en.wikipedia.org/wiki/Unicode_block). 
+classes, the definitions of which can be found on
-If the language requires very specific punctuation
+[Wikipedia](https://en.wikipedia.org/wiki/Unicode_block). If the language
-rules, you should consider overwriting the default regular expressions with your
+requires very specific punctuation rules, you should consider overwriting the
-own in the language's `Defaults`.
+default regular expressions with your own in the language's `Defaults`.
 
 </Infobox>
 
@@ -121,9 +121,9 @@ spaCy, named according to the language's
 code and resources specific to Spanish are placed into a directory
 `spacy/lang/es`, which can be imported as `spacy.lang.es`.
 
-To get started, you can use our
+To get started, you can check out the
-[templates](https://github.com/explosion/spacy-dev-resources/templates/new_language)
+[existing languages](https://github.com/explosion/spacy/tree/master/spacy/lang).
-for the most important files. Here's what the class template looks like:
+Here's what the class could look like:
 
 ```python
 ### __init__.py (excerpt)
@@ -631,13 +631,13 @@ of using deep learning for NLP with limited labeled data. The vectors are also
 useful by themselves – they power the `.similarity` methods in spaCy. For best
 results, you should pre-process the text with spaCy before training the Word2vec
 model. This ensures your tokenization will match. You can use our
-[word vectors training script](https://github.com/explosion/spacy-dev-resources/tree/master/training/word_vectors.py),
+[word vectors training script](https://github.com/explosion/spacy/tree/master/bin/train_word_vectors.py),
 which pre-processes the text with your language-specific tokenizer and trains
 the model using [Gensim](https://radimrehurek.com/gensim/). The `vectors.bin`
 file should consist of one word and vector per line.
 
 ```python
-https://github.com/explosion/spacy-dev-resources/tree/master/training/word_vectors.py
+https://github.com/explosion/spacy/tree/master/bin/train_word_vectors.py
 ```
 
 If you don't have a large sample of text available, you can also convert word

website/meta/universe.json

@@ -524,6 +524,22 @@
             },
             "category": ["standalone", "research"]
         },
+        {
+            "id": "scispacy",
+            "title": "scispaCy",
+            "slogan": "A full spaCy pipeline and models for scientific/biomedical documents",
+            "github": "allenai/scispacy",
+            "pip": "scispacy",
+            "thumb": "https://i.imgur.com/dJQSclW.png",
+            "url": "https://allenai.github.io/scispacy/",
+            "author": " Allen Institute for Artificial Intelligence",
+            "author_links": {
+                "github": "allenai",
+                "twitter": "allenai_org",
+                "website": "http://allenai.org"
+            },
+            "category": ["models", "research"]
+        },
         {
             "id": "textacy",
             "slogan": "NLP, before and after spaCy",
@@ -851,6 +867,22 @@
             },
             "category": ["courses"]
         },
+        {
+            "type": "education",
+            "id": "datacamp-advanced-nlp",
+            "title": "Advanced Natural Language Processing with spaCy",
+            "slogan": "Datacamp, 2019",
+            "description": "If you're working with a lot of text, you'll eventually want to know more about it. For example, what's it about? What do the words mean in context? Who is doing what to whom? What companies and products are mentioned? Which texts are similar to each other? In this course, you'll learn how to use spaCy, a fast-growing industry standard library for NLP in Python, to build advanced natural language understanding systems, using both rule-based and machine learning approaches.",
+            "url": "https://www.datacamp.com/courses/advanced-nlp-with-spacy",
+            "thumb": "https://i.imgur.com/0Zks7c0.jpg",
+            "author": "Ines Montani",
+            "author_links": {
+                "twitter": "_inesmontani",
+                "github": "ines",
+                "website": "https://ines.io"
+            },
+            "category": ["courses"]
+        },
         {
             "type": "education",
             "id": "learning-path-spacy",
@@ -910,6 +942,7 @@
             "description": "Most NLP projects rely crucially on the quality of annotations used for training and evaluating models. In this episode, Matt and Ines of Explosion AI tell us how Prodigy can improve data annotation and model development workflows. Prodigy is an annotation tool implemented as a python library, and it comes with a web application and a command line interface. A developer can define input data streams and design simple annotation interfaces. Prodigy can help break down complex annotation decisions into a series of binary decisions, and it provides easy integration with spaCy models. Developers can specify how models should be modified as new annotations come in in an active learning framework.",
             "soundcloud": "559200912",
             "thumb": "https://i.imgur.com/hOBQEzc.jpg",
+            "url": "https://soundcloud.com/nlp-highlights/78-where-do-corpora-come-from-with-matt-honnibal-and-ines-montani",
             "author": "Matt Gardner, Waleed Ammar (Allen AI)",
             "author_links": {
                 "website": "https://soundcloud.com/nlp-highlights"
@@ -925,12 +958,28 @@
             "iframe": "https://www.pythonpodcast.com/wp-content/plugins/podlove-podcasting-plugin-for-wordpress/lib/modules/podlove_web_player/player_v4/dist/share.html?episode=https://www.pythonpodcast.com/?podlove_player4=176",
             "iframe_height": 200,
             "thumb": "https://i.imgur.com/rpo6BuY.png",
+            "url": "https://www.podcastinit.com/episode-87-spacy-with-matthew-honnibal/",
             "author": "Tobias Macey",
             "author_links": {
                 "website": "https://www.podcastinit.com"
             },
             "category": ["podcasts"]
         },
+        {
+            "type": "education",
+            "id": "talk-python-podcast",
+            "title": "Talk Python 202: Building a software business",
+            "slogan": "March 2019",
+            "description": "One core question around open source is how do you fund it? Well, there is always that PayPal donate button. But that's been a tremendous failure for many projects. Often the go-to answer is consulting. But what if you don't want to trade time for money? You could take things up a notch and change the equation, exchanging value for money. That's what Ines Montani and her co-founder did when they started Explosion AI with spaCy as the foundation.",
+            "thumb": "https://i.imgur.com/q1twuK8.png",
+            "url": "https://talkpython.fm/episodes/show/202/building-a-software-business",
+            "soundcloud": "588364857",
+            "author": "Michael Kennedy",
+            "author_links": {
+                "website": "https://talkpython.fm/"
+            },
+            "category": ["podcasts"]
+        },
         {
             "id": "adam_qas",
             "title": "ADAM: Question Answering System",

website/package-lock.json

@@ -1833,9 +1833,9 @@
       }
     },
     "acorn": {
-      "version": "6.1.0",
+      "version": "6.1.1",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.1.1.tgz",
-      "integrity": "sha512-MW/FjM+IvU9CgBzjO3UIPCE2pyEwUsoFl+VGdczOPEdxfGFjuKny/gN54mOuX7Qxmb9Rg9MCn2oKiSUeW+pjrw=="
+      "integrity": "sha512-jPTiwtOxaHNaAPg/dmrJ/beuzLRnXtB0kQPQ8JpotKJgTB6rX6c8mlf315941pyjBSaPg8NHXS9fhP4u17DpGA=="
     },
     "acorn-dynamic-import": {
       "version": "3.0.0",
@@ -5958,9 +5958,9 @@
       "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
     },
     "eslint": {
-      "version": "5.14.1",
+      "version": "5.15.1",
-      "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.14.1.tgz",
+      "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.15.1.tgz",
-      "integrity": "sha512-CyUMbmsjxedx8B0mr79mNOqetvkbij/zrXnFeK2zc3pGRn3/tibjiNAv/3UxFEyfMDjh+ZqTrJrEGBFiGfD5Og==",
+      "integrity": "sha512-NTcm6vQ+PTgN3UBsALw5BMhgO6i5EpIjQF/Xb5tIh3sk9QhrFafujUOczGz4J24JBlzWclSB9Vmx8d+9Z6bFCg==",
       "requires": {
         "@babel/code-frame": "^7.0.0",
         "ajv": "^6.9.1",
@@ -5968,7 +5968,7 @@
         "cross-spawn": "^6.0.5",
         "debug": "^4.0.1",
         "doctrine": "^3.0.0",
-        "eslint-scope": "^4.0.0",
+        "eslint-scope": "^4.0.2",
         "eslint-utils": "^1.3.1",
         "eslint-visitor-keys": "^1.0.0",
         "espree": "^5.0.1",
@@ -6001,9 +6001,9 @@
       },
       "dependencies": {
         "ajv": {
-          "version": "6.9.2",
+          "version": "6.10.0",
-          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.9.2.tgz",
+          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz",
-          "integrity": "sha512-4UFy0/LgDo7Oa/+wOAlj44tp9K78u38E5/359eSrqEp1Z5PdVfimCcs7SluXMP755RUQu6d2b4AvF0R1C9RZjg==",
+          "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==",
           "requires": {
             "fast-deep-equal": "^2.0.1",
             "fast-json-stable-stringify": "^2.0.0",
@@ -6037,9 +6037,9 @@
           }
         },
         "eslint-scope": {
-          "version": "4.0.0",
+          "version": "4.0.2",
-          "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.0.tgz",
+          "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.2.tgz",
-          "integrity": "sha512-1G6UTDi7Jc1ELFwnR58HV4fK9OQK4S6N985f166xqXxpjU6plxFISJa2Ba9KCQuFa8RCnj/lSFJbHo7UFDBnUA==",
+          "integrity": "sha512-5q1+B/ogmHl8+paxtOKx38Z8LtWkVGuNt3+GQNErqwLl6ViNp/gdJGMCjZNxZ8j/VYjDNZ2Fo+eQc1TAVPIzbg==",
           "requires": {
             "esrecurse": "^4.1.0",
             "estraverse": "^4.1.1"
@@ -6448,52 +6448,6 @@
         }
       }
     },
-    "expand-range": {
-      "version": "1.8.2",
-      "resolved": "http://registry.npmjs.org/expand-range/-/expand-range-1.8.2.tgz",
-      "integrity": "sha1-opnv/TNf4nIeuujiV+x5ZE/IUzc=",
-      "requires": {
-        "fill-range": "^2.1.0"
-      },
-      "dependencies": {
-        "fill-range": {
-          "version": "2.2.4",
-          "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-2.2.4.tgz",
-          "integrity": "sha512-cnrcCbj01+j2gTG921VZPnHbjmdAf8oQV/iGeV2kZxGSyfYjjTyY79ErsK1WJWMpw6DaApEX72binqJE+/d+5Q==",
-          "requires": {
-            "is-number": "^2.1.0",
-            "isobject": "^2.0.0",
-            "randomatic": "^3.0.0",
-            "repeat-element": "^1.1.2",
-            "repeat-string": "^1.5.2"
-          }
-        },
-        "is-number": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-2.1.0.tgz",
-          "integrity": "sha1-Afy7s5NGOlSPL0ZszhbezknbkI8=",
-          "requires": {
-            "kind-of": "^3.0.2"
-          }
-        },
-        "isobject": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz",
-          "integrity": "sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk=",
-          "requires": {
-            "isarray": "1.0.0"
-          }
-        },
-        "kind-of": {
-          "version": "3.2.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-          "requires": {
-            "is-buffer": "^1.1.5"
-          }
-        }
-      }
-    },
     "expand-template": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
@@ -6818,11 +6772,6 @@
       "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
       "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
     },
-    "filename-regex": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/filename-regex/-/filename-regex-2.0.1.tgz",
-      "integrity": "sha1-wcS5vuPglyXdsQa3XB4wH+LxiyY="
-    },
     "filename-reserved-regex": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-2.0.0.tgz",
@@ -7130,468 +7079,6 @@
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
       "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
     },
-    "fsevents": {
-      "version": "1.2.4",
-      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.4.tgz",
-      "integrity": "sha512-z8H8/diyk76B7q5wg+Ud0+CqzcAF3mBBI/bA5ne5zrRUUIvNkJY//D3BqyH571KuAC4Nr7Rw7CjWX4r0y9DvNg==",
-      "optional": true,
-      "requires": {
-        "nan": "^2.9.2",
-        "node-pre-gyp": "^0.10.0"
-      },
-      "dependencies": {
-        "abbrev": {
-          "version": "1.1.1",
-          "bundled": true,
-          "optional": true
-        },
-        "ansi-regex": {
-          "version": "2.1.1",
-          "bundled": true
-        },
-        "aproba": {
-          "version": "1.2.0",
-          "bundled": true,
-          "optional": true
-        },
-        "are-we-there-yet": {
-          "version": "1.1.4",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "delegates": "^1.0.0",
-            "readable-stream": "^2.0.6"
-          }
-        },
-        "balanced-match": {
-          "version": "1.0.0",
-          "bundled": true
-        },
-        "brace-expansion": {
-          "version": "1.1.11",
-          "bundled": true,
-          "requires": {
-            "balanced-match": "^1.0.0",
-            "concat-map": "0.0.1"
-          }
-        },
-        "chownr": {
-          "version": "1.0.1",
-          "bundled": true,
-          "optional": true
-        },
-        "code-point-at": {
-          "version": "1.1.0",
-          "bundled": true
-        },
-        "concat-map": {
-          "version": "0.0.1",
-          "bundled": true
-        },
-        "console-control-strings": {
-          "version": "1.1.0",
-          "bundled": true
-        },
-        "core-util-is": {
-          "version": "1.0.2",
-          "bundled": true,
-          "optional": true
-        },
-        "debug": {
-          "version": "2.6.9",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "ms": "2.0.0"
-          }
-        },
-        "deep-extend": {
-          "version": "0.5.1",
-          "bundled": true,
-          "optional": true
-        },
-        "delegates": {
-          "version": "1.0.0",
-          "bundled": true,
-          "optional": true
-        },
-        "detect-libc": {
-          "version": "1.0.3",
-          "bundled": true,
-          "optional": true
-        },
-        "fs-minipass": {
-          "version": "1.2.5",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "minipass": "^2.2.1"
-          }
-        },
-        "fs.realpath": {
-          "version": "1.0.0",
-          "bundled": true,
-          "optional": true
-        },
-        "gauge": {
-          "version": "2.7.4",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "aproba": "^1.0.3",
-            "console-control-strings": "^1.0.0",
-            "has-unicode": "^2.0.0",
-            "object-assign": "^4.1.0",
-            "signal-exit": "^3.0.0",
-            "string-width": "^1.0.1",
-            "strip-ansi": "^3.0.1",
-            "wide-align": "^1.1.0"
-          }
-        },
-        "glob": {
-          "version": "7.1.2",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "fs.realpath": "^1.0.0",
-            "inflight": "^1.0.4",
-            "inherits": "2",
-            "minimatch": "^3.0.4",
-            "once": "^1.3.0",
-            "path-is-absolute": "^1.0.0"
-          }
-        },
-        "has-unicode": {
-          "version": "2.0.1",
-          "bundled": true,
-          "optional": true
-        },
-        "iconv-lite": {
-          "version": "0.4.21",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "safer-buffer": "^2.1.0"
-          }
-        },
-        "ignore-walk": {
-          "version": "3.0.1",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "minimatch": "^3.0.4"
-          }
-        },
-        "inflight": {
-          "version": "1.0.6",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "once": "^1.3.0",
-            "wrappy": "1"
-          }
-        },
-        "inherits": {
-          "version": "2.0.3",
-          "bundled": true
-        },
-        "ini": {
-          "version": "1.3.5",
-          "bundled": true,
-          "optional": true
-        },
-        "is-fullwidth-code-point": {
-          "version": "1.0.0",
-          "bundled": true,
-          "requires": {
-            "number-is-nan": "^1.0.0"
-          }
-        },
-        "isarray": {
-          "version": "1.0.0",
-          "bundled": true,
-          "optional": true
-        },
-        "minimatch": {
-          "version": "3.0.4",
-          "bundled": true,
-          "requires": {
-            "brace-expansion": "^1.1.7"
-          }
-        },
-        "minimist": {
-          "version": "0.0.8",
-          "bundled": true
-        },
-        "minipass": {
-          "version": "2.2.4",
-          "bundled": true,
-          "requires": {
-            "safe-buffer": "^5.1.1",
-            "yallist": "^3.0.0"
-          }
-        },
-        "minizlib": {
-          "version": "1.1.0",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "minipass": "^2.2.1"
-          }
-        },
-        "mkdirp": {
-          "version": "0.5.1",
-          "bundled": true,
-          "requires": {
-            "minimist": "0.0.8"
-          }
-        },
-        "ms": {
-          "version": "2.0.0",
-          "bundled": true,
-          "optional": true
-        },
-        "needle": {
-          "version": "2.2.0",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "debug": "^2.1.2",
-            "iconv-lite": "^0.4.4",
-            "sax": "^1.2.4"
-          }
-        },
-        "node-pre-gyp": {
-          "version": "0.10.0",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "detect-libc": "^1.0.2",
-            "mkdirp": "^0.5.1",
-            "needle": "^2.2.0",
-            "nopt": "^4.0.1",
-            "npm-packlist": "^1.1.6",
-            "npmlog": "^4.0.2",
-            "rc": "^1.1.7",
-            "rimraf": "^2.6.1",
-            "semver": "^5.3.0",
-            "tar": "^4"
-          }
-        },
-        "nopt": {
-          "version": "4.0.1",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "abbrev": "1",
-            "osenv": "^0.1.4"
-          }
-        },
-        "npm-bundled": {
-          "version": "1.0.3",
-          "bundled": true,
-          "optional": true
-        },
-        "npm-packlist": {
-          "version": "1.1.10",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "ignore-walk": "^3.0.1",
-            "npm-bundled": "^1.0.1"
-          }
-        },
-        "npmlog": {
-          "version": "4.1.2",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "are-we-there-yet": "~1.1.2",
-            "console-control-strings": "~1.1.0",
-            "gauge": "~2.7.3",
-            "set-blocking": "~2.0.0"
-          }
-        },
-        "number-is-nan": {
-          "version": "1.0.1",
-          "bundled": true
-        },
-        "object-assign": {
-          "version": "4.1.1",
-          "bundled": true,
-          "optional": true
-        },
-        "once": {
-          "version": "1.4.0",
-          "bundled": true,
-          "requires": {
-            "wrappy": "1"
-          }
-        },
-        "os-homedir": {
-          "version": "1.0.2",
-          "bundled": true,
-          "optional": true
-        },
-        "os-tmpdir": {
-          "version": "1.0.2",
-          "bundled": true,
-          "optional": true
-        },
-        "osenv": {
-          "version": "0.1.5",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "os-homedir": "^1.0.0",
-            "os-tmpdir": "^1.0.0"
-          }
-        },
-        "path-is-absolute": {
-          "version": "1.0.1",
-          "bundled": true,
-          "optional": true
-        },
-        "process-nextick-args": {
-          "version": "2.0.0",
-          "bundled": true,
-          "optional": true
-        },
-        "rc": {
-          "version": "1.2.7",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "deep-extend": "^0.5.1",
-            "ini": "~1.3.0",
-            "minimist": "^1.2.0",
-            "strip-json-comments": "~2.0.1"
-          },
-          "dependencies": {
-            "minimist": {
-              "version": "1.2.0",
-              "bundled": true,
-              "optional": true
-            }
-          }
-        },
-        "readable-stream": {
-          "version": "2.3.6",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "core-util-is": "~1.0.0",
-            "inherits": "~2.0.3",
-            "isarray": "~1.0.0",
-            "process-nextick-args": "~2.0.0",
-            "safe-buffer": "~5.1.1",
-            "string_decoder": "~1.1.1",
-            "util-deprecate": "~1.0.1"
-          }
-        },
-        "rimraf": {
-          "version": "2.6.2",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "glob": "^7.0.5"
-          }
-        },
-        "safe-buffer": {
-          "version": "5.1.1",
-          "bundled": true
-        },
-        "safer-buffer": {
-          "version": "2.1.2",
-          "bundled": true,
-          "optional": true
-        },
-        "sax": {
-          "version": "1.2.4",
-          "bundled": true,
-          "optional": true
-        },
-        "semver": {
-          "version": "5.5.0",
-          "bundled": true,
-          "optional": true
-        },
-        "set-blocking": {
-          "version": "2.0.0",
-          "bundled": true,
-          "optional": true
-        },
-        "signal-exit": {
-          "version": "3.0.2",
-          "bundled": true,
-          "optional": true
-        },
-        "string-width": {
-          "version": "1.0.2",
-          "bundled": true,
-          "requires": {
-            "code-point-at": "^1.0.0",
-            "is-fullwidth-code-point": "^1.0.0",
-            "strip-ansi": "^3.0.0"
-          }
-        },
-        "string_decoder": {
-          "version": "1.1.1",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "safe-buffer": "~5.1.0"
-          }
-        },
-        "strip-ansi": {
-          "version": "3.0.1",
-          "bundled": true,
-          "requires": {
-            "ansi-regex": "^2.0.0"
-          }
-        },
-        "strip-json-comments": {
-          "version": "2.0.1",
-          "bundled": true,
-          "optional": true
-        },
-        "tar": {
-          "version": "4.4.1",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "chownr": "^1.0.1",
-            "fs-minipass": "^1.2.5",
-            "minipass": "^2.2.4",
-            "minizlib": "^1.1.0",
-            "mkdirp": "^0.5.0",
-            "safe-buffer": "^5.1.1",
-            "yallist": "^3.0.2"
-          }
-        },
-        "util-deprecate": {
-          "version": "1.0.2",
-          "bundled": true,
-          "optional": true
-        },
-        "wide-align": {
-          "version": "1.1.2",
-          "bundled": true,
-          "optional": true,
-          "requires": {
-            "string-width": "^1.0.2"
-          }
-        },
-        "wrappy": {
-          "version": "1.0.2",
-          "bundled": true
-        },
-        "yallist": {
-          "version": "3.0.2",
-          "bundled": true
-        }
-      }
-    },
     "fstream": {
       "version": "1.0.11",
       "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.11.tgz",
@@ -8322,14 +7809,14 @@
       }
     },
     "gatsby-source-filesystem": {
-      "version": "2.0.20",
+      "version": "2.0.24",
-      "resolved": "https://registry.npmjs.org/gatsby-source-filesystem/-/gatsby-source-filesystem-2.0.20.tgz",
+      "resolved": "https://registry.npmjs.org/gatsby-source-filesystem/-/gatsby-source-filesystem-2.0.24.tgz",
-      "integrity": "sha512-nS2hBsqKEQIJ5Yd+g9p++FcsfmvbQmZlBUzx04VPBYZBu2LuLA/ZxQkmdiTNnbDQ18KJw0Zu2PnmUerPnEMqyg==",
+      "integrity": "sha512-KzyHzuXni9hOiZFDgeoH5ABJZqb59fSJNGr2C4U6B1AlGXFMucFK45Fh3V8axtpi833bIbCb9rGmK+tvL4Qb1w==",
       "requires": {
         "@babel/runtime": "^7.0.0",
         "better-queue": "^3.8.7",
         "bluebird": "^3.5.0",
-        "chokidar": "^1.7.0",
+        "chokidar": "^2.1.2",
         "file-type": "^10.2.0",
         "fs-extra": "^5.0.0",
         "got": "^7.1.0",
@@ -8343,83 +7830,6 @@
         "xstate": "^3.1.0"
       },
       "dependencies": {
-        "anymatch": {
-          "version": "1.3.2",
-          "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-1.3.2.tgz",
-          "integrity": "sha512-0XNayC8lTHQ2OI8aljNCN3sSx6hsr/1+rlcDAotXJR7C1oZZHCNsfpbKwMjRA3Uqb5tF1Rae2oloTr4xpq+WjA==",
-          "requires": {
-            "micromatch": "^2.1.5",
-            "normalize-path": "^2.0.0"
-          }
-        },
-        "arr-diff": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-2.0.0.tgz",
-          "integrity": "sha1-jzuCf5Vai9ZpaX5KQlasPOrjVs8=",
-          "requires": {
-            "arr-flatten": "^1.0.1"
-          }
-        },
-        "array-unique": {
-          "version": "0.2.1",
-          "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.2.1.tgz",
-          "integrity": "sha1-odl8yvy8JiXMcPrc6zalDFiwGlM="
-        },
-        "braces": {
-          "version": "1.8.5",
-          "resolved": "https://registry.npmjs.org/braces/-/braces-1.8.5.tgz",
-          "integrity": "sha1-uneWLhLf+WnWt2cR6RS3N4V79qc=",
-          "requires": {
-            "expand-range": "^1.8.1",
-            "preserve": "^0.2.0",
-            "repeat-element": "^1.1.2"
-          }
-        },
-        "chokidar": {
-          "version": "1.7.0",
-          "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-1.7.0.tgz",
-          "integrity": "sha1-eY5ol3gVHIB2tLNg5e3SjNortGg=",
-          "requires": {
-            "anymatch": "^1.3.0",
-            "async-each": "^1.0.0",
-            "fsevents": "^1.0.0",
-            "glob-parent": "^2.0.0",
-            "inherits": "^2.0.1",
-            "is-binary-path": "^1.0.0",
-            "is-glob": "^2.0.0",
-            "path-is-absolute": "^1.0.0",
-            "readdirp": "^2.0.0"
-          }
-        },
-        "expand-brackets": {
-          "version": "0.1.5",
-          "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-0.1.5.tgz",
-          "integrity": "sha1-3wcoTjQqgHzXM6xa9yQR5YHRF3s=",
-          "requires": {
-            "is-posix-bracket": "^0.1.0"
-          }
-        },
-        "extglob": {
-          "version": "0.3.2",
-          "resolved": "https://registry.npmjs.org/extglob/-/extglob-0.3.2.tgz",
-          "integrity": "sha1-Lhj/PS9JqydlzskCPwEdqo2DSaE=",
-          "requires": {
-            "is-extglob": "^1.0.0"
-          }
-        },
-        "file-type": {
-          "version": "10.7.1",
-          "resolved": "https://registry.npmjs.org/file-type/-/file-type-10.7.1.tgz",
-          "integrity": "sha512-kUc4EE9q3MH6kx70KumPOvXLZLEJZzY9phEVg/bKWyGZ+OA9KoKZzFR4HS0yDmNv31sJkdf4hbTERIfplF9OxQ=="
-        },
-        "glob-parent": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-2.0.0.tgz",
-          "integrity": "sha1-gTg9ctsFT8zPUzbaqQLxgvbtuyg=",
-          "requires": {
-            "is-glob": "^2.0.0"
-          }
-        },
         "got": {
           "version": "7.1.0",
           "resolved": "https://registry.npmjs.org/got/-/got-7.1.0.tgz",
@@ -8441,47 +7851,6 @@
             "url-to-options": "^1.0.1"
           }
         },
-        "is-extglob": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-1.0.0.tgz",
-          "integrity": "sha1-rEaBd8SUNAWgkvyPKXYMb/xiBsA="
-        },
-        "is-glob": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz",
-          "integrity": "sha1-0Jb5JqPe1WAPP9/ZEZjLCIjC2GM=",
-          "requires": {
-            "is-extglob": "^1.0.0"
-          }
-        },
-        "kind-of": {
-          "version": "3.2.2",
-          "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
-          "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=",
-          "requires": {
-            "is-buffer": "^1.1.5"
-          }
-        },
-        "micromatch": {
-          "version": "2.3.11",
-          "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-2.3.11.tgz",
-          "integrity": "sha1-hmd8l9FyCzY0MdBNDRUpO9OMFWU=",
-          "requires": {
-            "arr-diff": "^2.0.0",
-            "array-unique": "^0.2.1",
-            "braces": "^1.8.2",
-            "expand-brackets": "^0.1.4",
-            "extglob": "^0.3.1",
-            "filename-regex": "^2.0.0",
-            "is-extglob": "^1.0.0",
-            "is-glob": "^2.0.1",
-            "kind-of": "^3.0.2",
-            "normalize-path": "^2.0.1",
-            "object.omit": "^2.0.0",
-            "parse-glob": "^3.0.4",
-            "regex-cache": "^0.4.2"
-          }
-        },
         "pify": {
           "version": "4.0.1",
           "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz",
@@ -8493,12 +7862,12 @@
           "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74="
         },
         "read-chunk": {
-          "version": "3.0.0",
+          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/read-chunk/-/read-chunk-3.0.0.tgz",
+          "resolved": "https://registry.npmjs.org/read-chunk/-/read-chunk-3.1.0.tgz",
-          "integrity": "sha512-8lBUVPjj9TC5bKLBacB+rpexM03+LWiYbv6ma3BeWmUYXGxqA1WNNgIZHq/iIsCrbFMzPhFbkOqdsyOFRnuoXg==",
+          "integrity": "sha512-ZdiZJXXoZYE08SzZvTipHhI+ZW0FpzxmFtLI3vIeMuRN9ySbIZ+SZawKogqJ7dxW9fJ/W73BNtxu4Zu/bZp+Ng==",
           "requires": {
-            "pify": "^4.0.0",
+            "pify": "^4.0.1",
-            "with-open-file": "^0.1.3"
+            "with-open-file": "^0.1.5"
           }
         }
       }
@@ -8742,38 +8111,6 @@
         "path-is-absolute": "^1.0.0"
       }
     },
-    "glob-base": {
-      "version": "0.3.0",
-      "resolved": "https://registry.npmjs.org/glob-base/-/glob-base-0.3.0.tgz",
-      "integrity": "sha1-27Fk9iIbHAscz4Kuoyi0l98Oo8Q=",
-      "requires": {
-        "glob-parent": "^2.0.0",
-        "is-glob": "^2.0.0"
-      },
-      "dependencies": {
-        "glob-parent": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-2.0.0.tgz",
-          "integrity": "sha1-gTg9ctsFT8zPUzbaqQLxgvbtuyg=",
-          "requires": {
-            "is-glob": "^2.0.0"
-          }
-        },
-        "is-extglob": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-1.0.0.tgz",
-          "integrity": "sha1-rEaBd8SUNAWgkvyPKXYMb/xiBsA="
-        },
-        "is-glob": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz",
-          "integrity": "sha1-0Jb5JqPe1WAPP9/ZEZjLCIjC2GM=",
-          "requires": {
-            "is-extglob": "^1.0.0"
-          }
-        }
-      }
-    },
     "glob-parent": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz",
@@ -10110,19 +9447,6 @@
       "resolved": "https://registry.npmjs.org/is-directory/-/is-directory-0.3.1.tgz",
       "integrity": "sha1-YTObbyR1/Hcv2cnYP1yFddwVSuE="
     },
-    "is-dotfile": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/is-dotfile/-/is-dotfile-1.0.3.tgz",
-      "integrity": "sha1-pqLzL/0t+wT1yiXs0Pa4PPeYoeE="
-    },
-    "is-equal-shallow": {
-      "version": "0.1.3",
-      "resolved": "https://registry.npmjs.org/is-equal-shallow/-/is-equal-shallow-0.1.3.tgz",
-      "integrity": "sha1-IjgJj8Ih3gvPpdnqxMRdY4qhxTQ=",
-      "requires": {
-        "is-primitive": "^2.0.0"
-      }
-    },
     "is-extendable": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
@@ -10263,16 +9587,6 @@
       "resolved": "https://registry.npmjs.org/is-png/-/is-png-1.1.0.tgz",
       "integrity": "sha1-1XSxK/J1wDUEVVcLDltXqwYgd84="
     },
-    "is-posix-bracket": {
-      "version": "0.1.1",
-      "resolved": "https://registry.npmjs.org/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz",
-      "integrity": "sha1-MzTceXdDaOkvAW5vvAqI9c1ua8Q="
-    },
-    "is-primitive": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/is-primitive/-/is-primitive-2.0.0.tgz",
-      "integrity": "sha1-IHurkWOEmcB7Kt8kCkGochADRXU="
-    },
     "is-promise": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz",
@@ -11162,11 +10476,6 @@
       "resolved": "https://registry.npmjs.org/marked/-/marked-0.4.0.tgz",
       "integrity": "sha512-tMsdNBgOsrUophCAFQl0XPe6Zqk/uy9gnue+jIIKhykO51hxyu6uNx7zBPy0+y/WKYVZZMspV9YeXLNdKk+iYw=="
     },
-    "math-random": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/math-random/-/math-random-1.0.1.tgz",
-      "integrity": "sha1-izqsWIuKZuSXXjzepn97sylgH6w="
-    },
     "md-attr-parser": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/md-attr-parser/-/md-attr-parser-1.2.1.tgz",
@@ -12230,15 +11539,6 @@
         "es-abstract": "^1.5.1"
       }
     },
-    "object.omit": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/object.omit/-/object.omit-2.0.1.tgz",
-      "integrity": "sha1-Gpx0SCnznbuFjHbKNXmuKlTr0fo=",
-      "requires": {
-        "for-own": "^0.1.4",
-        "is-extendable": "^0.1.1"
-      }
-    },
     "object.pick": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz",
@@ -12579,32 +11879,6 @@
         "path-root": "^0.1.1"
       }
     },
-    "parse-glob": {
-      "version": "3.0.4",
-      "resolved": "https://registry.npmjs.org/parse-glob/-/parse-glob-3.0.4.tgz",
-      "integrity": "sha1-ssN2z7EfNVE7rdFz7wu246OIORw=",
-      "requires": {
-        "glob-base": "^0.3.0",
-        "is-dotfile": "^1.0.0",
-        "is-extglob": "^1.0.0",
-        "is-glob": "^2.0.0"
-      },
-      "dependencies": {
-        "is-extglob": {
-          "version": "1.0.0",
-          "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-1.0.0.tgz",
-          "integrity": "sha1-rEaBd8SUNAWgkvyPKXYMb/xiBsA="
-        },
-        "is-glob": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz",
-          "integrity": "sha1-0Jb5JqPe1WAPP9/ZEZjLCIjC2GM=",
-          "requires": {
-            "is-extglob": "^1.0.0"
-          }
-        }
-      }
-    },
     "parse-headers": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.1.tgz",
@@ -14769,11 +14043,6 @@
       "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz",
       "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw="
     },
-    "preserve": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/preserve/-/preserve-0.2.0.tgz",
-      "integrity": "sha1-gV7R9uvGWSb4ZbMQwHE7yzMVzks="
-    },
     "prettier": {
       "version": "1.16.4",
       "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.16.4.tgz",
@@ -14982,23 +14251,6 @@
       "resolved": "http://registry.npmjs.org/ramda/-/ramda-0.21.0.tgz",
       "integrity": "sha1-oAGr7bP/YQd9T/HVd9RN536NCjU="
     },
-    "randomatic": {
-      "version": "3.1.1",
-      "resolved": "https://registry.npmjs.org/randomatic/-/randomatic-3.1.1.tgz",
-      "integrity": "sha512-TuDE5KxZ0J461RVjrJZCJc+J+zCkTb1MbH9AQUq68sMhOMcy9jLcb3BrZKgp9q9Ncltdg4QVqWrH02W2EFFVYw==",
-      "requires": {
-        "is-number": "^4.0.0",
-        "kind-of": "^6.0.0",
-        "math-random": "^1.0.1"
-      },
-      "dependencies": {
-        "is-number": {
-          "version": "4.0.0",
-          "resolved": "https://registry.npmjs.org/is-number/-/is-number-4.0.0.tgz",
-          "integrity": "sha512-rSklcAIlf1OmFdyAqbnWTLVelsQ58uvZ66S/ZyawjWqIviTWCjg2PzVGw8WUA+nNuPTqb4wgA+NszrJ+08LlgQ=="
-        }
-      }
-    },
     "randombytes": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
@@ -15458,14 +14710,6 @@
         "private": "^0.1.6"
       }
     },
-    "regex-cache": {
-      "version": "0.4.4",
-      "resolved": "https://registry.npmjs.org/regex-cache/-/regex-cache-0.4.4.tgz",
-      "integrity": "sha512-nVIZwtCjkC9YgvWkpM55B5rBhBYRZhAaJbgcFYXXsHnbZ9UZI9nnVWYZpBlCqv9ho2eZryPnWrZGsOdPwVWXWQ==",
-      "requires": {
-        "is-equal-shallow": "^0.1.3"
-      }
-    },
     "regex-not": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz",
@@ -17710,9 +16954,9 @@
       },
       "dependencies": {
         "ajv": {
-          "version": "6.9.2",
+          "version": "6.10.0",
-          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.9.2.tgz",
+          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz",
-          "integrity": "sha512-4UFy0/LgDo7Oa/+wOAlj44tp9K78u38E5/359eSrqEp1Z5PdVfimCcs7SluXMP755RUQu6d2b4AvF0R1C9RZjg==",
+          "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==",
           "requires": {
             "fast-deep-equal": "^2.0.1",
             "fast-json-stable-stringify": "^2.0.0",
@@ -17721,26 +16965,26 @@
           }
         },
         "ansi-regex": {
-          "version": "4.0.0",
+          "version": "4.1.0",
-          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.0.0.tgz",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
-          "integrity": "sha512-iB5Dda8t/UqpPI/IjsejXu5jOGDrzn41wJyljwPH65VCIbk6+1BzFIMJGFwTNrYXT1CrD+B4l19U7awiQ8rk7w=="
+          "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg=="
         },
         "string-width": {
-          "version": "3.0.0",
+          "version": "3.1.0",
-          "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.0.0.tgz",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz",
-          "integrity": "sha512-rr8CUxBbvOZDUvc5lNIJ+OC1nPVpz+Siw9VBtUjB9b6jZehZLFt0JMCZzShFHIsI8cbhm0EsNIfWJMFV3cu3Ew==",
+          "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==",
           "requires": {
             "emoji-regex": "^7.0.1",
             "is-fullwidth-code-point": "^2.0.0",
-            "strip-ansi": "^5.0.0"
+            "strip-ansi": "^5.1.0"
           }
         },
         "strip-ansi": {
-          "version": "5.0.0",
+          "version": "5.1.0",
-          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.0.0.tgz",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.1.0.tgz",
-          "integrity": "sha512-Uu7gQyZI7J7gn5qLn1Np3G9vcYGTVqB+lFTytnDJv83dd8T22aGH451P3jueT2/QemInJDfxHB5Tde5OzgG1Ow==",
+          "integrity": "sha512-TjxrkPONqO2Z8QDCpeE2j6n0M6EwxzyDgzEeGp+FbdvaJAt//ClYi6W5my+3ROlC/hZX2KACUwDfK49Ka5eDvg==",
           "requires": {
-            "ansi-regex": "^4.0.0"
+            "ansi-regex": "^4.1.0"
           }
         }
       }

website/package.json

@@ -35,7 +35,7 @@
         "gatsby-remark-prismjs": "^3.2.4",
         "gatsby-remark-smartypants": "^2.0.8",
         "gatsby-remark-unwrap-images": "^1.0.1",
-        "gatsby-source-filesystem": "^2.0.20",
+        "gatsby-source-filesystem": "^2.0.24",
         "gatsby-transformer-remark": "^2.2.5",
         "gatsby-transformer-sharp": "^2.1.13",
         "html-to-react": "^1.3.4",

website/src/components/accordion.js

@@ -7,7 +7,7 @@ import Link from './link'
 import classes from '../styles/accordion.module.sass'
 
 const Accordion = ({ title, id, expanded, children }) => {
-    const anchorId = id ? id : slugify(title)
+    const anchorId = id || slugify(title)
     const [isExpanded, setIsExpanded] = useState(expanded)
     const contentClassNames = classNames(classes.content, {
         [classes.hidden]: !isExpanded,

website/src/components/github.js

@@ -33,10 +33,11 @@ const GitHubCode = ({ url, lang, errorMsg, className }) => {
                 })
                 .catch(err => {
                     setCode(errorMsg)
+                    console.error(err)
                 })
             setInitialized(true)
         }
-    }, [])
+    }, [initialized, rawUrl, errorMsg])
 
     const highlighted = lang === 'none' || !code ? code : highlightCode(lang, code)
 

website/src/components/progress.js

@@ -34,8 +34,7 @@ const Progress = () => {
         setOffset(getOffset())
     }
 
-    useEffect(
+    useEffect(() => {
-        () => {
         if (!initialized && progressRef.current) {
             handleResize()
             setInitialized(true)
@@ -47,9 +46,7 @@ const Progress = () => {
             window.removeEventListener('scroll', handleScroll)
             window.removeEventListener('resize', handleResize)
         }
-        },
+    }, [initialized, progressRef])
-        [progressRef]
-    )
 
     const { height, vh } = offset
     const total = 100 - ((height - scrollY - vh) / height) * 100

website/src/components/quickstart.js

@@ -8,6 +8,12 @@ import Icon from './icon'
 import { H2 } from './typography'
 import classes from '../styles/quickstart.module.sass'
 
+function getNewChecked(optionId, checkedForId, multiple) {
+    if (!multiple) return [optionId]
+    if (checkedForId.includes(optionId)) return checkedForId.filter(opt => opt !== optionId)
+    return [...checkedForId, optionId]
+}
+
 const Quickstart = ({ data, title, description, id, children }) => {
     const [styles, setStyles] = useState({})
     const [checked, setChecked] = useState({})
@@ -38,7 +44,7 @@ const Quickstart = ({ data, title, description, id, children }) => {
             setStyles(initialStyles)
             setInitialized(true)
         }
-    })
+    }, [data, initialized])
 
     return !data.length ? null : (
         <Section id={id}>
@@ -76,13 +82,11 @@ const Quickstart = ({ data, title, description, id, children }) => {
                                             onChange={() => {
                                                 const newChecked = {
                                                     ...checked,
-                                                    [id]: !multiple
+                                                    [id]: getNewChecked(
-                                                        ? [option.id]
+                                                        option.id,
-                                                        : checkedForId.includes(option.id)
+                                                        checkedForId,
-                                                        ? checkedForId.filter(
+                                                        multiple
-                                                              opt => opt !== option.id
+                                                    ),
-                                                          )
-                                                        : [...checkedForId, option.id],
                                                 }
                                                 setChecked(newChecked)
                                                 setStyles({

website/src/components/search.js

@@ -7,10 +7,10 @@ import classes from '../styles/search.module.sass'
 
 const Search = ({ id, placeholder, settings }) => {
     const { apiKey, indexName } = settings
-    const [isInitialized, setIsInitialized] = useState(false)
+    const [initialized, setInitialized] = useState(false)
     useEffect(() => {
-        if (!isInitialized) {
+        if (!initialized) {
-            setIsInitialized(true)
+            setInitialized(true)
             window.docsearch({
                 apiKey,
                 indexName,
@@ -18,7 +18,7 @@ const Search = ({ id, placeholder, settings }) => {
                 debug: false,
             })
         }
-    }, window.docsearch)
+    }, [initialized, apiKey, indexName, id])
     return (
         <form className={classes.root}>
             <label htmlFor={id} className={classes.icon}>

website/src/components/section.js

@@ -15,7 +15,7 @@ const Section = ({ id, className, ...props }) => {
         if (inView && relId) {
             window.dispatchEvent(new CustomEvent('inview', { detail: relId }))
         }
-    })
+    }, [inView, relId])
     return <section ref={ref} id={id} className={sectionClassNames} {...props} />
 }
 

website/src/components/sidebar.js

@@ -28,9 +28,9 @@ const Sidebar = ({ items, pageMenu, slug }) => {
     const [initialized, setInitialized] = useState(false)
     const [activeSection, setActiveSection] = useState(null)
     const activeRef = useRef()
-    const handleInView = ({ detail }) => setActiveSection(detail)
 
     useEffect(() => {
+        const handleInView = ({ detail }) => setActiveSection(detail)
         window.addEventListener('inview', handleInView, { passive: true })
         if (!initialized) {
             if (activeRef && activeRef.current) {
@@ -41,7 +41,7 @@ const Sidebar = ({ items, pageMenu, slug }) => {
         return () => {
             window.removeEventListener('inview', handleInView)
         }
-    }, [])
+    }, [initialized])
 
     return (
         <menu className={classNames('sidebar', classes.root)}>

website/src/templates/index.js

@@ -88,6 +88,34 @@ const AlertSpace = () => {
 }
 
 class Layout extends React.Component {
+    static defaultProps = {
+        scope: {},
+    }
+
+    static propTypes = {
+        data: PropTypes.shape({
+            mdx: PropTypes.shape({
+                code: PropTypes.shape({
+                    body: PropTypes.string.isRequired,
+                }).isRequired,
+            }),
+        }).isRequired,
+        scope: PropTypes.object.isRequired,
+        pageContext: PropTypes.shape({
+            title: PropTypes.string,
+            section: PropTypes.string,
+            teaser: PropTypes.string,
+            source: PropTypes.string,
+            isIndex: PropTypes.bool.isRequired,
+            theme: PropTypes.string,
+            next: PropTypes.shape({
+                title: PropTypes.string.isRequired,
+                slug: PropTypes.string.isRequired,
+            }),
+        }),
+        children: PropTypes.node,
+    }
+
     constructor(props) {
         super(props)
         // NB: Compiling the scope here instead of in render() is super
@@ -148,37 +176,7 @@ class Layout extends React.Component {
     }
 }
 
-Layout.defaultProps = {
+export default withMDXScope(Layout)
-    scope: {},
-}
-
-Layout.propTypes = {
-    data: PropTypes.shape({
-        mdx: PropTypes.shape({
-            code: PropTypes.shape({
-                body: PropTypes.string.isRequired,
-            }).isRequired,
-        }),
-    }).isRequired,
-    scope: PropTypes.object.isRequired,
-    pageContext: PropTypes.shape({
-        title: PropTypes.string,
-        section: PropTypes.string,
-        teaser: PropTypes.string,
-        source: PropTypes.string,
-        isIndex: PropTypes.bool.isRequired,
-        theme: PropTypes.string,
-        next: PropTypes.shape({
-            title: PropTypes.string.isRequired,
-            slug: PropTypes.string.isRequired,
-        }),
-    }),
-    children: PropTypes.node,
-}
-
-Layout = withMDXScope(Layout)
-
-export default Layout
 
 export const pageQuery = graphql`
     query($slug: String!) {

website/src/templates/models.js

@@ -120,10 +120,11 @@ const Model = ({ name, langId, langName, baseUrl, repo, compatibility, hasExampl
                 })
                 .catch(err => {
                     setIsError(true)
+                    console.error(err)
                 })
             setInitialized(true)
         }
-    })
+    }, [initialized, version, baseUrl, name])
 
     const releaseTag = meta.fullName ? `/tag/${meta.fullName}` : ''
     const releaseUrl = `https://github.com/${repo}/releases/${releaseTag}`
@@ -133,6 +134,7 @@ const Model = ({ name, langId, langName, baseUrl, repo, compatibility, hasExampl
     const author = !meta.url ? meta.author : <Link to={meta.url}>{meta.author}</Link>
     const licenseUrl = licenses[meta.license] ? licenses[meta.license].url : null
     const license = licenseUrl ? <Link to={licenseUrl}>{meta.license}</Link> : meta.license
+    const hasInteractiveCode = size === 'sm' && hasExamples
 
     const rows = [
         { label: 'Language', tag: langId, content: langName },
@@ -213,7 +215,7 @@ const Model = ({ name, langId, langName, baseUrl, repo, compatibility, hasExampl
                     )}
                 </tbody>
             </Table>
-            <Grid cols={2} gutterBottom={false}>
+            <Grid cols={2} gutterBottom={hasInteractiveCode}>
                 {accuracy &&
                     accuracy.map(({ label, items }, i) =>
                         !items ? null : (
@@ -241,7 +243,7 @@ const Model = ({ name, langId, langName, baseUrl, repo, compatibility, hasExampl
                     )}
             </Grid>
             {meta.notes && <p>{meta.notes}</p>}
-            {size === 'sm' && hasExamples && (
+            {hasInteractiveCode && (
                 <CodeBlock title="Try out the model" lang="python" executable={true}>
                     {[
                         `import spacy`,
@@ -275,7 +277,7 @@ const Models = ({ pageContext, repo, children }) => {
                 .catch(err => console.error(err))
             setInitialized(true)
         }
-    })
+    }, [initialized, baseUrl])
 
     return (
         <>

website/src/widgets/changelog.js

@@ -73,10 +73,11 @@ const Changelog = () => {
                 .catch(err => {
                     setIsLoading(false)
                     setIsError(true)
+                    console.error(err)
                 })
             setInitialized(true)
         }
-    }, [])
+    }, [initialized])
 
     const error = (
         <Infobox title="Unable to load changelog from GitHub" variant="danger">

website/src/widgets/landing.js

@@ -2,9 +2,18 @@ import React from 'react'
 import PropTypes from 'prop-types'
 import { StaticQuery, graphql } from 'gatsby'
 
-import { LandingHeader, LandingTitle, LandingSubtitle, LandingGrid } from '../components/landing'
+import {
-import { LandingCard, LandingButton, LandingDemo } from '../components/landing'
+    LandingHeader,
-import { LandingBannerGrid, LandingBanner, LandingLogos } from '../components/landing'
+    LandingTitle,
+    LandingSubtitle,
+    LandingGrid,
+    LandingCard,
+    LandingButton,
+    LandingDemo,
+    LandingBannerGrid,
+    LandingBanner,
+    LandingLogos,
+} from '../components/landing'
 import { H2 } from '../components/typography'
 import { Ul, Li } from '../components/list'
 import Button from '../components/button'