Merge branch 'develop' of https://github.com/explosion/spaCy into develop

spacy/lang/de/syntax_iterators.py

@@ -15,9 +15,9 @@ def noun_chunks(obj):
     # and not just "eine Tasse", same for "das Thema Familie".
     labels = ['sb', 'oa', 'da', 'nk', 'mo', 'ag', 'ROOT', 'root', 'cj', 'pd', 'og', 'app']
     doc = obj.doc # Ensure works on both Doc and Span.
-    np_label = doc.vocab.strings['NP']
-    np_deps = set(doc.vocab.strings[label] for label in labels)
-    close_app = doc.vocab.strings['nk']
+    np_label = doc.vocab.strings.add('NP')
+    np_deps = set(doc.vocab.strings.add(label) for label in labels)
+    close_app = doc.vocab.strings.add('nk')
 
     rbracket = 0
     for i, word in enumerate(obj):

spacy/lang/en/__init__.py

@@ -31,7 +31,7 @@ class EnglishDefaults(Language.Defaults):
     lemma_rules = dict(LEMMA_RULES)
     lemma_index = dict(LEMMA_INDEX)
     lemma_exc = dict(LEMMA_EXC)
-    sytax_iterators = dict(SYNTAX_ITERATORS)
+    syntax_iterators = dict(SYNTAX_ITERATORS)
 
 
 class English(Language):

spacy/lang/en/syntax_iterators.py

@@ -11,9 +11,9 @@ def noun_chunks(obj):
     labels = ['nsubj', 'dobj', 'nsubjpass', 'pcomp', 'pobj',
               'attr', 'ROOT']
     doc = obj.doc # Ensure works on both Doc and Span.
-    np_deps = [doc.vocab.strings[label] for label in labels]
-    conj = doc.vocab.strings['conj']
-    np_label = doc.vocab.strings['NP']
+    np_deps = [doc.vocab.strings.add(label) for label in labels]
+    conj = doc.vocab.strings.add('conj')
+    np_label = doc.vocab.strings.add('NP')
     seen = set()
     for i, word in enumerate(obj):
         if word.pos not in (NOUN, PROPN, PRON):

spacy/lang/hu/punctuation.py

@@ -9,7 +9,8 @@ LIST_ICONS = [r'[\p{So}--[°]]']
 _currency = r'\$|¢|£|€|¥|฿'
 _quotes = QUOTES.replace("'", '')
 
-_prefixes = ([r'\+'] + LIST_PUNCT + LIST_ELLIPSES + LIST_QUOTES + LIST_ICONS)
+_prefixes = ([r'\+'] + LIST_PUNCT + LIST_ELLIPSES + LIST_QUOTES + LIST_ICONS +
+             [r'[,.:](?=[{a}])'.format(a=ALPHA)])
 
 _suffixes = (LIST_PUNCT + LIST_ELLIPSES + LIST_QUOTES + LIST_ICONS +
              [r'(?<=[0-9])\+',
@@ -21,7 +22,7 @@ _suffixes = (LIST_PUNCT + LIST_ELLIPSES + LIST_QUOTES + LIST_ICONS +
 
 _infixes = (LIST_ELLIPSES + LIST_ICONS +
             [r'(?<=[{}])\.(?=[{}])'.format(ALPHA_LOWER, ALPHA_UPPER),
-             r'(?<=[{a}]),(?=[{a}])'.format(a=ALPHA),
+             r'(?<=[{a}])[,!?](?=[{a}])'.format(a=ALPHA),
              r'(?<=[{a}"])[:<>=](?=[{a}])'.format(a=ALPHA),
              r'(?<=[{a}])--(?=[{a}])'.format(a=ALPHA),
              r'(?<=[{a}]),(?=[{a}])'.format(a=ALPHA),

spacy/language.py

@@ -184,6 +184,35 @@ class Language(object):
                 flat_list.append(pipe)
         self.pipeline = flat_list
 
+    # Conveniences to access pipeline components
+    @property
+    def tensorizer(self):
+        return self.get_component('tensorizer')
+
+    @property
+    def tagger(self):
+        return self.get_component('tagger')
+
+    @property
+    def parser(self):
+        return self.get_component('parser')
+
+    @property
+    def entity(self):
+        return self.get_component('ner')
+
+    @property
+    def matcher(self):
+        return self.get_component('matcher')
+
+    def get_component(self, name): 
+        if self.pipeline in (True, None):
+            return None
+        for proc in self.pipeline:
+            if hasattr(proc, 'name') and proc.name.endswith(name):
+                return proc
+        return None
+
     def __call__(self, text, disable=[]):
         """'Apply the pipeline to some text. The text can span multiple sentences,
         and can contain arbtrary whitespace. Alignment into the original string

spacy/morphology.pxd

@@ -30,6 +30,7 @@ cdef class Morphology:
     cdef public object n_tags
     cdef public object reverse_index
     cdef public object tag_names
+    cdef public object exc
 
     cdef RichTagC* rich_tags
     cdef PreshMapArray _cache

spacy/morphology.pyx

@@ -33,7 +33,7 @@ def _normalize_props(props):
 
 
 cdef class Morphology:
-    def __init__(self, StringStore string_store, tag_map, lemmatizer):
+    def __init__(self, StringStore string_store, tag_map, lemmatizer, exc=None):
         self.mem = Pool()
         self.strings = string_store
         self.tag_map = {}
@@ -53,9 +53,14 @@ cdef class Morphology:
             self.rich_tags[i].pos = attrs[POS]
             self.reverse_index[self.rich_tags[i].name] = i
         self._cache = PreshMapArray(self.n_tags)
+        self.exc = {}
+        if exc is not None:
+            for (tag_str, orth_str), attrs in exc.items():
+                self.add_special_case(tag_str, orth_str, attrs)
 
     def __reduce__(self):
-        return (Morphology, (self.strings, self.tag_map, self.lemmatizer), None, None)
+        return (Morphology, (self.strings, self.tag_map, self.lemmatizer,
+                             self.exc), None, None)
 
     cdef int assign_tag(self, TokenC* token, tag) except -1:
         if isinstance(tag, basestring):
@@ -106,6 +111,7 @@ cdef class Morphology:
             tag (unicode): The part-of-speech tag to key the exception.
             orth (unicode): The word-form to key the exception.
         """
+        self.exc[(tag_str, orth_str)] = dict(attrs)
         tag = self.strings.add(tag_str)
         tag_id = self.reverse_index[tag]
         orth = self.strings[orth_str]

spacy/pipeline.pyx

@@ -233,7 +233,9 @@ class NeuralTagger(object):
         for i, doc in enumerate(docs):
             doc_tag_ids = batch_tag_ids[i]
             for j, tag_id in enumerate(doc_tag_ids):
-                vocab.morphology.assign_tag_id(&doc.c[j], tag_id)
+                # Don't clobber preset POS tags
+                if doc.c[j].tag == 0 and doc.c[j].pos == 0:
+                    vocab.morphology.assign_tag_id(&doc.c[j], tag_id)
                 idx += 1
         doc.is_tagged = True
 
@@ -285,7 +287,8 @@ class NeuralTagger(object):
         cdef Vocab vocab = self.vocab
         if new_tag_map:
             vocab.morphology = Morphology(vocab.strings, new_tag_map,
-                                          vocab.morphology.lemmatizer)
+                                          vocab.morphology.lemmatizer,
+                                          exc=vocab.morphology.exc)
         token_vector_width = pipeline[0].model.nO
         if self.model is True:
             self.model = self.Model(self.vocab.morphology.n_tags, token_vector_width)
@@ -321,7 +324,9 @@ class NeuralTagger(object):
             tag_map = msgpack.loads(b, encoding='utf8')
             self.vocab.morphology = Morphology(
                 self.vocab.strings, tag_map=tag_map,
-                lemmatizer=self.vocab.morphology.lemmatizer)
+                lemmatizer=self.vocab.morphology.lemmatizer,
+                exc=self.vocab.morphology.exc)
+ 
         deserialize = OrderedDict((
             ('vocab', lambda b: self.vocab.from_bytes(b)),
             ('tag_map', load_tag_map),
@@ -353,7 +358,9 @@ class NeuralTagger(object):
                 tag_map = msgpack.loads(file_.read(), encoding='utf8')
             self.vocab.morphology = Morphology(
                 self.vocab.strings, tag_map=tag_map,
-                lemmatizer=self.vocab.morphology.lemmatizer)
+                lemmatizer=self.vocab.morphology.lemmatizer,
+                exc=self.vocab.morphology.exc)
+ 
 
         deserialize = OrderedDict((
             ('vocab', lambda p: self.vocab.from_disk(p)),

spacy/syntax/nn_parser.pyx

@@ -164,6 +164,7 @@ cdef class precompute_hiddens:
         return best, backprop
 
 
+
 cdef void sum_state_features(float* output,
         const float* cached, const int* token_ids, int B, int F, int O) nogil:
     cdef int idx, b, f, i

spacy/tests/conftest.py

@@ -13,7 +13,7 @@ from .. import util
 
 _languages = ['bn', 'da', 'de', 'en', 'es', 'fi', 'fr', 'he', 'hu', 'it', 'nb',
               'nl', 'pl', 'pt', 'sv', 'xx']
-_models = {'en': ['en_core_web_sm', 'en_core_web_md'],
+_models = {'en': ['en_core_web_sm', 'en_depent_web_sm', 'en_core_web_md'],
            'de': ['de_core_news_md'],
            'fr': ['fr_depvec_web_lg'],
            'xx': ['xx_ent_web_md']}
@@ -22,48 +22,48 @@ _models = {'en': ['en_core_web_sm', 'en_core_web_md'],
 # only used for tests that require loading the models
 # in all other cases, use specific instances
 
-@pytest.fixture(params=_models['en'], scope="session")
+@pytest.fixture(params=_models['en'], scope='session')
 def EN(request):
     return load_test_model(request.param)
 
 
-@pytest.fixture(params=_models['de'], scope="session")
+@pytest.fixture(params=_models['de'], scope='session')
 def DE(request):
     return load_test_model(request.param)
 
 
-@pytest.fixture(params=_models['fr'], scope="session")
+@pytest.fixture(params=_models['fr'], scope='session')
 def FR(request):
     return load_test_model(request.param)
 
 
-@pytest.fixture(params=_languages)
+@pytest.fixture(params=_languages, scope='module')
 def tokenizer(request):
     lang = util.get_lang_class(request.param)
     return lang.Defaults.create_tokenizer()
 
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def en_tokenizer():
     return util.get_lang_class('en').Defaults.create_tokenizer()
 
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def en_vocab():
     return util.get_lang_class('en').Defaults.create_vocab()
 
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def en_parser():
     return util.get_lang_class('en').Defaults.create_parser()
 
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def es_tokenizer():
     return util.get_lang_class('es').Defaults.create_tokenizer()
 
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def de_tokenizer():
     return util.get_lang_class('de').Defaults.create_tokenizer()
 
@@ -73,31 +73,31 @@ def fr_tokenizer():
     return util.get_lang_class('fr').Defaults.create_tokenizer()
 
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def hu_tokenizer():
     return util.get_lang_class('hu').Defaults.create_tokenizer()
 
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def fi_tokenizer():
     return util.get_lang_class('fi').Defaults.create_tokenizer()
 
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def sv_tokenizer():
     return util.get_lang_class('sv').Defaults.create_tokenizer()
 
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def bn_tokenizer():
     return util.get_lang_class('bn').Defaults.create_tokenizer()
 
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def he_tokenizer():
     return util.get_lang_class('he').Defaults.create_tokenizer()
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def nb_tokenizer():
     return util.get_lang_class('nb').Defaults.create_tokenizer()
 
@@ -107,7 +107,7 @@ def stringstore():
     return StringStore()
 
 
-@pytest.fixture
+@pytest.fixture(scope='module')
 def en_entityrecognizer():
      return util.get_lang_class('en').Defaults.create_entity()
 

spacy/tests/lang/en/test_lemmatizer.py

@@ -40,7 +40,8 @@ def test_en_lemmatizer_punct(en_lemmatizer):
 @pytest.mark.models('en')
 def test_en_lemmatizer_lemma_assignment(EN):
     text = "Bananas in pyjamas are geese."
-    doc = EN.tokenizer(text)
+    doc = EN.make_doc(text)
+    EN.tensorizer(doc)
     assert all(t.lemma_ == '' for t in doc)
     EN.tagger(doc)
     assert all(t.lemma_ != '' for t in doc)

spacy/tests/lang/en/test_ner.py

@@ -26,6 +26,7 @@ def test_en_ner_consistency_bug(EN):
     EN.entity(tokens)
 
 
+@pytest.mark.skip
 @pytest.mark.models('en')
 def test_en_ner_unit_end_gazetteer(EN):
     '''Test a bug in the interaction between the NER model and the gazetteer'''

spacy/tests/lang/hu/test_tokenizer.py

@@ -5,11 +5,11 @@ import pytest
 
 DEFAULT_TESTS = [
     ('N. kormányzósági\nszékhely.', ['N.', 'kormányzósági', 'székhely', '.']),
-    ('A .hu egy tld.', ['A', '.hu', 'egy', 'tld', '.']),
+    pytest.param('A .hu egy tld.', ['A', '.hu', 'egy', 'tld', '.'], marks=pytest.mark.xfail),
     ('Az egy.ketto pelda.', ['Az', 'egy.ketto', 'pelda', '.']),
     ('A pl. rovidites.', ['A', 'pl.', 'rovidites', '.']),
     ('A S.M.A.R.T. szo.', ['A', 'S.M.A.R.T.', 'szo', '.']),
-    ('A .hu.', ['A', '.hu', '.']),
+    pytest.param('A .hu.', ['A', '.hu', '.'], marks=pytest.mark.xfail),
     ('Az egy.ketto.', ['Az', 'egy.ketto', '.']),
     ('A pl.', ['A', 'pl.']),
     ('A S.M.A.R.T.', ['A', 'S.M.A.R.T.']),
@@ -18,7 +18,9 @@ DEFAULT_TESTS = [
     ('Valami ...van...', ['Valami', '...', 'van', '...']),
     ('Valami...', ['Valami', '...']),
     ('Valami ...', ['Valami', '...']),
-    ('Valami ... más.', ['Valami', '...', 'más', '.'])
+    ('Valami ... más.', ['Valami', '...', 'más', '.']),
+    ('Soha nem lesz!', ['Soha', 'nem', 'lesz', '!']),
+    ('Soha nem lesz?', ['Soha', 'nem', 'lesz', '?'])
 ]
 
 HYPHEN_TESTS = [
@@ -225,11 +227,11 @@ QUOTE_TESTS = [
 
 DOT_TESTS = [
     ('N. kormányzósági\nszékhely.', ['N.', 'kormányzósági', 'székhely', '.']),
-    ('A .hu egy tld.', ['A', '.hu', 'egy', 'tld', '.']),
+    pytest.param('A .hu egy tld.', ['A', '.hu', 'egy', 'tld', '.'], marks=pytest.mark.xfail),
     ('Az egy.ketto pelda.', ['Az', 'egy.ketto', 'pelda', '.']),
     ('A pl. rövidítés.', ['A', 'pl.', 'rövidítés', '.']),
     ('A S.M.A.R.T. szó.', ['A', 'S.M.A.R.T.', 'szó', '.']),
-    ('A .hu.', ['A', '.hu', '.']),
+    pytest.param('A .hu.', ['A', '.hu', '.'], marks=pytest.mark.xfail),
     ('Az egy.ketto.', ['Az', 'egy.ketto', '.']),
     ('A pl.', ['A', 'pl.']),
     ('A S.M.A.R.T.', ['A', 'S.M.A.R.T.']),
@@ -241,6 +243,24 @@ DOT_TESTS = [
     ('Valami ... más.', ['Valami', '...', 'más', '.'])
 ]
 
+TYPO_TESTS = [
+    (
+    'Ez egy mondat vége.Ez egy másik eleje.', ['Ez', 'egy', 'mondat', 'vége', '.', 'Ez', 'egy', 'másik', 'eleje', '.']),
+    ('Ez egy mondat vége .Ez egy másik eleje.',
+     ['Ez', 'egy', 'mondat', 'vége', '.', 'Ez', 'egy', 'másik', 'eleje', '.']),
+    (
+    'Ez egy mondat vége!ez egy másik eleje.', ['Ez', 'egy', 'mondat', 'vége', '!', 'ez', 'egy', 'másik', 'eleje', '.']),
+    ('Ez egy mondat vége !ez egy másik eleje.',
+     ['Ez', 'egy', 'mondat', 'vége', '!', 'ez', 'egy', 'másik', 'eleje', '.']),
+    (
+    'Ez egy mondat vége?Ez egy másik eleje.', ['Ez', 'egy', 'mondat', 'vége', '?', 'Ez', 'egy', 'másik', 'eleje', '.']),
+    ('Ez egy mondat vége ?Ez egy másik eleje.',
+     ['Ez', 'egy', 'mondat', 'vége', '?', 'Ez', 'egy', 'másik', 'eleje', '.']),
+    ('egy,kettő', ['egy', ',', 'kettő']),
+    ('egy ,kettő', ['egy', ',', 'kettő']),
+    ('egy :kettő', ['egy', ':', 'kettő']),
+]
+
 WIKI_TESTS = [
     ('!"', ['!', '"']),
     ('lány"a', ['lány', '"', 'a']),
@@ -253,7 +273,7 @@ WIKI_TESTS = [
     ('cérium(IV)-oxid', ['cérium', '(', 'IV', ')', '-oxid'])
 ]
 
-TESTCASES = DEFAULT_TESTS + DOT_TESTS + QUOTE_TESTS + NUMBER_TESTS + HYPHEN_TESTS + WIKI_TESTS
+TESTCASES = DEFAULT_TESTS + DOT_TESTS + QUOTE_TESTS + NUMBER_TESTS + HYPHEN_TESTS + WIKI_TESTS + TYPO_TESTS
 
 
 @pytest.mark.parametrize('text,expected_tokens', TESTCASES)

spacy/tests/regression/test_issue429.py

@@ -19,6 +19,7 @@ def test_issue429(EN):
     matcher = Matcher(EN.vocab)
     matcher.add('TEST', merge_phrases, [{'ORTH': 'a'}])
     doc = EN.make_doc('a b c')
+    EN.tensorizer(doc)
     EN.tagger(doc)
     matcher(doc)
     EN.entity(doc)

spacy/tests/regression/test_issue514.py

@@ -6,6 +6,7 @@ from ..util import get_doc
 import pytest
 
 
+@pytest.mark.skip
 @pytest.mark.models('en')
 def test_issue514(EN):
     """Test serializing after adding entity"""

spacy/tests/regression/test_issue589.py

@@ -7,6 +7,7 @@ from ..util import get_doc
 import pytest
 
 
+@pytest.mark.xfail
 def test_issue589():
     vocab = Vocab()
     vocab.strings.set_frozen(True)

spacy/tests/regression/test_issue704.py

@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import pytest
 
 
+@pytest.mark.xfail
 @pytest.mark.models('en')
 def test_issue704(EN):
     """Test that sentence boundaries are detected correctly."""

spacy/tests/regression/test_issue910.py

@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 import json
-import os
 import random
 import contextlib
 import shutil
@@ -9,7 +8,6 @@ import tempfile
 from pathlib import Path
 
 
-import pathlib
 from ...gold import GoldParse
 from ...pipeline import EntityRecognizer
 from ...lang.en import English
@@ -57,19 +55,13 @@ def additional_entity_types():
 
 @contextlib.contextmanager
 def temp_save_model(model):
-    model_dir = Path(tempfile.mkdtemp())
-    # store the fine tuned model
-    with (model_dir / "config.json").open('w') as file_:
-        data = json.dumps(model.cfg)
-        if not isinstance(data, unicode):
-            data = data.decode('utf8')
-        file_.write(data)
-    model.model.dump((model_dir / 'model').as_posix())
+    model_dir = tempfile.mkdtemp()
+    model.to_disk(model_dir)
     yield model_dir
     shutil.rmtree(model_dir.as_posix())
 
 
-
+@pytest.mark.xfail
 @pytest.mark.models('en')
 def test_issue910(EN, train_data, additional_entity_types):
     '''Test that adding entities and resuming training works passably OK.
@@ -79,24 +71,27 @@ def test_issue910(EN, train_data, additional_entity_types):
     2) There's no way to set the learning rate for the weight update, so we
         end up out-of-scale, causing it to learn too fast.
     '''
-    doc = EN(u"I am looking for a restaurant in Berlin")
+    nlp = EN
+    doc = nlp(u"I am looking for a restaurant in Berlin")
     ents_before_train = [(ent.label_, ent.text) for ent in doc.ents]
     # Fine tune the ner model
     for entity_type in additional_entity_types:
         nlp.entity.add_label(entity_type)
 
-    nlp.entity.model.learn_rate = 0.001
+    sgd = Adam(nlp.entity.model[0].ops, 0.001)
     for itn in range(10):
         random.shuffle(train_data)
         for raw_text, entity_offsets in train_data:
             doc = nlp.make_doc(raw_text)
             nlp.tagger(doc)
+            nlp.tensorizer(doc)
             gold = GoldParse(doc, entities=entity_offsets)
-            loss = nlp.entity.update(doc, gold)
+            loss = nlp.entity.update(doc, gold, sgd=sgd, drop=0.5)
 
     with temp_save_model(nlp.entity) as model_dir:
         # Load the fine tuned model
-        loaded_ner = EntityRecognizer.load(model_dir, nlp.vocab)
+        loaded_ner = EntityRecognizer(nlp.vocab)
+        loaded_ner.from_disk(model_dir)
 
     for raw_text, entity_offsets in train_data:
         doc = nlp.make_doc(raw_text)

spacy/tests/regression/test_issue995.py

@@ -4,7 +4,7 @@ import pytest
 
 
 @pytest.mark.models('en')
-def test_issue955(EN, doc):
+def test_issue955(EN):
     '''Test that we don't have any nested noun chunks'''
     doc = EN('Does flight number three fifty-four require a connecting flight'
              ' to get to Boston?')

spacy/tokens/doc.pyx

@@ -65,8 +65,13 @@ cdef attr_t get_token_attr(const TokenC* token, attr_id_t feat_name) nogil:
         return Lexeme.get_struct_attr(token.lex, feat_name)
 
 def _get_chunker(lang):
-    cls = util.get_lang_class(lang)
-    return cls.Defaults.syntax_iterators.get('noun_chunks')
+    try:
+        cls = util.get_lang_class(lang)
+    except ImportError:
+        return None
+    except KeyError:
+        return None
+    return cls.Defaults.syntax_iterators.get(u'noun_chunks')
 
 cdef class Doc:
     """A sequence of Token objects. Access sentences and named entities, export

website/_includes/_page-docs.jade

@@ -22,12 +22,12 @@ main.o-main.o-main--sidebar.o-main--aside
             +infobox("⚠️ You are viewing the spaCy v2.0.0 alpha docs")
                 strong This page is part of the alpha documentation for spaCy v2.0.
                 |  It does not reflect the state of the latest stable release.
-                |  Because v2.0 is still under development, the actual
-                |  implementation may differ from the intended state described
-                |  here.
-                |  #[+a("#") See here] for more information on how to install
-                |  and test the new version. To read the official docs for
-                |  v1.x, #[+a("https://spacy.io/docs") go here].
+                |  Because v2.0 is still under development, the implementation
+                |  may differ from the intended state described here. See the
+                |  #[+a(gh("spaCy") + "/releases/tag/v2.0.0-alpha") release notes]
+                |  for details on how to install and test the new version. To
+                |  read the official docs for spaCy v1.x,
+                |  #[+a("https://spacy.io/docs") go here].
 
         !=yield
 

website/docs/api/cli.jade

@@ -209,8 +209,8 @@ p
         +cell Number of sentences (default: #[code 0]).
 
     +row
-        +cell #[code --use-gpu], #[code -G]
-        +cell flag
+        +cell #[code --use-gpu], #[code -g]
+        +cell option
         +cell Use GPU.
 
     +row

website/docs/usage/adding-languages.jade

@@ -42,6 +42,7 @@ p
         +item #[+a("#tokenizer-exceptions") Tokenizer exceptions]
         +item #[+a("#norm-exceptions") Norm exceptions]
         +item #[+a("#lex-attrs") Lexical attributes]
+        +item #[+a("#syntax-iterators") Syntax iterators]
         +item #[+a("#lemmatizer") Lemmatizer]
         +item #[+a("#tag-map") Tag map]
         +item #[+a("#morph-rules") Morph rules]
@@ -104,6 +105,13 @@ p
         +cell dict
         +cell Attribute ID mapped to function.
 
+    +row
+        +cell #[code SYNTAX_ITERATORS]
+        +cell dict
+        +cell
+            |  Iterator ID mapped to function. Currently only supports
+            |  #[code 'noun_chunks'].
+
     +row
         +cell #[code LOOKUP]
         +cell dict
@@ -341,9 +349,12 @@ p
     |  a token's norm equals its lowercase text. If the lowercase spelling of a
     |  word exists, norms should always be in lowercase.
 
-+aside-code("Accessing norms").
-    doc = nlp(u"I can't")
-    assert [t.norm_ for t in doc] == ['i', 'can', 'not']
++aside-code("Norms vs. lemmas").
+    doc = nlp(u"I'm gonna realise")
+    norms = [token.norm_ for token in doc]
+    lemmas = [token.lemma_ for token in doc]
+    assert norms == ['i', 'am', 'going', 'to', 'realize']
+    assert lemmas == ['i', 'be', 'go', 'to', 'realise']
 
 p
     |  spaCy usually tries to normalise words with different spellings to a single,
@@ -449,6 +460,33 @@ p
     |  #[code lex_attr_getters.update(LEX_ATTRS)], only the new custom functions
     |  are overwritten.
 
++h(3, "syntax-iterators") Syntax iterators
+
+p
+    |  Syntax iterators are functions that compute views of a #[code Doc]
+    |  object based on its syntax. At the moment, this data is only used for
+    |  extracting
+    |  #[+a("/docs/usage/dependency-parse#noun-chunks") noun chunks], which
+    |  are available as the #[+api("doc#noun_chunks") #[code Doc.noun_chunks]]
+    |  property. Because base noun phrases work differently across languages,
+    |  the rules to compute them are part of the individual language's data. If
+    |  a language does not include a noun chunks iterator, the property won't
+    |  be available. For examples, see the existing syntax iterators:
+
++aside-code("Noun chunks example").
+    doc = nlp(u'A phrase with another phrase occurs.')
+    chunks = list(doc.noun_chunks)
+    assert chunks[0].text == "A phrase"
+    assert chunks[1].text == "another phrase"
+
++table(["Language", "Source"])
+    for lang, lang_id in {en: "English", de: "German", es: "Spanish"}
+        +row
+            +cell=lang
+            +cell
+                +src(gh("spaCy", "spacy/lang/" + lang_id + "/syntax_iterators.py"))
+                    |  lang/#{lang_id}/syntax_iterators.py
+
 +h(3, "lemmatizer") Lemmatizer
 
 p
@@ -604,6 +642,8 @@ p
 
 +h(2, "vocabulary") Building the vocabulary
 
++under-construction
+
 p
     |  spaCy expects that common words will be cached in a
     |  #[+api("vocab") #[code Vocab]] instance. The vocabulary caches lexical
@@ -697,6 +737,8 @@ p
 
 +h(3, "word-vectors") Training the word vectors
 
++under-construction
+
 p
     |  #[+a("https://en.wikipedia.org/wiki/Word2vec") Word2vec] and related
     |  algorithms let you train useful word similarity models from unlabelled
@@ -731,6 +773,8 @@ p
 
 +h(2, "train-tagger-parser") Training the tagger and parser
 
++under-construction
+
 p
     |  You can now train the model using a corpus for your language annotated
     |  with #[+a("http://universaldependencies.org/") Universal Dependencies].