Merge branch 'develop'

bin/parser/train.py

@@ -66,8 +66,8 @@ def score_model(scorer, nlp, raw_text, annot_tuples, verbose=False):
 
 def train(Language, train_data, dev_data, model_dir, tagger_cfg, parser_cfg, entity_cfg,
         n_iter=15, seed=0, gold_preproc=False, n_sents=0, corruption_level=0):
-    print("Itn.\tP.Loss\tUAS\tNER F.\tTag %\tToken %")
+    print("Itn.\tN weight\tN feats\tUAS\tNER F.\tTag %\tToken %")
-    format_str = '{:d}\t{:d}\t{uas:.3f}\t{ents_f:.3f}\t{tags_acc:.3f}\t{token_acc:.3f}'
+    format_str = '{:d}\t{:d}\t{:d}\t{uas:.3f}\t{ents_f:.3f}\t{tags_acc:.3f}\t{token_acc:.3f}'
     with Language.train(model_dir, train_data,
             tagger_cfg, parser_cfg, entity_cfg) as trainer:
         loss = 0
@@ -76,11 +76,13 @@ def train(Language, train_data, dev_data, model_dir, tagger_cfg, parser_cfg, ent
             for doc, gold in epoch:
                 trainer.update(doc, gold)
             dev_scores = trainer.evaluate(dev_data, gold_preproc=gold_preproc)
-            print(format_str.format(itn, loss, **dev_scores.scores))
+            print(format_str.format(itn, trainer.nlp.parser.model.nr_weight,
+                trainer.nlp.parser.model.nr_active_feat, **dev_scores.scores))
 
 
 def evaluate(Language, gold_tuples, model_dir, gold_preproc=False, verbose=False,
              beam_width=None, cand_preproc=None):
+    print("Load parser", model_dir)
     nlp = Language(path=model_dir)
     if nlp.lang == 'de':
         nlp.vocab.morphology.lemmatizer = lambda string,pos: set([string])
@@ -145,21 +147,25 @@ def write_parses(Language, dev_loc, model_dir, out_loc):
     verbose=("Verbose error reporting", "flag", "v", bool),
     debug=("Debug mode", "flag", "d", bool),
     pseudoprojective=("Use pseudo-projective parsing", "flag", "p", bool),
+    L1=("L1 regularization penalty", "option", "L", float),
 )
 def main(language, train_loc, dev_loc, model_dir, n_sents=0, n_iter=15, out_loc="", verbose=False,
-         debug=False, corruption_level=0.0, gold_preproc=False, eval_only=False, pseudoprojective=False):
+         debug=False, corruption_level=0.0, gold_preproc=False, eval_only=False, pseudoprojective=False,
+         L1=1e-6):
     parser_cfg = dict(locals())
     tagger_cfg = dict(locals())
     entity_cfg = dict(locals())
 
     lang = spacy.util.get_lang_class(language)
-    
+
     parser_cfg['features'] = lang.Defaults.parser_features
     entity_cfg['features'] = lang.Defaults.entity_features
 
     if not eval_only:
         gold_train = list(read_json_file(train_loc))
         gold_dev = list(read_json_file(dev_loc))
+        if n_sents > 0:
+            gold_train = gold_train[:n_sents]
         train(lang, gold_train, gold_dev, model_dir, tagger_cfg, parser_cfg, entity_cfg,
               n_sents=n_sents, gold_preproc=gold_preproc, corruption_level=corruption_level,
               n_iter=n_iter)

requirements.txt

@@ -10,3 +10,4 @@ six
 ujson>=1.35
 cloudpickle
 sputnik>=0.9.2,<0.10.0
+dill>=0.2,<0.3

setup.py

@@ -241,7 +241,8 @@ def setup_package():
                 'cloudpickle',
                 'pathlib',
                 'sputnik>=0.9.2,<0.10.0',
-                'ujson>=1.35'],
+                'ujson>=1.35',
+                'dill>=0.2,<0.3'],
             classifiers=[
                 'Development Status :: 5 - Production/Stable',
                 'Environment :: Console',

spacy/attrs.pyx

@@ -93,7 +93,7 @@ NAMES = [key for key, value in sorted(IDS.items(), key=lambda item: item[1])]
 
 def intify_attrs(stringy_attrs, strings_map=None, _do_deprecated=False):
     '''Normalize a dictionary of attributes, converting them to ints.
-    
+
     Arguments:
         stringy_attrs (dict):
             Dictionary keyed by attribute string names. Values can be ints or strings.
@@ -125,7 +125,9 @@ def intify_attrs(stringy_attrs, strings_map=None, _do_deprecated=False):
             'VerbForm', 'PronType', 'Aspect', 'Tense', 'PartType', 'Poss',
             'Hyph', 'ConjType', 'NumType', 'Foreign', 'VerbType', 'NounType',
             'Number', 'PronType', 'AdjType', 'Person', 'Variant', 'AdpType',
-            'Reflex', 'Negative', 'Mood', 'Aspect', 'Case']
+            'Reflex', 'Negative', 'Mood', 'Aspect', 'Case',
+            'Polarity', # U20
+        ]
         for key in morph_keys:
             if key in stringy_attrs:
                 stringy_attrs.pop(key)

spacy/cfile.pxd

@@ -4,6 +4,20 @@ from cymem.cymem cimport Pool
 cdef class CFile:
     cdef FILE* fp
     cdef bint is_open
+    cdef Pool mem
+    cdef int size # For compatibility with subclass
+    cdef int _capacity # For compatibility with subclass
+
+    cdef int read_into(self, void* dest, size_t number, size_t elem_size) except -1
+
+    cdef int write_from(self, void* src, size_t number, size_t elem_size) except -1
+
+    cdef void* alloc_read(self, Pool mem, size_t number, size_t elem_size) except *
+
+
+
+cdef class StringCFile(CFile):
+    cdef unsigned char* data
  
     cdef int read_into(self, void* dest, size_t number, size_t elem_size) except -1
 

spacy/cfile.pyx

@@ -1,4 +1,5 @@
 from libc.stdio cimport fopen, fclose, fread, fwrite, FILE
+from libc.string cimport memcpy
 
 
 cdef class CFile:
@@ -9,6 +10,7 @@ cdef class CFile:
             mode_str = mode
         if hasattr(loc, 'as_posix'):
             loc = loc.as_posix()
+        self.mem = Pool()
         cdef bytes bytes_loc = loc.encode('utf8') if type(loc) == unicode else loc
         self.fp = fopen(<char*>bytes_loc, mode_str)
         if self.fp == NULL:
@@ -45,3 +47,42 @@ cdef class CFile:
         cdef bytes py_bytes = value.encode('utf8')
         cdef char* chars = <char*>py_bytes
         self.write(sizeof(char), len(py_bytes), chars)
+
+
+cdef class StringCFile:
+    def __init__(self, mode, bytes data=b'', on_open_error=None):
+        self.mem = Pool()
+        self.is_open = 'w' in mode
+        self._capacity = max(len(data), 8)
+        self.size = len(data)
+        self.data = <unsigned char*>self.mem.alloc(1, self._capacity)
+        for i in range(len(data)):
+            self.data[i] = data[i]
+
+    def close(self):
+        self.is_open = False
+
+    def string_data(self):
+        return (self.data-self.size)[:self.size]
+
+    cdef int read_into(self, void* dest, size_t number, size_t elem_size) except -1:
+        memcpy(dest, self.data, elem_size * number)
+        self.data += elem_size * number
+
+    cdef int write_from(self, void* src, size_t elem_size, size_t number) except -1:
+        write_size = number * elem_size
+        if (self.size + write_size) >= self._capacity:
+            self._capacity = (self.size + write_size) * 2
+            self.data = <unsigned char*>self.mem.realloc(self.data, self._capacity)
+        memcpy(&self.data[self.size], src, elem_size * number)
+        self.size += write_size
+
+    cdef void* alloc_read(self, Pool mem, size_t number, size_t elem_size) except *:
+        cdef void* dest = mem.alloc(number, elem_size)
+        self.read_into(dest, number, elem_size)
+        return dest
+
+    def write_unicode(self, unicode value):
+        cdef bytes py_bytes = value.encode('utf8')
+        cdef char* chars = <char*>py_bytes
+        self.write(sizeof(char), len(py_bytes), chars)

spacy/de/tag_map.py

@@ -41,7 +41,7 @@ TAG_MAP = {
     "PRF":      {POS: PRON, "PronType": "prs", "Reflex": "yes"},
     "PTKA":     {POS: PART},
     "PTKANT":   {POS: PART, "PartType": "res"},
-    "PTKNEG":   {POS: PART, "Negative": "yes"},
+    "PTKNEG":   {POS: PART, "Polarity": "Neg"},
     "PTKVZ":    {POS: PART, "PartType": "vbp"},
     "PTKZU":    {POS: PART, "PartType": "inf"},
     "PWAT":     {POS: DET, "PronType": "int"},

spacy/en/__init__.py

@@ -2,6 +2,7 @@
 from __future__ import unicode_literals, print_function
 
 from os import path
+from pathlib import Path
 
 from ..util import match_best_version
 from ..util import get_data_path
@@ -13,6 +14,11 @@ from ..attrs import LANG
 
 from .language_data import *
 
+try:
+    basestring
+except NameError:
+    basestring = str
+
 
 class English(Language):
     lang = 'en'
@@ -43,14 +49,15 @@ def _fix_deprecated_glove_vectors_loading(overrides):
         data_path = get_data_path()
     else:
         path = overrides['path']
+        if isinstance(path, basestring):
+            path = Path(path)
         data_path = path.parent
     vec_path = None
     if 'add_vectors' not in overrides:
         if 'vectors' in overrides:
             vec_path = match_best_version(overrides['vectors'], None, data_path)
             if vec_path is None:
-                raise IOError(
+                return overrides
-                    'Could not load data pack %s from %s' % (overrides['vectors'], data_path))
         else:
             vec_path = match_best_version('en_glove_cc_300_1m_vectors', None, data_path)
         if vec_path is not None:

spacy/en/tag_map.py

@@ -16,7 +16,7 @@ TAG_MAP = {
     "$":        {POS: SYM, "Other": {"SymType": "currency"}},
     "#":        {POS: SYM, "Other": {"SymType": "numbersign"}},
     "AFX":      {POS: ADJ,  "Hyph": "yes"},
-    "CC":       {POS: CONJ, "ConjType": "coor"},
+    "CC":       {POS: CCONJ, "ConjType": "coor"},
     "CD":       {POS: NUM, "NumType": "card"},
     "DT":       {POS: DET},
     "EX":       {POS: ADV, "AdvType": "ex"},

spacy/language.py

@@ -5,7 +5,7 @@ import pathlib
 from contextlib import contextmanager
 import shutil
 
-import ujson as json
+import ujson
 
 
 try:
@@ -13,6 +13,10 @@ try:
 except NameError:
     basestring = str
 
+try:
+    unicode
+except NameError:
+    unicode = str
 
 from .tokenizer import Tokenizer
 from .vocab import Vocab
@@ -226,12 +230,21 @@ class Language(object):
         parser_cfg['actions'] = ArcEager.get_actions(gold_parses=gold_tuples)
         entity_cfg['actions'] = BiluoPushDown.get_actions(gold_parses=gold_tuples)
 
-        with (dep_model_dir / 'config.json').open('w') as file_:
+        with (dep_model_dir / 'config.json').open('wb') as file_:
-            json.dump(parser_cfg, file_)
+            data = ujson.dumps(parser_cfg)
-        with (ner_model_dir / 'config.json').open('w') as file_:
+            if isinstance(data, unicode):
-            json.dump(entity_cfg, file_)
+                data = data.encode('utf8')
-        with (pos_model_dir / 'config.json').open('w') as file_:
+            file_.write(data)
-            json.dump(tagger_cfg, file_)
+        with (ner_model_dir / 'config.json').open('wb') as file_:
+            data = ujson.dumps(entity_cfg)
+            if isinstance(data, unicode):
+                data = data.encode('utf8')
+            file_.write(data)
+        with (pos_model_dir / 'config.json').open('wb') as file_:
+            data = ujson.dumps(tagger_cfg)
+            if isinstance(data, unicode):
+                data = data.encode('utf8')
+            file_.write(data)
 
         self = cls(
                 path=path,
@@ -252,7 +265,7 @@ class Language(object):
         self.entity = self.Defaults.create_entity(self)
         self.pipeline = self.Defaults.create_pipeline(self)
         yield Trainer(self, gold_tuples)
-        self.end_training()
+        self.end_training(path=path)
 
     def __init__(self, **overrides):
         if 'data_dir' in overrides and 'path' not in overrides:
@@ -391,12 +404,14 @@ class Language(object):
         else:
             entity_iob_freqs = []
             entity_type_freqs = []
-        with (path / 'vocab' / 'serializer.json').open('w') as file_:
+        with (path / 'vocab' / 'serializer.json').open('wb') as file_:
-            file_.write(
+            data = ujson.dumps([
-                json.dumps([
+                        (TAG, tagger_freqs),
-                    (TAG, tagger_freqs),
+                        (DEP, dep_freqs),
-                    (DEP, dep_freqs),
+                        (ENT_IOB, entity_iob_freqs),
-                    (ENT_IOB, entity_iob_freqs),
+                        (ENT_TYPE, entity_type_freqs),
-                    (ENT_TYPE, entity_type_freqs),
+                        (HEAD, head_freqs)
-                    (HEAD, head_freqs)
+                    ])
-                ]))
+            if isinstance(data, unicode):
+                data = data.encode('utf8')
+            file_.write(data)

spacy/language_data/tag_map.py

@@ -19,6 +19,7 @@ TAG_MAP = {
     "AUX":      {POS: AUX},
     "X":        {POS: X},
     "CONJ":     {POS: CONJ},
+    "CCONJ":    {POS: CCONJ}, # U20
     "ADJ":      {POS: ADJ},
     "VERB":     {POS: VERB},
     "PART":     {POS: PART}

spacy/morphology.pxd

@@ -37,7 +37,7 @@ cdef class Morphology:
     cdef int assign_tag(self, TokenC* token, tag) except -1
 
     cdef int assign_tag_id(self, TokenC* token, int tag_id) except -1
-    
+
     cdef int assign_feature(self, uint64_t* morph, univ_morph_t feat_id, bint value) except -1
 
 
@@ -80,6 +80,7 @@ cpdef enum univ_morph_t:
     Definite_two
     Definite_def
     Definite_red
+    Definite_cons # U20
     Definite_ind
     Degree_cmp
     Degree_comp
@@ -103,6 +104,8 @@ cpdef enum univ_morph_t:
     Negative_neg
     Negative_pos
     Negative_yes
+    Polarity_neg # U20
+    Polarity_pos # U20
     Number_com
     Number_dual
     Number_none
@@ -151,6 +154,7 @@ cpdef enum univ_morph_t:
     VerbForm_partPres
     VerbForm_sup
     VerbForm_trans
+    VerbForm_conv # U20
     VerbForm_gdv # la
     Voice_act
     Voice_cau

spacy/morphology.pyx

@@ -192,6 +192,7 @@ IDS = {
     "Definite_two": Definite_two,
     "Definite_def": Definite_def,
     "Definite_red": Definite_red,
+    "Definite_cons": Definite_cons, # U20
     "Definite_ind": Definite_ind,
     "Degree_cmp": Degree_cmp,
     "Degree_comp": Degree_comp,
@@ -215,6 +216,8 @@ IDS = {
     "Negative_neg": Negative_neg,
     "Negative_pos": Negative_pos,
     "Negative_yes": Negative_yes,
+    "Polarity_neg": Polarity_neg, # U20
+    "Polarity_pos": Polarity_pos, # U20
     "Number_com": Number_com,
     "Number_dual": Number_dual,
     "Number_none": Number_none,
@@ -263,6 +266,7 @@ IDS = {
     "VerbForm_partPres": VerbForm_partPres,
     "VerbForm_sup": VerbForm_sup,
     "VerbForm_trans": VerbForm_trans,
+    "VerbForm_conv": VerbForm_conv, # U20
     "VerbForm_gdv ": VerbForm_gdv, # la,
     "Voice_act": Voice_act,
     "Voice_cau": Voice_cau,

spacy/parts_of_speech.pxd

@@ -7,6 +7,7 @@ cpdef enum univ_pos_t:
     ADV
     AUX
     CONJ
+    CCONJ # U20
     DET
     INTJ
     NOUN

spacy/parts_of_speech.pyx

@@ -7,7 +7,8 @@ IDS = {
     "ADP": ADP,
     "ADV": ADV,
     "AUX": AUX,
-    "CONJ": CONJ,
+    "CONJ": CONJ, # U20
+    "CCONJ": CCONJ,
     "DET": DET,
     "INTJ": INTJ,
     "NOUN": NOUN,

spacy/strings.pyx

@@ -3,7 +3,7 @@ from __future__ import unicode_literals, absolute_import
 
 cimport cython
 from libc.string cimport memcpy
-from libc.stdint cimport uint64_t
+from libc.stdint cimport uint64_t, uint32_t
 
 from murmurhash.mrmr cimport hash64, hash32
 
@@ -12,22 +12,19 @@ from preshed.maps cimport map_iter, key_t
 from .typedefs cimport hash_t
 from libc.stdint cimport uint32_t
 
-try:
+import ujson
-    import ujson as json
-except ImportError:
-    import json
 
 
 cpdef hash_t hash_string(unicode string) except 0:
     chars = string.encode('utf8')
-    return _hash_utf8(chars, len(chars))
+    return hash_utf8(chars, len(chars))
 
 
-cdef hash_t _hash_utf8(char* utf8_string, int length):
+cdef hash_t hash_utf8(char* utf8_string, int length) nogil:
     return hash64(utf8_string, length, 1)
 
 
-cdef uint32_t _hash32_utf8(char* utf8_string, int length):
+cdef uint32_t hash32_utf8(char* utf8_string, int length) nogil:
     return hash32(utf8_string, length, 1)
 
 
@@ -48,11 +45,11 @@ cdef unicode _decode(const Utf8Str* string):
         return string.p[i:length + i].decode('utf8')
 
 
-cdef Utf8Str _allocate(Pool mem, const unsigned char* chars, int length) except *:
+cdef Utf8Str _allocate(Pool mem, const unsigned char* chars, uint32_t length) except *:
     cdef int n_length_bytes
     cdef int i
     cdef Utf8Str string
-    assert length != 0
+    cdef uint32_t ulength = length
     if length < sizeof(string.s):
         string.s[0] = <unsigned char>length
         memcpy(&string.s[1], chars, length)
@@ -98,6 +95,14 @@ cdef class StringStore:
         def __get__(self):
             return self.size -1
 
+    def __reduce__(self):
+        # TODO: OOV words, for the is_frozen stuff?
+        if self.is_frozen:
+            raise NotImplementedError(
+                "Currently missing support for pickling StringStore when "
+                "is_frozen=True")
+        return (StringStore, (list(self),))
+
     def __len__(self):
         """The number of strings in the store.
 
@@ -149,7 +154,7 @@ cdef class StringStore:
                 # pretty bad.
                 # We could also get unlucky here, and hash into a value that
                 # collides with the 'real' strings. 
-                return _hash32_utf8(byte_string, len(byte_string))
+                return hash32_utf8(byte_string, len(byte_string))
             else:
                 return utf8str - self.c
 
@@ -200,7 +205,7 @@ cdef class StringStore:
     cdef const Utf8Str* _intern_utf8(self, char* utf8_string, int length):
         # TODO: This function's API/behaviour is an unholy mess...
         # 0 means missing, but we don't bother offsetting the index.
-        cdef hash_t key = _hash_utf8(utf8_string, length)
+        cdef hash_t key = hash_utf8(utf8_string, length)
         cdef Utf8Str* value = <Utf8Str*>self._map.get(key)
         if value is not NULL:
             return value
@@ -209,7 +214,7 @@ cdef class StringStore:
             return value
         if self.is_frozen:
             # OOV store uses 32 bit hashes. Pretty ugly :(
-            key32 = _hash32_utf8(utf8_string, length)
+            key32 = hash32_utf8(utf8_string, length)
             # Important: Make the OOV store own the memory. That way it's trivial
             # to flush them all.
             value = <Utf8Str*>self._oov.mem.alloc(1, sizeof(Utf8Str))
@@ -232,7 +237,7 @@ cdef class StringStore:
         Returns:
             None
         """
-        string_data = json.dumps(list(self))
+        string_data = ujson.dumps(list(self))
         if not isinstance(string_data, unicode):
             string_data = string_data.decode('utf8')
         # TODO: OOV?
@@ -246,7 +251,7 @@ cdef class StringStore:
         Returns:
             None
         """
-        strings = json.load(file_)
+        strings = ujson.load(file_)
         if strings == ['']:
             return None
         cdef unicode string
@@ -271,7 +276,7 @@ cdef class StringStore:
             # Find array index with pointer arithmetic
             offset = ((<Utf8Str*>value) - self.c)
             keys[offset] = key
-        
+
         self._resize_at *= 2
         cdef size_t new_size = self._resize_at * sizeof(Utf8Str)
         self.c = <Utf8Str*>self.mem.realloc(self.c, new_size)

spacy/symbols.pxd

@@ -13,7 +13,7 @@ cpdef enum symbol_t:
     LIKE_EMAIL
     IS_STOP
     IS_OOV
-    
+
     FLAG14 = 14
     FLAG15
     FLAG16
@@ -90,6 +90,7 @@ cpdef enum symbol_t:
     ADV
     AUX
     CONJ
+    CCONJ # U20
     DET
     INTJ
     NOUN
@@ -107,11 +108,14 @@ cpdef enum symbol_t:
 
     Animacy_anim
     Animacy_inam
+    Animacy_hum # U20
     Aspect_freq
     Aspect_imp
     Aspect_mod
     Aspect_none
     Aspect_perf
+    Aspect_iter # U20
+    Aspect_hab # U20
     Case_abe
     Case_abl
     Case_abs
@@ -120,10 +124,12 @@ cpdef enum symbol_t:
     Case_all
     Case_cau
     Case_com
+    Case_cmp # U20
     Case_dat
     Case_del
     Case_dis
     Case_ela
+    Case_equ # U20
     Case_ess
     Case_gen
     Case_ill
@@ -142,7 +148,9 @@ cpdef enum symbol_t:
     Definite_two
     Definite_def
     Definite_red
+    Definite_cons # U20
     Definite_ind
+    Definite_spec # U20
     Degree_cmp
     Degree_comp
     Degree_none
@@ -151,6 +159,8 @@ cpdef enum symbol_t:
     Degree_abs
     Degree_com
     Degree_dim # du
+    Degree_equ # U20
+    Evident_nfh # U20
     Gender_com
     Gender_fem
     Gender_masc
@@ -162,16 +172,21 @@ cpdef enum symbol_t:
     Mood_pot
     Mood_sub
     Mood_opt
+    Mood_prp # U20
+    Mood_adm # U20
     Negative_neg
     Negative_pos
     Negative_yes
+    Polarity_neg # U20
+    Polarity_pos # U20
     Number_com
     Number_dual
     Number_none
     Number_plur
     Number_sing
     Number_ptan # bg
-    Number_count # bg
+    Number_count # bg, U20
+    Number_tri # U20
     NumType_card
     NumType_dist
     NumType_frac
@@ -197,7 +212,8 @@ cpdef enum symbol_t:
     PronType_rel
     PronType_tot
     PronType_clit
-    PronType_exc # es, ca, it, fa
+    PronType_exc # es, ca, it, fa, U20
+    PronType_emp # U20
     Reflex_yes
     Tense_fut
     Tense_imp
@@ -213,12 +229,17 @@ cpdef enum symbol_t:
     VerbForm_partPres
     VerbForm_sup
     VerbForm_trans
+    VerbForm_conv # U20
     VerbForm_gdv # la
+    VerbForm_vnoun # U20
     Voice_act
     Voice_cau
     Voice_pass
-    Voice_mid # gkc
+    Voice_mid # gkc, U20
     Voice_int # hb
+    Voice_antip # U20
+    Voice_dir # U20
+    Voice_inv # U20
     Abbr_yes # cz, fi, sl, U
     AdpType_prep # cz, U
     AdpType_post # U
@@ -284,6 +305,10 @@ cpdef enum symbol_t:
     Number_psee_plur # U
     Number_psor_sing # cz, fi, sl, U
     Number_psor_plur # cz, fi, sl, U
+    Number_pauc # U20
+    Number_grpa # U20
+    Number_grpl # U20
+    Number_inv # U20
     NumForm_digit # cz, sl, U
     NumForm_roman # cz, sl, U
     NumForm_word # cz, sl, U
@@ -311,6 +336,8 @@ cpdef enum symbol_t:
     Person_psor_one # fi, U
     Person_psor_two # fi, U
     Person_psor_three # fi, U
+    Person_zero # U20
+    Person_four # U20
     Polite_inf # bq, U
     Polite_pol # bq, U
     Polite_abs_inf # bq, U
@@ -319,6 +346,10 @@ cpdef enum symbol_t:
     Polite_erg_pol # bq, U
     Polite_dat_inf # bq, U
     Polite_dat_pol # bq, U
+    Polite_infm # U20
+    Polite_form # U20
+    Polite_form_elev # U20
+    Polite_form_humb # U20
     Prefix_yes # U
     PrepCase_npr # cz
     PrepCase_pre # U
@@ -383,6 +414,7 @@ cpdef enum symbol_t:
     ccomp
     complm
     conj
+    cop # U20
     csubj
     csubjpass
     dep
@@ -405,6 +437,8 @@ cpdef enum symbol_t:
     num
     number
     oprd
+    obj # U20
+    obl # U20
     parataxis
     partmod
     pcomp

spacy/symbols.pyx

@@ -91,6 +91,7 @@ IDS = {
     "ADV": ADV,
     "AUX": AUX,
     "CONJ": CONJ,
+    "CCONJ": CCONJ, # U20
     "DET": DET,
     "INTJ": INTJ,
     "NOUN": NOUN,
@@ -108,11 +109,14 @@ IDS = {
 
     "Animacy_anim": Animacy_anim,
     "Animacy_inam": Animacy_inam,
+    "Animacy_hum": Animacy_hum, # U20
     "Aspect_freq": Aspect_freq,
     "Aspect_imp": Aspect_imp,
     "Aspect_mod": Aspect_mod,
     "Aspect_none": Aspect_none,
     "Aspect_perf": Aspect_perf,
+    "Aspect_iter": Aspect_iter, # U20
+    "Aspect_hab": Aspect_hab, # U20
     "Case_abe": Case_abe,
     "Case_abl": Case_abl,
     "Case_abs": Case_abs,
@@ -121,10 +125,12 @@ IDS = {
     "Case_all": Case_all,
     "Case_cau": Case_cau,
     "Case_com": Case_com,
+    "Case_cmp": Case_cmp, # U20
     "Case_dat": Case_dat,
     "Case_del": Case_del,
     "Case_dis": Case_dis,
     "Case_ela": Case_ela,
+    "Case_equ": Case_equ, # U20
     "Case_ess": Case_ess,
     "Case_gen": Case_gen,
     "Case_ill": Case_ill,
@@ -143,7 +149,9 @@ IDS = {
     "Definite_two": Definite_two,
     "Definite_def": Definite_def,
     "Definite_red": Definite_red,
+    "Definite_cons": Definite_cons, # U20
     "Definite_ind": Definite_ind,
+    "Definite_spec": Definite_spec, # U20
     "Degree_cmp": Degree_cmp,
     "Degree_comp": Degree_comp,
     "Degree_none": Degree_none,
@@ -152,6 +160,8 @@ IDS = {
     "Degree_abs": Degree_abs,
     "Degree_com": Degree_com,
     "Degree_dim ": Degree_dim, # du
+    "Degree_equ": Degree_equ, # U20
+    "Evident_nfh": Evident_nfh, # U20
     "Gender_com": Gender_com,
     "Gender_fem": Gender_fem,
     "Gender_masc": Gender_masc,
@@ -163,16 +173,21 @@ IDS = {
     "Mood_pot": Mood_pot,
     "Mood_sub": Mood_sub,
     "Mood_opt": Mood_opt,
+    "Mood_prp": Mood_prp, # U20
+    "Mood_adm": Mood_adm, # U20
     "Negative_neg": Negative_neg,
     "Negative_pos": Negative_pos,
     "Negative_yes": Negative_yes,
+    "Polarity_neg": Polarity_neg, # U20
+    "Polarity_pos": Polarity_pos, # U20
     "Number_com": Number_com,
     "Number_dual": Number_dual,
     "Number_none": Number_none,
     "Number_plur": Number_plur,
     "Number_sing": Number_sing,
     "Number_ptan ": Number_ptan, # bg
-    "Number_count ": Number_count, # bg
+    "Number_count ": Number_count, # bg, U20
+    "Number_tri": Number_tri, # U20
     "NumType_card": NumType_card,
     "NumType_dist": NumType_dist,
     "NumType_frac": NumType_frac,
@@ -198,7 +213,8 @@ IDS = {
     "PronType_rel": PronType_rel,
     "PronType_tot": PronType_tot,
     "PronType_clit": PronType_clit,
-    "PronType_exc ": PronType_exc, # es, ca, it, fa,
+    "PronType_exc": PronType_exc, # es, ca, it, fa, U20
+    "PronType_emp": PronType_emp, # U20
     "Reflex_yes": Reflex_yes,
     "Tense_fut": Tense_fut,
     "Tense_imp": Tense_imp,
@@ -214,12 +230,17 @@ IDS = {
     "VerbForm_partPres": VerbForm_partPres,
     "VerbForm_sup": VerbForm_sup,
     "VerbForm_trans": VerbForm_trans,
+    "VerbForm_conv": VerbForm_conv, # U20
     "VerbForm_gdv ": VerbForm_gdv, # la,
+    "VerbForm_vnoun": VerbForm_vnoun, # U20
     "Voice_act": Voice_act,
     "Voice_cau": Voice_cau,
     "Voice_pass": Voice_pass,
-    "Voice_mid ": Voice_mid, # gkc,
+    "Voice_mid ": Voice_mid, # gkc, U20
     "Voice_int ": Voice_int, # hb,
+    "Voice_antip": Voice_antip, # U20
+    "Voice_dir": Voice_dir, # U20
+    "Voice_inv": Voice_inv, # U20
     "Abbr_yes ": Abbr_yes, # cz, fi, sl, U,
     "AdpType_prep ": AdpType_prep, # cz, U,
     "AdpType_post ": AdpType_post, # U,
@@ -285,6 +306,10 @@ IDS = {
     "Number_psee_plur ": Number_psee_plur, # U,
     "Number_psor_sing ": Number_psor_sing, # cz, fi, sl, U,
     "Number_psor_plur ": Number_psor_plur, # cz, fi, sl, U,
+    "Number_pauc": Number_pauc, # U20
+    "Number_grpa": Number_grpa, # U20
+    "Number_grpl": Number_grpl, # U20
+    "Number_inv": Number_inv, # U20
     "NumForm_digit ": NumForm_digit, # cz, sl, U,
     "NumForm_roman ": NumForm_roman, # cz, sl, U,
     "NumForm_word ": NumForm_word, # cz, sl, U,
@@ -312,6 +337,8 @@ IDS = {
     "Person_psor_one ": Person_psor_one, # fi, U,
     "Person_psor_two ": Person_psor_two, # fi, U,
     "Person_psor_three ": Person_psor_three, # fi, U,
+    "Person_zero ": Person_zero, # U20
+    "Person_four ": Person_four, # U20
     "Polite_inf ": Polite_inf, # bq, U,
     "Polite_pol ": Polite_pol, # bq, U,
     "Polite_abs_inf ": Polite_abs_inf, # bq, U,
@@ -320,6 +347,10 @@ IDS = {
     "Polite_erg_pol ": Polite_erg_pol, # bq, U,
     "Polite_dat_inf ": Polite_dat_inf, # bq, U,
     "Polite_dat_pol ": Polite_dat_pol, # bq, U,
+    "Polite_infm ": Polite_infm, # U20
+    "Polite_form ": Polite_form, # U20
+    "Polite_form_elev ": Polite_form_elev, # U20
+    "Polite_form_humb ": Polite_form_humb, # U20
     "Prefix_yes ": Prefix_yes, # U,
     "PrepCase_npr ": PrepCase_npr, # cz,
     "PrepCase_pre ": PrepCase_pre, # U,
@@ -384,6 +415,7 @@ IDS = {
     "ccomp": ccomp,
     "complm": complm,
     "conj": conj,
+    "cop": cop, # U20
     "csubj": csubj,
     "csubjpass": csubjpass,
     "dep": dep,
@@ -406,6 +438,8 @@ IDS = {
     "num": num,
     "number": number,
     "oprd": oprd,
+    "obj": obj, # U20
+    "obl": obl, # U20
     "parataxis": parataxis,
     "partmod": partmod,
     "pcomp": pcomp,

spacy/syntax/nonproj.pyx

@@ -8,7 +8,7 @@ from spacy.attrs import DEP, HEAD
 def ancestors(tokenid, heads):
     # returns all words going from the word up the path to the root
     # the path to root cannot be longer than the number of words in the sentence
-    # this function ends after at most len(heads) steps 
+    # this function ends after at most len(heads) steps
     # because it would otherwise loop indefinitely on cycles
     head = tokenid
     cnt = 0
@@ -180,7 +180,7 @@ class PseudoProjectivity:
             next_queue = []
             for qtoken in queue:
                 for child in qtoken.children:
-                    if child.is_space: continue                        
+                    if child.is_space: continue
                     if child == token: continue
                     if child.dep_ == headlabel:
                         return child

spacy/syntax/parser.pyx

@@ -68,7 +68,7 @@ def get_templates(name):
 
 
 cdef class ParserModel(AveragedPerceptron):
-    cdef void set_featuresC(self, ExampleC* eg, const StateC* state) nogil: 
+    cdef void set_featuresC(self, ExampleC* eg, const StateC* state) nogil:
         fill_context(eg.atoms, state)
         eg.nr_feat = self.extracter.set_features(eg.features, eg.atoms)
 
@@ -124,6 +124,8 @@ cdef class Parser:
         elif 'features' not in cfg:
             cfg['features'] = self.feature_templates
         self.model = ParserModel(cfg['features'])
+        self.model.l1_penalty = cfg.get('L1', 0.0)
+
         self.cfg = cfg
 
     def __reduce__(self):
@@ -232,7 +234,7 @@ cdef class Parser:
         free(eg.scores)
         free(eg.is_valid)
         return 0
-  
+
     def update(self, Doc tokens, GoldParse gold):
         """Update the statistical model.
 
@@ -258,15 +260,20 @@ cdef class Parser:
             self.model.set_featuresC(&eg.c, stcls.c)
             self.moves.set_costs(eg.c.is_valid, eg.c.costs, stcls, gold)
             self.model.set_scoresC(eg.c.scores, eg.c.features, eg.c.nr_feat)
-            self.model.updateC(&eg.c)
+            self.model.time += 1
             guess = VecVec.arg_max_if_true(eg.c.scores, eg.c.is_valid, eg.c.nr_class)
+            if eg.c.costs[guess] > 0:
+                best = arg_max_if_gold(eg.c.scores, eg.c.costs, eg.c.nr_class)
+                for feat in eg.c.features[:eg.c.nr_feat]:
+                    self.model.update_weight_ftrl(feat.key, best, -feat.value * eg.c.costs[guess])
+                    self.model.update_weight_ftrl(feat.key, guess, feat.value * eg.c.costs[guess])
 
-            action = self.moves.c[eg.guess]
+            action = self.moves.c[guess]
             action.do(stcls.c, action.label)
-            loss += eg.costs[eg.guess]
+            loss += eg.costs[guess]
-            eg.fill_scores(0, eg.nr_class)
+            eg.fill_scores(0, eg.c.nr_class)
-            eg.fill_costs(0, eg.nr_class)
+            eg.fill_costs(0, eg.c.nr_class)
-            eg.fill_is_valid(1, eg.nr_class)
+            eg.fill_is_valid(1, eg.c.nr_class)
         return loss
 
     def step_through(self, Doc doc):
@@ -296,7 +303,7 @@ cdef class Parser:
         # Doesn't set label into serializer -- subclasses override it to do that.
         for action in self.moves.action_types:
             self.moves.add_action(action, label)
-                
+
 
 cdef class StepwiseState:
     cdef readonly StateClass stcls
@@ -385,6 +392,14 @@ class ParserStateError(ValueError):
             "Please include the text that the parser failed on, which is:\n"
             "%s" % repr(doc.text))
 
+cdef int arg_max_if_gold(const weight_t* scores, const weight_t* costs, int n) nogil:
+    cdef int best = -1
+    for i in range(n):
+        if costs[i] <= 0:
+            if best == -1 or scores[i] > scores[best]:
+                best = i
+    return best
+
 
 cdef int _arg_max_clas(const weight_t* scores, int move, const Transition* actions,
                        int nr_class) except -1:

spacy/tagger.pyx

@@ -13,13 +13,13 @@ from thinc.linalg cimport VecVec
 from .typedefs cimport attr_t
 from .tokens.doc cimport Doc
 from .attrs cimport TAG
-from .parts_of_speech cimport NO_TAG, ADJ, ADV, ADP, CONJ, DET, NOUN, NUM, PRON
+from .parts_of_speech cimport NO_TAG, ADJ, ADV, ADP, CCONJ, DET, NOUN, NUM, PRON
 from .parts_of_speech cimport VERB, X, PUNCT, EOL, SPACE
 from .gold cimport GoldParse
 
 from .attrs cimport *
 
- 
+
 cpdef enum:
     P2_orth
     P2_cluster
@@ -71,7 +71,7 @@ cpdef enum:
 
 cdef class TaggerModel(AveragedPerceptron):
     cdef void set_featuresC(self, ExampleC* eg, const TokenC* tokens, int i) except *:
-        
+
         _fill_from_token(&eg.atoms[P2_orth], &tokens[i-2])
         _fill_from_token(&eg.atoms[P1_orth], &tokens[i-1])
         _fill_from_token(&eg.atoms[W_orth], &tokens[i])
@@ -152,6 +152,7 @@ cdef class Tagger:
             model = TaggerModel(cfg.get('features', self.feature_templates))
         self.vocab = vocab
         self.model = model
+        self.model.l1_penalty = 0.0
         # TODO: Move this to tag map
         self.freqs = {TAG: defaultdict(int)}
         for tag in self.tag_names:
@@ -191,7 +192,7 @@ cdef class Tagger:
                                   nr_class=self.vocab.morphology.n_tags,
                                   nr_feat=self.model.nr_feat)
         for i in range(tokens.length):
-            if tokens.c[i].pos == 0:                
+            if tokens.c[i].pos == 0:
                 self.model.set_featuresC(&eg.c, tokens.c, i)
                 self.model.set_scoresC(eg.c.scores,
                     eg.c.features, eg.c.nr_feat)
@@ -217,7 +218,7 @@ cdef class Tagger:
         for doc in stream:
             self(doc)
             yield doc
-    
+
     def update(self, Doc tokens, GoldParse gold):
         """Update the statistical model, with tags supplied for the given document.
 
@@ -251,7 +252,7 @@ cdef class Tagger:
             self.model.updateC(&eg.c)
 
             self.vocab.morphology.assign_tag_id(&tokens.c[i], eg.guess)
-            
+
             correct += eg.cost == 0
             self.freqs[TAG][tokens.c[i].tag] += 1
             eg.fill_scores(0, eg.c.nr_class)

spacy/tests/regression/test_issue792.py

@@ -4,9 +4,15 @@ from __future__ import unicode_literals
 import pytest
 
 
-@pytest.mark.xfail
 @pytest.mark.parametrize('text', ["This is a string ", "This is a string\u0020"])
 def test_issue792(en_tokenizer, text):
-    """Test for Issue #792: Trailing whitespace is removed after parsing."""
+    """Test for Issue #792: Trailing whitespace is removed after tokenization."""
     doc = en_tokenizer(text)
-    assert doc.text_with_ws == text
+    assert ''.join([token.text_with_ws for token in doc]) == text
+
+
+@pytest.mark.parametrize('text', ["This is a string", "This is a string\n"])
+def test_control_issue792(en_tokenizer, text):
+    """Test base case for Issue #792: Non-trailing whitespace"""
+    doc = en_tokenizer(text)
+    assert ''.join([token.text_with_ws for token in doc]) == text

spacy/tests/regression/test_issue850.py

@@ -0,0 +1,52 @@
+'''
+Test Matcher matches with '*' operator and Boolean flag
+'''
+from __future__ import unicode_literals
+from __future__ import print_function
+import pytest
+
+from ...matcher import Matcher
+from ...vocab import Vocab
+from ...attrs import LOWER
+from ...tokens import Doc
+
+
+def test_basic_case():
+    matcher = Matcher(Vocab(
+                lex_attr_getters={LOWER: lambda string: string.lower()}))
+    IS_ANY_TOKEN = matcher.vocab.add_flag(lambda x: True)
+    matcher.add_pattern(
+        "FarAway",
+        [
+            {LOWER: "bob"},
+            {'OP': '*', LOWER: 'and'},
+            {LOWER: 'frank'}
+        ])
+    doc = Doc(matcher.vocab, words=['bob', 'and', 'and', 'frank'])
+    match = matcher(doc)
+    assert len(match) == 1
+    ent_id, label, start, end = match[0]
+    assert start == 0
+    assert end == 4
+
+
+@pytest.mark.xfail
+def test_issue850():
+    '''The problem here is that the variable-length pattern matches the
+    succeeding token. We then don't handle the ambiguity correctly.'''
+    matcher = Matcher(Vocab(
+                lex_attr_getters={LOWER: lambda string: string.lower()}))
+    IS_ANY_TOKEN = matcher.vocab.add_flag(lambda x: True)
+    matcher.add_pattern(
+        "FarAway",
+        [
+            {LOWER: "bob"},
+            {'OP': '*', IS_ANY_TOKEN: True},
+            {LOWER: 'frank'}
+        ])
+    doc = Doc(matcher.vocab, words=['bob', 'and', 'and', 'frank'])
+    match = matcher(doc)
+    assert len(match) == 1
+    ent_id, label, start, end = match[0]
+    assert start == 0
+    assert end == 4

spacy/tests/regression/test_issue859.py

@@ -0,0 +1,12 @@
+# encoding: utf8
+from __future__ import unicode_literals
+
+import pytest
+
+
+@pytest.mark.parametrize('text', ["aaabbb@ccc.com\nThank you!",
+                                  "aaabbb@ccc.com \nThank you!"])
+def test_issue859(en_tokenizer, text):
+    """Test that no extra space is added in doc.text method."""
+    doc = en_tokenizer(text)
+    assert doc.text == text

spacy/tests/test_pickles.py

@@ -0,0 +1,40 @@
+from __future__ import unicode_literals
+
+import io
+import pytest
+import dill as pickle
+
+from ..strings import StringStore
+from ..vocab import Vocab
+from ..attrs import NORM
+
+
+def test_pickle_string_store():
+    sstore = StringStore()
+    hello = sstore['hello']
+    bye = sstore['bye']
+    bdata = pickle.dumps(sstore, protocol=-1)
+    unpickled = pickle.loads(bdata)
+    assert unpickled['hello'] == hello
+    assert unpickled['bye'] == bye
+    assert len(sstore) == len(unpickled)
+
+
+@pytest.mark.xfail
+def test_pickle_vocab():
+    vocab = Vocab(lex_attr_getters={int(NORM): lambda string: string[:-1]})
+    dog = vocab[u'dog']
+    cat = vocab[u'cat']
+    assert dog.norm_ == 'do'
+    assert cat.norm_ == 'ca'
+
+    bdata = pickle.dumps(vocab)
+    unpickled = pickle.loads(bdata)
+
+    assert unpickled[u'dog'].orth == dog.orth
+    assert unpickled[u'cat'].orth == cat.orth
+    assert unpickled[u'dog'].norm == dog.norm
+    assert unpickled[u'cat'].norm == cat.norm
+    dog_ = unpickled[u'dog']
+    cat_ = unpickled[u'cat']
+    assert dog_.norm != cat_.norm

spacy/tokenizer.pyx

@@ -163,7 +163,6 @@ cdef class Tokenizer:
                     start = i
                 in_ws = not in_ws
             i += 1
-        i += 1
         if start < i:
             span = string[start:]
             key = hash_string(span)
@@ -275,7 +274,10 @@ cdef class Tokenizer:
             if cache_hit:
                 pass
             elif self.token_match and self.token_match(string): 
-                tokens.push_back(self.vocab.get(tokens.mem, string), not suffixes.size())
+                # We're always saying 'no' to spaces here -- the caller will
+                # fix up the outermost one, with reference to the original.
+                # See Issue #859
+                tokens.push_back(self.vocab.get(tokens.mem, string), False)
             else:
                 matches = self.find_infix(string)
                 if not matches:

spacy/tokens/doc.pyx

@@ -16,7 +16,7 @@ from ..typedefs cimport attr_t, flags_t
 from ..attrs cimport attr_id_t
 from ..attrs cimport ID, ORTH, NORM, LOWER, SHAPE, PREFIX, SUFFIX, LENGTH, CLUSTER
 from ..attrs cimport POS, LEMMA, TAG, DEP, HEAD, SPACY, ENT_IOB, ENT_TYPE
-from ..parts_of_speech cimport CONJ, PUNCT, NOUN
+from ..parts_of_speech cimport CCONJ, PUNCT, NOUN
 from ..parts_of_speech cimport univ_pos_t
 from ..lexeme cimport Lexeme
 from .span cimport Span
@@ -59,13 +59,13 @@ cdef attr_t get_token_attr(const TokenC* token, attr_id_t feat_name) nogil:
 
 cdef class Doc:
     """
-    A sequence of `Token` objects. Access sentences and named entities, 
+    A sequence of `Token` objects. Access sentences and named entities,
-    export annotations to numpy arrays, losslessly serialize to compressed 
+    export annotations to numpy arrays, losslessly serialize to compressed
     binary strings.
 
     Aside: Internals
-        The `Doc` object holds an array of `TokenC` structs. 
+        The `Doc` object holds an array of `TokenC` structs.
-        The Python-level `Token` and `Span` objects are views of this 
+        The Python-level `Token` and `Span` objects are views of this
         array, i.e. they don't own the data themselves.
 
     Code: Construction 1
@@ -80,13 +80,13 @@ cdef class Doc:
         Create a Doc object.
 
         Aside: Implementation
-            This method of constructing a `Doc` object is usually only used 
+            This method of constructing a `Doc` object is usually only used
-            for deserialization. Standard usage is to construct the document via 
+            for deserialization. Standard usage is to construct the document via
             a call to the language object.
 
         Arguments:
             vocab:
-                A Vocabulary object, which must match any models you want to 
+                A Vocabulary object, which must match any models you want to
                 use (e.g. tokenizer, parser, entity recognizer).
 
             words:
@@ -156,19 +156,19 @@ cdef class Doc:
         if self.length == 0:
             self.is_tagged = True
             self.is_parsed = True
-    
+
     def __getitem__(self, object i):
         '''
         doc[i]
-            Get the Token object at position i, where i is an integer. 
+            Get the Token object at position i, where i is an integer.
-            Negative indexing is supported, and follows the usual Python 
+            Negative indexing is supported, and follows the usual Python
             semantics, i.e. doc[-2] is doc[len(doc) - 2].
         doc[start : end]]
             Get a `Span` object, starting at position `start`
             and ending at position `end`, where `start` and
             `end` are token indices. For instance,
-            `doc[2:5]` produces a span consisting of 
+            `doc[2:5]` produces a span consisting of
-            tokens 2, 3 and 4. Stepped slices (e.g. `doc[start : end : step]`) 
+            tokens 2, 3 and 4. Stepped slices (e.g. `doc[start : end : step]`)
             are not supported, as `Span` objects must be contiguous (cannot have gaps).
             You can use negative indices and open-ended ranges, which have their
             normal Python semantics.
@@ -188,11 +188,11 @@ cdef class Doc:
     def __iter__(self):
         '''
         for token in doc
-            Iterate over `Token`  objects, from which the annotations can 
+            Iterate over `Token`  objects, from which the annotations can
-            be easily accessed. This is the main way of accessing Token 
+            be easily accessed. This is the main way of accessing Token
-            objects, which are the main way annotations are accessed from 
+            objects, which are the main way annotations are accessed from
-            Python. If faster-than-Python speeds are required, you can 
+            Python. If faster-than-Python speeds are required, you can
-            instead access the annotations as a numpy array, or access the 
+            instead access the annotations as a numpy array, or access the
             underlying C data directly from Cython.
         '''
         cdef int i
@@ -251,13 +251,13 @@ cdef class Doc:
         def __get__(self):
             if 'has_vector' in self.user_hooks:
                 return self.user_hooks['has_vector'](self)
- 
+
             return any(token.has_vector for token in self)
 
     property vector:
         '''
         A real-valued meaning representation. Defaults to an average of the token vectors.
-        
+
         Type: numpy.ndarray[ndim=1, dtype='float32']
         '''
         def __get__(self):
@@ -285,14 +285,14 @@ cdef class Doc:
                     norm += value * value
                 self._vector_norm = sqrt(norm) if norm != 0 else 0
             return self._vector_norm
-        
+
         def __set__(self, value):
-            self._vector_norm = value 
+            self._vector_norm = value
 
     @property
     def string(self):
         return self.text
-    
+
     property text:
         '''A unicode representation of the document text.'''
         def __get__(self):
@@ -306,7 +306,7 @@ cdef class Doc:
     property ents:
         '''
         Yields named-entity `Span` objects, if the entity recognizer
-        has been applied to the document. Iterate over the span to get 
+        has been applied to the document. Iterate over the span to get
         individual Token objects, or access the label:
 
         Example:
@@ -352,7 +352,7 @@ cdef class Doc:
             cdef int i
             for i in range(self.length):
                 self.c[i].ent_type = 0
-                # At this point we don't know whether the NER has run over the 
+                # At this point we don't know whether the NER has run over the
                 # Doc. If the ent_iob is missing, leave it missing.
                 if self.c[i].ent_iob != 0:
                     self.c[i].ent_iob = 2 # Means O. Non-O are set from ents.
@@ -384,9 +384,9 @@ cdef class Doc:
     property noun_chunks:
         '''
         Yields base noun-phrase #[code Span] objects, if the document
-        has been syntactically parsed. A base noun phrase, or 
+        has been syntactically parsed. A base noun phrase, or
-        'NP chunk', is a noun phrase that does not permit other NPs to 
+        'NP chunk', is a noun phrase that does not permit other NPs to
-        be nested within it – so no NP-level coordination, no prepositional 
+        be nested within it – so no NP-level coordination, no prepositional
         phrases, and no relative clauses. For example:
         '''
         def __get__(self):
@@ -422,7 +422,7 @@ cdef class Doc:
         def __get__(self):
             if 'sents' in self.user_hooks:
                 return self.user_hooks['sents'](self)
- 
+
             if not self.is_parsed:
                 raise ValueError(
                     "sentence boundary detection requires the dependency parse, which "
@@ -465,8 +465,8 @@ cdef class Doc:
     @cython.boundscheck(False)
     cpdef np.ndarray to_array(self, object py_attr_ids):
         """
-        Given a list of M attribute IDs, export the tokens to a numpy 
+        Given a list of M attribute IDs, export the tokens to a numpy
-        `ndarray` of shape (N, M), where `N` is the length 
+        `ndarray` of shape (N, M), where `N` is the length
         of the document. The values will be 32-bit integers.
 
         Example:
@@ -474,7 +474,7 @@ cdef class Doc:
             doc = nlp(text)
             # All strings mapped to integers, for easy export to numpy
             np_array = doc.to_array([attrs.LOWER, attrs.POS, attrs.ENT_TYPE, attrs.IS_ALPHA])
-                
+
         Arguments:
             attr_ids (list[int]): A list of attribute ID ints.
 
@@ -520,7 +520,7 @@ cdef class Doc:
         cdef int i
         cdef attr_t attr
         cdef size_t count
-        
+
         if counts is None:
             counts = PreshCounter()
             output_dict = True
@@ -570,7 +570,7 @@ cdef class Doc:
         cdef TokenC* tokens = self.c
         cdef int length = len(array)
         cdef attr_t[:] values
-        for col, attr_id in enumerate(attrs): 
+        for col, attr_id in enumerate(attrs):
             values = array[:, col]
             if attr_id == HEAD:
                 for i in range(length):
@@ -612,11 +612,11 @@ cdef class Doc:
         '''Deserialize, loading from bytes.'''
         self.vocab.serializer.unpack_into(data[4:], self)
         return self
-    
+
     @staticmethod
     def read_bytes(file_):
         '''
-        A static method, used to read serialized #[code Doc] objects from 
+        A static method, used to read serialized #[code Doc] objects from
         a file. For example:
 
         Example:
@@ -673,7 +673,7 @@ cdef class Doc:
                 "Expected either 3 arguments (deprecated), or 0 (use keyword arguments). "
                 "Arguments supplied:\n%s\n"
                 "Keyword arguments:%s\n" % (len(args), repr(args), repr(attributes)))
- 
+
         cdef int start = token_by_start(self.c, self.length, start_idx)
         if start == -1:
             return None
@@ -784,7 +784,7 @@ cdef int set_children_from_heads(TokenC* tokens, int length) except -1:
             if child.l_edge < head.l_edge:
                 head.l_edge = child.l_edge
             head.l_kids += 1
-        
+
     # Set right edges --- same as above, but iterate in reverse
     for i in range(length-1, -1, -1):
         child = &tokens[i]
@@ -798,4 +798,4 @@ cdef int set_children_from_heads(TokenC* tokens, int length) except -1:
     for i in range(length):
         if tokens[i].head == 0 and tokens[i].dep != 0:
             tokens[tokens[i].l_edge].sent_start = True
-            
+

spacy/tokens/token.pyx

@@ -20,7 +20,7 @@ from .. import parts_of_speech
 from ..attrs cimport LEMMA
 from ..attrs cimport ID, ORTH, NORM, LOWER, SHAPE, PREFIX, SUFFIX, LENGTH, CLUSTER
 from ..attrs cimport POS, LEMMA, TAG, DEP
-from ..parts_of_speech cimport CONJ, PUNCT
+from ..parts_of_speech cimport CCONJ, PUNCT
 
 from ..attrs cimport IS_ALPHA, IS_ASCII, IS_DIGIT, IS_LOWER, IS_PUNCT, IS_SPACE
 from ..attrs cimport IS_BRACKET
@@ -84,7 +84,7 @@ cdef class Token:
 
     cpdef bint check_flag(self, attr_id_t flag_id) except -1:
         '''Check the value of a boolean flag.
-        
+
         Arguments:
             flag_id (int): The ID of the flag attribute.
         Returns:
@@ -225,7 +225,7 @@ cdef class Token:
     property vector:
         '''
         A real-valued meaning representation.
-        
+
         Type: numpy.ndarray[ndim=1, dtype='float32']
         '''
         def __get__(self):
@@ -343,7 +343,7 @@ cdef class Token:
         '''
         def __get__(self):
             cdef const TokenC* head_ptr = self.c
-            # guard against infinite loop, no token can have 
+            # guard against infinite loop, no token can have
             # more ancestors than tokens in the tree
             cdef int i = 0
             while head_ptr.head != 0 and i < self.doc.length:
@@ -370,7 +370,7 @@ cdef class Token:
 
     property head:
         '''The syntactic parent, or "governor", of this token.
-        
+
         Returns: Token
         '''
         def __get__(self):
@@ -390,7 +390,7 @@ cdef class Token:
 
             # is the new head a descendant of the old head
             cdef bint is_desc = old_head.is_ancestor_of(new_head)
-            
+
             cdef int new_edge
             cdef Token anc, child
 
@@ -420,7 +420,7 @@ cdef class Token:
                         if anc.c.l_edge <= new_edge:
                             break
                         anc.c.l_edge = new_edge
-            
+
             elif self.c.head < 0: # right dependent
                 old_head.c.r_kids -= 1
                 # do the same thing as for l_edge
@@ -435,7 +435,7 @@ cdef class Token:
                             if child.c.r_edge > new_edge:
                                 new_edge = child.c.r_edge
                         old_head.c.r_edge = new_edge
-                    
+
                     for anc in old_head.ancestors:
                         if anc.c.r_edge >= new_edge:
                             break
@@ -598,19 +598,19 @@ cdef class Token:
     property is_punct:
         def __get__(self): return Lexeme.c_check_flag(self.c.lex, IS_PUNCT)
 
-    property is_space: 
+    property is_space:
         def __get__(self): return Lexeme.c_check_flag(self.c.lex, IS_SPACE)
-    
+
-    property is_bracket: 
+    property is_bracket:
         def __get__(self): return Lexeme.c_check_flag(self.c.lex, IS_BRACKET)
 
-    property is_quote: 
+    property is_quote:
         def __get__(self): return Lexeme.c_check_flag(self.c.lex, IS_QUOTE)
 
-    property is_left_punct: 
+    property is_left_punct:
         def __get__(self): return Lexeme.c_check_flag(self.c.lex, IS_LEFT_PUNCT)
 
-    property is_right_punct: 
+    property is_right_punct:
         def __get__(self): return Lexeme.c_check_flag(self.c.lex, IS_RIGHT_PUNCT)
 
     property like_url:

spacy/vocab.pyx

@@ -9,11 +9,16 @@ import bz2
 import ujson as json
 import re
 
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
 from .lexeme cimport EMPTY_LEXEME
 from .lexeme cimport Lexeme
 from .strings cimport hash_string
 from .typedefs cimport attr_t
-from .cfile cimport CFile
+from .cfile cimport CFile, StringCFile
 from .lemmatizer import Lemmatizer
 from .attrs import intify_attrs
 from .tokens.token cimport Token
@@ -346,17 +351,18 @@ cdef class Vocab:
                 Token.set_struct_attr(token, attr_id, value)
         return tokens
 
-    def dump(self, loc):
+    def dump(self, loc=None):
-        """Save the lexemes binary data to the given location.
+        """Save the lexemes binary data to the given location, or
+        return a byte-string with the data if loc is None.
 
         Arguments:
-            loc (Path): The path to save to.
+            loc (Path or None): The path to save to, or None.
         """
-        if hasattr(loc, 'as_posix'):
+        cdef CFile fp
-            loc = loc.as_posix()
+        if loc is None:
-        cdef bytes bytes_loc = loc.encode('utf8') if type(loc) == unicode else loc
+            fp = StringCFile('wb')
-
+        else:
-        cdef CFile fp = CFile(bytes_loc, 'wb')
+            fp = CFile(loc, 'wb')
         cdef size_t st
         cdef size_t addr
         cdef hash_t key
@@ -378,6 +384,8 @@ cdef class Vocab:
             fp.write_from(&lexeme.l2_norm, sizeof(lexeme.l2_norm), 1)
             fp.write_from(&lexeme.lang, sizeof(lexeme.lang), 1)
         fp.close()
+        if loc is None:
+            return fp.string_data()
 
     def load_lexemes(self, loc):
         '''Load the binary vocabulary data from the given location.
@@ -427,6 +435,60 @@ cdef class Vocab:
             i += 1
         fp.close()
 
+    def _deserialize_lexemes(self, CFile fp):
+        '''Load the binary vocabulary data from the given CFile.
+        '''
+        cdef LexemeC* lexeme
+        cdef hash_t key
+        cdef unicode py_str
+        cdef attr_t orth
+        assert sizeof(orth) == sizeof(lexeme.orth)
+        i = 0
+        cdef int todo = fp.size
+        cdef int lex_size = sizeof(lexeme.flags)
+        lex_size += sizeof(lexeme.id)
+        lex_size += sizeof(lexeme.length)
+        lex_size += sizeof(lexeme.orth)
+        lex_size += sizeof(lexeme.lower)
+        lex_size += sizeof(lexeme.norm)
+        lex_size += sizeof(lexeme.shape)
+        lex_size += sizeof(lexeme.prefix)
+        lex_size += sizeof(lexeme.suffix)
+        lex_size += sizeof(lexeme.cluster)
+        lex_size += sizeof(lexeme.prob)
+        lex_size += sizeof(lexeme.sentiment)
+        lex_size += sizeof(lexeme.l2_norm)
+        lex_size += sizeof(lexeme.lang)
+        while True:
+            if todo < lex_size:
+                break
+            todo -= lex_size
+            lexeme = <LexemeC*>self.mem.alloc(sizeof(LexemeC), 1)
+            # Copy data from the file into the lexeme
+            fp.read_into(&lexeme.flags, 1, sizeof(lexeme.flags))
+            fp.read_into(&lexeme.id, 1, sizeof(lexeme.id))
+            fp.read_into(&lexeme.length, 1, sizeof(lexeme.length))
+            fp.read_into(&lexeme.orth, 1, sizeof(lexeme.orth))
+            fp.read_into(&lexeme.lower, 1, sizeof(lexeme.lower))
+            fp.read_into(&lexeme.norm, 1, sizeof(lexeme.norm))
+            fp.read_into(&lexeme.shape, 1, sizeof(lexeme.shape))
+            fp.read_into(&lexeme.prefix, 1, sizeof(lexeme.prefix))
+            fp.read_into(&lexeme.suffix, 1, sizeof(lexeme.suffix))
+            fp.read_into(&lexeme.cluster, 1, sizeof(lexeme.cluster))
+            fp.read_into(&lexeme.prob, 1, sizeof(lexeme.prob))
+            fp.read_into(&lexeme.sentiment, 1, sizeof(lexeme.sentiment))
+            fp.read_into(&lexeme.l2_norm, 1, sizeof(lexeme.l2_norm))
+            fp.read_into(&lexeme.lang, 1, sizeof(lexeme.lang))
+
+            lexeme.vector = EMPTY_VEC
+            py_str = self.strings[lexeme.orth]
+            key = hash_string(py_str)
+            self._by_hash.set(key, lexeme)
+            self._by_orth.set(lexeme.orth, lexeme)
+            self.length += 1
+            i += 1
+        fp.close()
+
     def dump_vectors(self, out_loc):
         '''Save the word vectors to a binary file.
 
@@ -553,6 +615,42 @@ cdef class Vocab:
         return vec_len
 
 
+def pickle_vocab(vocab):
+    sstore = vocab.strings
+    morph = vocab.morphology
+    length = vocab.length
+    serializer = vocab._serializer
+    data_dir = vocab.data_dir
+    lex_attr_getters = vocab.lex_attr_getters
+
+    lexemes_data = vocab.dump()
+    vectors_length = vocab.vectors_length
+
+    return (unpickle_vocab,
+        (sstore, morph, serializer, data_dir, lex_attr_getters,
+            lexemes_data, length, vectors_length))
+
+
+def unpickle_vocab(sstore, morphology, serializer, data_dir,
+        lex_attr_getters, bytes lexemes_data, int length, int vectors_length):
+    cdef Vocab vocab = Vocab()
+    vocab.length = length
+    vocab.vectors_length = vectors_length
+    vocab.strings = sstore
+    cdef CFile fp = StringCFile('r', data=lexemes_data)
+    vocab.morphology = morphology
+    vocab._serializer = serializer
+    vocab.data_dir = data_dir
+    vocab.lex_attr_getters = lex_attr_getters
+    vocab._deserialize_lexemes(fp)
+    vocab.length = length
+    vocab.vectors_length = vectors_length
+    return vocab
+
+
+copy_reg.pickle(Vocab, pickle_vocab, unpickle_vocab)
+
+
 def write_binary_vectors(in_loc, out_loc):
     cdef CFile out_file = CFile(out_loc, 'wb')
     cdef Address mem