Merge remote-tracking branch 'refs/remotes/honnibal/master'

Conflicts:
	setup.py

.gitignore

@@ -91,3 +91,4 @@ coverage.xml
 # Sphinx documentation
 docs/_build/
 docs/_themes/
+setup.py

.travis.yml

@@ -24,4 +24,4 @@ install:
 
 # run tests
 script:
-  - "py.test tests/ website/tests/ -x"
+  - "py.test tests/ -x"

bin/init_model.py

@@ -27,8 +27,8 @@ from pathlib import Path
 
 from shutil import copyfile
 from shutil import copytree
-import codecs
 from collections import defaultdict
+import io
 
 from spacy.vocab import Vocab
 from spacy.vocab import write_binary_vectors
@@ -61,7 +61,7 @@ def _read_clusters(loc):
         print("Warning: Clusters file not found")
         return {}
     clusters = {}
-    for line in codecs.open(str(loc), 'r', 'utf8'):
+    for line in io.open(str(loc), 'r', encoding='utf8'):
         try:
             cluster, word, freq = line.split()
         except ValueError:
@@ -88,7 +88,7 @@ def _read_probs(loc):
         print("Probabilities file not found. Trying freqs.")
         return {}, 0.0
     probs = {}
-    for i, line in enumerate(codecs.open(str(loc), 'r', 'utf8')):
+    for i, line in enumerate(io.open(str(loc), 'r', encoding='utf8')):
         prob, word = line.split()
         prob = float(prob)
         probs[word] = prob

bin/ner_tag.py

@@ -1,11 +1,11 @@
-import codecs
+import io
 import plac
 
 from spacy.en import English
 
 
 def main(text_loc):
-    with codecs.open(text_loc, 'r', 'utf8') as file_:
+    with io.open(text_loc, 'r', encoding='utf8') as file_:
         text = file_.read()
     NLU = English()
     for paragraph in text.split('\n\n'):

bin/parser/train.py

@@ -6,7 +6,7 @@ from __future__ import print_function
 import os
 from os import path
 import shutil
-import codecs
+import io
 import random
 
 import plac
@@ -177,7 +177,7 @@ def write_parses(Language, dev_loc, model_dir, out_loc):
     nlp = Language(data_dir=model_dir)
     gold_tuples = read_json_file(dev_loc)
     scorer = Scorer()
-    out_file = codecs.open(out_loc, 'w', 'utf8')
+    out_file = io.open(out_loc, 'w', 'utf8')
     for raw_text, sents in gold_tuples:
         sents = _merge_sents(sents)
         for annot_tuples, brackets in sents:
@@ -229,7 +229,6 @@ def main(train_loc, dev_loc, model_dir, n_sents=0, n_iter=15, out_loc="", verbos
     print('POS', scorer.tags_acc)
     print('UAS', scorer.uas)
     print('LAS', scorer.las)
-    print('SBD', scorer.sbd_acc)
 
     print('NER P', scorer.ents_p)
     print('NER R', scorer.ents_r)

bin/prepare_treebank.py

@@ -27,7 +27,7 @@ import json
 from os import path
 import os
 import re
-import codecs
+import io
 from collections import defaultdict
 
 from spacy.munge import read_ptb
@@ -122,7 +122,7 @@ def read_file(*pieces):
     if not path.exists(loc):
         return None
     else:
-        return codecs.open(loc, 'r', 'utf8').read().strip()
+        return io.open(loc, 'r', encoding='utf8').read().strip()
 
 
 def get_file_names(section_dir, subsection):

examples/multi_word_matches.py

@@ -22,74 +22,77 @@ our pattern set stays very small (exact size depends on the maximum length we're
 looking for, as the query language currently has no quantifiers)
 """
 from __future__ import print_function, unicode_literals, division
+from ast import literal_eval
+from bz2 import BZ2File
+import time
+import math
+import codecs
 
 import plac
 
 from preshed.maps import PreshMap
+from preshed.counter import PreshCounter
 from spacy.strings import hash_string
 from spacy.en import English
-from spacy.matcher import Matcher
-
-from spacy.attrs import FLAG63 as U_ENT
-from spacy.attrs import FLAG62 as L_ENT
-from spacy.attrs import FLAG61 as I_ENT
-from spacy.attrs import FLAG60 as B_ENT
+from spacy.matcher import PhraseMatcher
 
 
-def get_bilou(length):
-    if length == 1:
-        return [U_ENT]
-    else:
-        return [B_ENT] + [I_ENT] * (length - 2) + [L_ENT]
+def read_gazetteer(tokenizer, loc, n=-1):
+    for i, line in enumerate(open(loc)):
+        phrase = literal_eval('u' + line.strip())
+        if ' (' in phrase and phrase.endswith(')'):
+            phrase = phrase.split(' (', 1)[0]
+        if i >= n:
+            break
+        phrase = tokenizer(phrase)
+        if all((t.is_lower and t.prob >= -10) for t in phrase):
+            continue
+        if len(phrase) >= 2:
+            yield phrase
 
 
-def make_matcher(vocab, max_length):
-    abstract_patterns = []
-    for length in range(1, max_length+1):
-        abstract_patterns.append([{tag: True} for tag in get_bilou(length)])
-    return Matcher(vocab, {'Candidate': ('CAND', {}, abstract_patterns)})
+def read_text(bz2_loc):
+    with BZ2File(bz2_loc) as file_:
+        for line in file_:
+            yield line.decode('utf8')
 
 
-def get_matches(matcher, pattern_ids, doc):
-    matches = []
-    for label, start, end in matcher(doc):
-        candidate = doc[start : end]
-        if pattern_ids[hash_string(candidate.text)] == True:
-            start = candidate[0].idx
-            end = candidate[-1].idx + len(candidate[-1])
-            matches.append((start, end, candidate.root.tag_, candidate.text))
-    return matches
+def get_matches(tokenizer, phrases, texts, max_length=6):
+    matcher = PhraseMatcher(tokenizer.vocab, phrases, max_length=max_length)
+    print("Match")
+    for text in texts:
+        doc = tokenizer(text)
+        matches = matcher(doc)
+        for mwe in doc.ents:
+            yield mwe
 
 
-def merge_matches(doc, matches):
-    for start, end, tag, text in matches:
-        doc.merge(start, end, tag, text, 'MWE')
-
-
-def main():
+def main(patterns_loc, text_loc, counts_loc, n=10000000):
     nlp = English(parser=False, tagger=False, entity=False)
-
-    gazetteer = [u'M.I.A.', 'Shiny Happy People', 'James E. Jones']
-    example_text = u'The artist M.I.A. did a cover of Shiny Happy People. People is not an entity.'
-    pattern_ids = PreshMap()
-    max_length = 0
-    for pattern_str in gazetteer:
-        pattern = nlp.tokenizer(pattern_str)
-        bilou_tags = get_bilou(len(pattern))
-        for word, tag in zip(pattern, bilou_tags):
-            lexeme = nlp.vocab[word.orth]
-            lexeme.set_flag(tag, True)
-        pattern_ids[hash_string(pattern.text)] = True
-        max_length = max(max_length, len(pattern))
-
-    matcher = make_matcher(nlp.vocab, max_length)
-
-    doc = nlp(example_text)
-    matches = get_matches(matcher, pattern_ids, doc)
-    merge_matches(doc, matches)
-    for token in doc:
-        print(token.text, token.ent_type_)
+    print("Make matcher")
+    phrases = read_gazetteer(nlp.tokenizer, patterns_loc, n=n)
+    counts = PreshCounter()
+    t1 = time.time()
+    for mwe in get_matches(nlp.tokenizer, phrases, read_text(text_loc)):
+        counts.inc(hash_string(mwe.text), 1)
+    t2 = time.time()
+    print("10m tokens in %d s" % (t2 - t1))
+    
+    with codecs.open(counts_loc, 'w', 'utf8') as file_:
+        for phrase in read_gazetteer(nlp.tokenizer, patterns_loc, n=n):
+            text = phrase.string
+            key = hash_string(text)
+            count = counts[key]
+            if count != 0:
+                file_.write('%d\t%s\n' % (count, text))
     
 
 if __name__ == '__main__':
-    plac.call(main)
+    if False:
+        import cProfile
+        import pstats
+        cProfile.runctx("plac.call(main)", globals(), locals(), "Profile.prof")
+        s = pstats.Stats("Profile.prof")
+        s.strip_dirs().sort_stats("time").print_stats()
+    else:
+        plac.call(main)

fabfile.py

@@ -47,7 +47,7 @@ def prebuild(build_dir='/tmp/build_spacy'):
         local('git clone %s .' % spacy_dir)
         local('virtualenv ' + build_venv)
         with prefix('cd %s && PYTHONPATH=`pwd` && . %s/bin/activate' % (build_dir, build_venv)):
-            local('pip install cython fabric fabtools')
+            local('pip install cython fabric fabtools pytest')
             local('pip install -r requirements.txt')
             local('fab clean make')
             local('cp -r %s/corpora/en/wordnet corpora/en/' % spacy_dir)

lang_data/en/generate_specials.py

@@ -1,3 +1,4 @@
+# -#- coding: utf-8 -*-
 import json
 
 contractions = {"n't", "'nt", "not", "'ve", "'d", "'ll", "'s", "'m", "'ma", "'re"}
@@ -114,6 +115,8 @@ hardcoded_specials = {
                 "'s":  [{"F": "'s", "L": "'s"}],
 
                 "'S":  [{"F": "'S", "L": "'s"}],
+                u"\u2018s": [{"F": u"\u2018s", "L": "'s"}],
+                u"\u2018S": [{"F": u"\u2018S", "L": "'s"}],
 
                 "'em": [{"F": "'em"}],
 
@@ -133,6 +136,8 @@ hardcoded_specials = {
 
                 "''": [{"F": "''"}],
 
+                "—": [{"F": "—", "L": "--", "pos": ":"}],
+
                 "Corp.": [{"F": "Corp."}],
                 "Inc.": [{"F": "Inc."}],
                 "Co.": [{"F": "Co."}],
@@ -336,7 +341,8 @@ hardcoded_specials = {
                 "E.G.": [{"F": "E.G."}],
                 "\n": [{"F": "\n", "pos": "SP"}],
                 "\t": [{"F": "\t", "pos": "SP"}],
-                " ": [{"F": " ", "pos": "SP"}]
+                " ": [{"F": " ", "pos": "SP"}],
+                u"\xa0": [{"F": u"\xa0", "pos": "SP", "L": "  "}]
 
 }
 
@@ -412,6 +418,6 @@ def generate_specials():
 
 if __name__ == "__main__":
     specials = generate_specials()
-    with open("specials.json", "w") as f:
-        json.dump(specials, f)
+    with open("specials.json", "w") as file_:
+        file_.write(json.dumps(specials, indent=2))
 

lang_data/en/lemma_rules.json

@@ -27,5 +27,12 @@
         ["est", ""],
         ["er", "e"],
         ["est", "e"]
+    ],
+
+    "punct": [
+        ["“", "\""],
+        ["”", "\""],
+        ["\u2018", "'"],
+        ["\u2019", "'"]
     ]
 }

lang_data/fi/tag_map.json

@@ -13,5 +13,7 @@
     "ADP": {"pos": "ADP"},
     "SYM": {"pos": "SYM"},
     "X": {"pos": "X"},
-    "INTJ": {"pos": "INTJ"}
+    "INTJ": {"pos": "INTJ"},
+    "DET": {"pos": "DET"},
+    "PART": {"pos": "PART"}
 }

lang_data/it/tag_map.json

@@ -2,43 +2,43 @@
 "S": {"pos": "NOUN"},
 "E":   {"pos": "ADP"},
 "RD":  {"pos": "DET"},
-"V":   {"pos": "VER"},
-"_":   {"pos": "_"},
+"V":   {"pos": "VERB"},
+"_":   {"pos": "NO_TAG"},
 "A":   {"pos": "ADJ"},
-"SP":  {"pos": "PROP"},
-"FF":  {"pos": "PUNC"},
-"FS":  {"pos": "PUNC"},
+"SP":  {"pos": "PROPN"},
+"FF":  {"pos": "PUNCT"},
+"FS":  {"pos": "PUNCT"},
 "B":   {"pos": "ADV"},
-"CC":  {"pos": "CON"},
-"FB":  {"pos": "PUNC"},
+"CC":  {"pos": "CONJ"},
+"FB":  {"pos": "PUNCT"},
 "VA":  {"pos": "AUX"},
-"PC":  {"pos": "PRO"},
+"PC":  {"pos": "PRON"},
 "N":   {"pos": "NUM"},
 "RI":  {"pos": "DET"},
-"PR":  {"pos": "PRO"},
-"CS":  {"pos": "SCON"},
+"PR":  {"pos": "PRON"},
+"CS":  {"pos": "SCONJ"},
 "BN":  {"pos": "ADV"},
 "AP":  {"pos": "DET"},
 "VM":  {"pos": "AUX"},
 "DI":  {"pos": "DET"},
-"FC":  {"pos": "PUNC"},
-"PI":  {"pos": "PRO"},
+"FC":  {"pos": "PUNCT"},
+"PI":  {"pos": "PRON"},
 "DD":  {"pos": "DET"},
 "DQ":  {"pos": "DET"},
-"PQ":  {"pos": "PRO"},
-"PD":  {"pos": "PRO"},
+"PQ":  {"pos": "PRON"},
+"PD":  {"pos": "PRON"},
 "NO":  {"pos": "ADJ"},
-"PE":  {"pos": "PRO"},
+"PE":  {"pos": "PRON"},
 "T":   {"pos": "DET"},
 "X":   {"pos": "SYM"},
 "SW":  {"pos": "X"},
-"NO":  {"pos": "PRO"},
-"I":   {"pos": "INT"},
+"NO":  {"pos": "PRON"},
+"I":   {"pos": "INTJ"},
 "X":   {"pos": "X"},
 "DR":  {"pos": "DET"},
 "EA":  {"pos": "ADP"},
-"PP":  {"pos": "PRO"},
+"PP":  {"pos": "PRON"},
 "X":   {"pos": "NUM"},
 "DE":  {"pos": "DET"},
-"X":   {"pos": "PAR"}
+"X":   {"pos": "PART"}
 }

setup.py

@@ -134,13 +134,17 @@ def run_setup(exts):
     headers_workaround.install_headers('numpy')
 
 
-VERSION = '0.93'
+VERSION = '0.94'
 def main(modules, is_pypy):
     language = "cpp"
     includes = ['.', path.join(sys.prefix, 'include')]
+<<<<<<< HEAD
 # This is gcc only. Also -03 is everywhere and is not recognized :()
 #    compile_args = ['-O3', '-Wno-strict-prototypes']
     compile_args = ['-Ox', '-EHsc']
+=======
+    compile_args = ['-O3', '-Wno-strict-prototypes', '-Wno-unused-function']
+>>>>>>> refs/remotes/honnibal/master
     link_args = []
 # It is not prefix !!!
     if sys.prefix == 'darwin':
@@ -159,9 +163,13 @@ MOD_NAMES = ['spacy.parts_of_speech', 'spacy.strings',
              'spacy.morphology', 'spacy.tagger',
              'spacy.syntax.stateclass', 
              'spacy._ml', 'spacy._theano',
+<<<<<<< HEAD
              'spacy.tokenizer', 
 #'spacy.en.attrs',
 #'spacy.en.pos', 
+=======
+             'spacy.tokenizer',
+>>>>>>> refs/remotes/honnibal/master
              'spacy.syntax.parser', 
              'spacy.syntax.transition_system',
              'spacy.syntax.arc_eager',

spacy/en/download.py

@@ -7,7 +7,7 @@ import wget
 import plac
 
 # TODO: Read this from the same source as the setup
-VERSION = '0.9.0'
+VERSION = '0.9.1'
 
 AWS_STORE = 'https://s3-us-west-1.amazonaws.com/media.spacynlp.com'
 

spacy/gold.pyx

@@ -1,5 +1,7 @@
 import numpy
-import codecs
+import io
+import json
+import ujson
 import random
 import re
 import os

spacy/lemmatizer.py

@@ -1,4 +1,4 @@
-from __future__ import unicode_literals
+from __future__ import unicode_literals, print_function
 from os import path
 import codecs
 
@@ -7,7 +7,7 @@ try:
 except ImportError:
     import json
 
-from .parts_of_speech import NOUN, VERB, ADJ
+from .parts_of_speech import NOUN, VERB, ADJ, PUNCT
 
 
 class Lemmatizer(object):
@@ -36,6 +36,8 @@ class Lemmatizer(object):
             pos = 'verb'
         elif pos == ADJ:
             pos = 'adj'
+        elif pos == PUNCT:
+            pos = 'punct'
         lemmas = lemmatize(string, self.index.get(pos, {}), self.exc.get(pos, {}), self.rules.get(pos, []))
         return lemmas
 
@@ -48,6 +50,9 @@ class Lemmatizer(object):
     def adj(self, string):
         return self(string, 'adj')
 
+    def punct(self, string):
+        return self(string, 'punct')
+
 
 def lemmatize(string, index, exceptions, rules):
     string = string.lower()
@@ -58,7 +63,7 @@ def lemmatize(string, index, exceptions, rules):
     for old, new in rules:
         if string.endswith(old):
             form = string[:len(string) - len(old)] + new
-            if form in index:
+            if form in index or not form.isalpha():
                 forms.append(form)
     if not forms:
         forms.append(string)

spacy/matcher.pyx

@@ -1,11 +1,18 @@
+# cython: profile=True
+from __future__ import unicode_literals
+
 from os import path
 
 from .typedefs cimport attr_t
+from .typedefs cimport hash_t
 from .attrs cimport attr_id_t
-from .structs cimport TokenC
+from .structs cimport TokenC, LexemeC
+from .lexeme cimport Lexeme
 
 from cymem.cymem cimport Pool
+from preshed.maps cimport PreshMap
 from libcpp.vector cimport vector
+from murmurhash.mrmr cimport hash64
 
 from .attrs cimport LENGTH, ENT_TYPE, ORTH, NORM, LEMMA, LOWER, SHAPE
 from .attrs cimport FLAG13, FLAG14, FLAG15, FLAG16, FLAG17, FLAG18, FLAG19, FLAG20, FLAG21, FLAG22, FLAG23, FLAG24, FLAG25
@@ -15,6 +22,38 @@ from .vocab cimport Vocab
 
 from libcpp.vector cimport vector
 
+from .attrs import FLAG61 as U_ENT
+
+from .attrs import FLAG60 as B2_ENT
+from .attrs import FLAG59 as B3_ENT
+from .attrs import FLAG58 as B4_ENT
+from .attrs import FLAG57 as B5_ENT
+from .attrs import FLAG56 as B6_ENT
+from .attrs import FLAG55 as B7_ENT
+from .attrs import FLAG54 as B8_ENT
+from .attrs import FLAG53 as B9_ENT
+from .attrs import FLAG52 as B10_ENT
+
+from .attrs import FLAG51 as I3_ENT
+from .attrs import FLAG50 as I4_ENT
+from .attrs import FLAG49 as I5_ENT
+from .attrs import FLAG48 as I6_ENT
+from .attrs import FLAG47 as I7_ENT
+from .attrs import FLAG46 as I8_ENT
+from .attrs import FLAG45 as I9_ENT
+from .attrs import FLAG44 as I10_ENT
+
+from .attrs import FLAG43 as L2_ENT
+from .attrs import FLAG42 as L3_ENT
+from .attrs import FLAG41 as L4_ENT
+from .attrs import FLAG40 as L5_ENT
+from .attrs import FLAG39 as L6_ENT
+from .attrs import FLAG38 as L7_ENT
+from .attrs import FLAG37 as L8_ENT
+from .attrs import FLAG36 as L9_ENT
+from .attrs import FLAG35 as L10_ENT
+
+
 try:
     import ujson as json
 except ImportError:
@@ -81,7 +120,33 @@ def _convert_strings(token_specs, string_store):
                 value = int(value)
             converted[-1].append((attr, value))
     return converted
-    
+
+
+def get_bilou(length):
+    if length == 1:
+        return [U_ENT]
+    elif length == 2:
+        return [B2_ENT, L2_ENT]
+    elif length == 3:
+        return [B3_ENT, I3_ENT, L3_ENT]
+    elif length == 4:
+        return [B4_ENT, I4_ENT, I4_ENT, L4_ENT]
+    elif length == 5:
+        return [B5_ENT, I5_ENT, I5_ENT, I5_ENT, L5_ENT]
+    elif length == 6:
+        return [B6_ENT, I6_ENT, I6_ENT, I6_ENT, I6_ENT, L6_ENT]
+    elif length == 7:
+        return [B7_ENT, I7_ENT, I7_ENT, I7_ENT, I7_ENT, I7_ENT, L7_ENT]
+    elif length == 8:
+        return [B8_ENT, I8_ENT, I8_ENT, I8_ENT, I8_ENT, I8_ENT, I8_ENT, L8_ENT]
+    elif length == 9:
+        return [B9_ENT, I9_ENT, I9_ENT, I9_ENT, I9_ENT, I9_ENT, I9_ENT, I9_ENT, L9_ENT]
+    elif length == 10:
+        return [B10_ENT, I10_ENT, I10_ENT, I10_ENT, I10_ENT, I10_ENT, I10_ENT,
+                I10_ENT, I10_ENT, L10_ENT]
+    else:
+        raise ValueError("Max length currently 10 for phrase matching")
+
 
 def map_attr_name(attr):
     attr = attr.upper()
@@ -95,32 +160,6 @@ def map_attr_name(attr):
         return SHAPE
     elif attr == 'NORM':
         return NORM
-    elif attr == 'FLAG13':
-        return FLAG13
-    elif attr == 'FLAG14':
-        return FLAG14
-    elif attr == 'FLAG15':
-        return FLAG15
-    elif attr == 'FLAG16':
-        return FLAG16
-    elif attr == 'FLAG17':
-        return FLAG17
-    elif attr == 'FLAG18':
-        return FLAG18
-    elif attr == 'FLAG19':
-        return FLAG19
-    elif attr == 'FLAG20':
-        return FLAG20
-    elif attr == 'FLAG21':
-        return FLAG21
-    elif attr == 'FLAG22':
-        return FLAG22
-    elif attr == 'FLAG23':
-        return FLAG23
-    elif attr == 'FLAG24':
-        return FLAG24
-    elif attr == 'FLAG25':
-        return FLAG25
     else:
         raise Exception("TODO: Finish supporting attr mapping %s" % attr)
 
@@ -163,7 +202,7 @@ cdef class Matcher:
             spec = _convert_strings(spec, self.vocab.strings)
             self.patterns.push_back(init_pattern(self.mem, spec, etype))
 
-    def __call__(self, Doc doc):
+    def __call__(self, Doc doc, acceptor=None):
         cdef vector[Pattern*] partials
         cdef int n_partials = 0
         cdef int q = 0
@@ -174,21 +213,94 @@ cdef class Matcher:
         for token_i in range(doc.length):
             token = &doc.data[token_i]
             q = 0
+            # Go over the open matches, extending or finalizing if able. Otherwise,
+            # we over-write them (q doesn't advance)
             for i in range(partials.size()):
                 state = partials.at(i)
                 if match(state, token):
                     if is_final(state):
-                        matches.append(get_entity(state, token, token_i))
+                        label, start, end = get_entity(state, token, token_i)
+                        if acceptor is None or acceptor(doc, label, start, end):
+                            matches.append((label, start, end))
                     else:
                         partials[q] = state + 1
                         q += 1
             partials.resize(q)
+            # Check whether we open any new patterns on this token
             for i in range(self.n_patterns):
                 state = self.patterns[i]
                 if match(state, token):
                     if is_final(state):
-                        matches.append(get_entity(state, token, token_i))
+                        label, start, end = get_entity(state, token, token_i)
+                        if acceptor is None or acceptor(doc, label, start, end):
+                            matches.append((label, start, end))
                     else:
                         partials.push_back(state + 1)
         doc.ents = [(e.label, e.start, e.end) for e in doc.ents] + matches
         return matches
+
+
+cdef class PhraseMatcher:
+    cdef Pool mem
+    cdef Vocab vocab
+    cdef Matcher matcher
+    cdef PreshMap phrase_ids
+
+    cdef int max_length
+    cdef attr_t* _phrase_key
+
+    def __init__(self, Vocab vocab, phrases, max_length=10):
+        self.mem = Pool()
+        self._phrase_key = <attr_t*>self.mem.alloc(max_length, sizeof(attr_t))
+        self.max_length = max_length
+        self.vocab = vocab
+        self.matcher = Matcher(self.vocab, {})
+        self.phrase_ids = PreshMap()
+        for phrase in phrases:
+            if len(phrase) < max_length:
+                self.add(phrase)
+
+        abstract_patterns = []
+        for length in range(1, max_length):
+            abstract_patterns.append([{tag: True} for tag in get_bilou(length)])
+        self.matcher.add('Candidate', 'MWE', {}, abstract_patterns)
+
+    def add(self, Doc tokens):
+        cdef int length = tokens.length
+        assert length < self.max_length
+        tags = get_bilou(length)
+        assert len(tags) == length, length
+        
+        cdef int i
+        for i in range(self.max_length):
+            self._phrase_key[i] = 0
+        for i, tag in enumerate(tags):
+            lexeme = self.vocab[tokens.data[i].lex.orth]
+            lexeme.set_flag(tag, True)
+            self._phrase_key[i] = lexeme.orth
+        cdef hash_t key = hash64(self._phrase_key, self.max_length * sizeof(attr_t), 0)
+        self.phrase_ids[key] = True
+
+    def __call__(self, Doc doc):
+        matches = []
+        for label, start, end in self.matcher(doc, acceptor=self.accept_match):
+            cand = doc[start : end]
+            start = cand[0].idx
+            end = cand[-1].idx + len(cand[-1])
+            matches.append((start, end, cand.root.tag_, cand.text, 'MWE'))
+        for match in matches:
+            doc.merge(*match)
+        return matches
+
+    def accept_match(self, Doc doc, int label, int start, int end):
+        assert (end - start) < self.max_length
+        cdef int i, j
+        for i in range(self.max_length):
+            self._phrase_key[i] = 0
+        for i, j in enumerate(range(start, end)):
+            self._phrase_key[i] = doc.data[j].lex.orth
+        cdef hash_t key = hash64(self._phrase_key, self.max_length * sizeof(attr_t), 0)
+        if self.phrase_ids.get(key):
+            return True
+        else:
+            return False

spacy/morphology.pyx

@@ -7,7 +7,7 @@ except ImportError:
     import json
 
 from .parts_of_speech import UNIV_POS_NAMES
-from .parts_of_speech cimport ADJ, VERB, NOUN
+from .parts_of_speech cimport ADJ, VERB, NOUN, PUNCT
 
 
 cdef class Morphology:
@@ -31,10 +31,7 @@ cdef class Morphology:
     cdef int assign_tag(self, TokenC* token, tag) except -1:
         cdef int tag_id
         if isinstance(tag, basestring):
-            try:
-                tag_id = self.reverse_index[self.strings[tag]]
-            except KeyError:
-                raise
+            tag_id = self.reverse_index[self.strings[tag]]
         else:
             tag_id = tag
         analysis = <MorphAnalysisC*>self._cache.get(tag_id, token.lex.orth)
@@ -84,7 +81,7 @@ cdef class Morphology:
         if self.lemmatizer is None:
             return orth
         cdef unicode py_string = self.strings[orth]
-        if pos != NOUN and pos != VERB and pos != ADJ:
+        if pos != NOUN and pos != VERB and pos != ADJ and pos != PUNCT:
             return orth
         cdef set lemma_strings
         cdef unicode lemma_string

spacy/orth.pyx

@@ -11,6 +11,7 @@ try:
 except ImportError:
     from text_unidecode import unidecode
 
+
 import re
 
 import math
@@ -165,7 +166,7 @@ cpdef unicode norm1(unicode string, lower_pc=0.0, upper_pc=0.0, title_pc=0.0):
 
 
 cpdef bytes asciied(unicode string):
-    cdef str stripped = unidecode(string)
+    stripped = unidecode(string)
     if not stripped:
         return b'???'
     return stripped.encode('ascii')

spacy/syntax/arc_eager.pyx

@@ -9,7 +9,8 @@ from .transition_system cimport do_func_t, get_cost_func_t
 from .transition_system cimport move_cost_func_t, label_cost_func_t
 from ..gold cimport GoldParse
 from ..gold cimport GoldParseC
-from ..attrs cimport TAG, HEAD, DEP, ENT_IOB, ENT_TYPE
+from ..attrs cimport TAG, HEAD, DEP, ENT_IOB, ENT_TYPE, IS_SPACE
+from ..lexeme cimport Lexeme
 
 from libc.stdint cimport uint32_t
 from libc.string cimport memcpy
@@ -379,8 +380,18 @@ cdef class ArcEager(TransitionSystem):
         st.fast_forward()
 
     cdef int finalize_state(self, StateClass st) nogil:
+        cdef int i
         for i in range(st.length):
-            if st._sent[i].head == 0 and st._sent[i].dep == 0:
+            # Always attach spaces to the previous word
+            if Lexeme.c_check_flag(st._sent[i].lex, IS_SPACE):
+                st._sent[i].head = -1 if (i >= 1) else 1
+                if st._sent[i].sent_start and st._sent[i].head == -1:
+                    st._sent[i].sent_start = False
+                    # If we had this space token as the start of a sentence,
+                    # move that sentence start forward one
+                    if (i + 1) < st.length and not st._sent[i+1].sent_start:
+                        st._sent[i+1].sent_start = True
+            elif st._sent[i].head == 0 and st._sent[i].dep == 0:
                 st._sent[i].dep = self.root_label
             # If we're not using the Break transition, we segment via root-labelled
             # arcs between the root words.

spacy/tokens/doc.pyx

@@ -21,6 +21,7 @@ from ..lexeme cimport Lexeme
 from .spans cimport Span
 from .token cimport Token
 from ..serialize.bits cimport BitArray
+from ..util import normalize_slice
 
 
 DEF PADDING = 5
@@ -81,20 +82,14 @@ cdef class Doc:
         self._vector = None
 
     def __getitem__(self, object i):
-        """Get a token.
+        """Get a Token or a Span from the Doc.
 
         Returns:
-            token (Token):
+            token (Token) or span (Span):
         """
         if isinstance(i, slice):
-            if i.step is not None:
-                raise ValueError("Stepped slices not supported in Span objects."
-                                 "Try: list(doc)[start:stop:step] instead.")
-            if i.start is None:
-                i = slice(0, i.stop)
-            if i.stop is None:
-                i = slice(i.start, len(self))
-            return Span(self, i.start, i.stop, label=0)
+            start, stop = normalize_slice(len(self), i.start, i.stop, i.step)
+            return Span(self, start, stop, label=0)
 
         if i < 0:
             i = self.length + i

spacy/tokens/spans.pyx

@@ -9,16 +9,16 @@ from ..structs cimport TokenC, LexemeC
 from ..typedefs cimport flags_t, attr_t
 from ..attrs cimport attr_id_t
 from ..parts_of_speech cimport univ_pos_t
+from ..util import normalize_slice
 
 
 cdef class Span:
     """A slice from a Doc object."""
     def __cinit__(self, Doc tokens, int start, int end, int label=0, vector=None,
                   vector_norm=None):
-        if start < 0:
-            start = tokens.length - start
-        if end < 0:
-            end = tokens.length - end
+        if not (0 <= start <= end <= len(tokens)):
+            raise IndexError
+
         self.doc = tokens
         self.start = start
         self.end = end
@@ -46,7 +46,13 @@ cdef class Span:
             return 0
         return self.end - self.start
 
-    def __getitem__(self, int i):
+    def __getitem__(self, object i):
+        if isinstance(i, slice):
+            start, end = normalize_slice(len(self), i.start, i.stop, i.step)
+            start += self.start
+            end += self.start
+            return Span(self.doc, start, end)
+
         if i < 0:
             return self.doc[self.end + i]
         else:

spacy/util.py

@@ -1,5 +1,5 @@
 from os import path
-import codecs
+import io
 import json
 import re
 from .attrs import TAG, HEAD, DEP, ENT_IOB, ENT_TYPE
@@ -7,8 +7,28 @@ from .attrs import TAG, HEAD, DEP, ENT_IOB, ENT_TYPE
 DATA_DIR = path.join(path.dirname(__file__), '..', 'data')
 
 
+def normalize_slice(length, start, stop, step=None):
+    if not (step is None or step == 1):
+        raise ValueError("Stepped slices not supported in Span objects."
+                         "Try: list(tokens)[start:stop:step] instead.")
+    if start is None:
+       start = 0
+    elif start < 0:
+       start += length
+    start = min(length, max(0, start))
+
+    if stop is None:
+       stop = length
+    elif stop < 0:
+       stop += length
+    stop = min(length, max(start, stop))
+
+    assert 0 <= start <= stop <= length
+    return start, stop
+
+
 def utf8open(loc, mode='r'):
-    return codecs.open(loc, mode, 'utf8')
+    return io.open(loc, mode, encoding='utf8')
 
 
 def read_lang_data(data_dir):

spacy/vocab.pyx

@@ -7,7 +7,7 @@ from libc.stdint cimport uint64_t
 
 import bz2
 from os import path
-import codecs
+import io
 import math
 import json
 

tests/matcher/test_matcher_bugfixes.py

@@ -3,6 +3,7 @@ import pytest
 
 from spacy.matcher import Matcher
 
+@pytest.mark.xfail
 def test_overlap_issue118(EN):
     '''Test a bug that arose from having overlapping matches'''
     doc = EN.tokenizer(u'how many points did lebron james score against the boston celtics last night')

tests/parser/test_parse_navigate.py

@@ -1,13 +1,13 @@
 from __future__ import unicode_literals
 from os import path
-import codecs
+import io
 
 import pytest
 
 
 @pytest.fixture
 def sun_text():
-    with codecs.open(path.join(path.dirname(__file__), '..', 'sun.txt'), 'r', 'utf8') as file_:
+    with io.open(path.join(path.dirname(__file__), 'sun.txt'), 'r', encoding='utf8') as file_:
         text = file_.read()
     return text
 

tests/tagger/test_lemmatizer.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 from spacy.lemmatizer import Lemmatizer, read_index, read_exc
@@ -34,3 +35,9 @@ def test_noun_lemmas(lemmatizer):
     assert do('planets') == set(['planet'])
     assert do('ring') == set(['ring'])
     assert do('axes') == set(['axis', 'axe', 'ax'])
+
+
+def test_smart_quotes(lemmatizer):
+    do = lemmatizer.punct
+    assert do('“') == set(['"'])
+    assert do('“') == set(['"'])

tests/tokens/test_array.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import pytest
 
-from spacy.en import attrs
+from spacy import attrs
 
 
 def test_attr_of_token(EN):

tests/tokens/test_token_api.py

@@ -1,8 +1,8 @@
 from __future__ import unicode_literals
 from spacy.en import English
-from spacy.en.attrs import IS_ALPHA, IS_ASCII, IS_DIGIT, IS_LOWER, IS_PUNCT
-from spacy.en.attrs import IS_SPACE, IS_TITLE, IS_UPPER, LIKE_URL, LIKE_NUM
-from spacy.en.attrs import IS_STOP
+from spacy.attrs import IS_ALPHA, IS_ASCII, IS_DIGIT, IS_LOWER, IS_PUNCT
+from spacy.attrs import IS_SPACE, IS_TITLE, IS_UPPER, LIKE_URL, LIKE_NUM
+from spacy.attrs import IS_STOP
 
 import pytest
 

tests/tokens/test_tokens_api.py

@@ -12,6 +12,72 @@ def test_getitem(EN):
     with pytest.raises(IndexError):
         tokens[len(tokens)]
 
+    def to_str(span):
+       return '/'.join(token.orth_ for token in span)
+
+    span = tokens[1:1]
+    assert not to_str(span)
+    span = tokens[1:4]
+    assert to_str(span) == 'it/back/!'
+    span = tokens[1:4:1]
+    assert to_str(span) == 'it/back/!'
+    with pytest.raises(ValueError):
+        tokens[1:4:2]
+    with pytest.raises(ValueError):
+        tokens[1:4:-1]
+
+    span = tokens[-3:6]
+    assert to_str(span) == 'He/pleaded'
+    span = tokens[4:-1]
+    assert to_str(span) == 'He/pleaded'
+    span = tokens[-5:-3]
+    assert to_str(span) == 'back/!'
+    span = tokens[5:4]
+    assert span.start == span.end == 5 and not to_str(span)
+    span = tokens[4:-3]
+    assert span.start == span.end == 4 and not to_str(span)
+
+    span = tokens[:]
+    assert to_str(span) == 'Give/it/back/!/He/pleaded/.'
+    span = tokens[4:]
+    assert to_str(span) == 'He/pleaded/.'
+    span = tokens[:4]
+    assert to_str(span) == 'Give/it/back/!'
+    span = tokens[:-3]
+    assert to_str(span) == 'Give/it/back/!'
+    span = tokens[-3:]
+    assert to_str(span) == 'He/pleaded/.'
+
+    span = tokens[4:50]
+    assert to_str(span) == 'He/pleaded/.'
+    span = tokens[-50:4]
+    assert to_str(span) == 'Give/it/back/!'
+    span = tokens[-50:-40]
+    assert span.start == span.end == 0 and not to_str(span)
+    span = tokens[40:50]
+    assert span.start == span.end == 7 and not to_str(span)
+
+    span = tokens[1:4]
+    assert span[0].orth_ == 'it'
+    subspan = span[:]
+    assert to_str(subspan) == 'it/back/!'
+    subspan = span[:2]
+    assert to_str(subspan) == 'it/back'
+    subspan = span[1:]
+    assert to_str(subspan) == 'back/!'
+    subspan = span[:-1]
+    assert to_str(subspan) == 'it/back'
+    subspan = span[-2:]
+    assert to_str(subspan) == 'back/!'
+    subspan = span[1:2]
+    assert to_str(subspan) == 'back'
+    subspan = span[-2:-1]
+    assert to_str(subspan) == 'back'
+    subspan = span[-50:50]
+    assert to_str(subspan) == 'it/back/!'
+    subspan = span[50:-50]
+    assert subspan.start == subspan.end == 4 and not to_str(subspan)
+
 
 @pytest.mark.models
 def test_serialize(EN):

tests/vocab/test_lexeme_flags.py

@@ -2,7 +2,7 @@ from __future__ import unicode_literals
 
 import pytest
 
-from spacy.en.attrs import *
+from spacy.attrs import *
 
 
 def test_is_alpha(en_vocab):

tests/website/test_api.py

@@ -26,6 +26,7 @@ def test_main_entry_point(nlp):
     doc = nlp(b'Some text'.decode('utf8')) # Encode to unicode first.
 
 
+@pytest.mark.models
 def test_sentence_spans(nlp):
     # from spacy.en import English
     # nlp = English()
@@ -33,6 +34,7 @@ def test_sentence_spans(nlp):
     assert [s.root.orth_ for s in doc.sents] == ["is", "'s"]
 
 
+@pytest.mark.models
 def test_entity_spans(nlp):
     # from spacy.en import English
     # nlp = English()
@@ -44,6 +46,7 @@ def test_entity_spans(nlp):
     assert ents[0].string == ents[0].string
 
 
+@pytest.mark.models
 def test_noun_chunk_spans(nlp):
     # from spacy.en import English
     # nlp = English()
@@ -56,11 +59,12 @@ def test_noun_chunk_spans(nlp):
     # NP three noun chunks <-- has
 
 
+@pytest.mark.models
 def test_count_by(nlp):
     # from spacy.en import English, attrs
     # nlp = English()
     import numpy
-    from spacy.en import attrs
+    from spacy import attrs
     tokens = nlp('apple apple orange banana')
     assert tokens.count_by(attrs.ORTH) == {2529: 2, 4117: 1, 6650: 1}
     assert repr(tokens.to_array([attrs.ORTH])) == repr(numpy.array([[2529],
@@ -76,7 +80,7 @@ def test_read_bytes(nlp):
         file_.write(nlp(u'This is a document.').to_bytes())
         file_.write(nlp(u'This is another.').to_bytes())
     docs = []
-    with open(loc) as file_:
+    with open(loc, 'rb') as file_:
         for byte_string in Doc.read_bytes(file_):
             docs.append(Doc(nlp.vocab).from_bytes(byte_string))
     assert len(docs) == 2
@@ -88,6 +92,7 @@ def test_token_span(doc):
     assert token.i == 4
 
 
+@pytest.mark.models
 def test_example_i_like_new_york1(nlp):
     toks = nlp('I like New York in Autumn.')
 
@@ -127,16 +132,19 @@ def dot(toks):
     return tok(toks, "dot")
 
 
+@pytest.mark.models
 def test_example_i_like_new_york3(toks, new, york):
     assert toks[new].head.orth_ == 'York'
     assert toks[york].head.orth_ == 'like'
 
 
+@pytest.mark.models
 def test_example_i_like_new_york4(toks, new, york):
     new_york = toks[new:york+1]
     assert new_york.root.orth_ == 'York'
 
 
+@pytest.mark.models
 def test_example_i_like_new_york5(toks, autumn, dot):
     assert toks[autumn].head.orth_ == 'in'
     assert toks[dot].head.orth_ == 'like'
@@ -144,6 +152,7 @@ def test_example_i_like_new_york5(toks, autumn, dot):
     assert autumn_dot.root.orth_ == 'Autumn'
 
 
+@pytest.mark.models
 def test_navigating_the_parse_tree_lefts(doc):
     # TODO: where does the span object come from?
     span = doc[:2]
@@ -151,6 +160,7 @@ def test_navigating_the_parse_tree_lefts(doc):
              if span.doc[i].head in span]
 
 
+@pytest.mark.models
 def test_navigating_the_parse_tree_rights(doc):
     span = doc[:2]
     rights = [span.doc[i] for i in range(span.end, len(span.doc))

tests/website/test_home.py

@@ -1,6 +1,6 @@
 from __future__ import unicode_literals
 import pytest
-import spacy.en
+import spacy
 
 
 @pytest.fixture()
@@ -17,11 +17,12 @@ def test_load_resources_and_process_text():
 @pytest.mark.models
 def test_get_tokens_and_sentences(doc):
     token = doc[0]
-    sentence = doc.sents.next()
+    sentence = next(doc.sents)
     assert token is sentence[0]
     assert sentence.text == 'Hello, world.'
 
 
+@pytest.mark.models
 def test_use_integer_ids_for_any_strings(nlp, token):
     hello_id = nlp.vocab.strings['Hello']
     hello_str = nlp.vocab.strings[hello_id]
@@ -45,7 +46,7 @@ def test_get_and_set_string_views_and_flags(nlp, token):
 
 
 def test_export_to_numpy_arrays(nlp, doc):
-    from spacy.en.attrs import ORTH, LIKE_URL, IS_OOV
+    from spacy.attrs import ORTH, LIKE_URL, IS_OOV
 
     attr_ids = [ORTH, LIKE_URL, IS_OOV]
     doc_array = doc.to_array(attr_ids)
@@ -68,6 +69,7 @@ def test_word_vectors(nlp):
     assert apples.similarity(oranges) > boots.similarity(hippos)
 
 
+@pytest.mark.models
 def test_part_of_speech_tags(nlp):
     from spacy.parts_of_speech import ADV
 

website/Makefile

@@ -12,9 +12,6 @@ site/index.html: src/jade/header.jade src/jade/*.jade
 site/docs/: src/jade/docs/*.jade src/jade/header.jade
 	jade -P src/jade/docs/index.jade --out $@
 
-site/license/: src/jade/license/*.jade src/jade/header.jade 
-	jade -P src/jade/license/index.jade --out $@
-
 site/blog/: src/jade/blog/*.jade site/blog/*/ site/tutorials/*/ src/jade/header.jade
 	jade -P src/jade/blog/index.jade --out $@
 

website/src/jade/blog/eli5-computers-learn-reading/index.jade

@@ -24,7 +24,7 @@ include ./meta.jade
 
     p These days we just show the computer lots and lots and lots of words. We gave up trying to get it to understand what a “dress” is. We let #[em dress] be just some letters. But if it is seen it around #[em girl] enough times (which is just some other letters, which are seen around some #[strong other] other letters), it can make good guesses.
 
-    p It doesn't always guess right, but we can tell how often it does, and we can think of ways t help it learn better. We have a number, and we can slowly make it bigger, a little bit by a little bit.
+    p It doesn't always guess right, but we can tell how often it does, and we can think of ways to help it learn better. We have a number, and we can slowly make it bigger, a little bit by a little bit.
 
     p (One thing I've learned is, people are great at making a number bigger, if you pay a lot of them to try. The key is to pick numbers where, if they make the number bigger, they can't help but have done something actually good. This is harder than it sounds. Some say no numbers are like this. I ask them to show me much good being done another way, but they never can.)
 

website/src/jade/home/_installation.jade

@@ -20,6 +20,11 @@ mixin Option(name, open)
     | $ conda install spacy
     | $ python -m spacy.en.download all
 
+  p Latest stable conda packages are available from the spacy channel:
+
+  pre.language-bash: code
+    | $ conda install -c https://conda.anaconda.org/spacy spacy
+
 +Option("pip and virtualenv", true)
     p With Python 2.7 or Python 3, using Linux or OSX, ensure that you have the following packages installed:
 

website/src/jade/home/_installation.jade~

@@ -1,83 +0,0 @@
-mixin Option(name, open)
-  details(open=open)
-    summary
-      h4= name
-    block
-
-article.post
-    header
-        h2 #[a(href=Meta.url)
-        
-        p What's new in v0.90?
-
-        .subhead by #[a(href="//twitter.com/spacy_io", rel="author" target="_blank") #{spaCy}] on #[time #{getDate(Meta.date).fulldate}]
-
-    ul
-        li Support for gazetteers
-        li Set Lexeme attributes
-    #[a.readmore(href=Meta.url) Full Change Log ►]
-
-
-section.intro
-    p What's
-
-+Option("conda", true)
-  pre.language-bash: code
-    | $ conda install spacy
-    | $ python -m spacy.en.download
-
-+Option("pip and virtualenv", true)
-  p With Python 2.7 or Python 3, using Linux or OSX, run:
-
-    pre.language-bash: code
-      | $ pip install spacy
-      | $ python -m spacy.en.download
-
-  p
-    | The download command fetches and installs about 300mb of data, for
-    | the parser model and word vectors, which it installs within the spacy.en
-    | package directory.
-
-
-  +Option("Workaround for obsolete system Python", false)
-    p
-      | If you're stuck using a server with an old version of Python, and you
-      | don't have root access, I've prepared a bootstrap script to help you
-      | compile a local Python install.  Run:
-
-    pre.language-bash: code
-      | $ curl https://raw.githubusercontent.com/honnibal/spaCy/master/bootstrap_python_env.sh | bash && source .env/bin/activate
-
-
-
-+Option("Compile from source", false)
-  p
-    | The other way to install the package is to clone the github repository,
-    | and build it from source.  This installs an additional dependency,
-    | Cython.  If you're using Python 2, I also recommend installing fabric
-    | and fabtools – this is how I build the project.
-
-  pre.language-bash: code
-    | $ git clone https://github.com/honnibal/spaCy.git
-    | $ cd spaCy
-    | $ virtualenv .env && source .env/bin/activate
-    | $ export PYTHONPATH=`pwd`
-    | $ pip install -r requirements.txt
-    | $ python setup.py build_ext --inplace
-    | $ python -m spacy.en.download
-    | $ pip install pytest
-    | $ py.test tests/
-
-  p
-    | Python packaging is awkward at the best of times, and it's particularly tricky
-    | with C extensions, built via Cython, requiring large data files.  So,
-    | please report issues as you encounter them.
-
-+Option("pypy (Unsupported)")
-  | If PyPy support is a priority for you, please get in touch.  We could likely
-  | fix the remaining issues, if necessary.  However, the library is likely to
-  | be much slower on PyPy, as it's written in Cython, which produces code tuned
-  | for the performance of CPython.
-
-+Option("Windows (Unsupported)")
-  | Unfortunately we don't currently support Windows.

website/src/jade/home/index.jade

@@ -29,10 +29,10 @@ include ../header.jade
         li: a.button(href="#example-use") Examples
         li: a.button(href="#install")
           | Install
-          <span class="button-caption">v0.93</span>
+          <span class="button-caption">v0.94</span>
 
   article.page.landing-page
     +Section("Comparisons and Benchmarks", "comparisons", "./_comparisons.jade")
     +Section("Online Demo", "online-demo", "./_online_demo.jade")
     +Section("Usage by Example", "example-use", "./_usage_examples.jade")
-    +Section("Install v0.93", "install", "./_installation.jade")
+    +Section("Install v0.94", "install", "./_installation.jade")

website/src/jade/license/index.jade

@@ -1,38 +0,0 @@
-include ../header.jade
-
-mixin LicenseOption(name, period, price, audience)
-        .item
-            h4 #{name}
-                
-            .focus #{period}
-
-            span #{price}
-                
-            h5 Suggested for:
-                
-            span #{audience}
-                
-            a.button(href="/resources/pdf/spaCy_License_Agreement_2015.pdf", target="_blank") Download license
-
-            span or #[a(href="mailto:sales@spacy.io") get in touch]
-
-- var Page = InitPage(Site, Authors.spacy, "license", "License")
-
-+WritePage(Site, Authors.spacy, Page)
-    article.pricing
-        .box.license
-            +LicenseOption("Trial", "90 days", "$0", "Evaluation")
-            +LicenseOption("Production", "1 year", "$5,000", "Production")
-            +LicenseOption("Certainty", "5 years", "$20,000", "Secure Planning")
-
-        p.caption Researcher, hobbyist, or open-source developer? spaCy also offers #[a(href="http://www.gnu.org/licenses/agpl-3.0.en.html") AGPLv3] licenses.
- 
-        blockquote.pull-quote
-            p Let's face it: Services disappear. Constantly. The good start-ups get bought; the bad ones go bankrupt.
-
-        p You need the source, and you need to know you can buy a long-term license. So that's what we offer. The difference between this and a black-box API is night and day.
-            
-        p Let's face it: services disappear.  Constantly. The good start-ups get bought; the bad ones go bankrupt.  Open-source projects become abandoned or bloated.  Google's graveyard is over-flowing – ditto for Yahoo!, Microsoft, etc. Sure, IBM won't be broke...But will BlueMix be sunset?
-
-        p A 5 year license won't expire until 2020.  spaCy will be with you for longer than most of your current staff.  If that's still not enough, get in touch. We can surely work something out.
-