diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000..4dd7b0a31 --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1 @@ +build: off diff --git a/.gitignore b/.gitignore index 8716a8ef0..2209f5b4a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,50 +1,45 @@ -# Vim -*.swp -*.sw* -Profile.prof -tmp/ -.dev -.denv -.pypyenv -.eggs -*.tgz -.sass-cache -.python-version - -MANIFEST - +# spaCy +spacy/data/ corpora/ models/ keys/ -spacy/syntax/*.cpp -spacy/syntax/*.html -spacy/en/*.cpp -spacy/tokens/*.cpp -spacy/serialize/*.cpp -spacy/en/data/* -spacy/*.cpp -spacy/ner/*.cpp -spacy/orthography/*.cpp -ext/murmurhash.cpp -ext/sparsehash.cpp +# Website +website/www/ +website/_deploy.sh +website/package.json +website/announcement.jade +website/.gitignore -/spacy/data/ - -_build/ -.env/ -tmp/ +# Cython / C extensions cythonize.json - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] - -# C extensions +spacy/*.html +*.cpp *.so -# Distribution / packaging +# Vim / VSCode / editors +*.swp +*.sw* +Profile.prof +.vscode +.sass-cache + +# Python .Python +.python-version +__pycache__/ +*.py[cod] +.env/ +.env2/ +.env3/ +.~env/ +.venv +venv/ +.dev +.denv +.pypyenv + +# Distribution / packaging env/ bin/ build/ @@ -59,6 +54,12 @@ var/ *.egg-info/ .installed.cfg *.egg +.eggs +MANIFEST + +# Temporary files +*.~* +tmp/ # Installer logs pip-log.txt @@ -87,25 +88,16 @@ coverage.xml *.log *.pot -# Windows local helper files +# Windows *.bat +Thumbs.db +Desktop.ini # Mac OS X *.DS_Store -# Temporary files / Dropbox hack -*.~* - # Komodo project files *.komodoproject -# Website -website/_deploy.sh -website/package.json -website/announcement.jade -website/www/ -website/.gitignore - -# Python virtualenv -venv -venv/* +# Other +*.tgz diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index b64dc8db3..c419a03cf 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -16,6 +16,7 @@ This is a list of everyone who has made significant contributions to spaCy, in a * Daniel Vila Suero, [@dvsrepo](https://github.com/dvsrepo) * Dmytro Sadovnychyi, [@sadovnychyi](https://github.com/sadovnychyi) * Eric Zhao, [@ericzhao28](https://github.com/ericzhao28) +* Francisco Aranda, [@frascuchon](https://github.com/frascuchon) * Greg Baker, [@solresol](https://github.com/solresol) * Grégory Howard, [@Gregory-Howard](https://github.com/Gregory-Howard) * György Orosz, [@oroszgy](https://github.com/oroszgy) @@ -24,6 +25,7 @@ This is a list of everyone who has made significant contributions to spaCy, in a * Ines Montani, [@ines](https://github.com/ines) * J Nicolas Schrading, [@NSchrading](https://github.com/NSchrading) * Janneke van der Zwaan, [@jvdzwaan](https://github.com/jvdzwaan) +* Jim Regan, [@jimregan](https://github.com/jimregan) * Jordan Suchow, [@suchow](https://github.com/suchow) * Josh Reeter, [@jreeter](https://github.com/jreeter) * Juan Miguel Cejuela, [@juanmirocks](https://github.com/juanmirocks) diff --git a/README.rst b/README.rst index 24b0c232a..0f3efc146 100644 --- a/README.rst +++ b/README.rst @@ -4,18 +4,22 @@ spaCy: Industrial-strength NLP spaCy is a library for advanced natural language processing in Python and Cython. spaCy is built on the very latest research, but it isn't researchware. It was designed from day one to be used in real products. spaCy currently supports -English, German and French, as well as tokenization for Spanish, Italian, +English, German, French and Spanish, as well as tokenization for Italian, Portuguese, Dutch, Swedish, Finnish, Norwegian, Hungarian, Bengali, Hebrew, Chinese and Japanese. It's commercial open-source software, released under the MIT license. -📊 **Help us improve the library!** `Take the spaCy user survey `_. +⭐️ **Test spaCy v2.0.0 alpha and the new models!** `Read the release notes here. `_ 💫 **Version 1.8 out now!** `Read the release notes here. `_ .. image:: https://img.shields.io/travis/explosion/spaCy/master.svg?style=flat-square :target: https://travis-ci.org/explosion/spaCy - :alt: Build Status + :alt: Travis Build Status + +.. image:: https://img.shields.io/appveyor/ci/explosion/spacy/master.svg?style=flat-square + :target: https://ci.appveyor.com/project/explosion/spacy + :alt: Appveyor Build Status .. image:: https://img.shields.io/github/release/explosion/spacy.svg?style=flat-square :target: https://github.com/explosion/spaCy/releases @@ -85,7 +89,7 @@ Features * GIL-free **multi-threading** * Efficient binary serialization * Easy **deep learning** integration -* Statistical models for **English** and **German** +* Statistical models for **English**, **German**, **French** and **Spanish** * State-of-the-art speed * Robust, rigorously evaluated accuracy diff --git a/examples/training/train_new_entity_type.py b/examples/training/train_new_entity_type.py index 6c432acdf..666a3bad4 100644 --- a/examples/training/train_new_entity_type.py +++ b/examples/training/train_new_entity_type.py @@ -52,6 +52,7 @@ def train_ner(nlp, train_data, output_dir): random.shuffle(train_data) loss = 0. for raw_text, entity_offsets in train_data: + doc = nlp.make_doc(raw_text) gold = GoldParse(doc, entities=entity_offsets) # By default, the GoldParse class assumes that the entities # described by offset are complete, and all other words should @@ -63,7 +64,6 @@ def train_ner(nlp, train_data, output_dir): #for i in range(len(gold.ner)): #if not gold.ner[i].endswith('ANIMAL'): # gold.ner[i] = '-' - doc = nlp.make_doc(raw_text) nlp.tagger(doc) # As of 1.9, spaCy's parser now lets you supply a dropout probability # This might help the model generalize better from only a few diff --git a/requirements.txt b/requirements.txt index 42910d1be..20c587841 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,9 +7,11 @@ thinc>=6.5.0,<6.6.0 murmurhash>=0.26,<0.27 plac<1.0.0,>=0.9.6 six +html5lib==1.0b8 ujson>=1.35 dill>=0.2,<0.3 requests>=2.13.0,<3.0.0 regex==2017.4.5 ftfy>=4.4.2,<5.0.0 pytest>=3.0.6,<4.0.0 +pip>=9.0.0,<10.0.0 diff --git a/setup.py b/setup.py index a112a6e80..89aaf8eba 100755 --- a/setup.py +++ b/setup.py @@ -197,6 +197,7 @@ def setup_package(): 'preshed>=1.0.0,<2.0.0', 'thinc>=6.5.0,<6.6.0', 'plac<1.0.0,>=0.9.6', + 'pip>=9.0.0,<10.0.0', 'six', 'pathlib', 'ujson>=1.35', diff --git a/spacy/about.py b/spacy/about.py index ad4a021c2..8c0e0afd3 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -10,7 +10,7 @@ __author__ = 'Matthew Honnibal' __email__ = 'matt@explosion.ai' __license__ = 'MIT' -__docs__ = 'https://spacy.io/docs/usage' +__docs_models__ = 'https://spacy.io/docs/usage' __download_url__ = 'https://github.com/explosion/spacy-models/releases/download' __compatibility__ = 'https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json' __shortcuts__ = 'https://raw.githubusercontent.com/explosion/spacy-models/master/shortcuts.json' diff --git a/spacy/cli/download.py b/spacy/cli/download.py index 0419de118..70ca64b22 100644 --- a/spacy/cli/download.py +++ b/spacy/cli/download.py @@ -79,5 +79,5 @@ def check_error_depr(model): "As of v1.7.0, the download all command is deprecated. Please " "download the models individually via spacy.download [model name] " "or pip install. For more info on this, see the documentation: " - "{d}".format(d=about.__docs__), + "{d}".format(d=about.__docs_models__), title="Deprecated command") diff --git a/spacy/cli/package.py b/spacy/cli/package.py index 102b07472..74086613a 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -47,7 +47,7 @@ def package(input_dir, output_dir, meta_path, force): def check_dirs(input_path, output_path, meta_path): if not input_path.exists(): - util.sys_exit(unicode_(input_path.as_poisx), title="Model directory not found") + util.sys_exit(unicode_(input_path.as_posix()), title="Model directory not found") if not output_path.exists(): util.sys_exit(unicode_(output_path), title="Output directory not found") if meta_path and not meta_path.exists(): diff --git a/spacy/deprecated.py b/spacy/deprecated.py index 65053089a..3f3c69b88 100644 --- a/spacy/deprecated.py +++ b/spacy/deprecated.py @@ -146,7 +146,7 @@ class ModelDownload(): "The spacy.{l}.download command is now deprecated. Please use " "python -m spacy download [model name or shortcut] instead. For more " "info and available models, see the documentation: {d}. " - "Downloading default '{l}' model now...".format(d=about.__docs__, l=lang), + "Downloading default '{l}' model now...".format(d=about.__docs_models__, l=lang), title="Warning: deprecated command") download(lang) diff --git a/spacy/en/tokenizer_exceptions.py b/spacy/en/tokenizer_exceptions.py index 3d009241b..d9aa01734 100644 --- a/spacy/en/tokenizer_exceptions.py +++ b/spacy/en/tokenizer_exceptions.py @@ -178,7 +178,7 @@ for word in ["who", "what", "when", "where", "why", "how", "there", "that"]: EXC[orth + "ve"] = [ {ORTH: orth, LEMMA: word}, - {ORTH: "'ve", LEMMA: "have", TAG: "VB"} + {ORTH: "ve", LEMMA: "have", TAG: "VB"} ] EXC[orth + "'d"] = [ diff --git a/spacy/es/tokenizer_exceptions.py b/spacy/es/tokenizer_exceptions.py index e60bcd104..f9c955338 100644 --- a/spacy/es/tokenizer_exceptions.py +++ b/spacy/es/tokenizer_exceptions.py @@ -6,36 +6,6 @@ from ..language_data import PRON_LEMMA, DET_LEMMA TOKENIZER_EXCEPTIONS = { - "al": [ - {ORTH: "a", LEMMA: "a", TAG: ADP}, - {ORTH: "el", LEMMA: "el", TAG: DET} - ], - - "consigo": [ - {ORTH: "con", LEMMA: "con"}, - {ORTH: "sigo", LEMMA: PRON_LEMMA, NORM: "sí"} - ], - - "conmigo": [ - {ORTH: "con", LEMMA: "con"}, - {ORTH: "migo", LEMMA: PRON_LEMMA, NORM: "mí"} - ], - - "contigo": [ - {ORTH: "con", LEMMA: "con"}, - {ORTH: "tigo", LEMMA: PRON_LEMMA, NORM: "ti"} - ], - - "del": [ - {ORTH: "de", LEMMA: "de", TAG: ADP}, - {ORTH: "l", LEMMA: "el", TAG: DET} - ], - - "pel": [ - {ORTH: "pe", LEMMA: "per", TAG: ADP}, - {ORTH: "l", LEMMA: "el", TAG: DET} - ], - "pal": [ {ORTH: "pa", LEMMA: "para"}, {ORTH: "l", LEMMA: DET_LEMMA, NORM: "el"} @@ -43,7 +13,7 @@ TOKENIZER_EXCEPTIONS = { "pala": [ {ORTH: "pa", LEMMA: "para"}, - {ORTH: "la", LEMMA: DET_LEMMA} + {ORTH: "la", LEMMA: DET_LEMMA, NORM: "la"} ], "aprox.": [ diff --git a/spacy/ja/__init__.py b/spacy/ja/__init__.py index 07e40ada6..1c85ded95 100644 --- a/spacy/ja/__init__.py +++ b/spacy/ja/__init__.py @@ -3,21 +3,39 @@ from __future__ import unicode_literals, print_function from os import path -from ..language import Language +from ..language import Language, BaseDefaults +from ..tokenizer import Tokenizer from ..attrs import LANG from ..tokens import Doc from .language_data import * - -class Japanese(Language): - lang = 'ja' - - def make_doc(self, text): +class JapaneseTokenizer(object): + def __init__(self, cls, nlp=None): + self.vocab = nlp.vocab if nlp is not None else cls.create_vocab(nlp) try: from janome.tokenizer import Tokenizer except ImportError: raise ImportError("The Japanese tokenizer requires the Janome library: " "https://github.com/mocobeta/janome") - words = [x.surface for x in Tokenizer().tokenize(text)] + self.tokenizer = Tokenizer() + + def __call__(self, text): + words = [x.surface for x in self.tokenizer.tokenize(text)] return Doc(self.vocab, words=words, spaces=[False]*len(words)) + +class JapaneseDefaults(BaseDefaults): + @classmethod + def create_tokenizer(cls, nlp=None): + return JapaneseTokenizer(cls, nlp) + +class Japanese(Language): + lang = 'ja' + + Defaults = JapaneseDefaults + + def make_doc(self, text): + words = self.tokenizer(text) + return Doc(self.vocab, words=words, spaces=[False]*len(words)) + + diff --git a/spacy/lexeme.pyx b/spacy/lexeme.pyx index 5d9ce7b98..05d8bddc6 100644 --- a/spacy/lexeme.pyx +++ b/spacy/lexeme.pyx @@ -24,6 +24,7 @@ from .attrs cimport IS_QUOTE from .attrs cimport IS_LEFT_PUNCT from .attrs cimport IS_RIGHT_PUNCT from .attrs cimport IS_OOV +from . import about memset(&EMPTY_LEXEME, 0, sizeof(LexemeC)) @@ -137,11 +138,10 @@ cdef class Lexeme: cdef int length = self.vocab.vectors_length if length == 0: raise ValueError( - "Word vectors set to length 0. This may be because the " - "data is not installed. If you haven't already, run" - "\npython -m spacy download %s\n" - "to install the data." % self.vocab.lang - ) + "Word vectors set to length 0. This may be because you " + "don't have a model installed or loaded, or because your " + "model doesn't include word vectors. For more info, see " + "the documentation: \n%s\n" % about.__docs_models__) vector_view = self.c.vector return numpy.asarray(vector_view) diff --git a/spacy/syntax/iterators.pyx b/spacy/syntax/iterators.pyx index e1c44da7f..b0d1c78ca 100644 --- a/spacy/syntax/iterators.pyx +++ b/spacy/syntax/iterators.pyx @@ -1,7 +1,7 @@ # coding: utf-8 from __future__ import unicode_literals -from ..parts_of_speech cimport NOUN, PROPN, PRON +from ..parts_of_speech cimport NOUN, PROPN, PRON, VERB, AUX def english_noun_chunks(obj): @@ -66,4 +66,55 @@ def german_noun_chunks(obj): yield word.left_edge.i, rbracket, np_label -CHUNKERS = {'en': english_noun_chunks, 'de': german_noun_chunks} +def es_noun_chunks(obj): + + doc = obj.doc + np_label = doc.vocab.strings['NP'] + + left_labels = ['det', 'fixed', 'neg'] #['nunmod', 'det', 'appos', 'fixed'] + right_labels = ['flat', 'fixed', 'compound', 'neg'] + stop_labels = ['punct'] + + np_left_deps = [doc.vocab.strings[label] for label in left_labels] + np_right_deps = [doc.vocab.strings[label] for label in right_labels] + stop_deps = [doc.vocab.strings[label] for label in stop_labels] + + def next_token(token): + try: + return token.nbor() + except: + return None + + def noun_bounds(root): + + def is_verb_token(token): + return token.pos in [VERB, AUX] + + left_bound = root + for token in reversed(list(root.lefts)): + if token.dep in np_left_deps: + left_bound = token + + right_bound = root + for token in root.rights: + if (token.dep in np_right_deps): + left, right = noun_bounds(token) + + if list(filter(lambda t: is_verb_token(t) or t.dep in stop_deps, doc[left_bound.i: right.i])): + break + else: + right_bound = right + + return left_bound, right_bound + + + token = doc[0] + while token and token.i < len(doc): + if token.pos in [PROPN, NOUN, PRON]: + left, right = noun_bounds(token) + yield left.i, right.i+1, np_label + token = right + token = next_token(token) + + +CHUNKERS = {'en': english_noun_chunks, 'de': german_noun_chunks, 'es': es_noun_chunks} diff --git a/spacy/tests/conftest.py b/spacy/tests/conftest.py index b8ada1d9a..6e00b1513 100644 --- a/spacy/tests/conftest.py +++ b/spacy/tests/conftest.py @@ -5,6 +5,7 @@ from ..en import English from ..de import German from ..es import Spanish from ..it import Italian +from ..ja import Japanese from ..fr import French from ..pt import Portuguese from ..nl import Dutch @@ -26,7 +27,7 @@ from pathlib import Path import os import pytest - +# These languages get run through generic tokenizer tests LANGUAGES = [English, German, Spanish, Italian, French, Portuguese, Dutch, Swedish, Hungarian, Finnish, Bengali, Norwegian] @@ -76,6 +77,12 @@ def fi_tokenizer(): return Finnish.Defaults.create_tokenizer() +@pytest.fixture +def ja_tokenizer(): + janome = pytest.importorskip("janome") + return Japanese.Defaults.create_tokenizer() + + @pytest.fixture def sv_tokenizer(): return Swedish.Defaults.create_tokenizer() diff --git a/spacy/tests/doc/test_doc_api.py b/spacy/tests/doc/test_doc_api.py index 064cbecfd..1bc534ecd 100644 --- a/spacy/tests/doc/test_doc_api.py +++ b/spacy/tests/doc/test_doc_api.py @@ -217,10 +217,13 @@ def test_doc_api_has_vector(en_tokenizer, text_file, text, vectors): assert doc.has_vector -def test_parse_tree(EN): +def test_parse_tree(en_tokenizer): + """Tests doc.print_tree() method.""" text = 'I like New York in Autumn.' - doc = EN(text, tag=True) - doc.from_array([HEAD], numpy.asarray([[1, 0, 1, -2, -3, -1, -5]], dtype='int32').T) + heads = [1, 0, 1, -2, -3, -1, -5] + tags = ['PRP', 'IN', 'NNP', 'NNP', 'IN', 'NNP', '.'] + tokens = en_tokenizer(text) + doc = get_doc(tokens.vocab, [t.text for t in tokens], heads=heads, tags=tags) # full method parse_tree(text) is a trivial composition trees = doc.print_tree() assert len(trees) > 0 diff --git a/spacy/tests/ja/__init__.py b/spacy/tests/ja/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/spacy/tests/ja/test_tokenizer.py b/spacy/tests/ja/test_tokenizer.py new file mode 100644 index 000000000..58700b353 --- /dev/null +++ b/spacy/tests/ja/test_tokenizer.py @@ -0,0 +1,17 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import pytest + +TOKENIZER_TESTS = [ + ("日本語だよ", ['日本語', 'だ', 'よ']), + ("東京タワーの近くに住んでいます。", ['東京', 'タワー', 'の', '近く', 'に', '住ん', 'で', 'い', 'ます', '。']), + ("吾輩は猫である。", ['吾輩', 'は', '猫', 'で', 'ある', '。']), + ("月に代わって、お仕置きよ!", ['月', 'に', '代わっ', 'て', '、', 'お仕置き', 'よ', '!']), + ("すもももももももものうち", ['すもも', 'も', 'もも', 'も', 'もも', 'の', 'うち']) +] + +@pytest.mark.parametrize('text,expected_tokens', TOKENIZER_TESTS) +def test_japanese_tokenizer(ja_tokenizer, text, expected_tokens): + tokens = [token.text for token in ja_tokenizer(text)] + assert tokens == expected_tokens diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index fe8e019ec..ca5a3d696 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -29,6 +29,7 @@ from ..serialize.bits cimport BitArray from ..util import normalize_slice from ..syntax.iterators import CHUNKERS from ..compat import is_config +from .. import about DEF PADDING = 5 @@ -403,9 +404,8 @@ cdef class Doc: if not self.is_parsed: raise ValueError( "noun_chunks requires the dependency parse, which " - "requires data to be installed. If you haven't done so, run: " - "\npython -m spacy download %s\n" - "to install the data" % self.vocab.lang) + "requires data to be installed. For more info, see the " + "documentation: \n%s\n" % about.__docs_models__) # Accumulate the result before beginning to iterate over it. This prevents # the tokenisation from being changed out from under us during the iteration. # The tricky thing here is that Span accepts its tokenisation changing, @@ -431,14 +431,14 @@ cdef class Doc: """ def __get__(self): if 'sents' in self.user_hooks: - return self.user_hooks['sents'](self) + yield from self.user_hooks['sents'](self) + return if not self.is_parsed: raise ValueError( - "sentence boundary detection requires the dependency parse, which " - "requires data to be installed. If you haven't done so, run: " - "\npython -m spacy download %s\n" - "to install the data" % self.vocab.lang) + "Sentence boundary detection requires the dependency parse, which " + "requires data to be installed. For more info, see the " + "documentation: \n%s\n" % about.__docs_models__) cdef int i start = 0 for i in range(1, self.length): diff --git a/spacy/tokens/printers.py b/spacy/tokens/printers.py index 487d74167..81c0243ee 100644 --- a/spacy/tokens/printers.py +++ b/spacy/tokens/printers.py @@ -1,13 +1,23 @@ -from copy import deepcopy +# coding: utf8 +from __future__ import unicode_literals + +from .doc import Doc +from ..symbols import HEAD, TAG, DEP, ENT_IOB, ENT_TYPE + def merge_ents(doc): - '''Helper: merge adjacent entities into single tokens; modifies the doc.''' + """ + Helper: merge adjacent entities into single tokens; modifies the doc. + """ for ent in doc.ents: ent.merge(ent.root.tag_, ent.text, ent.label_) return doc + def format_POS(token, light, flat): - '''helper: form the POS output for a token''' + """ + Helper: form the POS output for a token. + """ subtree = dict([ ("word", token.text), ("lemma", token.lemma_), # trigger @@ -25,17 +35,22 @@ def format_POS(token, light, flat): subtree.pop("modifiers") return subtree -def POS_tree(root, light, flat): - '''Helper: generate a POS tree for a root token. - The doc must have merge_ents(doc) ran on it. - ''' + +def POS_tree(root, light=False, flat=False): + """ + Helper: generate a POS tree for a root token. The doc must have + merge_ents(doc) ran on it. + """ subtree = format_POS(root, light=light, flat=flat) for c in root.children: subtree["modifiers"].append(POS_tree(c)) return subtree + def parse_tree(doc, light=False, flat=False): - """Makes a copy of the doc, then construct a syntactic parse tree, similar to the one used in displaCy. Generates the POS tree for all sentences in a doc + """ + Makes a copy of the doc, then construct a syntactic parse tree, similar to + the one used in displaCy. Generates the POS tree for all sentences in a doc. Args: doc: The doc for parsing. @@ -50,6 +65,8 @@ def parse_tree(doc, light=False, flat=False): [{'modifiers': [{'modifiers': [], 'NE': 'PERSON', 'word': 'Bob', 'arc': 'nsubj', 'POS_coarse': 'PROPN', 'POS_fine': 'NNP', 'lemma': 'Bob'}, {'modifiers': [], 'NE': 'PERSON', 'word': 'Alice', 'arc': 'dobj', 'POS_coarse': 'PROPN', 'POS_fine': 'NNP', 'lemma': 'Alice'}, {'modifiers': [{'modifiers': [], 'NE': '', 'word': 'the', 'arc': 'det', 'POS_coarse': 'DET', 'POS_fine': 'DT', 'lemma': 'the'}], 'NE': '', 'word': 'pizza', 'arc': 'dobj', 'POS_coarse': 'NOUN', 'POS_fine': 'NN', 'lemma': 'pizza'}, {'modifiers': [], 'NE': '', 'word': '.', 'arc': 'punct', 'POS_coarse': 'PUNCT', 'POS_fine': '.', 'lemma': '.'}], 'NE': '', 'word': 'brought', 'arc': 'ROOT', 'POS_coarse': 'VERB', 'POS_fine': 'VBD', 'lemma': 'bring'}, {'modifiers': [{'modifiers': [], 'NE': 'PERSON', 'word': 'Alice', 'arc': 'nsubj', 'POS_coarse': 'PROPN', 'POS_fine': 'NNP', 'lemma': 'Alice'}, {'modifiers': [{'modifiers': [], 'NE': '', 'word': 'the', 'arc': 'det', 'POS_coarse': 'DET', 'POS_fine': 'DT', 'lemma': 'the'}], 'NE': '', 'word': 'pizza', 'arc': 'dobj', 'POS_coarse': 'NOUN', 'POS_fine': 'NN', 'lemma': 'pizza'}, {'modifiers': [], 'NE': '', 'word': '.', 'arc': 'punct', 'POS_coarse': 'PUNCT', 'POS_fine': '.', 'lemma': '.'}], 'NE': '', 'word': 'ate', 'arc': 'ROOT', 'POS_coarse': 'VERB', 'POS_fine': 'VBD', 'lemma': 'eat'}] """ doc_clone = Doc(doc.vocab, words=[w.text for w in doc]) - doc_clone.from_array(doc.to_array([HEAD, DEP, TAG, ENT_IOB, ENT_TYPE]) + doc_clone = Doc(doc.vocab, words=[w.text for w in doc]) + doc_clone.from_array([HEAD, TAG, DEP, ENT_IOB, ENT_TYPE], + doc.to_array([HEAD, TAG, DEP, ENT_IOB, ENT_TYPE])) merge_ents(doc_clone) # merge the entities into single tokens first return [POS_tree(sent.root, light=light, flat=flat) for sent in doc_clone.sents] diff --git a/spacy/tokens/span.pyx b/spacy/tokens/span.pyx index fb1e5c732..09927ab4c 100644 --- a/spacy/tokens/span.pyx +++ b/spacy/tokens/span.pyx @@ -16,6 +16,7 @@ from ..util import normalize_slice from ..attrs cimport IS_PUNCT, IS_SPACE from ..lexeme cimport Lexeme from ..compat import is_config +from .. import about cdef class Span: @@ -221,9 +222,8 @@ cdef class Span: if not self.doc.is_parsed: raise ValueError( "noun_chunks requires the dependency parse, which " - "requires data to be installed. If you haven't done so, run: " - "\npython -m spacy download %s\n" - "to install the data" % self.vocab.lang) + "requires data to be installed. For more info, see the " + "documentation: \n%s\n" % about.__docs_models__) # Accumulate the result before beginning to iterate over it. This prevents # the tokenisation from being changed out from under us during the iteration. # The tricky thing here is that Span accepts its tokenisation changing, diff --git a/spacy/tokens/token.pyx b/spacy/tokens/token.pyx index f146f5cd6..daef48233 100644 --- a/spacy/tokens/token.pyx +++ b/spacy/tokens/token.pyx @@ -26,6 +26,7 @@ from ..attrs cimport IS_TITLE, IS_UPPER, LIKE_URL, LIKE_NUM, LIKE_EMAIL, IS_STOP from ..attrs cimport IS_OOV from ..lexeme cimport Lexeme from ..compat import is_config +from .. import about cdef class Token: @@ -237,11 +238,10 @@ cdef class Token: cdef int length = self.vocab.vectors_length if length == 0: raise ValueError( - "Word vectors set to length 0. This may be because the " - "data is not installed. If you haven't already, run" - "\npython -m spacy download %s\n" - "to install the data." % self.vocab.lang - ) + "Word vectors set to length 0. This may be because you " + "don't have a model installed or loaded, or because your " + "model doesn't include word vectors. For more info, see " + "the documentation: \n%s\n" % about.__docs_models__) vector_view = self.c.lex.vector return numpy.asarray(vector_view) diff --git a/spacy/zh/__init__.py b/spacy/zh/__init__.py index 1847a7d8d..0f407dec6 100644 --- a/spacy/zh/__init__.py +++ b/spacy/zh/__init__.py @@ -8,4 +8,5 @@ class Chinese(Language): def make_doc(self, text): import jieba words = list(jieba.cut(text, cut_all=True)) + words=[x for x in words if x] return Doc(self.vocab, words=words, spaces=[False]*len(words)) diff --git a/website/_harp.json b/website/_harp.json index 672640405..cb476541a 100644 --- a/website/_harp.json +++ b/website/_harp.json @@ -14,8 +14,8 @@ "SPACY_VERSION": "1.8", "LATEST_NEWS": { - "url": "https://survey.spacy.io/", - "title": "Take the spaCy user survey and help us improve the library!" + "url": "/docs/usage/models", + "title": "The first official Spanish model is here!" }, "SOCIAL": { @@ -55,7 +55,33 @@ } }, - "V_CSS": "1.6", + "QUICKSTART": [ + { "id": "os", "title": "Operating system", "options": [ + { "id": "mac", "title": "macOS / OSX", "checked": true }, + { "id": "windows", "title": "Windows" }, + { "id": "linux", "title": "Linux" }] + }, + { "id": "package", "title": "Package manager", "options": [ + { "id": "pip", "title": "pip", "checked": true }, + { "id": "conda", "title": "conda" }, + { "id": "source", "title": "from source" }] + }, + { "id": "python", "title": "Python version", "options": [ + { "id": 2, "title": "2.x" }, + { "id": 3, "title": "3.x", "checked": true }] + }, + { "id": "config", "title": "Configuration", "multiple": true, "options": [ + {"id": "venv", "title": "virtualenv", "help": "Use a virtual environment and install spaCy into a user directory" }] + }, + { "id": "model", "title": "Models", "multiple": true, "options": [ + { "id": "en", "title": "English", "meta": "50MB" }, + { "id": "de", "title": "German", "meta": "645MB" }, + { "id": "fr", "title": "French", "meta": "1.33GB" }, + { "id": "es", "title": "Spanish", "meta": "377MB"}] + } + ], + + "V_CSS": "1.7", "V_JS": "1.2", "DEFAULT_SYNTAX": "python", "ANALYTICS": "UA-58931649-1", diff --git a/website/_includes/_mixins-base.jade b/website/_includes/_mixins-base.jade index 94b1bfd84..106f8f1ca 100644 --- a/website/_includes/_mixins-base.jade +++ b/website/_includes/_mixins-base.jade @@ -121,6 +121,47 @@ mixin badge(name) img(src=site.badge alt="{name} version" height="20") +//- Quickstart widget + quickstart.js with manual markup, inspired by PyTorch's "Getting started" + groups - [object] option groups, uses global variable QUICKSTART + headline - [string] optional text to be rendered as widget headline + +mixin quickstart(groups, headline) + .c-quickstart.o-block-small#qs + .c-quickstart__content + if headline + +h(2)=headline + for group in groups + .c-quickstart__group.u-text-small(data-qs-group=group.id) + .c-quickstart__legend=group.title + .c-quickstart__fields + for option in group.options + input.c-quickstart__input(class="c-quickstart__input--" + (group.multiple ? "check" : "radio") type=group.multiple ? "checkbox" : "radio" name=group.id id="qs-#{option.id}" value=option.id checked=option.checked) + label.c-quickstart__label(for="qs-#{option.id}")=option.title + if option.meta + | #[span.c-quickstart__label__meta (#{option.meta})] + if option.help + | #[+help(option.help).c-quickstart__label__meta] + + pre.c-code-block + code.c-code-block__content.c-quickstart__code(data-qs-results="") + block + + .c-quickstart__info.u-text-tiny.o-block.u-text-right + | Like this widget? Check out #[+a("https://github.com/ines/quickstart").u-link quickstart.js]! + + +//- Quickstart code item + data [object] - Rendering conditions (keyed by option group ID, value: option) + +mixin qs(data) + - args = {} + for value, setting in data + - args['data-qs-' + setting] = value + span.c-quickstart__line&attributes(args) + block + + //- Logo mixin logo() diff --git a/website/_includes/_mixins.jade b/website/_includes/_mixins.jade index 2f89b0ec4..aeee54f52 100644 --- a/website/_includes/_mixins.jade +++ b/website/_includes/_mixins.jade @@ -47,6 +47,14 @@ mixin api(path) | #[+icon("book", 18).o-icon--inline.u-color-subtle] +//- Help icon with tooltip + tooltip - [string] Tooltip text + +mixin help(tooltip) + span(data-tooltip=tooltip)&attributes(attributes) + +icon("help", 16).i-icon--inline + + //- Aside for text label - [string] aside title (optional) diff --git a/website/_includes/_scripts.jade b/website/_includes/_scripts.jade index 544cf0977..b31c7119e 100644 --- a/website/_includes/_scripts.jade +++ b/website/_includes/_scripts.jade @@ -1,9 +1,13 @@ //- 💫 INCLUDES > SCRIPTS -script(src="/assets/js/main.js?v#{V_JS}", type="text/javascript") -script(src="/assets/js/prism.js", type="text/javascript") +script(src="/assets/js/main.js?v#{V_JS}") +script(src="/assets/js/prism.js") if SECTION == "docs" + if quickstart + script(src="/assets/js/quickstart.js") + script var qs = new Quickstart("#qs"); + script. ((window.gitter = {}).chat = {}).options = { useStyles: false, diff --git a/website/assets/css/_components/_code.sass b/website/assets/css/_components/_code.sass index 83462ef72..b2ba9c60e 100644 --- a/website/assets/css/_components/_code.sass +++ b/website/assets/css/_components/_code.sass @@ -18,7 +18,7 @@ .c-code-block__content display: block - font: normal normal 1.1rem/#{2} $font-code + font: normal 600 1.1rem/#{2} $font-code padding: 1em 2em diff --git a/website/assets/css/_components/_quickstart.sass b/website/assets/css/_components/_quickstart.sass new file mode 100644 index 000000000..a3e0bff9c --- /dev/null +++ b/website/assets/css/_components/_quickstart.sass @@ -0,0 +1,90 @@ +//- 💫 CSS > COMPONENTS > QUICKSTART + +.c-quickstart + border: 1px solid $color-subtle + border-radius: 2px + display: none + background: $color-subtle-light + + &:not([style]) + .c-quickstart__info + display: none + +.c-quickstart__content + padding: 2rem 3rem + +.c-quickstart__input + @include size(0) + opacity: 0 + position: absolute + left: -9999px + +.c-quickstart__label + cursor: pointer + background: $color-back + border: 1px solid $color-subtle + border-radius: 2px + display: inline-block + padding: 0.75rem 1.25rem + margin: 0 0.5rem 0.5rem 0 + font-weight: bold + + &:hover + background: lighten($color-theme-light, 5) + + .c-quickstart__input:focus + & + border: 1px solid $color-theme + + .c-quickstart__input--radio:checked + & + color: $color-back + border-color: $color-theme + background: $color-theme + + .c-quickstart__input--check + &:before + content: "" + background: $color-back + display: inline-block + width: 20px + height: 20px + border: 1px solid $color-subtle + vertical-align: middle + margin-right: 1rem + cursor: pointer + border-radius: 50% + + .c-quickstart__input--check:checked + &:before + background: $color-theme url(data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIyNCIgaGVpZ2h0PSIyNCIgdmlld0JveD0iMCAwIDI0IDI0Ij4gICAgPHBhdGggZmlsbD0iI2ZmZiIgZD0iTTkgMTYuMTcybDEwLjU5NC0xMC41OTQgMS40MDYgMS40MDYtMTIgMTItNS41NzgtNS41NzggMS40MDYtMS40MDZ6Ii8+PC9zdmc+) + background-size: contain + border-color: $color-theme + +.c-quickstart__label__meta + font-weight: normal + color: $color-subtle-dark + +.c-quickstart__group + @include breakpoint(min, md) + display: flex + flex-flow: row nowrap + + &:not(:last-child) + margin-bottom: 1rem + +.c-quickstart__fields + flex: 100% + +.c-quickstart__legend + color: $color-subtle-dark + margin-right: 2rem + padding-top: 0.75rem + flex: 1 1 35% + font-weight: bold + +.c-quickstart__line + display: block + + &:before + color: $color-theme + margin-right: 1em + content: "$" + +.c-quickstart__code + font-size: 1.6rem diff --git a/website/assets/css/_components/_tooltips.sass b/website/assets/css/_components/_tooltips.sass new file mode 100644 index 000000000..a19456aa5 --- /dev/null +++ b/website/assets/css/_components/_tooltips.sass @@ -0,0 +1,29 @@ +//- 💫 CSS > COMPONENTS > TOOLTIPS + +[data-tooltip] + position: relative + + @include breakpoint(min, sm) + &:before + @include position(absolute, top, left, 125%, 50%) + display: inline-block + content: attr(data-tooltip) + background: $color-front + border-radius: 2px + color: $color-back + font-family: inherit + font-size: 1.3rem + line-height: 1.25 + opacity: 0 + padding: 0.5em 0.75em + transform: translateX(-50%) translateY(-2px) + transition: opacity 0.1s ease-out, transform 0.1s ease-out + visibility: hidden + min-width: 200px + max-width: 300px + z-index: 200 + + &:hover:before + opacity: 1 + transform: translateX(-50%) translateY(0) + visibility: visible diff --git a/website/assets/css/_variables.sass b/website/assets/css/_variables.sass index 5f9453ea6..ad0739838 100644 --- a/website/assets/css/_variables.sass +++ b/website/assets/css/_variables.sass @@ -27,6 +27,7 @@ $font-code: 'Source Code Pro', Consolas, 'Andale Mono', Menlo, Monaco, Courier, // Colors $colors: ( blue: #09a3d5, red: #d9515d ) +$colors-light: (blue: #cceaf4, red: #f9d7da) $color-back: #fff !default $color-front: #1a1e23 !default @@ -34,7 +35,7 @@ $color-dark: lighten($color-front, 20) !default $color-theme: map-get($colors, $theme) $color-theme-dark: darken(map-get($colors, $theme), 5) -$color-theme-light: saturate(lighten(map-get($colors, $theme), 35), 5) +$color-theme-light: map-get($colors-light, $theme) $color-subtle: #ddd !default $color-subtle-light: #f6f6f6 !default diff --git a/website/assets/css/style.sass b/website/assets/css/style.sass index a8d2edad4..809598663 100644 --- a/website/assets/css/style.sass +++ b/website/assets/css/style.sass @@ -32,3 +32,5 @@ $theme: blue !default @import _components/navigation @import _components/sidebar @import _components/tables +@import _components/tooltips +@import _components/quickstart diff --git a/website/assets/img/graphics.svg b/website/assets/img/graphics.svg index c24473b4c..a449c3d04 100644 --- a/website/assets/img/graphics.svg +++ b/website/assets/img/graphics.svg @@ -1,5 +1,16 @@ + + spaCy v2.0.0 alpha + + + + + + + + + spaCy user survey 2017 diff --git a/website/assets/img/icons.svg b/website/assets/img/icons.svg index f62901592..e970bb52c 100644 --- a/website/assets/img/icons.svg +++ b/website/assets/img/icons.svg @@ -27,5 +27,8 @@ + + + diff --git a/website/assets/js/quickstart.js b/website/assets/js/quickstart.js new file mode 100644 index 000000000..d062aa91f --- /dev/null +++ b/website/assets/js/quickstart.js @@ -0,0 +1,8 @@ +/** + * quickstart.js + * A micro-form for user-specific installation instructions + * + * @author Ines Montani + * @version 0.0.1 + * @license MIT + */'use strict';var _createClass=function(){function a(b,c){for(var e,d=0;d['+this.dpfx+'-'+c+']'+e+' {display: none}';this._$('['+this.dpfx+'-style="'+c+'"]').textContent=g}},{key:'updateContainer',value:function updateContainer(){if(!this._$('['+this.dpfx+'-results]')){var b=this.childNodes(this.container,'pre'),c=b?b[0]:this._c('pre',this.pfx+'-code'),d=this.childNodes(c,'code')||this.childNodes(this.container,'code'),e=d?d[0]:this._c('code',this.pfx+'-results');e.setAttribute(this.dpfx+'-results','');var f=this.childNodes(e,'span')||this.childNodes(c,'span')||this.childNodes(this.container,'span');f&&f.forEach(function(g){return e.appendChild(g)}),c.appendChild(e),this.container.appendChild(c)}}},{key:'createGroup',value:function createGroup(b){var d=this,c=this._c('fieldset',this.pfx+'-group');c.setAttribute(this.dpfx+'-group',b.id),c.innerHTML=this.createStyles(b.id).outerHTML,c.innerHTML+=''+b.title+'',c.innerHTML+=b.options.map(function(e){var f=b.multiple?'checkbox':'radio';return''}).join(''),this.container.insertBefore(c,this.container.firstChild),this.initGroup(c,b.id)}},{key:'createStyles',value:function createStyles(b){var c=this._c('style');return c.setAttribute(this.dpfx+'-style',b),c.textContent='['+this.dpfx+'-results]>['+this.dpfx+'-'+b+'] {display: none}',c}},{key:'childNodes',value:function childNodes(b,c){var d=c.toUpperCase();if(!b.hasChildNodes)return!1;var e=[].concat(_toConsumableArray(b.childNodes)).filter(function(f){return f.nodeName===d});return!!e.length&&e}},{key:'_$',value:function _$(b){return document.querySelector(b)}},{key:'_$$',value:function _$$(b){return[].concat(_toConsumableArray(document.querySelectorAll(b)))}},{key:'_c',value:function _c(b,c){var d=document.createElement(b);return c&&(d.className=c),d}}]),a}(); diff --git a/website/docs/api/_annotation/_dep-labels.jade b/website/docs/api/_annotation/_dep-labels.jade index 9e1e89324..427b2f53a 100644 --- a/website/docs/api/_annotation/_dep-labels.jade +++ b/website/docs/api/_annotation/_dep-labels.jade @@ -1,10 +1,5 @@ //- 💫 DOCS > API > ANNOTATION > DEPENDENCY LABELS -+infobox("Tip") - | In spaCy v1.8.3+, you can also use #[code spacy.explain()] to get the - | description for the string representation of a label. For example, - | #[code spacy.explain("prt")] will return "particle". - +h(3, "dependency-parsing-english") English dependency labels p diff --git a/website/docs/api/_annotation/_named-entities.jade b/website/docs/api/_annotation/_named-entities.jade index 68b3bd17d..476659d4a 100644 --- a/website/docs/api/_annotation/_named-entities.jade +++ b/website/docs/api/_annotation/_named-entities.jade @@ -1,10 +1,5 @@ //- 💫 DOCS > API > ANNOTATION > NAMED ENTITIES -+infobox("Tip") - | In spaCy v1.8.3+, you can also use #[code spacy.explain()] to get the - | description for the string representation of an entity label. For example, - | #[code spacy.explain("LANGUAGE")] will return "any named language". - +table([ "Type", "Description" ]) +row +cell #[code PERSON] diff --git a/website/docs/api/_annotation/_pos-tags.jade b/website/docs/api/_annotation/_pos-tags.jade index d3ceef777..ea3a225bf 100644 --- a/website/docs/api/_annotation/_pos-tags.jade +++ b/website/docs/api/_annotation/_pos-tags.jade @@ -1,10 +1,5 @@ //- 💫 DOCS > API > ANNOTATION > POS TAGS -+infobox("Tip") - | In spaCy v1.8.3+, you can also use #[code spacy.explain()] to get the - | description for the string representation of a tag. For example, - | #[code spacy.explain("RB")] will return "adverb". - +h(3, "pos-tagging-english") English part-of-speech tag scheme p diff --git a/website/docs/api/doc.jade b/website/docs/api/doc.jade index 72fe34f8c..1c2911f52 100644 --- a/website/docs/api/doc.jade +++ b/website/docs/api/doc.jade @@ -103,7 +103,7 @@ p Get a #[code Token] object. doc = nlp(u'Give it back! He pleaded.') assert doc[0].text == 'Give' assert doc[-1].text == '.' - span = doc[1:1] + span = doc[1:3] assert span.text == 'it back' +table(["Name", "Type", "Description"]) @@ -272,7 +272,7 @@ p Import the document contents from a binary string. p | Retokenize the document, such that the span at | #[code doc.text[start_idx : end_idx]] is merged into a single token. If - | #[code start_idx] and #[end_idx] do not mark start and end token + | #[code start_idx] and #[code end_idx] do not mark start and end token | boundaries, the document remains unchanged. +table(["Name", "Type", "Description"]) diff --git a/website/docs/api/token.jade b/website/docs/api/token.jade index 7a09f9d11..45c8a8f43 100644 --- a/website/docs/api/token.jade +++ b/website/docs/api/token.jade @@ -67,6 +67,16 @@ p An individual token — i.e. a word, punctuation symbol, whitespace, etc. +cell unicode +cell Base form of the word, with no inflectional suffixes. + +row + +cell #[code orth] + +cell int + +cell word's string. + + +row + +cell #[code orth_] + +cell unicode + +cell word's string. + +row +cell #[code lower] +cell int @@ -238,11 +248,6 @@ p An individual token — i.e. a word, punctuation symbol, whitespace, etc. +cell #[code text_with_ws] +cell unicode +cell Text content, with trailing space character if present. - - +row - +cell #[code whitespace] - +cell int - +cell Trailing space character if present. +row +cell #[code whitespace_] +cell unicode diff --git a/website/docs/api/vocab.jade b/website/docs/api/vocab.jade index 7490bccf4..c036c650b 100644 --- a/website/docs/api/vocab.jade +++ b/website/docs/api/vocab.jade @@ -124,7 +124,7 @@ p +cell #[code Lexeme] +cell The lexeme indicated by the given ID. -+h(2, "iter") Span.__iter__ ++h(2, "iter") Vocab.__iter__ +tag method p Iterate over the lexemes in the vocabulary. diff --git a/website/docs/usage/_data.json b/website/docs/usage/_data.json index 78e8b3e27..703a185d6 100644 --- a/website/docs/usage/_data.json +++ b/website/docs/usage/_data.json @@ -33,6 +33,7 @@ "index": { "title": "Install spaCy", + "quickstart": true, "next": "models" }, diff --git a/website/docs/usage/_models-list.jade b/website/docs/usage/_models-list.jade index 942de28c4..36de137e5 100644 --- a/website/docs/usage/_models-list.jade +++ b/website/docs/usage/_models-list.jade @@ -25,3 +25,4 @@ p +model-row("en_vectors_glove_md", "English", [1, 0, 0, 1], "727 MB", "CC BY-SA") +model-row("de_core_news_md", "German", [1, 1, 1, 1], "645 MB", "CC BY-SA", true, true) +model-row("fr_depvec_web_lg", "French", [1, 1, 0, 1], "1.33 GB", "CC BY-NC", true, true) + +model-row("es_core_web_md", "Spanish", [1, 1, 1, 1], "377 MB", "CC BY-SA", true, true) diff --git a/website/docs/usage/customizing-tokenizer.jade b/website/docs/usage/customizing-tokenizer.jade index d43fb438f..354a56c22 100644 --- a/website/docs/usage/customizing-tokenizer.jade +++ b/website/docs/usage/customizing-tokenizer.jade @@ -113,7 +113,7 @@ p else: tokens.append(substring) substring = '' - tokens.extend(suffixes) + tokens.extend(reversed(suffixes)) return tokens p @@ -214,7 +214,7 @@ p def __call__(self, text): words = text.split(' ') # All tokens 'own' a subsequent space character in this tokenizer - spaces = [True] * len(word) + spaces = [True] * len(words) return Doc(self.vocab, words=words, spaces=spaces) p diff --git a/website/docs/usage/deep-learning.jade b/website/docs/usage/deep-learning.jade index fec01b4ba..739cf858c 100644 --- a/website/docs/usage/deep-learning.jade +++ b/website/docs/usage/deep-learning.jade @@ -36,7 +36,7 @@ p | to #[code spacy.load()]. The function should take a | #[code spacy.language.Language] object as its only argument, and return | a sequence of callables. Each callable should accept a - | #[+api("docs") #[code Doc]] object, modify it in place, and return + | #[+api("doc") #[code Doc]] object, modify it in place, and return | #[code None]. p diff --git a/website/docs/usage/index.jade b/website/docs/usage/index.jade index 48fe6b783..9ad2fde5f 100644 --- a/website/docs/usage/index.jade +++ b/website/docs/usage/index.jade @@ -12,6 +12,40 @@ p | #[a(href="#source-ubuntu") Ubuntu], #[a(href="#source-osx") macOS/OS X] | and #[a(href="#source-windows") Windows] for details. ++quickstart(QUICKSTART, "Quickstart") + +qs({config: 'venv', python: 2}) python -m pip install -U virtualenv + +qs({config: 'venv', python: 3}) python -m pip install -U venv + +qs({config: 'venv', python: 2}) virtualenv .env + +qs({config: 'venv', python: 3}) venv .env + +qs({config: 'venv', os: 'mac'}) source .env/bin/activate + +qs({config: 'venv', os: 'linux'}) source .env/bin/activate + +qs({config: 'venv', os: 'windows'}) .env\Scripts\activate + + +qs({package: 'pip'}) pip install -U spacy + + +qs({package: 'conda'}) conda config --add channels conda-forge + +qs({package: 'conda'}) conda install spacy + + +qs({package: 'source'}) git clone https://github.com/explosion/spaCy + +qs({package: 'source'}) cd spaCy + +qs({package: 'source'}) pip install -r requirements.txt + +qs({package: 'source'}) pip install -e . + + +qs({model: 'en'}) python -m spacy download en + +qs({model: 'de'}) python -m spacy download de + +qs({model: 'fr'}) python -m spacy download fr + +qs({model: 'es'}) python -m spacy download es + ++h(2, "installation") Installation instructions + ++h(3, "pip") pip + +badge("pipy") + +p Using pip, spaCy releases are currently only available as source packages. + ++code(false, "bash"). + pip install -U spacy + +aside("Download models") | After installation you need to download a language model. For more info | and available models, see the #[+a("/docs/usage/models") docs on models]. @@ -22,14 +56,6 @@ p >>> import spacy >>> nlp = spacy.load('en') -+h(2, "pip") pip - +badge("pipy") - -p Using pip, spaCy releases are currently only available as source packages. - -+code(false, "bash"). - pip install -U spacy - p | When using pip it is generally recommended to install packages in a | #[code virtualenv] to avoid modifying system state: @@ -39,7 +65,7 @@ p source .env/bin/activate pip install spacy -+h(2, "conda") conda ++h(3, "conda") conda +badge("conda") p diff --git a/website/docs/usage/language-processing-pipeline.jade b/website/docs/usage/language-processing-pipeline.jade index c372dfbf4..3ddf9179c 100644 --- a/website/docs/usage/language-processing-pipeline.jade +++ b/website/docs/usage/language-processing-pipeline.jade @@ -17,10 +17,10 @@ p | trying to do. +code. - import spacy # See "Installing spaCy" - nlp = spacy.load('en') # You are here. - doc = nlp(u'Hello, spacy!') # See "Using the pipeline" - print((w.text, w.pos_) for w in doc) # See "Doc, Span and Token" + import spacy # See "Installing spaCy" + nlp = spacy.load('en') # You are here. + doc = nlp(u'Hello, spacy!') # See "Using the pipeline" + print([(w.text, w.pos_) for w in doc]) # See "Doc, Span and Token" +aside("Why do we have to preload?") | Loading the models takes ~200x longer than diff --git a/website/docs/usage/lightning-tour.jade b/website/docs/usage/lightning-tour.jade index 967d0c61e..2fd390d26 100644 --- a/website/docs/usage/lightning-tour.jade +++ b/website/docs/usage/lightning-tour.jade @@ -83,7 +83,7 @@ p +h(2, "examples-word-vectors") Word vectors +code. - doc = nlp("Apples and oranges are similar. Boots and hippos aren't.") + doc = nlp(u"Apples and oranges are similar. Boots and hippos aren't.") apples = doc[0] oranges = doc[2] @@ -148,24 +148,20 @@ p +code. def put_spans_around_tokens(doc, get_classes): - '''Given some function to compute class names, put each token in a - span element, with the appropriate classes computed. - - All whitespace is preserved, outside of the spans. (Yes, I know HTML - won't display it. But the point is no information is lost, so you can - calculate what you need, e.g.
tags,

tags, etc.) - ''' + """Given some function to compute class names, put each token in a + span element, with the appropriate classes computed. All whitespace is + preserved, outside of the spans. (Of course, HTML won't display more than + one whitespace character it – but the point is, no information is lost + and you can calculate what you need, e.g. <br />, <p> etc.) + """ output = [] - template = '{word}{space}' + html = '<span class="{classes}">{word}</span>{space}' for token in doc: if token.is_space: - output.append(token.orth_) + output.append(token.text) else: - output.append( - template.format( - classes=' '.join(get_classes(token)), - word=token.orth_, - space=token.whitespace_)) + classes = ' '.join(get_classes(token)) + output.append(html.format(classes=classes, word=token.text, space=token.whitespace_)) string = ''.join(output) string = string.replace('\n', '') string = string.replace('\t', ' ') diff --git a/website/docs/usage/models.jade b/website/docs/usage/models.jade index 9bb75ba9a..30863720c 100644 --- a/website/docs/usage/models.jade +++ b/website/docs/usage/models.jade @@ -203,7 +203,7 @@ p p | If you've trained your own model, for example for | #[+a("/docs/usage/adding-languages") additional languages] or - | #[+a("/docs/usage/train-ner") custom named entities], you can save its + | #[+a("/docs/usage/training-ner") custom named entities], you can save its | state using the #[code Language.save_to_directory()] method. To make the | model more convenient to deploy, we recommend wrapping it as a Python | package. diff --git a/website/docs/usage/rule-based-matching.jade b/website/docs/usage/rule-based-matching.jade index aea943a61..db7c70608 100644 --- a/website/docs/usage/rule-based-matching.jade +++ b/website/docs/usage/rule-based-matching.jade @@ -19,11 +19,11 @@ p Here's a minimal example. We first add a pattern that specifies three tokens: p | Once we've added the pattern, we can use the #[code matcher] as a | callable, to receive a list of #[code (ent_id, start, end)] tuples. - | Note that #[code LOWER] and #[code IS_PUNCT] are data attributes - | of #[code spacy.attrs]. +code. from spacy.matcher import Matcher + from spacy.attrs import IS_PUNCT, LOWER + matcher = Matcher(nlp.vocab) matcher.add_pattern("HelloWorld", [{LOWER: "hello"}, {IS_PUNCT: True}, {LOWER: "world"}]) diff --git a/website/docs/usage/saving-loading.jade b/website/docs/usage/saving-loading.jade index c4eb08f04..8978cce7a 100644 --- a/website/docs/usage/saving-loading.jade +++ b/website/docs/usage/saving-loading.jade @@ -28,7 +28,7 @@ p | and walk you through generating the meta data. You can also create the | meta.json manually and place it in the model data directory, or supply a | path to it using the #[code --meta] flag. For more info on this, see the - | #[+a("/docs/usage/cli/#package") #[code package] command] documentation. + | #[+a("/docs/usage/cli#package") #[code package] command] documentation. +aside-code("meta.json", "json"). { diff --git a/website/docs/usage/training-ner.jade b/website/docs/usage/training-ner.jade index 78eb4905e..52eedd21e 100644 --- a/website/docs/usage/training-ner.jade +++ b/website/docs/usage/training-ner.jade @@ -150,8 +150,8 @@ p for itn in range(20): random.shuffle(train_data) for raw_text, entity_offsets in train_data: - gold = GoldParse(doc, entities=entity_offsets) doc = nlp.make_doc(raw_text) + gold = GoldParse(doc, entities=entity_offsets) nlp.tagger(doc) loss = nlp.entity.update(doc, gold) nlp.end_training() diff --git a/website/index.jade b/website/index.jade index 17b564b42..df5428316 100644 --- a/website/index.jade +++ b/website/index.jade @@ -11,7 +11,7 @@ include _includes/_mixins h2.c-landing__title.o-block.u-heading-1 | in Python - +landing-badge("https://survey.spacy.io", "usersurvey", "Take the user survey!") + +landing-badge(gh("spaCy") + "/releases/tag/v2.0.0-alpha", "v2alpha", "Try spaCy v2.0.0 alpha!") +grid.o-content +grid-col("third").o-card