spaCy/website/docs/usage/lightning-tour.jade

183 lines
5.3 KiB
Plaintext
Raw Normal View History

2016-10-31 21:04:15 +03:00
//- 💫 DOCS > USAGE > LIGHTNING TOUR
include ../../_includes/_mixins
p
2016-12-25 17:23:30 +03:00
| The following examples and code snippets give you an overview of spaCy's
2016-10-31 21:04:15 +03:00
| functionality and its usage.
2017-03-17 15:11:00 +03:00
+h(2, "models") Install and load models
+code(false, "bash").
2017-03-18 17:24:42 +03:00
python -m spacy download en
2017-03-17 15:11:00 +03:00
+code.
import spacy
nlp = spacy.load('en')
2016-10-31 21:04:15 +03:00
+h(2, "examples-resources") Load resources and process text
+code.
import spacy
en_nlp = spacy.load('en')
de_nlp = spacy.load('de')
en_doc = en_nlp(u'Hello, world. Here are two sentences.')
de_doc = de_nlp(u'ich bin ein Berliner.')
+h(2, "multi-threaded") Multi-threaded generator
+code.
texts = [u'One document.', u'...', u'Lots of documents']
# .pipe streams input, and produces streaming output
iter_texts = (texts[i % 3] for i in xrange(100000000))
for i, doc in enumerate(nlp.pipe(iter_texts, batch_size=50, n_threads=4)):
assert doc.is_parsed
if i == 100:
break
+h(2, "examples-tokens-sentences") Get tokens and sentences
+code.
token = doc[0]
sentence = next(doc.sents)
assert token is sentence[0]
assert sentence.text == 'Hello, world.'
+h(2, "examples-integer-ids") Use integer IDs for any string
+code.
hello_id = nlp.vocab.strings['Hello']
hello_str = nlp.vocab.strings[hello_id]
assert token.orth == hello_id == 3125
assert token.orth_ == hello_str == 'Hello'
+h(2, "examples-string-views-flags") Get and set string views and flags
+code.
assert token.shape_ == 'Xxxxx'
for lexeme in nlp.vocab:
if lexeme.is_alpha:
lexeme.shape_ = 'W'
elif lexeme.is_digit:
lexeme.shape_ = 'D'
elif lexeme.is_punct:
lexeme.shape_ = 'P'
else:
lexeme.shape_ = 'M'
assert token.shape_ == 'W'
+h(2, "examples-numpy-arrays") Export to numpy arrays
+code.
from spacy.attrs import ORTH, LIKE_URL, IS_OOV
attr_ids = [ORTH, LIKE_URL, IS_OOV]
doc_array = doc.to_array(attr_ids)
assert doc_array.shape == (len(doc), len(attr_ids))
assert doc[0].orth == doc_array[0, 0]
assert doc[1].orth == doc_array[1, 0]
assert doc[0].like_url == doc_array[0, 1]
assert list(doc_array[:, 1]) == [t.like_url for t in doc]
+h(2, "examples-word-vectors") Word vectors
+code.
2017-07-07 23:18:17 +03:00
doc = nlp(u"Apples and oranges are similar. Boots and hippos aren't.")
2016-10-31 21:04:15 +03:00
apples = doc[0]
oranges = doc[2]
boots = doc[6]
hippos = doc[8]
assert apples.similarity(oranges) > boots.similarity(hippos)
+h(2, "examples-pos-tags") Part-of-speech tags
+code.
from spacy.parts_of_speech import ADV
def is_adverb(token):
return token.pos == spacy.parts_of_speech.ADV
# These are data-specific, so no constants are provided. You have to look
# up the IDs from the StringStore.
NNS = nlp.vocab.strings['NNS']
NNPS = nlp.vocab.strings['NNPS']
def is_plural_noun(token):
return token.tag == NNS or token.tag == NNPS
def print_coarse_pos(token):
print(token.pos_)
def print_fine_pos(token):
print(token.tag_)
+h(2, "examples-dependencies") Syntactic dependencies
+code.
def dependency_labels_to_root(token):
'''Walk up the syntactic tree, collecting the arc labels.'''
dep_labels = []
while token.head is not token:
dep_labels.append(token.dep)
token = token.head
return dep_labels
+h(2, "examples-entities") Named entities
+code.
def iter_products(docs):
for doc in docs:
for ent in doc.ents:
if ent.label_ == 'PRODUCT':
yield ent
def word_is_in_entity(word):
return word.ent_type != 0
def count_parent_verb_by_person(docs):
counts = defaultdict(lambda: defaultdict(int))
2016-10-31 21:04:15 +03:00
for doc in docs:
for ent in doc.ents:
if ent.label_ == 'PERSON' and ent.root.head.pos == VERB:
counts[ent.orth_][ent.root.head.lemma_] += 1
return counts
+h(2, "examples-inline") Calculate inline mark-up on original string
+code.
def put_spans_around_tokens(doc, get_classes):
2017-06-01 18:53:41 +03:00
"""Given some function to compute class names, put each token in a
span element, with the appropriate classes computed. All whitespace is
preserved, outside of the spans. (Of course, HTML won't display more than
one whitespace character it but the point is, no information is lost
and you can calculate what you need, e.g. <br />, <p> etc.)
"""
2016-10-31 21:04:15 +03:00
output = []
2017-06-01 18:53:41 +03:00
html = '<span class="{classes}">{word}</span>{space}'
2016-10-31 21:04:15 +03:00
for token in doc:
if token.is_space:
2017-06-01 18:53:41 +03:00
output.append(token.text)
2016-10-31 21:04:15 +03:00
else:
2017-06-01 18:53:41 +03:00
classes = ' '.join(get_classes(token))
output.append(html.format(classes=classes, word=token.text, space=token.whitespace_))
2016-10-31 21:04:15 +03:00
string = ''.join(output)
string = string.replace('\n', '')
string = string.replace('\t', ' ')
return string
+h(2, "examples-binary") Efficient binary serialization
+code.
import spacy
from spacy.tokens.doc import Doc
byte_string = doc.to_bytes()
open('moby_dick.bin', 'wb').write(byte_string)
nlp = spacy.load('en')
for byte_string in Doc.read_bytes(open('moby_dick.bin', 'rb')):
doc = Doc(nlp.vocab)
doc.from_bytes(byte_string)