mirror of
https://github.com/explosion/spaCy.git
synced 2025-11-04 01:48:04 +03:00
Remove old tests for old website example code
This commit is contained in:
Ines Montani
parent
eef94e3ee2
commit
d5c72c40eb
|
|
@ -1,20 +0,0 @@
|
||||||
from __future__ import unicode_literals
|
|
||||||
import pytest
|
|
||||||
import os
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(scope='session')
|
|
||||||
def nlp():
|
|
||||||
from spacy.en import English
|
|
||||||
if os.environ.get('SPACY_DATA'):
|
|
||||||
data_dir = os.environ.get('SPACY_DATA')
|
|
||||||
else:
|
|
||||||
data_dir = True
|
|
||||||
return English(path=data_dir)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture()
|
|
||||||
def doc(nlp):
|
|
||||||
for word in ['Hello', ',', 'world', '.', 'Here', 'are', 'two', 'sentences', '.']:
|
|
||||||
_ = nlp.vocab[word]
|
|
||||||
return nlp('Hello, world. Here are two sentences.')
|
|
||||||
|
|
@ -1,172 +0,0 @@
|
||||||
from __future__ import unicode_literals
|
|
||||||
import pytest
|
|
||||||
from spacy.attrs import HEAD
|
|
||||||
import numpy
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.xfail
|
|
||||||
def test_example_war_and_peace(nlp):
|
|
||||||
# from spacy.en import English
|
|
||||||
from spacy._doc_examples import download_war_and_peace
|
|
||||||
|
|
||||||
unprocessed_unicode = download_war_and_peace()
|
|
||||||
|
|
||||||
# nlp = English()
|
|
||||||
# TODO: ImportError: No module named _doc_examples
|
|
||||||
doc = nlp(unprocessed_unicode)
|
|
||||||
|
|
||||||
|
|
||||||
def test_main_entry_point(nlp):
|
|
||||||
# from spacy.en import English
|
|
||||||
# nlp = English()
|
|
||||||
doc = nlp('Some text.') # Applies tagger, parser, entity
|
|
||||||
doc = nlp('Some text.', parse=False) # Applies tagger and entity, not parser
|
|
||||||
doc = nlp('Some text.', entity=False) # Applies tagger and parser, not entity
|
|
||||||
doc = nlp('Some text.', tag=False) # Does not apply tagger, entity or parser
|
|
||||||
doc = nlp('') # Zero-length tokens, not an error
|
|
||||||
# doc = nlp(b'Some text') <-- Error: need unicode
|
|
||||||
doc = nlp(b'Some text'.decode('utf8')) # Encode to unicode first.
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_sentence_spans(nlp):
|
|
||||||
# from spacy.en import English
|
|
||||||
# nlp = English()
|
|
||||||
doc = nlp("This is a sentence. Here's another...")
|
|
||||||
assert [s.root.orth_ for s in doc.sents] == ["is", "'s"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_entity_spans(nlp):
|
|
||||||
# from spacy.en import English
|
|
||||||
# nlp = English()
|
|
||||||
tokens = nlp('Mr. Best flew to New York on Saturday morning.')
|
|
||||||
ents = list(tokens.ents)
|
|
||||||
assert ents[0].label == 346
|
|
||||||
assert ents[0].label_ == 'PERSON'
|
|
||||||
assert ents[0].orth_ == 'Best'
|
|
||||||
assert ents[0].string == ents[0].string
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_noun_chunk_spans(nlp):
|
|
||||||
# from spacy.en import English
|
|
||||||
# nlp = English()
|
|
||||||
doc = nlp('The sentence in this example has three noun chunks.')
|
|
||||||
for chunk in doc.noun_chunks:
|
|
||||||
print(chunk.label, chunk.orth_, '<--', chunk.root.head.orth_)
|
|
||||||
|
|
||||||
# NP The sentence <-- has
|
|
||||||
# NP this example <-- in
|
|
||||||
# NP three noun chunks <-- has
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_count_by(nlp):
|
|
||||||
# from spacy.en import English, attrs
|
|
||||||
# nlp = English()
|
|
||||||
import numpy
|
|
||||||
from spacy import attrs
|
|
||||||
tokens = nlp('apple apple orange banana')
|
|
||||||
assert tokens.count_by(attrs.ORTH) == {3699: 2, 3750: 1, 5965: 1}
|
|
||||||
assert repr(tokens.to_array([attrs.ORTH])) == repr(numpy.array([[3699],
|
|
||||||
[3699],
|
|
||||||
[3750],
|
|
||||||
[5965]], dtype=numpy.int32))
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_read_bytes(nlp):
|
|
||||||
from spacy.tokens.doc import Doc
|
|
||||||
loc = 'test_serialize.bin'
|
|
||||||
with open(loc, 'wb') as file_:
|
|
||||||
file_.write(nlp(u'This is a document.').to_bytes())
|
|
||||||
file_.write(nlp(u'This is another.').to_bytes())
|
|
||||||
docs = []
|
|
||||||
with open(loc, 'rb') as file_:
|
|
||||||
for byte_string in Doc.read_bytes(file_):
|
|
||||||
docs.append(Doc(nlp.vocab).from_bytes(byte_string))
|
|
||||||
assert len(docs) == 2
|
|
||||||
|
|
||||||
|
|
||||||
def test_token_span(doc):
|
|
||||||
span = doc[4:6]
|
|
||||||
token = span[0]
|
|
||||||
assert token.i == 4
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_example_i_like_new_york1(nlp):
|
|
||||||
toks = nlp('I like New York in Autumn.')
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def toks(nlp):
|
|
||||||
doc = nlp('I like New York in Autumn.')
|
|
||||||
doc.from_array([HEAD], numpy.asarray([[1, 0, 1, -2, -3, -1, -5]], dtype='int32').T)
|
|
||||||
return doc
|
|
||||||
|
|
||||||
|
|
||||||
def test_example_i_like_new_york2(toks):
|
|
||||||
i, like, new, york, in_, autumn, dot = range(len(toks))
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def tok(toks, tok):
|
|
||||||
i, like, new, york, in_, autumn, dot = range(len(toks))
|
|
||||||
return locals()[tok]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def new(toks):
|
|
||||||
return tok(toks, "new")
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def york(toks):
|
|
||||||
return tok(toks, "york")
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def autumn(toks):
|
|
||||||
return tok(toks, "autumn")
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def dot(toks):
|
|
||||||
return tok(toks, "dot")
|
|
||||||
|
|
||||||
|
|
||||||
def test_example_i_like_new_york3(toks, new, york):
|
|
||||||
assert toks[new].head.orth_ == 'York'
|
|
||||||
assert toks[york].head.orth_ == 'like'
|
|
||||||
|
|
||||||
|
|
||||||
def test_example_i_like_new_york4(toks, new, york):
|
|
||||||
new_york = toks[new:york+1]
|
|
||||||
assert new_york.root.orth_ == 'York'
|
|
||||||
|
|
||||||
|
|
||||||
def test_example_i_like_new_york5(toks, autumn, dot):
|
|
||||||
assert toks[autumn].head.orth_ == 'in'
|
|
||||||
assert toks[dot].head.orth_ == 'like'
|
|
||||||
autumn_dot = toks[autumn:]
|
|
||||||
assert autumn_dot.root.orth_ == 'Autumn'
|
|
||||||
|
|
||||||
|
|
||||||
def test_navigating_the_parse_tree_lefts(doc):
|
|
||||||
# TODO: where does the span object come from?
|
|
||||||
span = doc[:2]
|
|
||||||
lefts = [span.doc[i] for i in range(0, span.start)
|
|
||||||
if span.doc[i].head in span]
|
|
||||||
|
|
||||||
|
|
||||||
def test_navigating_the_parse_tree_rights(doc):
|
|
||||||
span = doc[:2]
|
|
||||||
rights = [span.doc[i] for i in range(span.end, len(span.doc))
|
|
||||||
if span.doc[i].head in span]
|
|
||||||
|
|
||||||
|
|
||||||
def test_string_store(doc):
|
|
||||||
string_store = doc.vocab.strings
|
|
||||||
for i, string in enumerate(string_store):
|
|
||||||
assert i == string_store[string]
|
|
||||||
|
|
@ -1,180 +0,0 @@
|
||||||
from __future__ import unicode_literals
|
|
||||||
import pytest
|
|
||||||
import spacy
|
|
||||||
import os
|
|
||||||
|
|
||||||
|
|
||||||
try:
|
|
||||||
xrange
|
|
||||||
except NameError:
|
|
||||||
xrange = range
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture()
|
|
||||||
def token(doc):
|
|
||||||
return doc[0]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_load_resources_and_process_text():
|
|
||||||
from spacy.en import English
|
|
||||||
nlp = English()
|
|
||||||
doc = nlp(u'Hello, world. Here are two sentences.')
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_get_tokens_and_sentences(doc):
|
|
||||||
token = doc[0]
|
|
||||||
sentence = next(doc.sents)
|
|
||||||
assert token is sentence[0]
|
|
||||||
assert sentence.text == 'Hello, world.'
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_use_integer_ids_for_any_strings(nlp, token):
|
|
||||||
hello_id = nlp.vocab.strings['Hello']
|
|
||||||
hello_str = nlp.vocab.strings[hello_id]
|
|
||||||
|
|
||||||
assert token.orth == hello_id == 3125
|
|
||||||
assert token.orth_ == hello_str == 'Hello'
|
|
||||||
|
|
||||||
|
|
||||||
def test_get_and_set_string_views_and_flags(nlp, token):
|
|
||||||
assert token.shape_ == 'Xxxxx'
|
|
||||||
for lexeme in nlp.vocab:
|
|
||||||
if lexeme.is_alpha:
|
|
||||||
lexeme.shape_ = 'W'
|
|
||||||
elif lexeme.is_digit:
|
|
||||||
lexeme.shape_ = 'D'
|
|
||||||
elif lexeme.is_punct:
|
|
||||||
lexeme.shape_ = 'P'
|
|
||||||
else:
|
|
||||||
lexeme.shape_ = 'M'
|
|
||||||
assert token.shape_ == 'W'
|
|
||||||
|
|
||||||
|
|
||||||
def test_export_to_numpy_arrays(nlp, doc):
|
|
||||||
from spacy.attrs import ORTH, LIKE_URL, IS_OOV
|
|
||||||
|
|
||||||
attr_ids = [ORTH, LIKE_URL, IS_OOV]
|
|
||||||
doc_array = doc.to_array(attr_ids)
|
|
||||||
assert doc_array.shape == (len(doc), len(attr_ids))
|
|
||||||
assert doc[0].orth == doc_array[0, 0]
|
|
||||||
assert doc[1].orth == doc_array[1, 0]
|
|
||||||
assert doc[0].like_url == doc_array[0, 1]
|
|
||||||
assert list(doc_array[:, 1]) == [t.like_url for t in doc]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_word_vectors(nlp):
|
|
||||||
doc = nlp("Apples and oranges are similar. Boots and hippos aren't.")
|
|
||||||
|
|
||||||
apples = doc[0]
|
|
||||||
oranges = doc[2]
|
|
||||||
boots = doc[6]
|
|
||||||
hippos = doc[8]
|
|
||||||
|
|
||||||
assert apples.similarity(oranges) > boots.similarity(hippos)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_part_of_speech_tags(nlp):
|
|
||||||
from spacy.parts_of_speech import ADV
|
|
||||||
|
|
||||||
def is_adverb(token):
|
|
||||||
return token.pos == spacy.parts_of_speech.ADV
|
|
||||||
|
|
||||||
# These are data-specific, so no constants are provided. You have to look
|
|
||||||
# up the IDs from the StringStore.
|
|
||||||
NNS = nlp.vocab.strings['NNS']
|
|
||||||
NNPS = nlp.vocab.strings['NNPS']
|
|
||||||
def is_plural_noun(token):
|
|
||||||
return token.tag == NNS or token.tag == NNPS
|
|
||||||
|
|
||||||
def print_coarse_pos(token):
|
|
||||||
print(token.pos_)
|
|
||||||
|
|
||||||
def print_fine_pos(token):
|
|
||||||
print(token.tag_)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_syntactic_dependencies():
|
|
||||||
def dependency_labels_to_root(token):
|
|
||||||
'''Walk up the syntactic tree, collecting the arc labels.'''
|
|
||||||
dep_labels = []
|
|
||||||
while token.head is not token:
|
|
||||||
dep_labels.append(token.dep)
|
|
||||||
token = token.head
|
|
||||||
return dep_labels
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_named_entities():
|
|
||||||
def iter_products(docs):
|
|
||||||
for doc in docs:
|
|
||||||
for ent in doc.ents:
|
|
||||||
if ent.label_ == 'PRODUCT':
|
|
||||||
yield ent
|
|
||||||
|
|
||||||
def word_is_in_entity(word):
|
|
||||||
return word.ent_type != 0
|
|
||||||
|
|
||||||
def count_parent_verb_by_person(docs):
|
|
||||||
counts = defaultdict(defaultdict(int))
|
|
||||||
for doc in docs:
|
|
||||||
for ent in doc.ents:
|
|
||||||
if ent.label_ == 'PERSON' and ent.root.head.pos == VERB:
|
|
||||||
counts[ent.orth_][ent.root.head.lemma_] += 1
|
|
||||||
return counts
|
|
||||||
|
|
||||||
|
|
||||||
def test_calculate_inline_mark_up_on_original_string():
|
|
||||||
def put_spans_around_tokens(doc, get_classes):
|
|
||||||
'''Given some function to compute class names, put each token in a
|
|
||||||
span element, with the appropriate classes computed.
|
|
||||||
|
|
||||||
All whitespace is preserved, outside of the spans. (Yes, I know HTML
|
|
||||||
won't display it. But the point is no information is lost, so you can
|
|
||||||
calculate what you need, e.g. <br /> tags, <p> tags, etc.)
|
|
||||||
'''
|
|
||||||
output = []
|
|
||||||
template = '<span classes="{classes}">{word}</span>{space}'
|
|
||||||
for token in doc:
|
|
||||||
if token.is_space:
|
|
||||||
output.append(token.orth_)
|
|
||||||
else:
|
|
||||||
output.append(
|
|
||||||
template.format(
|
|
||||||
classes=' '.join(get_classes(token)),
|
|
||||||
word=token.orth_,
|
|
||||||
space=token.whitespace_))
|
|
||||||
string = ''.join(output)
|
|
||||||
string = string.replace('\n', '')
|
|
||||||
string = string.replace('\t', ' ')
|
|
||||||
return string
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_efficient_binary_serialization(doc):
|
|
||||||
from spacy.tokens.doc import Doc
|
|
||||||
|
|
||||||
byte_string = doc.to_bytes()
|
|
||||||
open('moby_dick.bin', 'wb').write(byte_string)
|
|
||||||
|
|
||||||
nlp = spacy.en.English()
|
|
||||||
for byte_string in Doc.read_bytes(open('moby_dick.bin', 'rb')):
|
|
||||||
doc = Doc(nlp.vocab)
|
|
||||||
doc.from_bytes(byte_string)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.models
|
|
||||||
def test_multithreading(nlp):
|
|
||||||
texts = [u'One document.', u'...', u'Lots of documents']
|
|
||||||
# .pipe streams input, and produces streaming output
|
|
||||||
iter_texts = (texts[i % 3] for i in xrange(100000000))
|
|
||||||
for i, doc in enumerate(nlp.pipe(iter_texts, batch_size=50, n_threads=4)):
|
|
||||||
assert doc.is_parsed
|
|
||||||
if i == 100:
|
|
||||||
break
|
|
||||||
|
|
||||||
Loading…
Reference in New Issue
Block a user