mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-14 13:47:13 +03:00
215 lines
7.1 KiB
Python
215 lines
7.1 KiB
Python
# coding: utf-8
|
|
from __future__ import unicode_literals
|
|
|
|
import pytest
|
|
import re
|
|
from spacy.matcher import Matcher, DependencyTreeMatcher
|
|
from spacy.tokens import Doc
|
|
from ..util import get_doc
|
|
|
|
|
|
@pytest.fixture
|
|
def matcher(en_vocab):
|
|
rules = {'JS': [[{'ORTH': 'JavaScript'}]],
|
|
'GoogleNow': [[{'ORTH': 'Google'}, {'ORTH': 'Now'}]],
|
|
'Java': [[{'LOWER': 'java'}]]}
|
|
matcher = Matcher(en_vocab)
|
|
for key, patterns in rules.items():
|
|
matcher.add(key, None, *patterns)
|
|
return matcher
|
|
|
|
|
|
def test_matcher_from_api_docs(en_vocab):
|
|
matcher = Matcher(en_vocab)
|
|
pattern = [{'ORTH': 'test'}]
|
|
assert len(matcher) == 0
|
|
matcher.add('Rule', None, pattern)
|
|
assert len(matcher) == 1
|
|
matcher.remove('Rule')
|
|
assert 'Rule' not in matcher
|
|
matcher.add('Rule', None, pattern)
|
|
assert 'Rule' in matcher
|
|
on_match, patterns = matcher.get('Rule')
|
|
assert len(patterns[0])
|
|
|
|
|
|
def test_matcher_from_usage_docs(en_vocab):
|
|
text = "Wow 😀 This is really cool! 😂 😂"
|
|
doc = Doc(en_vocab, words=text.split(' '))
|
|
pos_emoji = ['😀', '😃', '😂', '🤣', '😊', '😍']
|
|
pos_patterns = [[{'ORTH': emoji}] for emoji in pos_emoji]
|
|
|
|
def label_sentiment(matcher, doc, i, matches):
|
|
match_id, start, end = matches[i]
|
|
if doc.vocab.strings[match_id] == 'HAPPY':
|
|
doc.sentiment += 0.1
|
|
span = doc[start : end]
|
|
token = span.merge()
|
|
token.vocab[token.text].norm_ = 'happy emoji'
|
|
|
|
matcher = Matcher(en_vocab)
|
|
matcher.add('HAPPY', label_sentiment, *pos_patterns)
|
|
matches = matcher(doc)
|
|
assert doc.sentiment != 0
|
|
assert doc[1].norm_ == 'happy emoji'
|
|
|
|
|
|
def test_matcher_len_contains(matcher):
|
|
assert len(matcher) == 3
|
|
matcher.add('TEST', None, [{'ORTH': 'test'}])
|
|
assert 'TEST' in matcher
|
|
assert 'TEST2' not in matcher
|
|
|
|
|
|
def test_matcher_no_match(matcher):
|
|
doc = Doc(matcher.vocab, words=["I", "like", "cheese", "."])
|
|
assert matcher(doc) == []
|
|
|
|
|
|
def test_matcher_match_start(matcher):
|
|
doc = Doc(matcher.vocab, words=["JavaScript", "is", "good"])
|
|
assert matcher(doc) == [(matcher.vocab.strings['JS'], 0, 1)]
|
|
|
|
|
|
def test_matcher_match_end(matcher):
|
|
words = ["I", "like", "java"]
|
|
doc = Doc(matcher.vocab, words=words)
|
|
assert matcher(doc) == [(doc.vocab.strings['Java'], 2, 3)]
|
|
|
|
|
|
def test_matcher_match_middle(matcher):
|
|
words = ["I", "like", "Google", "Now", "best"]
|
|
doc = Doc(matcher.vocab, words=words)
|
|
assert matcher(doc) == [(doc.vocab.strings['GoogleNow'], 2, 4)]
|
|
|
|
|
|
def test_matcher_match_multi(matcher):
|
|
words = ["I", "like", "Google", "Now", "and", "java", "best"]
|
|
doc = Doc(matcher.vocab, words=words)
|
|
assert matcher(doc) == [(doc.vocab.strings['GoogleNow'], 2, 4),
|
|
(doc.vocab.strings['Java'], 5, 6)]
|
|
|
|
|
|
def test_matcher_empty_dict(en_vocab):
|
|
"""Test matcher allows empty token specs, meaning match on any token."""
|
|
matcher = Matcher(en_vocab)
|
|
doc = Doc(matcher.vocab, words=["a", "b", "c"])
|
|
matcher.add('A.C', None, [{'ORTH': 'a'}, {}, {'ORTH': 'c'}])
|
|
matches = matcher(doc)
|
|
assert len(matches) == 1
|
|
assert matches[0][1:] == (0, 3)
|
|
matcher = Matcher(en_vocab)
|
|
matcher.add('A.', None, [{'ORTH': 'a'}, {}])
|
|
matches = matcher(doc)
|
|
assert matches[0][1:] == (0, 2)
|
|
|
|
|
|
def test_matcher_operator_shadow(en_vocab):
|
|
matcher = Matcher(en_vocab)
|
|
doc = Doc(matcher.vocab, words=["a", "b", "c"])
|
|
pattern = [{'ORTH': 'a'}, {"IS_ALPHA": True, "OP": "+"}, {'ORTH': 'c'}]
|
|
matcher.add('A.C', None, pattern)
|
|
matches = matcher(doc)
|
|
assert len(matches) == 1
|
|
assert matches[0][1:] == (0, 3)
|
|
|
|
|
|
def test_matcher_match_zero(matcher):
|
|
words1 = 'He said , " some words " ...'.split()
|
|
words2 = 'He said , " some three words " ...'.split()
|
|
pattern1 = [{'ORTH': '"'},
|
|
{'OP': '!', 'IS_PUNCT': True},
|
|
{'OP': '!', 'IS_PUNCT': True},
|
|
{'ORTH': '"'}]
|
|
pattern2 = [{'ORTH': '"'},
|
|
{'IS_PUNCT': True},
|
|
{'IS_PUNCT': True},
|
|
{'IS_PUNCT': True},
|
|
{'ORTH': '"'}]
|
|
matcher.add('Quote', None, pattern1)
|
|
doc = Doc(matcher.vocab, words=words1)
|
|
assert len(matcher(doc)) == 1
|
|
doc = Doc(matcher.vocab, words=words2)
|
|
assert len(matcher(doc)) == 0
|
|
matcher.add('Quote', None, pattern2)
|
|
assert len(matcher(doc)) == 0
|
|
|
|
|
|
def test_matcher_match_zero_plus(matcher):
|
|
words = 'He said , " some words " ...'.split()
|
|
pattern = [{'ORTH': '"'},
|
|
{'OP': '*', 'IS_PUNCT': False},
|
|
{'ORTH': '"'}]
|
|
matcher = Matcher(matcher.vocab)
|
|
matcher.add('Quote', None, pattern)
|
|
doc = Doc(matcher.vocab, words=words)
|
|
assert len(matcher(doc)) == 1
|
|
|
|
|
|
def test_matcher_match_one_plus(matcher):
|
|
control = Matcher(matcher.vocab)
|
|
control.add('BasicPhilippe', None, [{'ORTH': 'Philippe'}])
|
|
doc = Doc(control.vocab, words=['Philippe', 'Philippe'])
|
|
m = control(doc)
|
|
assert len(m) == 2
|
|
matcher.add('KleenePhilippe', None, [{'ORTH': 'Philippe', 'OP': '1'},
|
|
{'ORTH': 'Philippe', 'OP': '+'}])
|
|
m = matcher(doc)
|
|
assert len(m) == 1
|
|
|
|
|
|
def test_matcher_any_token_operator(en_vocab):
|
|
"""Test that patterns with "any token" {} work with operators."""
|
|
matcher = Matcher(en_vocab)
|
|
matcher.add('TEST', None, [{'ORTH': 'test'}, {'OP': '*'}])
|
|
doc = Doc(en_vocab, words=['test', 'hello', 'world'])
|
|
matches = [doc[start:end].text for _, start, end in matcher(doc)]
|
|
assert len(matches) == 3
|
|
assert matches[0] == 'test'
|
|
assert matches[1] == 'test hello'
|
|
assert matches[2] == 'test hello world'
|
|
|
|
|
|
@pytest.fixture
|
|
def text():
|
|
return u"The quick brown fox jumped over the lazy fox"
|
|
|
|
@pytest.fixture
|
|
def heads():
|
|
return [3,2,1,1,0,-1,2,1,-3]
|
|
|
|
@pytest.fixture
|
|
def deps():
|
|
return ['det', 'amod', 'amod', 'nsubj', 'prep', 'pobj', 'det', 'amod']
|
|
|
|
@pytest.fixture
|
|
def dependency_tree_matcher(en_vocab):
|
|
is_brown_yellow = lambda text: bool(re.compile(r'brown|yellow|over').match(text))
|
|
IS_BROWN_YELLOW = en_vocab.add_flag(is_brown_yellow)
|
|
pattern1 = [
|
|
{'SPEC': {'NODE_NAME': 'fox'}, 'PATTERN': {'ORTH': 'fox'}},
|
|
{'SPEC': {'NODE_NAME': 'q', 'NBOR_RELOP': '>', 'NBOR_NAME': 'fox'},'PATTERN': {'LOWER': u'quick'}},
|
|
{'SPEC': {'NODE_NAME': 'r', 'NBOR_RELOP': '>', 'NBOR_NAME': 'fox'}, 'PATTERN': {IS_BROWN_YELLOW: True}}
|
|
]
|
|
|
|
pattern2 = [
|
|
{'SPEC': {'NODE_NAME': 'jumped'}, 'PATTERN': {'ORTH': 'jumped'}},
|
|
{'SPEC': {'NODE_NAME': 'fox', 'NBOR_RELOP': '>', 'NBOR_NAME': 'jumped'},'PATTERN': {'LOWER': u'fox'}},
|
|
{'SPEC': {'NODE_NAME': 'over', 'NBOR_RELOP': '>', 'NBOR_NAME': 'fox'}, 'PATTERN': {IS_BROWN_YELLOW: True}}
|
|
]
|
|
matcher = DependencyTreeMatcher(en_vocab)
|
|
matcher.add('pattern1', None, pattern1)
|
|
matcher.add('pattern2', None, pattern2)
|
|
return matcher
|
|
|
|
|
|
|
|
def test_dependency_tree_matcher_compile(dependency_tree_matcher):
|
|
assert len(dependency_tree_matcher) == 2
|
|
|
|
def test_dependency_tree_matcher(dependency_tree_matcher,text,heads,deps):
|
|
doc = get_doc(dependency_tree_matcher.vocab,text.split(),heads=heads,deps=deps)
|
|
matches = dependency_tree_matcher(doc)
|
|
assert len(matches) == 2
|
|
|