spaCy/spacy/tests/test_matcher.py

114 lines
3.6 KiB
Python
Raw Normal View History

from __future__ import unicode_literals
import pytest
from spacy.strings import StringStore
from spacy.matcher import *
2015-09-06 06:40:10 +03:00
from spacy.attrs import LOWER
from spacy.tokens.doc import Doc
from spacy.vocab import Vocab
from spacy.en import English
@pytest.fixture
def matcher():
2015-08-06 15:33:35 +03:00
patterns = {
'JS': ['PRODUCT', {}, [[{'ORTH': 'JavaScript'}]]],
2015-08-06 17:09:28 +03:00
'GoogleNow': ['PRODUCT', {}, [[{'ORTH': 'Google'}, {'ORTH': 'Now'}]]],
2015-09-06 06:40:10 +03:00
'Java': ['PRODUCT', {}, [[{'LOWER': 'java'}]]],
2015-08-06 15:33:35 +03:00
}
return Matcher(Vocab(lex_attr_getters=English.Defaults.lex_attr_getters), patterns)
def test_compile(matcher):
assert matcher.n_patterns == 3
2015-08-06 15:33:35 +03:00
def test_no_match(matcher):
2016-10-17 02:53:24 +03:00
doc = Doc(matcher.vocab, words=['I', 'like', 'cheese', '.'])
assert matcher(doc) == []
def test_match_start(matcher):
2016-10-17 02:53:24 +03:00
doc = Doc(matcher.vocab, words=['JavaScript', 'is', 'good'])
assert matcher(doc) == [(matcher.vocab.strings['JS'],
matcher.vocab.strings['PRODUCT'], 0, 1)]
def test_match_end(matcher):
2016-10-17 02:53:24 +03:00
doc = Doc(matcher.vocab, words=['I', 'like', 'java'])
assert matcher(doc) == [(doc.vocab.strings['Java'],
doc.vocab.strings['PRODUCT'], 2, 3)]
def test_match_middle(matcher):
2016-10-17 02:53:24 +03:00
doc = Doc(matcher.vocab, words=['I', 'like', 'Google', 'Now', 'best'])
assert matcher(doc) == [(doc.vocab.strings['GoogleNow'],
doc.vocab.strings['PRODUCT'], 2, 4)]
def test_match_multi(matcher):
2016-10-17 02:53:24 +03:00
doc = Doc(matcher.vocab, words='I like Google Now and java best'.split())
assert matcher(doc) == [(doc.vocab.strings['GoogleNow'],
doc.vocab.strings['PRODUCT'], 2, 4),
(doc.vocab.strings['Java'],
doc.vocab.strings['PRODUCT'], 5, 6)]
def test_match_zero(matcher):
matcher.add('Quote', '', {}, [
[
{'ORTH': '"'},
{'OP': '!', 'IS_PUNCT': True},
{'OP': '!', 'IS_PUNCT': True},
{'ORTH': '"'}
]])
2016-10-17 02:53:24 +03:00
doc = Doc(matcher.vocab, words='He said , " some words " ...'.split())
assert len(matcher(doc)) == 1
2016-10-17 02:53:24 +03:00
doc = Doc(matcher.vocab, words='He said , " some three words " ...'.split())
assert len(matcher(doc)) == 0
matcher.add('Quote', '', {}, [
[
{'ORTH': '"'},
{'IS_PUNCT': True},
{'IS_PUNCT': True},
{'IS_PUNCT': True},
{'ORTH': '"'}
]])
assert len(matcher(doc)) == 0
def test_match_zero_plus(matcher):
matcher.add('Quote', '', {}, [
[
{'ORTH': '"'},
{'OP': '*', 'IS_PUNCT': False},
{'ORTH': '"'}
]])
2016-10-17 02:53:24 +03:00
doc = Doc(matcher.vocab, words='He said , " some words " ...'.split())
assert len(matcher(doc)) == 1
2015-08-06 15:33:35 +03:00
2016-11-08 19:10:32 +03:00
def test_phrase_matcher():
vocab = Vocab(lex_attr_getters=English.Defaults.lex_attr_getters)
matcher = PhraseMatcher(vocab, [Doc(vocab, words='Google Now'.split())])
doc = Doc(vocab, words=['I', 'like', 'Google', 'Now', 'best'])
assert len(matcher(doc)) == 1
#@pytest.mark.models
#def test_match_preserved(EN):
# patterns = {
# 'JS': ['PRODUCT', {}, [[{'ORTH': 'JavaScript'}]]],
# 'GoogleNow': ['PRODUCT', {}, [[{'ORTH': 'Google'}, {'ORTH': 'Now'}]]],
# 'Java': ['PRODUCT', {}, [[{'LOWER': 'java'}]]],
# }
# matcher = Matcher(EN.vocab, patterns)
# doc = EN.tokenizer('I like java.')
# EN.tagger(doc)
# assert len(doc.ents) == 0
# doc = EN.tokenizer('I like java.')
# doc.ents += tuple(matcher(doc))
# assert len(doc.ents) == 1
# EN.tagger(doc)
# EN.entity(doc)
# assert len(doc.ents) == 1