explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-04-20 00:51:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

Update Tokenizer.explain for special cases with whitespace

Browse Source 
Update `Tokenizer.explain` to skip special case matches if the exact
text has not been matched due to intervening whitespace.

Enable fuzzy `Tokenizer.explain` tests with additional whitespace
normalization.
This commit is contained in:
Adriane Boyd 2023-10-25 15:07:51 +02:00
parent d717123819
commit e69348ca02
2 changed files with 14 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
spacy/tests/tokenizer/test_explain.py
View File

@ -112,7 +112,6 @@ def sentence_strategy(draw: hypothesis.strategies.DrawFn, max_n_words: int = 4)
return " ".join([token for token_pair in sentence for token in token_pair])
@pytest.mark.xfail
@pytest.mark.parametrize("lang", LANGUAGES)
@hypothesis.given(sentence=sentence_strategy())
def test_tokenizer_explain_fuzzy(lang: str, sentence: str) -> None:
@ -123,6 +122,9 @@ def test_tokenizer_explain_fuzzy(lang: str, sentence: str) -> None:
"""
tokenizer: Tokenizer = spacy.blank(lang).tokenizer
tokens = [t.text for t in tokenizer(sentence) if not t.is_space]
# Tokenizer.explain is not intended to handle whitespace or control
# characters in the same way as Tokenizer
sentence = re.sub(r"\s+", " ", sentence).strip()
tokens = [t.text for t in tokenizer(sentence)]
debug_tokens = [t[1] for t in tokenizer.explain(sentence)]
assert tokens == debug_tokens, f"{tokens}, {debug_tokens}, {sentence}"

13
spacy/tokenizer.pyx
View File

@ -730,9 +730,16 @@ cdef class Tokenizer:
if i in spans_by_start:
span = spans_by_start[i]
exc = [d[ORTH] for d in special_cases[span.label_]]
for j, orth in enumerate(exc):
final_tokens.append((f"SPECIAL-{j + 1}", self.vocab.strings[orth]))
i += len(span)
# The phrase matcher can overmatch for tokens separated by
# spaces in the text but not in the underlying rule, so skip
# cases where the texts aren't identical
if span.text != "".join([self.vocab.strings[orth] for orth in exc]):
final_tokens.append(tokens[i])
i += 1
else:
for j, orth in enumerate(exc):
final_tokens.append((f"SPECIAL-{j + 1}", self.vocab.strings[orth]))
i += len(span)
else:
final_tokens.append(tokens[i])
i += 1