mirror of
https://github.com/explosion/spaCy.git
synced 2025-11-04 09:57:26 +03:00
2c876eb672
* Expose tokenizer rules as a property
Expose the tokenizer rules property in the same way as the other core
properties. (The cache resetting is overkill, but consistent with
`from_bytes` for now.)
Add tests and update Tokenizer API docs.
* Update Hungarian punctuation to remove empty string
Update Hungarian punctuation definitions so that `_units` does not match
an empty string.
* Use _load_special_tokenization consistently
Use `_load_special_tokenization()` and have it to handle `None` checks.
* Fix precedence of `token_match` vs. special cases
Remove `token_match` check from `_split_affixes()` so that special cases
have precedence over `token_match`. `token_match` is checked only before
infixes are split.
* Add `make_debug_doc()` to the Tokenizer
Add `make_debug_doc()` to the Tokenizer as a working implementation of
the pseudo-code in the docs.
Add a test (marked as slow) that checks that `nlp.tokenizer()` and
`nlp.tokenizer.make_debug_doc()` return the same non-whitespace tokens
for all languages that have `examples.sentences` that can be imported.
* Update tokenization usage docs
Update pseudo-code and algorithm description to correspond to
`nlp.tokenizer.make_debug_doc()` with example debugging usage.
Add more examples for customizing tokenizers while preserving the
existing defaults.
Minor edits / clarifications.
* Revert "Update Hungarian punctuation to remove empty string"
This reverts commit f0a577f7a5.
* Rework `make_debug_doc()` as `explain()`
Rework `make_debug_doc()` as `explain()`, which returns a list of
`(pattern_string, token_string)` tuples rather than a non-standard
`Doc`. Update docs and tests accordingly, leaving the visualization for
future work.
* Handle cases with bad tokenizer patterns
Detect when tokenizer patterns match empty prefixes and suffixes so that
`explain()` does not hang on bad patterns.
* Remove unused displacy image
* Add tokenizer.explain() to usage docs
157 lines
3.8 KiB
Python
157 lines
3.8 KiB
Python
# coding: utf-8
|
|
from __future__ import unicode_literals
|
|
|
|
import pytest
|
|
import re
|
|
from spacy.lang.en import English
|
|
from spacy.tokenizer import Tokenizer
|
|
from spacy.util import compile_prefix_regex, compile_suffix_regex
|
|
from spacy.util import compile_infix_regex
|
|
|
|
|
|
@pytest.fixture
|
|
def custom_en_tokenizer(en_vocab):
|
|
prefix_re = compile_prefix_regex(English.Defaults.prefixes)
|
|
suffix_re = compile_suffix_regex(English.Defaults.suffixes)
|
|
custom_infixes = [
|
|
r"\.\.\.+",
|
|
r"(?<=[0-9])-(?=[0-9])",
|
|
r"[0-9]+(,[0-9]+)+",
|
|
r"[\[\]!&:,()\*—–\/-]",
|
|
]
|
|
infix_re = compile_infix_regex(custom_infixes)
|
|
token_match_re = re.compile("a-b")
|
|
return Tokenizer(
|
|
en_vocab,
|
|
English.Defaults.tokenizer_exceptions,
|
|
prefix_re.search,
|
|
suffix_re.search,
|
|
infix_re.finditer,
|
|
token_match=token_match_re.match,
|
|
)
|
|
|
|
|
|
def test_en_customized_tokenizer_handles_infixes(custom_en_tokenizer):
|
|
sentence = "The 8 and 10-county definitions are not used for the greater Southern California Megaregion."
|
|
context = [word.text for word in custom_en_tokenizer(sentence)]
|
|
assert context == [
|
|
"The",
|
|
"8",
|
|
"and",
|
|
"10",
|
|
"-",
|
|
"county",
|
|
"definitions",
|
|
"are",
|
|
"not",
|
|
"used",
|
|
"for",
|
|
"the",
|
|
"greater",
|
|
"Southern",
|
|
"California",
|
|
"Megaregion",
|
|
".",
|
|
]
|
|
# the trailing '-' may cause Assertion Error
|
|
sentence = "The 8- and 10-county definitions are not used for the greater Southern California Megaregion."
|
|
context = [word.text for word in custom_en_tokenizer(sentence)]
|
|
assert context == [
|
|
"The",
|
|
"8",
|
|
"-",
|
|
"and",
|
|
"10",
|
|
"-",
|
|
"county",
|
|
"definitions",
|
|
"are",
|
|
"not",
|
|
"used",
|
|
"for",
|
|
"the",
|
|
"greater",
|
|
"Southern",
|
|
"California",
|
|
"Megaregion",
|
|
".",
|
|
]
|
|
|
|
|
|
def test_en_customized_tokenizer_handles_token_match(custom_en_tokenizer):
|
|
sentence = "The 8 and 10-county definitions a-b not used for the greater Southern California Megaregion."
|
|
context = [word.text for word in custom_en_tokenizer(sentence)]
|
|
assert context == [
|
|
"The",
|
|
"8",
|
|
"and",
|
|
"10",
|
|
"-",
|
|
"county",
|
|
"definitions",
|
|
"a-b",
|
|
"not",
|
|
"used",
|
|
"for",
|
|
"the",
|
|
"greater",
|
|
"Southern",
|
|
"California",
|
|
"Megaregion",
|
|
".",
|
|
]
|
|
|
|
|
|
def test_en_customized_tokenizer_handles_rules(custom_en_tokenizer):
|
|
sentence = "The 8 and 10-county definitions are not used for the greater Southern California Megaregion. :)"
|
|
context = [word.text for word in custom_en_tokenizer(sentence)]
|
|
assert context == [
|
|
"The",
|
|
"8",
|
|
"and",
|
|
"10",
|
|
"-",
|
|
"county",
|
|
"definitions",
|
|
"are",
|
|
"not",
|
|
"used",
|
|
"for",
|
|
"the",
|
|
"greater",
|
|
"Southern",
|
|
"California",
|
|
"Megaregion",
|
|
".",
|
|
":)",
|
|
]
|
|
|
|
|
|
def test_en_customized_tokenizer_handles_rules_property(custom_en_tokenizer):
|
|
sentence = "The 8 and 10-county definitions are not used for the greater Southern California Megaregion. :)"
|
|
rules = custom_en_tokenizer.rules
|
|
del rules[":)"]
|
|
custom_en_tokenizer.rules = rules
|
|
context = [word.text for word in custom_en_tokenizer(sentence)]
|
|
assert context == [
|
|
"The",
|
|
"8",
|
|
"and",
|
|
"10",
|
|
"-",
|
|
"county",
|
|
"definitions",
|
|
"are",
|
|
"not",
|
|
"used",
|
|
"for",
|
|
"the",
|
|
"greater",
|
|
"Southern",
|
|
"California",
|
|
"Megaregion",
|
|
".",
|
|
":",
|
|
")",
|
|
]
|