mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-21 19:39:13 +03:00
Passing all old tests.
This commit is contained in:
parent
63037e79af
commit
9505c6a72b
|
@ -35,8 +35,7 @@ TOKENIZER_INFIXES = (
|
||||||
r'(?<=[{a}"])[:<>=](?=[{a}])'.format(a=ALPHA),
|
r'(?<=[{a}"])[:<>=](?=[{a}])'.format(a=ALPHA),
|
||||||
r'(?<=[{a}])--(?=[{a}])'.format(a=ALPHA),
|
r'(?<=[{a}])--(?=[{a}])'.format(a=ALPHA),
|
||||||
r'(?<=[{a}]),(?=[{a}])'.format(a=ALPHA),
|
r'(?<=[{a}]),(?=[{a}])'.format(a=ALPHA),
|
||||||
r'(?<=[0-9{a}])({q})(?=[\-{a}])'.format(a=ALPHA, q=QUOTES),
|
r'(?<=[0-9{a}])(({q})|[\)\]])(?=\-[{a}])'.format(a=ALPHA, q=QUOTES),
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
__all__ = ["TOKENIZER_PREFIXES", "TOKENIZER_SUFFIXES", "TOKENIZER_INFIXES"]
|
__all__ = ["TOKENIZER_PREFIXES", "TOKENIZER_SUFFIXES", "TOKENIZER_INFIXES"]
|
||||||
|
|
|
@ -248,7 +248,7 @@ WIKI_TESTS = [
|
||||||
('"(...)"–sokkal ', ['"', '(', '...', ')', '"', '–sokkal']),
|
('"(...)"–sokkal ', ['"', '(', '...', ')', '"', '–sokkal']),
|
||||||
]
|
]
|
||||||
|
|
||||||
TESTCASES = DEFAULT_TESTS + DOT_TESTS + QUOTE_TESTS + NUMBER_TESTS + HYPHEN_TESTS # + WIKI_TESTS
|
TESTCASES = DEFAULT_TESTS + DOT_TESTS + QUOTE_TESTS + NUMBER_TESTS + HYPHEN_TESTS + WIKI_TESTS
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize('text,expected_tokens', TESTCASES)
|
@pytest.mark.parametrize('text,expected_tokens', TESTCASES)
|
||||||
|
|
Loading…
Reference in New Issue
Block a user