explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-09-21 19:39:13 +03:00
Code Issues Packages Projects Releases Wiki Activity

Passing all old tests.

Browse Source
This commit is contained in:
Gyorgy Orosz 2017-01-14 20:26:32 +01:00
parent 63037e79af
commit 9505c6a72b
2 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
spacy/hu/punctuation.py
View File

@ -35,8 +35,7 @@ TOKENIZER_INFIXES = (
r'(?<=[{a}"])[:<>=](?=[{a}])'.format(a=ALPHA),
r'(?<=[{a}])--(?=[{a}])'.format(a=ALPHA),
r'(?<=[{a}]),(?=[{a}])'.format(a=ALPHA),
r'(?<=[0-9{a}])({q})(?=[\-{a}])'.format(a=ALPHA, q=QUOTES),
r'(?<=[0-9{a}])(({q})|[\)\]])(?=\-[{a}])'.format(a=ALPHA, q=QUOTES),
]
)
__all__ = ["TOKENIZER_PREFIXES", "TOKENIZER_SUFFIXES", "TOKENIZER_INFIXES"]

2
spacy/tests/hu/test_tokenizer.py
View File

@ -248,7 +248,7 @@ WIKI_TESTS = [
('"(...)"–sokkal ', ['"', '(', '...', ')', '"', '–sokkal']),
]
TESTCASES = DEFAULT_TESTS + DOT_TESTS + QUOTE_TESTS + NUMBER_TESTS + HYPHEN_TESTS # + WIKI_TESTS
TESTCASES = DEFAULT_TESTS + DOT_TESTS + QUOTE_TESTS + NUMBER_TESTS + HYPHEN_TESTS + WIKI_TESTS
@pytest.mark.parametrize('text,expected_tokens', TESTCASES)