explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-26 18:06:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Modernize tokenizer tests for whitespace

Browse Source
This commit is contained in:
Ines Montani 2017-01-04 00:46:35 +01:00
parent aafc894285
commit 667051375d
1 changed files with 21 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
spacy/tests/tokenizer/test_whitespace.py
View File

@ -1,37 +1,46 @@
"""Test that tokens are created correctly for whitespace."""
from __future__ import unicode_literals
import pytest
def test_single_space(en_tokenizer):
tokens = en_tokenizer('hello possums')
@pytest.mark.parametrize('text', ["hello possums"])
def test_tokenizer_splits_single_space(en_tokenizer, text):
tokens = en_tokenizer(text)
assert len(tokens) == 2
def test_double_space(en_tokenizer):
tokens = en_tokenizer('hello possums')
@pytest.mark.parametrize('text', ["hello possums"])
def test_tokenizer_splits_double_space(en_tokenizer, text):
tokens = en_tokenizer(text)
assert len(tokens) == 3
assert tokens[1].orth_ == ' '
assert tokens[1].text == " "
def test_newline(en_tokenizer):
@pytest.mark.parametrize('text', ["hello\npossums"])
def test_tokenizer_splits_newline(en_tokenizer, text):
tokens = en_tokenizer(text)
assert len(tokens) == 3
assert tokens[1].text == "\n"
@pytest.mark.parametrize('text', ["hello \npossums"])
def test_tokenizer_splits_newline_space(en_tokenizer, text):
tokens = en_tokenizer('hello \npossums')
assert len(tokens) == 3
def test_newline_space(en_tokenizer):
tokens = en_tokenizer('hello \npossums')
@pytest.mark.parametrize('text', ["hello \npossums"])
def test_tokenizer_splits_newline_double_space(en_tokenizer, text):
tokens = en_tokenizer(text)
assert len(tokens) == 3
def test_newline_double_space(en_tokenizer):
tokens = en_tokenizer('hello \npossums')
assert len(tokens) == 3
def test_newline_space_wrap(en_tokenizer):
tokens = en_tokenizer('hello \n possums')
@pytest.mark.parametrize('text', ["hello \n possums"])
def test_tokenizer_splits_newline_space_wrap(en_tokenizer, text):
tokens = en_tokenizer(text)
assert len(tokens) == 3