mirror of
https://github.com/explosion/spaCy.git
synced 2024-12-25 17:36:30 +03:00
* Add test for Issue #154: Additional whitespace introduced when string ends with a whitespace token.
This commit is contained in:
parent
3dc398b727
commit
223d2b3484
|
@ -142,6 +142,12 @@ def test_ie(en_tokenizer):
|
|||
assert len(tokens) == 6
|
||||
assert tokens[3].orth_ == "i.e."
|
||||
|
||||
|
||||
def test_two_whitespace(en_tokenizer):
|
||||
orig_str = u'there are 2 spaces after this '
|
||||
tokens = en_tokenizer(orig_str)
|
||||
assert repr(tokens.text_with_ws) == repr(orig_str)
|
||||
|
||||
#def test_cnts7():
|
||||
# text = 'But then the 6,000-year ice age came...'
|
||||
# tokens = EN.tokenize(text)
|
||||
|
|
Loading…
Reference in New Issue
Block a user