explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-11-04 01:48:04 +03:00
Code Issues Packages Projects Releases Wiki Activity

* Add test for ). in tokenizer

Browse Source
This commit is contained in:
Matthew Honnibal 2015-01-22 22:25:18 +11:00
parent a27b23cc8f
commit 93d4bd6c2e
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
tests/test_tokenizer.py
View File

@ -110,6 +110,11 @@ def test_cnts6(EN):
words = [t.string for t in tokens] words = [t.string for t in tokens]
assert len(words) == 6 assert len(words) == 6
def test_bracket_period(EN):
text = u'(And a 6a.m. run through Washington Park).'
tokens = EN(text)
assert tokens[len(tokens) - 1].string == u'.'
#def test_cnts7(): #def test_cnts7():
# text = 'But then the 6,000-year ice age came...' # text = 'But then the 6,000-year ice age came...'
# tokens = EN.tokenize(text) # tokens = EN.tokenize(text)