Add test for french tokenizer

This commit is contained in:
Matthew Honnibal 2016-11-02 20:40:31 +01:00
parent b012ae3044
commit c09a8ce5bb

View File

@ -5,3 +5,8 @@ from ...fr import French
def test_load_french():
nlp = French()
doc = nlp(u'Parlez-vous français?')
assert doc[0].text == u'Parlez'
assert doc[1].text == u'-'
assert doc[2].text == u'vouz'
assert doc[3].text == u'français'
assert doc[4].text == u'?'