explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-26 01:46:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Excuse emoji failure on narrow unicode builds

Browse Source
This commit is contained in:
Matthew Honnibal 2017-09-16 16:21:13 +02:00
parent 11f2a05ede
commit 8c945310fb
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
spacy/tests/tokenizer/test_exceptions.py
View File

@ -1,6 +1,7 @@
# coding: utf-8
from __future__ import unicode_literals
import sys
import pytest
@ -37,9 +38,10 @@ def test_tokenizer_excludes_false_pos_emoticons(tokenizer, text, length):
tokens = tokenizer(text)
assert len(tokens) == length
@pytest.mark.parametrize('text,length', [('can you still dunk?🍕🍔😵LOL', 8),
('i💙you', 3), ('🤘🤘yay!', 4)])
def test_tokenizer_handles_emoji(tokenizer, text, length):
tokens = tokenizer(text)
assert len(tokens) == length
# These break on narrow unicode builds, e.g. Windows
if sys.maxunicode >= 1114111:
tokens = tokenizer(text)
assert len(tokens) == length