mirror of
https://github.com/explosion/spaCy.git
synced 2025-07-13 17:52:31 +03:00
Excuse emoji failure on narrow unicode builds
This commit is contained in:
parent
11f2a05ede
commit
8c945310fb
|
@ -1,6 +1,7 @@
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
from __future__ import unicode_literals
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
import sys
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
@ -37,9 +38,10 @@ def test_tokenizer_excludes_false_pos_emoticons(tokenizer, text, length):
|
||||||
tokens = tokenizer(text)
|
tokens = tokenizer(text)
|
||||||
assert len(tokens) == length
|
assert len(tokens) == length
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize('text,length', [('can you still dunk?🍕🍔😵LOL', 8),
|
@pytest.mark.parametrize('text,length', [('can you still dunk?🍕🍔😵LOL', 8),
|
||||||
('i💙you', 3), ('🤘🤘yay!', 4)])
|
('i💙you', 3), ('🤘🤘yay!', 4)])
|
||||||
def test_tokenizer_handles_emoji(tokenizer, text, length):
|
def test_tokenizer_handles_emoji(tokenizer, text, length):
|
||||||
|
# These break on narrow unicode builds, e.g. Windows
|
||||||
|
if sys.maxunicode >= 1114111:
|
||||||
tokens = tokenizer(text)
|
tokens = tokenizer(text)
|
||||||
assert len(tokens) == length
|
assert len(tokens) == length
|
||||||
|
|
Loading…
Reference in New Issue
Block a user