explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-02-09 16:10:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

Apply emoticon exceptions to tokenizer

Browse Source
This commit is contained in:
Ines Montani 2016-12-07 21:11:59 +01:00
parent 71f0f34cb3
commit 0d07d7fc80
2 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
spacy/en/language_data.py
View File

@ -2,6 +2,7 @@
from __future__ import unicode_literals from __future__ import unicode_literals
import re import re
from ..symbols import * from ..symbols import *
from ..language_data import EMOTICONS
PRON_LEMMA = "-PRON-" PRON_LEMMA = "-PRON-"
@ -2017,9 +2018,10 @@ self_map = [
"z." "z."
] ]
overlap = set(TOKENIZER_EXCEPTIONS.keys()).intersection(set(self_map)) for orths in [self_map, EMOTICONS]:
assert not overlap, overlap overlap = set(TOKENIZER_EXCEPTIONS.keys()).intersection(set(orths))
TOKENIZER_EXCEPTIONS.update({orth: [{ORTH: orth}] for orth in self_map}) assert not overlap, overlap
TOKENIZER_EXCEPTIONS.update({orth: [{ORTH: orth}] for orth in orths})
TOKENIZER_PREFIXES = r''' TOKENIZER_PREFIXES = r'''

1
spacy/language_data/__init__.py
View File

@ -0,0 +1 @@
from .emoticons import *