explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-12 10:16:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix tokenization of 'i.' for Danish.

Browse Source
This commit is contained in:
Søren Lind Kristiansen 2017-11-24 11:16:53 +01:00
parent 726fb2d0b5
commit ac8116510d
1 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
spacy/lang/da/tokenizer_exceptions.py
View File

@ -1,7 +1,7 @@
# encoding: utf8 # encoding: utf8
from __future__ import unicode_literals from __future__ import unicode_literals
from ...symbols import ORTH, LEMMA, NORM from ...symbols import ORTH, LEMMA, NORM, TAG, ADP, PUNCT
_exc = {} _exc = {}
@ -28,5 +28,12 @@ for orth in [
"t.o.m.", "vha.", ""]: "t.o.m.", "vha.", ""]:
_exc[orth] = [{ORTH: orth}] _exc[orth] = [{ORTH: orth}]
_custom_base_exc = {
"i.": [
{ORTH: "i", LEMMA: "i", NORM: "i"},
{ORTH: ".", TAG: PUNCT}]
}
_exc.update(_custom_base_exc)
TOKENIZER_EXCEPTIONS = _exc TOKENIZER_EXCEPTIONS = _exc