explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-28 19:06:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(spanish sentence segmentation): remove tokenizer exceptions the break sentence segmentation. Aligned with training corpus

Browse Source
This commit is contained in:
Francisco Aranda 2017-06-02 08:19:57 +02:00
parent 5b385e7d78
commit 70a2180199
1 changed files with 2 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
spacy/es/tokenizer_exceptions.py
View File

@ -6,44 +6,15 @@ from ..language_data import PRON_LEMMA, DET_LEMMA
TOKENIZER_EXCEPTIONS = { TOKENIZER_EXCEPTIONS = {
"al": [
{ORTH: "a", LEMMA: "a", TAG: ADP},
{ORTH: "el", LEMMA: "el", TAG: DET}
],
"consigo": [
{ORTH: "con", LEMMA: "con"},
{ORTH: "sigo", LEMMA: PRON_LEMMA, NORM: "sí"}
],
"conmigo": [
{ORTH: "con", LEMMA: "con"},
{ORTH: "migo", LEMMA: PRON_LEMMA, NORM: "mí"}
],
"contigo": [
{ORTH: "con", LEMMA: "con"},
{ORTH: "tigo", LEMMA: PRON_LEMMA, NORM: "ti"}
],
"del": [
{ORTH: "de", LEMMA: "de", TAG: ADP},
{ORTH: "l", LEMMA: "el", TAG: DET}
],
"pel": [
{ORTH: "pe", LEMMA: "per", TAG: ADP},
{ORTH: "l", LEMMA: "el", TAG: DET}
],
"pal": [ "pal": [
{ORTH: "pa", LEMMA: "para"}, {ORTH: "pa", LEMMA: "para"},
{ORTH: "l", LEMMA: DET_LEMMA, NORM: "el"} {ORTH: "el", LEMMA: DET_LEMMA, NORM: "el"}
], ],
"pala": [ "pala": [
{ORTH: "pa", LEMMA: "para"}, {ORTH: "pa", LEMMA: "para"},
{ORTH: "la", LEMMA: DET_LEMMA} {ORTH: "la", LEMMA: DET_LEMMA, NORM: "la"}
], ],
"aprox.": [ "aprox.": [