explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-26 09:56:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1566 from MathiasDesch/master (resolves #1248)

Browse Source 
Add exceptions to tokenizer and norm
This commit is contained in:
Ines Montani 2017-11-13 19:05:22 +01:00 committed by GitHub
commit ea6c85c67a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 26 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -97,3 +97,6 @@ Desktop.ini
# Other # Other
*.tgz *.tgz
# Pycharm project files
*.idea

8
spacy/lang/en/norm_exceptions.py
View File

@ -459,6 +459,8 @@ _exc = {
"disorganised": "disorganized", "disorganised": "disorganized",
"distil": "distill", "distil": "distill",
"distils": "distills", "distils": "distills",
"doin": "doing",
"doin'": "doing",
"dramatisation": "dramatization", "dramatisation": "dramatization",
"dramatisations": "dramatizations", "dramatisations": "dramatizations",
"dramatise": "dramatize", "dramatise": "dramatize",
@ -687,6 +689,8 @@ _exc = {
"globalises": "globalizes", "globalises": "globalizes",
"globalising": "globalizing", "globalising": "globalizing",
"glueing ": "gluing ", "glueing ": "gluing ",
"goin": "going",
"goin'":"going",
"goitre": "goiter", "goitre": "goiter",
"goitres": "goiters", "goitres": "goiters",
"gonorrhoea": "gonorrhea", "gonorrhoea": "gonorrhea",
@ -733,6 +737,8 @@ _exc = {
"harmonised": "harmonized", "harmonised": "harmonized",
"harmonises": "harmonizes", "harmonises": "harmonizes",
"harmonising": "harmonizing", "harmonising": "harmonizing",
"havin": "having",
"havin'": "having",
"homoeopath": "homeopath", "homoeopath": "homeopath",
"homoeopathic": "homeopathic", "homoeopathic": "homeopathic",
"homoeopaths": "homeopaths", "homoeopaths": "homeopaths",
@ -924,6 +930,8 @@ _exc = {
"localised": "localized", "localised": "localized",
"localises": "localizes", "localises": "localizes",
"localising": "localizing", "localising": "localizing",
"lovin": "loving",
"lovin'": "loving",
"louvre": "louver", "louvre": "louver",
"louvred": "louvered", "louvred": "louvered",
"louvres": "louvers ", "louvres": "louvers ",

15
spacy/lang/en/tokenizer_exceptions.py
View File

@ -387,6 +387,21 @@ for exc_data in [
{ORTH: "O'clock", LEMMA: "o'clock", NORM: "o'clock"}, {ORTH: "O'clock", LEMMA: "o'clock", NORM: "o'clock"},
{ORTH: "lovin'", LEMMA: "love", NORM: "loving"}, {ORTH: "lovin'", LEMMA: "love", NORM: "loving"},
{ORTH: "Lovin'", LEMMA: "love", NORM: "loving"}, {ORTH: "Lovin'", LEMMA: "love", NORM: "loving"},
{ORTH: "lovin", LEMMA: "love", NORM: "loving"},
{ORTH: "Lovin", LEMMA: "love", NORM: "loving"},
{ORTH: "havin'", LEMMA: "have", NORM: "having"},
{ORTH: "Havin'", LEMMA: "have", NORM: "having"},
{ORTH: "havin", LEMMA: "have", NORM: "having"},
{ORTH: "Havin", LEMMA: "have", NORM: "having"},
{ORTH: "doin'", LEMMA: "do", NORM: "doing"},
{ORTH: "Doin'", LEMMA: "do", NORM: "doing"},
{ORTH: "doin", LEMMA: "do", NORM: "doing"},
{ORTH: "Doin", LEMMA: "do", NORM: "doing"},
{ORTH: "goin'", LEMMA: "go", NORM: "going"},
{ORTH: "Goin'", LEMMA: "go", NORM: "going"},
{ORTH: "goin", LEMMA: "go", NORM: "going"},
{ORTH: "Goin", LEMMA: "go", NORM: "going"},
{ORTH: "Mt.", LEMMA: "Mount", NORM: "Mount"}, {ORTH: "Mt.", LEMMA: "Mount", NORM: "Mount"},
{ORTH: "Ak.", LEMMA: "Alaska", NORM: "Alaska"}, {ORTH: "Ak.", LEMMA: "Alaska", NORM: "Alaska"},