explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-12 10:16:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Bring English tag_map in line with UD Treebank

Browse Source 
I wrote a small script to read the UD English training data and check
that our tag map and morph rules were resulting in the best POS map.
This hadn't been done for some time, and there have been various changes
to the UD schema since it has been done. After these changes we should
see much better agreement between our POS assignments and the UD POS
tags.
This commit is contained in:
Matthew Honnibal 2019-03-21 13:53:44 +01:00
parent 0c82a5ddb2
commit 04395ffa49
2 changed files with 174 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
spacy/lang/en/morph_rules.py
View File

@ -1,13 +1,97 @@
# coding: utf8 # coding: utf8
from __future__ import unicode_literals from __future__ import unicode_literals
from ...symbols import LEMMA, PRON_LEMMA from ...symbols import LEMMA, PRON_LEMMA, AUX
_subordinating_conjunctions = [
"that",
"if",
"as",
"because",
"of",
"for",
"before",
"in",
"while",
"after",
"since",
"like",
"with",
"so",
"to",
"by",
"on",
"about",
"than",
"whether",
"although",
"from",
"though",
"until",
"unless",
"once",
"without",
"at",
"into",
"cause",
"over",
"upon",
"till",
"whereas",
"beyond",
"whilst",
"except",
"despite",
"wether",
"then",
"but",
"becuse",
"whie",
"below",
"against",
"it",
"w/out",
"toward",
"albeit",
"save",
"besides",
"becouse",
"coz",
"til",
"ask",
"i'd",
"out",
"near",
"seince",
"towards",
"tho",
"sice",
"will",
]
_relative_pronouns = ["this", "that", "those", "these"]
MORPH_RULES = { MORPH_RULES = {
"DT": {word: {"POS": "PRON"} for word in _relative_pronouns},
"IN": {word: {"POS": "SCONJ"} for word in _subordinating_conjunctions},
"NN": {
"something": {"POS": "PRON"},
"anyone": {"POS": "PRON"},
"anything": {"POS": "PRON"},
"nothing": {"POS": "PRON"},
"someone": {"POS": "PRON"},
"everything": {"POS": "PRON"},
"everyone": {"POS": "PRON"},
"everybody": {"POS": "PRON"},
"nobody": {"POS": "PRON"},
"somebody": {"POS": "PRON"},
"anybody": {"POS": "PRON"},
"any1": {"POS": "PRON"},
},
"PRP": { "PRP": {
"I": { "I": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "One", "Person": "One",
"Number": "Sing", "Number": "Sing",
@ -15,14 +99,16 @@ MORPH_RULES = {
}, },
"me": { "me": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "One", "Person": "One",
"Number": "Sing", "Number": "Sing",
"Case": "Acc", "Case": "Acc",
}, },
"you": {LEMMA: PRON_LEMMA, "PronType": "Prs", "Person": "Two"}, "you": {LEMMA: PRON_LEMMA, "POS": "PRON", "PronType": "Prs", "Person": "Two"},
"he": { "he": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Sing", "Number": "Sing",
@ -31,6 +117,7 @@ MORPH_RULES = {
}, },
"him": { "him": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Sing", "Number": "Sing",
@ -39,6 +126,7 @@ MORPH_RULES = {
}, },
"she": { "she": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Sing", "Number": "Sing",
@ -47,6 +135,7 @@ MORPH_RULES = {
}, },
"her": { "her": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Sing", "Number": "Sing",
@ -55,6 +144,7 @@ MORPH_RULES = {
}, },
"it": { "it": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Sing", "Number": "Sing",
@ -62,6 +152,7 @@ MORPH_RULES = {
}, },
"we": { "we": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "One", "Person": "One",
"Number": "Plur", "Number": "Plur",
@ -69,6 +160,7 @@ MORPH_RULES = {
}, },
"us": { "us": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "One", "Person": "One",
"Number": "Plur", "Number": "Plur",
@ -76,6 +168,7 @@ MORPH_RULES = {
}, },
"they": { "they": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Plur", "Number": "Plur",
@ -83,6 +176,7 @@ MORPH_RULES = {
}, },
"them": { "them": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Plur", "Number": "Plur",
@ -90,6 +184,7 @@ MORPH_RULES = {
}, },
"mine": { "mine": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "One", "Person": "One",
"Number": "Sing", "Number": "Sing",
@ -98,6 +193,7 @@ MORPH_RULES = {
}, },
"his": { "his": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Sing", "Number": "Sing",
@ -107,6 +203,7 @@ MORPH_RULES = {
}, },
"hers": { "hers": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Sing", "Number": "Sing",
@ -116,6 +213,7 @@ MORPH_RULES = {
}, },
"its": { "its": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Sing", "Number": "Sing",
@ -125,6 +223,7 @@ MORPH_RULES = {
}, },
"ours": { "ours": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "One", "Person": "One",
"Number": "Plur", "Number": "Plur",
@ -133,6 +232,7 @@ MORPH_RULES = {
}, },
"yours": { "yours": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Two", "Person": "Two",
"Number": "Plur", "Number": "Plur",
@ -141,6 +241,7 @@ MORPH_RULES = {
}, },
"theirs": { "theirs": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Plur", "Number": "Plur",
@ -149,6 +250,7 @@ MORPH_RULES = {
}, },
"myself": { "myself": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "One", "Person": "One",
"Number": "Sing", "Number": "Sing",
@ -157,6 +259,7 @@ MORPH_RULES = {
}, },
"yourself": { "yourself": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Two", "Person": "Two",
"Case": "Acc", "Case": "Acc",
@ -164,6 +267,7 @@ MORPH_RULES = {
}, },
"himself": { "himself": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Sing", "Number": "Sing",
@ -173,6 +277,7 @@ MORPH_RULES = {
}, },
"herself": { "herself": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Sing", "Number": "Sing",
@ -182,6 +287,7 @@ MORPH_RULES = {
}, },
"itself": { "itself": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Sing", "Number": "Sing",
@ -191,6 +297,7 @@ MORPH_RULES = {
}, },
"themself": { "themself": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Sing", "Number": "Sing",
@ -199,6 +306,7 @@ MORPH_RULES = {
}, },
"ourselves": { "ourselves": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "One", "Person": "One",
"Number": "Plur", "Number": "Plur",
@ -207,6 +315,7 @@ MORPH_RULES = {
}, },
"yourselves": { "yourselves": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Two", "Person": "Two",
"Case": "Acc", "Case": "Acc",
@ -214,6 +323,7 @@ MORPH_RULES = {
}, },
"themselves": { "themselves": {
LEMMA: PRON_LEMMA, LEMMA: PRON_LEMMA,
"POS": "PRON",
"PronType": "Prs", "PronType": "Prs",
"Person": "Three", "Person": "Three",
"Number": "Plur", "Number": "Plur",
@ -269,9 +379,17 @@ MORPH_RULES = {
"Poss": "Yes", "Poss": "Yes",
}, },
}, },
"RB": {word: {"POS": "PART"} for word in ["not", "n't", "nt", "n’t"]},
"VB": {
word: {"POS": "AUX"}
for word in ["be", "have", "do", "get", "of", "am", "are", "'ve"]
},
"VBN": {"been": {LEMMA: "be", "POS": "AUX"}},
"VBG": {"being": {LEMMA: "be", "POS": "AUX"}},
"VBZ": { "VBZ": {
"am": { "am": {
LEMMA: "be", LEMMA: "be",
"POS": "AUX",
"VerbForm": "Fin", "VerbForm": "Fin",
"Person": "One", "Person": "One",
"Tense": "Pres", "Tense": "Pres",
@ -279,6 +397,7 @@ MORPH_RULES = {
}, },
"are": { "are": {
LEMMA: "be", LEMMA: "be",
"POS": "AUX",
"VerbForm": "Fin", "VerbForm": "Fin",
"Person": "Two", "Person": "Two",
"Tense": "Pres", "Tense": "Pres",
@ -286,6 +405,7 @@ MORPH_RULES = {
}, },
"is": { "is": {
LEMMA: "be", LEMMA: "be",
"POS": "AUX",
"VerbForm": "Fin", "VerbForm": "Fin",
"Person": "Three", "Person": "Three",
"Tense": "Pres", "Tense": "Pres",
@ -293,6 +413,7 @@ MORPH_RULES = {
}, },
"'re": { "'re": {
LEMMA: "be", LEMMA: "be",
"POS": "AUX",
"VerbForm": "Fin", "VerbForm": "Fin",
"Person": "Two", "Person": "Two",
"Tense": "Pres", "Tense": "Pres",
@ -300,26 +421,65 @@ MORPH_RULES = {
}, },
"'s": { "'s": {
LEMMA: "be", LEMMA: "be",
"POS": "AUX",
"VerbForm": "Fin", "VerbForm": "Fin",
"Person": "Three", "Person": "Three",
"Tense": "Pres", "Tense": "Pres",
"Mood": "Ind", "Mood": "Ind",
}, },
"has": {"POS": "AUX"},
"does": {"POS": "AUX"},
}, },
"VBP": { "VBP": {
"are": {LEMMA: "be", "VerbForm": "Fin", "Tense": "Pres", "Mood": "Ind"}, "are": {
"'re": {LEMMA: "be", "VerbForm": "Fin", "Tense": "Pres", "Mood": "Ind"}, LEMMA: "be",
"POS": "AUX",
"VerbForm": "Fin",
"Tense": "Pres",
"Mood": "Ind",
},
"'re": {
LEMMA: "be",
"POS": "AUX",
"VerbForm": "Fin",
"Tense": "Pres",
"Mood": "Ind",
},
"am": { "am": {
LEMMA: "be", LEMMA: "be",
"POS": "AUX",
"VerbForm": "Fin", "VerbForm": "Fin",
"Person": "One", "Person": "One",
"Tense": "Pres", "Tense": "Pres",
"Mood": "Ind", "Mood": "Ind",
}, },
"do": {"POS": "AUX"},
"have": {"POS": "AUX"},
"'m": {"POS": "AUX", LEMMA: "be"},
"'ve": {"POS": "AUX"},
"'re": {"POS": "AUX", LEMMA: "be"},
"'s": {"POS": "AUX"},
"is": {"POS": "AUX"},
"'d": {"POS": "AUX"},
}, },
"VBD": { "VBD": {
"was": {LEMMA: "be", "VerbForm": "Fin", "Tense": "Past", "Number": "Sing"}, "was": {
"were": {LEMMA: "be", "VerbForm": "Fin", "Tense": "Past", "Number": "Plur"}, LEMMA: "be",
"POS": "AUX",
"VerbForm": "Fin",
"Tense": "Past",
"Number": "Sing",
},
"were": {
LEMMA: "be",
"POS": "AUX",
"VerbForm": "Fin",
"Tense": "Past",
"Number": "Plur",
},
"did": {"POS": "AUX"},
"had": {"POS": "AUX"},
"'d": {"POS": "AUX"},
}, },
} }

16
spacy/lang/en/tag_map.py
View File

@ -2,7 +2,7 @@
from __future__ import unicode_literals from __future__ import unicode_literals
from ...symbols import POS, PUNCT, SYM, ADJ, CCONJ, NUM, DET, ADV, ADP, X, VERB from ...symbols import POS, PUNCT, SYM, ADJ, CCONJ, NUM, DET, ADV, ADP, X, VERB
from ...symbols import NOUN, PROPN, PART, INTJ, SPACE, PRON from ...symbols import NOUN, PROPN, PART, INTJ, SPACE, PRON, AUX
TAG_MAP = { TAG_MAP = {
@ -20,15 +20,15 @@ TAG_MAP = {
"CC": {POS: CCONJ, "ConjType": "coor"}, "CC": {POS: CCONJ, "ConjType": "coor"},
"CD": {POS: NUM, "NumType": "card"}, "CD": {POS: NUM, "NumType": "card"},
"DT": {POS: DET}, "DT": {POS: DET},
"EX": {POS: ADV, "AdvType": "ex"}, "EX": {POS: PRON, "AdvType": "ex"},
"FW": {POS: X, "Foreign": "yes"}, "FW": {POS: X, "Foreign": "yes"},
"HYPH": {POS: PUNCT, "PunctType": "dash"}, "HYPH": {POS: PUNCT, "PunctType": "dash"},
"IN": {POS: ADP}, "IN": {POS: ADP},
"JJ": {POS: ADJ, "Degree": "pos"}, "JJ": {POS: ADJ, "Degree": "pos"},
"JJR": {POS: ADJ, "Degree": "comp"}, "JJR": {POS: ADJ, "Degree": "comp"},
"JJS": {POS: ADJ, "Degree": "sup"}, "JJS": {POS: ADJ, "Degree": "sup"},
"LS": {POS: PUNCT, "NumType": "ord"}, "LS": {POS: X, "NumType": "ord"},
"MD": {POS: VERB, "VerbType": "mod"}, "MD": {POS: AUX, "VerbType": "mod"},
"NIL": {POS: ""}, "NIL": {POS: ""},
"NN": {POS: NOUN, "Number": "sing"}, "NN": {POS: NOUN, "Number": "sing"},
"NNP": {POS: PROPN, "NounType": "prop", "Number": "sing"}, "NNP": {POS: PROPN, "NounType": "prop", "Number": "sing"},
@ -37,11 +37,11 @@ TAG_MAP = {
"PDT": {POS: DET, "AdjType": "pdt", "PronType": "prn"}, "PDT": {POS: DET, "AdjType": "pdt", "PronType": "prn"},
"POS": {POS: PART, "Poss": "yes"}, "POS": {POS: PART, "Poss": "yes"},
"PRP": {POS: PRON, "PronType": "prs"}, "PRP": {POS: PRON, "PronType": "prs"},
"PRP$": {POS: DET, "PronType": "prs", "Poss": "yes"}, "PRP$": {POS: PRON, "PronType": "prs", "Poss": "yes"},
"RB": {POS: ADV, "Degree": "pos"}, "RB": {POS: ADV, "Degree": "pos"},
"RBR": {POS: ADV, "Degree": "comp"}, "RBR": {POS: ADV, "Degree": "comp"},
"RBS": {POS: ADV, "Degree": "sup"}, "RBS": {POS: ADV, "Degree": "sup"},
"RP": {POS: PART}, "RP": {POS: ADP},
"SP": {POS: SPACE}, "SP": {POS: SPACE},
"SYM": {POS: SYM}, "SYM": {POS: SYM},
"TO": {POS: PART, "PartType": "inf", "VerbForm": "inf"}, "TO": {POS: PART, "PartType": "inf", "VerbForm": "inf"},
@ -58,9 +58,9 @@ TAG_MAP = {
"Number": "sing", "Number": "sing",
"Person": 3, "Person": 3,
}, },
"WDT": {POS: DET, "PronType": "int|rel"}, "WDT": {POS: PRON, "PronType": "int|rel"},
"WP": {POS: PRON, "PronType": "int|rel"}, "WP": {POS: PRON, "PronType": "int|rel"},
"WP$": {POS: DET, "Poss": "yes", "PronType": "int|rel"}, "WP$": {POS: PRON, "Poss": "yes", "PronType": "int|rel"},
"WRB": {POS: ADV, "PronType": "int|rel"}, "WRB": {POS: ADV, "PronType": "int|rel"},
"ADD": {POS: X}, "ADD": {POS: X},
"NFP": {POS: PUNCT}, "NFP": {POS: PUNCT},