# encoding: utf8 from __future__ import unicode_literals import re STOP_WORDS = set() TOKENIZER_PREFIXES = map(re.escape, r''' , " ( [ { * < > $ £ „ “ ' `` ` # US$ C$ A$ a- ‘ .... ... ‚ » _ § '''.strip().split('\n')) TOKENIZER_SUFFIXES = r''' , \" \) \] \} \* \! \? % \$ > : ; ' ” “ « _ '' 's 'S ’s ’S ’ ‘ ° € \.\. \.\.\. \.\.\.\. (?<=[a-zäöüßÖÄÜ)\]"'´«‘’%\)²“”])\. \-\- ´ (?<=[0-9])km² (?<=[0-9])m² (?<=[0-9])cm² (?<=[0-9])mm² (?<=[0-9])km³ (?<=[0-9])m³ (?<=[0-9])cm³ (?<=[0-9])mm³ (?<=[0-9])ha (?<=[0-9])km (?<=[0-9])m (?<=[0-9])cm (?<=[0-9])mm (?<=[0-9])µm (?<=[0-9])nm (?<=[0-9])yd (?<=[0-9])in (?<=[0-9])ft (?<=[0-9])kg (?<=[0-9])g (?<=[0-9])mg (?<=[0-9])µg (?<=[0-9])t (?<=[0-9])lb (?<=[0-9])oz (?<=[0-9])m/s (?<=[0-9])km/h (?<=[0-9])mph (?<=[0-9])°C (?<=[0-9])°K (?<=[0-9])°F (?<=[0-9])hPa (?<=[0-9])Pa (?<=[0-9])mbar (?<=[0-9])mb (?<=[0-9])T (?<=[0-9])G (?<=[0-9])M (?<=[0-9])K (?<=[0-9])kb '''.strip().split('\n') TOKENIZER_INFIXES = r''' \.\.\. (?<=[a-z])\.(?=[A-Z]) (?<=[a-zöäüßA-ZÖÄÜ"]):(?=[a-zöäüßA-ZÖÄÜ]) (?<=[a-zöäüßA-ZÖÄÜ"])>(?=[a-zöäüßA-ZÖÄÜ]) (?<=[a-zöäüßA-ZÖÄÜ"])<(?=[a-zöäüßA-ZÖÄÜ]) (?<=[a-zöäüßA-ZÖÄÜ"])=(?=[a-zöäüßA-ZÖÄÜ]) '''.strip().split('\n') TOKENIZER_EXCEPTIONS = { "''": [ { "F": "''" } ], "'S": [ { "F": "'S", "L": "es" } ], "'n": [ { "F": "'n", "L": "ein" } ], "'ne": [ { "F": "'ne", "L": "eine" } ], "'nen": [ { "F": "'nen", "L": "einen" } ], "'s": [ { "F": "'s", "L": "es" } ], "(:": [ { "F": "(:" } ], "(=": [ { "F": "(=" } ], "(^_^)": [ { "F": "(^_^)" } ], "-_-": [ { "F": "-_-" } ], "-__-": [ { "F": "-__-" } ], ":')": [ { "F": ":')" } ], ":(": [ { "F": ":(" } ], ":((": [ { "F": ":((" } ], ":(((": [ { "F": ":(((" } ], ":)": [ { "F": ":)" } ], ":))": [ { "F": ":))" } ], ":-)": [ { "F": ":-)" } ], ":-/": [ { "F": ":-/" } ], ":-P": [ { "F": ":-P" } ], ":/": [ { "F": ":/" } ], ":0": [ { "F": ":0" } ], ":3": [ { "F": ":3" } ], ":>": [ { "F": ":>" } ], ":O": [ { "F": ":O" } ], ":P": [ { "F": ":P" } ], ":Y": [ { "F": ":Y" } ], ":]": [ { "F": ":]" } ], ":p": [ { "F": ":p" } ], ";(": [ { "F": ";(" } ], ";)": [ { "F": ";)" } ], ";-)": [ { "F": ";-)" } ], ";-p": [ { "F": ";-p" } ], ";D": [ { "F": ";D" } ], ";p": [ { "F": ";p" } ], "<3": [ { "F": "<3" } ], "<33": [ { "F": "<33" } ], "<333": [ { "F": "<333" } ], "": [ { "F": "SP" } ], "=)": [ { "F": "=)" } ], "=3": [ { "F": "=3" } ], "=D": [ { "F": "=D" } ], "=[[": [ { "F": "=[[" } ], "=]": [ { "F": "=]" } ], "A.C.": [ { "F": "A.C." } ], "A.D.": [ { "F": "A.D." } ], "A.G.": [ { "F": "A.G." } ], "Abb.": [ { "F": "Abb." } ], "Abk.": [ { "F": "Abk." } ], "Abs.": [ { "F": "Abs." } ], "Abt.": [ { "F": "Abt." } ], "Apr.": [ { "F": "Apr." } ], "Aug.": [ { "F": "Aug." } ], "B.A.": [ { "F": "B.A." } ], "B.Sc.": [ { "F": "B.Sc." } ], "Bd.": [ { "F": "Bd." } ], "Betr.": [ { "F": "Betr." } ], "Bf.": [ { "F": "Bf." } ], "Bhf.": [ { "F": "Bhf." } ], "Biol.": [ { "F": "Biol." } ], "Bsp.": [ { "F": "Bsp." } ], "Chr.": [ { "F": "Chr." } ], "Cie.": [ { "F": "Cie." } ], "Co.": [ { "F": "Co." } ], "D.C.": [ { "F": "D.C." } ], "Dez.": [ { "F": "Dez." } ], "Di.": [ { "F": "Di." } ], "Dipl.": [ { "F": "Dipl." } ], "Dipl.-Ing.": [ { "F": "Dipl.-Ing." } ], "Do.": [ { "F": "Do." } ], "Dr.": [ { "F": "Dr." } ], "Fa.": [ { "F": "Fa." } ], "Fam.": [ { "F": "Fam." } ], "Feb.": [ { "F": "Feb." } ], "Fr.": [ { "F": "Fr." } ], "Frl.": [ { "F": "Frl." } ], "G.m.b.H.": [ { "F": "G.m.b.H." } ], "Gebr.": [ { "F": "Gebr." } ], "Hbf.": [ { "F": "Hbf." } ], "Hg.": [ { "F": "Hg." } ], "Hr.": [ { "F": "Hr." } ], "Hrgs.": [ { "F": "Hrgs." } ], "Hrn.": [ { "F": "Hrn." } ], "Hrsg.": [ { "F": "Hrsg." } ], "Ing.": [ { "F": "Ing." } ], "Jan.": [ { "F": "Jan." } ], "Jh.": [ { "F": "Jh." } ], "Jhd.": [ { "F": "Jhd." } ], "Jr.": [ { "F": "Jr." } ], "Jul.": [ { "F": "Jul." } ], "Jun.": [ { "F": "Jun." } ], "K.O.": [ { "F": "K.O." } ], "L.A.": [ { "F": "L.A." } ], "M.A.": [ { "F": "M.A." } ], "M.Sc.": [ { "F": "M.Sc." } ], "Mi.": [ { "F": "Mi." } ], "Mio.": [ { "F": "Mio." } ], "Mo.": [ { "F": "Mo." } ], "Mr.": [ { "F": "Mr." } ], "Mrd.": [ { "F": "Mrd." } ], "Mrz.": [ { "F": "Mrz." } ], "MwSt.": [ { "F": "MwSt." } ], "M\u00e4r.": [ { "F": "M\u00e4r." } ], "N.Y.": [ { "F": "N.Y." } ], "N.Y.C.": [ { "F": "N.Y.C." } ], "Nov.": [ { "F": "Nov." } ], "Nr.": [ { "F": "Nr." } ], "O.K.": [ { "F": "O.K." } ], "Okt.": [ { "F": "Okt." } ], "Orig.": [ { "F": "Orig." } ], "P.S.": [ { "F": "P.S." } ], "Pkt.": [ { "F": "Pkt." } ], "Prof.": [ { "F": "Prof." } ], "R.I.P.": [ { "F": "R.I.P." } ], "Red.": [ { "F": "Red." } ], "S'": [ { "F": "S'", "L": "sie" } ], "Sa.": [ { "F": "Sa." } ], "Sep.": [ { "F": "Sep." } ], "Sept.": [ { "F": "Sept." } ], "So.": [ { "F": "So." } ], "St.": [ { "F": "St." } ], "Std.": [ { "F": "Std." } ], "Str.": [ { "F": "Str." } ], "Tel.": [ { "F": "Tel." } ], "Tsd.": [ { "F": "Tsd." } ], "U.S.": [ { "F": "U.S." } ], "U.S.A.": [ { "F": "U.S.A." } ], "U.S.S.": [ { "F": "U.S.S." } ], "Univ.": [ { "F": "Univ." } ], "V_V": [ { "F": "V_V" } ], "Vol.": [ { "F": "Vol." } ], "\\\")": [ { "F": "\\\")" } ], "\\n": [ { "F": "\\n", "L": "", "pos": "SP" } ], "\\t": [ { "F": "\\t", "L": "", "pos": "SP" } ], "^_^": [ { "F": "^_^" } ], "a.": [ { "F": "a." } ], "a.D.": [ { "F": "a.D." } ], "a.M.": [ { "F": "a.M." } ], "a.Z.": [ { "F": "a.Z." } ], "abzgl.": [ { "F": "abzgl." } ], "adv.": [ { "F": "adv." } ], "al.": [ { "F": "al." } ], "allg.": [ { "F": "allg." } ], "auf'm": [ { "F": "auf", "L": "auf" }, { "F": "'m", "L": "dem" } ], "b.": [ { "F": "b." } ], "betr.": [ { "F": "betr." } ], "biol.": [ { "F": "biol." } ], "bspw.": [ { "F": "bspw." } ], "bzgl.": [ { "F": "bzgl." } ], "bzw.": [ { "F": "bzw." } ], "c.": [ { "F": "c." } ], "ca.": [ { "F": "ca." } ], "co.": [ { "F": "co." } ], "d.": [ { "F": "d." } ], "d.h.": [ { "F": "d.h." } ], "dgl.": [ { "F": "dgl." } ], "du's": [ { "F": "du", "L": "du" }, { "F": "'s", "L": "es" } ], "e.": [ { "F": "e." } ], "e.V.": [ { "F": "e.V." } ], "e.g.": [ { "F": "e.g." } ], "ebd.": [ { "F": "ebd." } ], "ehem.": [ { "F": "ehem." } ], "eigtl.": [ { "F": "eigtl." } ], "engl.": [ { "F": "engl." } ], "entspr.": [ { "F": "entspr." } ], "er's": [ { "F": "er", "L": "er" }, { "F": "'s", "L": "es" } ], "erm.": [ { "F": "erm." } ], "etc.": [ { "F": "etc." } ], "ev.": [ { "F": "ev." } ], "evtl.": [ { "F": "evtl." } ], "f.": [ { "F": "f." } ], "frz.": [ { "F": "frz." } ], "g.": [ { "F": "g." } ], "geb.": [ { "F": "geb." } ], "gegr.": [ { "F": "gegr." } ], "gem.": [ { "F": "gem." } ], "ggf.": [ { "F": "ggf." } ], "ggfs.": [ { "F": "ggfs." } ], "gg\u00fc.": [ { "F": "gg\u00fc." } ], "h.": [ { "F": "h." } ], "h.c.": [ { "F": "h.c." } ], "hinter'm": [ { "F": "hinter", "L": "hinter" }, { "F": "'m", "L": "dem" } ], "hrsg.": [ { "F": "hrsg." } ], "i.": [ { "F": "i." } ], "i.A.": [ { "F": "i.A." } ], "i.G.": [ { "F": "i.G." } ], "i.O.": [ { "F": "i.O." } ], "i.Tr.": [ { "F": "i.Tr." } ], "i.V.": [ { "F": "i.V." } ], "i.d.R.": [ { "F": "i.d.R." } ], "i.e.": [ { "F": "i.e." } ], "ich's": [ { "F": "ich", "L": "ich" }, { "F": "'s", "L": "es" } ], "ihr's": [ { "F": "ihr", "L": "ihr" }, { "F": "'s", "L": "es" } ], "incl.": [ { "F": "incl." } ], "inkl.": [ { "F": "inkl." } ], "insb.": [ { "F": "insb." } ], "j.": [ { "F": "j." } ], "jr.": [ { "F": "jr." } ], "jun.": [ { "F": "jun." } ], "jur.": [ { "F": "jur." } ], "k.": [ { "F": "k." } ], "kath.": [ { "F": "kath." } ], "l.": [ { "F": "l." } ], "lat.": [ { "F": "lat." } ], "lt.": [ { "F": "lt." } ], "m.": [ { "F": "m." } ], "m.E.": [ { "F": "m.E." } ], "m.M.": [ { "F": "m.M." } ], "max.": [ { "F": "max." } ], "min.": [ { "F": "min." } ], "mind.": [ { "F": "mind." } ], "mtl.": [ { "F": "mtl." } ], "n.": [ { "F": "n." } ], "n.Chr.": [ { "F": "n.Chr." } ], "nat.": [ { "F": "nat." } ], "o.": [ { "F": "o." } ], "o.O": [ { "F": "o.O" } ], "o.a.": [ { "F": "o.a." } ], "o.g.": [ { "F": "o.g." } ], "o.k.": [ { "F": "o.k." } ], "o.\u00c4.": [ { "F": "o.\u00c4." } ], "o.\u00e4.": [ { "F": "o.\u00e4." } ], "o_O": [ { "F": "o_O" } ], "o_o": [ { "F": "o_o" } ], "orig.": [ { "F": "orig." } ], "p.": [ { "F": "p." } ], "p.a.": [ { "F": "p.a." } ], "p.s.": [ { "F": "p.s." } ], "pers.": [ { "F": "pers." } ], "phil.": [ { "F": "phil." } ], "q.": [ { "F": "q." } ], "q.e.d.": [ { "F": "q.e.d." } ], "r.": [ { "F": "r." } ], "rer.": [ { "F": "rer." } ], "r\u00f6m.": [ { "F": "r\u00f6m." } ], "s'": [ { "F": "s'", "L": "sie" } ], "s.": [ { "F": "s." } ], "s.o.": [ { "F": "s.o." } ], "sen.": [ { "F": "sen." } ], "sie's": [ { "F": "sie", "L": "sie" }, { "F": "'s", "L": "es" } ], "sog.": [ { "F": "sog." } ], "std.": [ { "F": "std." } ], "stellv.": [ { "F": "stellv." } ], "t.": [ { "F": "t." } ], "t\u00e4gl.": [ { "F": "t\u00e4gl." } ], "u.": [ { "F": "u." } ], "u.U.": [ { "F": "u.U." } ], "u.a.": [ { "F": "u.a." } ], "u.s.w.": [ { "F": "u.s.w." } ], "u.v.m.": [ { "F": "u.v.m." } ], "unter'm": [ { "F": "unter", "L": "unter" }, { "F": "'m", "L": "dem" } ], "usf.": [ { "F": "usf." } ], "usw.": [ { "F": "usw." } ], "uvm.": [ { "F": "uvm." } ], "v.": [ { "F": "v." } ], "v.Chr.": [ { "F": "v.Chr." } ], "v.a.": [ { "F": "v.a." } ], "v.l.n.r.": [ { "F": "v.l.n.r." } ], "vgl.": [ { "F": "vgl." } ], "vllt.": [ { "F": "vllt." } ], "vlt.": [ { "F": "vlt." } ], "vor'm": [ { "F": "vor", "L": "vor" }, { "F": "'m", "L": "dem" } ], "vs.": [ { "F": "vs." } ], "w.": [ { "F": "w." } ], "wir's": [ { "F": "wir", "L": "wir" }, { "F": "'s", "L": "es" } ], "wiss.": [ { "F": "wiss." } ], "x.": [ { "F": "x." } ], "xD": [ { "F": "xD" } ], "xDD": [ { "F": "xDD" } ], "y.": [ { "F": "y." } ], "z.": [ { "F": "z." } ], "z.B.": [ { "F": "z.B." } ], "z.Bsp.": [ { "F": "z.Bsp." } ], "z.T.": [ { "F": "z.T." } ], "z.Z.": [ { "F": "z.Z." } ], "z.Zt.": [ { "F": "z.Zt." } ], "z.b.": [ { "F": "z.b." } ], "zzgl.": [ { "F": "zzgl." } ], "\u00e4.": [ { "F": "\u00e4." } ], "\u00f6.": [ { "F": "\u00f6." } ], "\u00f6sterr.": [ { "F": "\u00f6sterr." } ], "\u00fc.": [ { "F": "\u00fc." } ], "\u00fcber'm": [ { "F": "\u00fcber", "L": "\u00fcber" }, { "F": "'m", "L": "dem" } ] } TAG_MAP = { "$(": {"pos": "PUNCT", "PunctType": "Brck"}, "$,": {"pos": "PUNCT", "PunctType": "Comm"}, "$.": {"pos": "PUNCT", "PunctType": "Peri"}, "ADJA": {"pos": "ADJ"}, "ADJD": {"pos": "ADJ", "Variant": "Short"}, "ADV": {"pos": "ADV"}, "APPO": {"pos": "ADP", "AdpType": "Post"}, "APPR": {"pos": "ADP", "AdpType": "Prep"}, "APPRART": {"pos": "ADP", "AdpType": "Prep", "PronType": "Art"}, "APZR": {"pos": "ADP", "AdpType": "Circ"}, "ART": {"pos": "DET", "PronType": "Art"}, "CARD": {"pos": "NUM", "NumType": "Card"}, "FM": {"pos": "X", "Foreign": "Yes"}, "ITJ": {"pos": "INTJ"}, "KOKOM": {"pos": "CONJ", "ConjType": "Comp"}, "KON": {"pos": "CONJ"}, "KOUI": {"pos": "SCONJ"}, "KOUS": {"pos": "SCONJ"}, "NE": {"pos": "PROPN"}, "NNE": {"pos": "PROPN"}, "NN": {"pos": "NOUN"}, "PAV": {"pos": "ADV", "PronType": "Dem"}, "PROAV": {"pos": "ADV", "PronType": "Dem"}, "PDAT": {"pos": "DET", "PronType": "Dem"}, "PDS": {"pos": "PRON", "PronType": "Dem"}, "PIAT": {"pos": "DET", "PronType": "Ind,Neg,Tot"}, "PIDAT": {"pos": "DET", "AdjType": "Pdt", "PronType": "Ind,Neg,Tot"}, "PIS": {"pos": "PRON", "PronType": "Ind,Neg,Tot"}, "PPER": {"pos": "PRON", "PronType": "Prs"}, "PPOSAT": {"pos": "DET", "Poss": "Yes", "PronType": "Prs"}, "PPOSS": {"pos": "PRON", "Poss": "Yes", "PronType": "Prs"}, "PRELAT": {"pos": "DET", "PronType": "Rel"}, "PRELS": {"pos": "PRON", "PronType": "Rel"}, "PRF": {"pos": "PRON", "PronType": "Prs", "Reflex": "Yes"}, "PTKA": {"pos": "PART"}, "PTKANT": {"pos": "PART", "PartType": "Res"}, "PTKNEG": {"pos": "PART", "Negative": "Neg"}, "PTKVZ": {"pos": "PART", "PartType": "Vbp"}, "PTKZU": {"pos": "PART", "PartType": "Inf"}, "PWAT": {"pos": "DET", "PronType": "Int"}, "PWAV": {"pos": "ADV", "PronType": "Int"}, "PWS": {"pos": "PRON", "PronType": "Int"}, "TRUNC": {"pos": "X", "Hyph": "Yes"}, "VAFIN": {"pos": "AUX", "Mood": "Ind", "VerbForm": "Fin"}, "VAIMP": {"pos": "AUX", "Mood": "Imp", "VerbForm": "Fin"}, "VAINF": {"pos": "AUX", "VerbForm": "Inf"}, "VAPP": {"pos": "AUX", "Aspect": "Perf", "VerbForm": "Part"}, "VMFIN": {"pos": "VERB", "Mood": "Ind", "VerbForm": "Fin", "VerbType": "Mod"}, "VMINF": {"pos": "VERB", "VerbForm": "Inf", "VerbType": "Mod"}, "VMPP": {"pos": "VERB", "Aspect": "Perf", "VerbForm": "Part", "VerbType": "Mod"}, "VVFIN": {"pos": "VERB", "Mood": "Ind", "VerbForm": "Fin"}, "VVIMP": {"pos": "VERB", "Mood": "Imp", "VerbForm": "Fin"}, "VVINF": {"pos": "VERB", "VerbForm": "Inf"}, "VVIZU": {"pos": "VERB", "VerbForm": "Inf"}, "VVPP": {"pos": "VERB", "Aspect": "Perf", "VerbForm": "Part"}, "XY": {"pos": "X"}, "SP": {"pos": "SPACE"} }