# -*- coding: utf-8 -*- from __future__ import unicode_literals from ...symbols import ORTH, LEMMA, NORM _exc = {} for token in ["Απ'", "ΑΠ'", "αφ'", "Αφ'"]: _exc[token] = [ {ORTH: token, LEMMA: "από", NORM: "από"} ] for token in ["Αλλ'", "αλλ'"]: _exc[token] = [ {ORTH: token, LEMMA: "αλλά", NORM: "αλλά"} ] for token in ["παρ'", "Παρ'", "ΠΑΡ'"]: _exc[token] = [ {ORTH: token, LEMMA: "παρά", NORM: "παρά"} ] for token in ["καθ'", "Καθ'"]: _exc[token] = [ {ORTH: token, LEMMA: "κάθε", NORM: "κάθε"} ] for token in ["κατ'", "Κατ'"]: _exc[token] = [ {ORTH: token, LEMMA: "κατά", NORM: "κατά"} ] for token in ["'ΣΟΥΝ", "'ναι", "'ταν", "'τανε", "'μαστε", "'μουνα", "'μουν"]: _exc[token] = [ {ORTH: token, LEMMA: "είμαι", NORM: "είμαι"} ] for token in ["Επ'", "επ'", "εφ'", "Εφ'"]: _exc[token] = [ {ORTH: token, LEMMA: "επί", NORM: "επί"} ] for token in ["Δι'", "δι'"]: _exc[token] = [ {ORTH: token, LEMMA: "δια", NORM: "δια"} ] for token in ["'χουν", "'χουμε", "'χαμε", "'χα", "'χε", "'χεις", "'χει"]: _exc[token] = [ {ORTH: token, LEMMA: "έχω", NORM: "έχω"} ] for token in ["υπ'", "Υπ'"]: _exc[token] = [ {ORTH: token, LEMMA: "υπό", NORM: "υπό"} ] for token in ["Μετ'", "ΜΕΤ'", "'μετ"]: _exc[token] = [ {ORTH: token, LEMMA: "μετά", NORM: "μετά"} ] for token in ["Μ'", "μ'"]: _exc[token] = [ {ORTH: token, LEMMA: "με", NORM: "με"} ] for token in ["Γι'", "ΓΙ'", "γι'"]: _exc[token] = [ {ORTH: token, LEMMA: "για", NORM: "για"} ] for token in ["Σ'", "σ'"]: _exc[token] = [ {ORTH: token, LEMMA: "σε", NORM: "σε"} ] for token in ["Θ'", "θ'"]: _exc[token] = [ {ORTH: token, LEMMA: "θα", NORM: "θα"} ] for token in ["Ν'", "ν'"]: _exc[token] = [ {ORTH: token, LEMMA: "να", NORM: "να"} ] for token in ["Τ'", "τ'"]: _exc[token] = [ {ORTH: token, LEMMA: "να", NORM: "να"} ] for token in ["'γω", "'σένα", "'μεις"]: _exc[token] = [ {ORTH: token, LEMMA: "εγώ", NORM: "εγώ"} ] for token in ["Τ'", "τ'"]: _exc[token] = [ {ORTH: token, LEMMA: "το", NORM: "το"} ] for token in ["Φέρ'", "Φερ'", "φέρ'", "φερ'"]: _exc[token] = [ {ORTH: token, LEMMA: "φέρνω", NORM: "φέρνω"} ] for token in ["'ρθούνε", "'ρθουν", "'ρθει", "'ρθεί", "'ρθε", "'ρχεται"]: _exc[token] = [ {ORTH: token, LEMMA: "έρχομαι", NORM: "έρχομαι"} ] for token in ["'πανε", "'λεγε", "'λεγαν", "'πε", "'λεγα"]: _exc[token] = [ {ORTH: token, LEMMA: "λέγω", NORM: "λέγω"} ] for token in ["Πάρ'", "πάρ'"]: _exc[token] = [ {ORTH: token, LEMMA: "παίρνω", NORM: "παίρνω"} ] for token in ["μέσ'", "Μέσ'", "μεσ'"]: _exc[token] = [ {ORTH: token, LEMMA: "μέσα", NORM: "μέσα"} ] for token in ["Δέσ'", "Δεσ'", "δεσ'"]: _exc[token] = [ {ORTH: token, LEMMA: "δένω", NORM: "δένω"} ] for token in ["'κανε", "Κάν'"]: _exc[token] = [ {ORTH: token, LEMMA: "κάνω", NORM: "κάνω"} ] _other_exc = { "κι": [ {ORTH: "κι", LEMMA: "και", NORM: "και"}, ], "Παίξ'": [ {ORTH: "Παίξ'", LEMMA: "παίζω", NORM: "παίζω"}, ], "Αντ'": [ {ORTH: "Αντ'", LEMMA: "αντί", NORM: "αντί"}, ], "ολ'": [ {ORTH: "ολ'", LEMMA: "όλος", NORM: "όλος"}, ], "ύστερ'": [ {ORTH: "ύστερ'", LEMMA: "ύστερα", NORM: "ύστερα"}, ], "'πρεπε": [ {ORTH: "'πρεπε", LEMMA: "πρέπει", NORM: "πρέπει"}, ], "Δύσκολ'": [ {ORTH: "Δύσκολ'", LEMMA: "δύσκολος", NORM: "δύσκολος"}, ], "'θελα": [ {ORTH: "'θελα", LEMMA: "θέλω", NORM: "θέλω"}, ], "'γραφα": [ {ORTH: "'γραφα", LEMMA: "γράφω", NORM: "γράφω"}, ], "'παιρνα": [ {ORTH: "'παιρνα", LEMMA: "παίρνω", NORM: "παίρνω"}, ], "'δειξε": [ {ORTH: "'δειξε", LEMMA: "δείχνω", NORM: "δείχνω"}, ], "όμουρφ'": [ {ORTH: "όμουρφ'", LEMMA: "όμορφος", NORM: "όμορφος"}, ], "κ'τσή": [ {ORTH: "κ'τσή", LEMMA: "κουτσός", NORM: "κουτσός"}, ], "μηδ'": [ {ORTH: "μηδ'", LEMMA: "μήδε", NORM: "μήδε"}, ], "'ξομολογήθηκε": [ {ORTH: "'ξομολογήθηκε", LEMMA: "εξομολογούμαι", NORM: "εξομολογούμαι"}, ], "'μας": [ {ORTH: "'μας", LEMMA: "εμάς", NORM: "εμάς"}, ], "'ξερες": [ {ORTH: "'ξερες", LEMMA: "ξέρω", NORM: "ξέρω"}, ], "έφθασ'": [ {ORTH: "έφθασ'", LEMMA: "φθάνω", NORM: "φθάνω"}, ], "εξ'": [ {ORTH: "εξ'", LEMMA: "εκ", NORM: "εκ"}, ], "δώσ'": [ {ORTH: "δώσ'", LEMMA: "δίνω", NORM: "δίνω"}, ], "τίποτ'": [ {ORTH: "τίποτ'", LEMMA: "τίποτα", NORM: "τίποτα"}, ], "Λήξ'": [ {ORTH: "Λήξ'", LEMMA: "λήγω", NORM: "λήγω"}, ], "άσ'": [ {ORTH: "άσ'", LEMMA: "αφήνω", NORM: "αφήνω"}, ], "Στ'": [ {ORTH: "Στ'", LEMMA: "στο", NORM: "στο"}, ], "Δωσ'": [ {ORTH: "Δωσ'", LEMMA: "δίνω", NORM: "δίνω"}, ], "Βάψ'": [ {ORTH: "Βάψ'", LEMMA: "βάφω", NORM: "βάφω"}, ], "Αλλ'": [ {ORTH: "Αλλ'", LEMMA: "αλλά", NORM: "αλλά"}, ], "Αμ'": [ {ORTH: "Αμ'", LEMMA: "άμα", NORM: "άμα"}, ], "Αγόρασ'": [ {ORTH: "Αγόρασ'", LEMMA: "αγοράζω", NORM: "αγοράζω"}, ], "'φύγε": [ {ORTH: "'φύγε", LEMMA: "φεύγω", NORM: "φεύγω"}, ], "'φερε": [ {ORTH: "'φερε", LEMMA: "φέρνω", NORM: "φέρνω"}, ], "'φαγε": [ {ORTH: "'φαγε", LEMMA: "τρώω", NORM: "τρώω"}, ], "'σπαγαν": [ {ORTH: "'σπαγαν", LEMMA: "σπάω", NORM: "σπάω"}, ], "'σκασε": [ {ORTH: "'σκασε", LEMMA: "σκάω", NORM: "σκάω"}, ], "'σβηνε": [ {ORTH: "'σβηνε", LEMMA: "σβήνω", NORM: "σβήνω"}, ], "'ριξε": [ {ORTH: "'ριξε", LEMMA: "ρίχνω", NORM: "ρίχνω"}, ], "'κλεβε": [ {ORTH: "'κλεβε", LEMMA: "κλέβω", NORM: "κλέβω"}, ], "'κει": [ {ORTH: "'κει", LEMMA: "εκεί", NORM: "εκεί"}, ], "'βλεπε": [ {ORTH: "'βλεπε", LEMMA: "βλέπω", NORM: "βλέπω"}, ], "'βγαινε": [ {ORTH: "'βγαινε", LEMMA: "βγαίνω", NORM: "βγαίνω"}, ] } _exc.update(_other_exc) for h in range(1, 12 + 1): for period in ["π.μ.", "πμ"]: _exc["%d%s" % (h, period)] = [ {ORTH: "%d" % h}, {ORTH: period, LEMMA: "π.μ.", NORM: "π.μ."}] for period in ["μ.μ.", "μμ"]: _exc["%d%s" % (h, period)] = [ {ORTH: "%d" % h}, {ORTH: period, LEMMA: "μ.μ.", NORM: "μ.μ."}] for exc_data in [ {ORTH: "ΑΓΡ.", LEMMA: "Αγροτικός", NORM: "Αγροτικός"}, {ORTH: "Αγ. Γρ.", LEMMA: "Αγία Γραφή", NORM: "Αγία Γραφή"}, {ORTH: "Αθ.", LEMMA: "Αθανάσιος", NORM: "Αθανάσιος"}, {ORTH: "Αλεξ.", LEMMA: "Αλέξανδρος", NORM: "Αλέξανδρος"}, {ORTH: "Απρ.", LEMMA: "Απρίλιος", NORM: "Απρίλιος"}, {ORTH: "Αύγ.", LEMMA: "Αύγουστος", NORM: "Αύγουστος"}, {ORTH: "Δεκ.", LEMMA: "Δεκέμβριος", NORM: "Δεκέμβριος"}, {ORTH: "Δημ.", LEMMA: "Δήμος", NORM: "Δήμος"}, {ORTH: "Ιαν.", LEMMA: "Ιανουάριος", NORM: "Ιανουάριος"}, {ORTH: "Ιούλ.", LEMMA: "Ιούλιος", NORM: "Ιούλιος"}, {ORTH: "Ιούν.", LEMMA: "Ιούνιος", NORM: "Ιούνιος"}, {ORTH: "Ιωαν.", LEMMA: "Ιωάννης", NORM: "Ιωάννης"}, {ORTH: "Μ. Ασία", LEMMA: "Μικρά Ασία", NORM: "Μικρά Ασία"}, {ORTH: "Μάρτ.", LEMMA: "Μάρτιος", NORM: "Μάρτιος"}, {ORTH: "Μάρτ'", LEMMA: "Μάρτιος", NORM: "Μάρτιος"}, {ORTH: "Νοέμβρ.", LEMMA: "Νοέμβριος", NORM: "Νοέμβριος"}, {ORTH: "Οκτ.", LEMMA: "Οκτώβριος", NORM: "Οκτώβριος"}, {ORTH: "Σεπτ.", LEMMA: "Σεπτέμβριος", NORM: "Σεπτέμβριος"}, {ORTH: "Φεβρ.", LEMMA: "Φεβρουάριος", NORM: "Φεβρουάριος"}, ]: _exc[exc_data[ORTH]] = [exc_data] for orth in [ "$ΗΠΑ", "Α'", "Α.Ε.", "Α.Ε.Β.Ε.", "Α.Ε.Ι.", "Α.Ε.Π.", "Α.Μ.Α.", "Α.Π.Θ.", "Α.Τ.", "Α.Χ.", "ΑΝ.", "Αγ.", "Αλ.", "Αν.", "Αντ.", "Απ.", "Β'", "Β)", "Β.Ζ.", "Β.Ι.Ο.", "Β.Κ.", "Β.Μ.Α.", "Βασ.", "Γ'", "Γ)", "Γ.Γ.", "Γ.Δ.", "Γκ.", "Δ.Ε.Η.", "Δ.Ε.Σ.Ε.", "Δ.Ν.", "Δ.Ο.Υ.", "Δ.Σ.", "Δ.Υ.", "ΔΙ.ΚΑ.Τ.Σ.Α.", "Δηλ.", "Διον.", "Ε.Α.", "Ε.Α.Κ.", "Ε.Α.Π.", "Ε.Ε.", "Ε.Κ.", "Ε.ΚΕ.ΠΙΣ.", "Ε.Λ.Α.", "Ε.Λ.Ι.Α.", "Ε.Π.Σ.", "Ε.Π.Τ.Α.", "Ε.Σ.Ε.Ε.Κ.", "Ε.Υ.Κ.", "ΕΕ.", "ΕΚ.", "ΕΛ.", "ΕΛ.ΑΣ.", "Εθν.", "Ελ.", "Εμ.", "Επ.", "Ευ.", "Η'", "Η.Π.Α.", "ΘΕ.", "Θεμ.", "Θεοδ.", "Θρ.", "Ι.Ε.Κ.", "Ι.Κ.Α.", "Ι.Κ.Υ.", "Ι.Σ.Θ.", "Ι.Χ.", "ΙΖ'", "ΙΧ.", "Κ.Α.Α.", "Κ.Α.Ε.", "Κ.Β.Σ.", "Κ.Δ.", "Κ.Ε.", "Κ.Ε.Κ.", "Κ.Ι.", "Κ.Κ.", "Κ.Ι.Θ.", "Κ.Ι.Θ.", "Κ.ΚΕΚ.", "Κ.Ο.", "Κ.Π.Ρ.", "ΚΑΤ.", "ΚΚ.", "Καν.", "Καρ.", "Κατ.", "Κυρ.", "Κων.", "Λ.Α.", "Λ.χ.", "Λ.Χ.", "Λεωφ.", "Λι.", "Μ.Δ.Ε.", "Μ.Ε.Ο.", "Μ.Ζ.", "Μ.Μ.Ε.", "Μ.Ο.", "Μεγ.", "Μιλτ.", "Μιχ.", "Ν.Δ.", "Ν.Ε.Α.", "Ν.Κ.", "Ν.Ο.", "Ν.Ο.Θ.", "Ν.Π.Δ.Δ.", "Ν.Υ.", "ΝΔ.", "Νικ.", "Ντ'", "Ντ.", "Ο'", "Ο.Α.", "Ο.Α.Ε.Δ.", "Ο.Δ.", "Ο.Ε.Ε.", "Ο.Ε.Ε.Κ.", "Ο.Η.Ε.", "Ο.Κ.", "Π.Δ.", "Π.Ε.Κ.Δ.Υ.", "Π.Ε.Π.", "Π.Μ.Σ.", "ΠΟΛ.", "Π.Χ.", "Παρ.", "Πλ.", "Πρ.", "Σ.Δ.Ο.Ε.", "Σ.Ε.", "Σ.Ε.Κ.", "Σ.Π.Δ.Ω.Β.", "Σ.Τ.", "Σαβ.", "Στ.", "ΣτΕ.", "Στρ.", "Τ.Α.", "Τ.Ε.Ε.", "Τ.Ε.Ι.", "ΤΡ.", "Τζ.", "Τηλ.", "Υ.Γ.", "ΥΓ.", "ΥΠ.Ε.Π.Θ.", "Φ.Α.Β.Ε.", "Φ.Κ.", "Φ.Σ.", "Φ.Χ.", "Φ.Π.Α.", "Φιλ.", "Χ.Α.Α.", "ΧΡ.", "Χ.Χ.", "Χαρ.", "Χιλ.", "Χρ.", "άγ.", "άρθρ.", "αι.", "αν.", "απ.", "αρ.", "αριθ.", "αριθμ.", "β'", "βλ.", "γ.γ.", "γεν.", "γραμμ.", "δ.δ.", "δ.σ.", "δηλ.", "δισ.", "δολ.", "δρχ.", "εκ.", "εκατ.", "ελ.", "θιν'", "κ.", "κ.ά.", "κ.α.", "κ.κ.", "κ.λπ.", "κ.ο.κ.", "κ.τ.λ.", "κλπ.", "κτλ.", "κυβ.", "λ.χ.", "μ.", "μ.Χ.", "μ.μ.", "μιλ.", "ντ'", "π.Χ.", "π.β.", "π.δ.", "π.μ.", "π.χ.", "σ.", "σ.α.λ.", "σ.σ.", "σελ.", "στρ.", "τ'ς", "τ.μ.", "τετ.", "τετρ.", "τηλ.", "τρισ.", "τόν.", "υπ.", "χ.μ.", "χγρ.", "χιλ.", "χλμ." ]: _exc[orth] = [{ORTH: orth}] TOKENIZER_EXCEPTIONS = _exc