from ...symbols import NORM, ORTH from ...util import update_exc from ..tokenizer_exceptions import BASE_EXCEPTIONS _exc = { "tayo'y": [{ORTH: "tayo"}, {ORTH: "'y", NORM: "ay"}], "isa'y": [{ORTH: "isa"}, {ORTH: "'y", NORM: "ay"}], "baya'y": [{ORTH: "baya"}, {ORTH: "'y", NORM: "ay"}], "sa'yo": [{ORTH: "sa"}, {ORTH: "'yo", NORM: "iyo"}], "ano'ng": [{ORTH: "ano"}, {ORTH: "'ng", NORM: "ang"}], "siya'y": [{ORTH: "siya"}, {ORTH: "'y", NORM: "ay"}], "nawa'y": [{ORTH: "nawa"}, {ORTH: "'y", NORM: "ay"}], "papa'no": [{ORTH: "papa'no", NORM: "papaano"}], "'di": [{ORTH: "'di", NORM: "hindi"}], } TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc)