explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-12-24 17:06:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add day of month tokenizer exceptions for Danish.

Browse Source
This commit is contained in:
Søren Lind Kristiansen 2017-11-24 15:03:24 +01:00
parent 0c276ed020
commit 6aa241bcec
2 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
spacy/lang/da/tokenizer_exceptions.py
View File

@ -117,6 +117,12 @@ for orth in [
"øv.", "øvr.", "årg.", "årh.", ""]: "øv.", "øvr.", "årg.", "årh.", ""]:
_exc[orth] = [{ORTH: orth}] _exc[orth] = [{ORTH: orth}]
# Dates
for h in range(1, 31 + 1):
for period in ["."]:
_exc["%d%s" % (h, period)] = [
{ORTH: "%d." % h}]
_custom_base_exc = { _custom_base_exc = {
"i.": [ "i.": [
{ORTH: "i", LEMMA: "i", NORM: "i"}, {ORTH: "i", LEMMA: "i", NORM: "i"},

5
spacy/tests/lang/da/test_exceptions.py
View File

@ -14,6 +14,11 @@ def test_da_tokenizer_handles_ambiguous_abbr(da_tokenizer, text):
tokens = da_tokenizer(text) tokens = da_tokenizer(text)
assert len(tokens) == 2 assert len(tokens) == 2
@pytest.mark.parametrize('text', ["1.", "10.", "31."])
def test_da_tokenizer_handles_dates(da_tokenizer, text):
tokens = da_tokenizer(text)
assert len(tokens) == 1
def test_da_tokenizer_handles_exc_in_text(da_tokenizer): def test_da_tokenizer_handles_exc_in_text(da_tokenizer):
text = "Det er bl.a. ikke meningen" text = "Det er bl.a. ikke meningen"
tokens = da_tokenizer(text) tokens = da_tokenizer(text)