explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-12 18:26:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Use hyphen to join parts of inflection in JA tokenizer

Browse Source
This commit is contained in:
Paul O'Leary McCann 2021-10-07 17:09:38 +09:00
parent 53b5f245ed
commit f975690cc9
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
spacy/lang/ja/__init__.py
View File

@ -94,7 +94,7 @@ class JapaneseTokenizer(DummyTokenizer):
DetailedToken( DetailedToken(
token.surface(), # orth token.surface(), # orth
"-".join([xx for xx in token.part_of_speech()[:4] if xx != "*"]), # tag "-".join([xx for xx in token.part_of_speech()[:4] if xx != "*"]), # tag
",".join([xx for xx in token.part_of_speech()[4:] if xx != "*"]), # inf "-".join([xx for xx in token.part_of_speech()[4:] if xx != "*"]), # inf
token.dictionary_form(), # lemma token.dictionary_form(), # lemma
token.normalized_form(), token.normalized_form(),
token.reading_form(), token.reading_form(),