Added further testcases.

2025-01-12 18:26:30 +03:00 · 2016-12-20 23:49:35 +01:00 · 2016-12-20 23:49:35 +01:00 · 3d5306acb9
commit 3d5306acb9
parent 23956e72ff
4 changed files with 261 additions and 4 deletions
--- a/spacy/hu/punctuations.py
+++ b/spacy/hu/punctuations.py
@ -32,7 +32,7 @@ _
 \-\-
 ´
 (?<=[0-9])\+
-(?<=[a-z0-9üóőúéáűí]\)\]”"'%\)§)\.
+(?<=[a-z0-9üóőúéáűí][\)\]”"'%\)§/])\.
 (?<=[0-9])km²
 (?<=[0-9])m²
 (?<=[0-9])cm²
--- a/spacy/tests/hu/tokenizer/test_default_token_it.txt
+++ b/spacy/tests/hu/tokenizer/test_default_token_it.txt
@ -0,0 +1,233 @@
+# TOKEN it
+
+tartomanynevek
+IN : .edu, .hu.
+OUT: <s><w>.edu</w><c>,</c><ws> </ws><w>.hu</w><c>.</c></s>
+
+url-ek
+IN : red-stars.com.
+OUT: <s><w>red-stars.com</w><c>.</c></s>
+IN : www.valami.com.
+OUT: <s><w>www.valami.com</w><c>.</c></s>
+
+url-ek toldalekolva
+IN : www.valami.com-ról.
+OUT: <s><w>www.valami.com-ról</w><c>.</c></s>
+IN : www.valami.comról.
+OUT: <s><w>www.valami.comról</w><c>.</c></s>
+
+osszetettebb url-ek
+IN : A https://www.valami.com/index.html van.
+OUT: <s><w>A</w><ws> </ws><w>https://www.valami.com/index.html</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A http://142.42.1.1/ van.
+OUT: <s><w>A</w><ws> </ws><w>http://142.42.1.1/</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A http://142.42.1.1/.
+OUT: <s><w>A</w><ws> </ws><w>http://142.42.1.1/</w><c>.</c></s>
+IN : A http://example.com/.
+OUT: <s><w>A</w><ws> </ws><w>http://example.com/</w><c>.</c></s>
+IN : A http://example.com/ van.
+OUT: <s><w>A</w><ws> </ws><w>http://example.com/</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A http://foo.com/blah_(wikipedia)#cite-1.
+OUT: <s><w>A</w><ws> </ws><w>http://foo.com/blah_(wikipedia)#cite-1</w><c>.</c></s>
+IN : A http://foo.com/blah_(wikipedia)#cite-1 van.
+OUT: <s><w>A</w><ws> </ws><w>http://foo.com/blah_(wikipedia)#cite-1</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A http://foo.com/blah_(wikipedia).
+OUT: <s><w>A</w><ws> </ws><w>http://foo.com/blah_(wikipedia)</w><c>.</c></s>
+IN : A http://foo.com/blah_(wikipedia) van.
+OUT: <s><w>A</w><ws> </ws><w>http://foo.com/blah_(wikipedia)</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A http://www.example.com/wpstyle/?bar=baz&inga=42&quux.
+OUT: <s><w>A</w><ws> </ws><w>http://www.example.com/wpstyle/?bar=baz&inga=42&quux</w><c>.</c></s>
+IN : A http://www.example.com/wpstyle/?bar=baz&inga=42&quux van.
+OUT: <s><w>A</w><ws> </ws><w>http://www.example.com/wpstyle/?bar=baz&inga=42&quux</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A http://example.com:8080.
+OUT: <s><w>A</w><ws> </ws><w>http://example.com:8080</w><c>.</c></s>
+IN : A http://example.com:8080 van.
+OUT: <s><w>A</w><ws> </ws><w>http://example.com:8080</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A http://foo.bar/?q=Test%20URL-encoded%20stuff.
+OUT: <s><w>A</w><ws> </ws><w>http://foo.bar/?q=Test%20URL-encoded%20stuff</w><c>.</c></s>
+IN : A http://foo.bar/?q=Test%20URL-encoded%20stuff van.
+OUT: <s><w>A</w><ws> </ws><w>http://foo.bar/?q=Test%20URL-encoded%20stuff</w><ws> </ws><w>van</w><c>.</c></s>
+# url with more '='
+IN : A www.kereso.elte.hu/nev=kiss,jozsef%kar=jog%tagozat=nappali.
+OUT: <s><w>A</w><ws> </ws><w>www.kereso.elte.hu/nev=kiss,jozsef%kar=jog%tagozat=nappali</w><c>.</c></s>
+IN : A www.kereso.elte.hu/nev=kiss,jozsef%kar=jog%tagozat=nappali van.
+OUT: <s><w>A</w><ws> </ws><w>www.kereso.elte.hu/nev=kiss,jozsef%kar=jog%tagozat=nappali</w><ws> </ws><w>van</w><c>.</c></s>
+
+email
+IN : A foo.bar@baz.com van.
+OUT: <s><w>A</w><ws> </ws><w>foo.bar@baz.com</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A foo.bar@baz.com.
+OUT: <s><w>A</w><ws> </ws><w>foo.bar@baz.com</w><c>.</c></s>
+IN : A foo.bar@baz.com-nak van.
+OUT: <s><w>A</w><ws> </ws><w>foo.bar@baz.com-nak</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A foo.bar@baz.com-nak.
+OUT: <s><w>A</w><ws> </ws><w>foo.bar@baz.com-nak</w><c>.</c></s>
+IN : A foo.bar@baz.comnak van.
+OUT: <s><w>A</w><ws> </ws><w>foo.bar@baz.comnak</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A foo.bar@baz.comnak.
+OUT: <s><w>A</w><ws> </ws><w>foo.bar@baz.comnak</w><c>.</c></s>
+IN : A mailto:foo.bar@baz.com van.
+OUT: <s><w>A</w><ws> </ws><w>mailto:foo.bar@baz.com</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A mailto:foo.bar@baz.com.
+OUT: <s><w>A</w><ws> </ws><w>mailto:foo.bar@baz.com</w><c>.</c></s>
+IN : A mailto:foo.bar@baz.com-nak van.
+OUT: <s><w>A</w><ws> </ws><w>mailto:foo.bar@baz.com-nak</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A mailto:foo.bar@baz.com-nak.
+OUT: <s><w>A</w><ws> </ws><w>mailto:foo.bar@baz.com-nak</w><c>.</c></s>
+IN : A mailto:foo.bar@baz.comnak van.
+OUT: <s><w>A</w><ws> </ws><w>mailto:foo.bar@baz.comnak</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A mailto:foo.bar@baz.comnak.
+OUT: <s><w>A</w><ws> </ws><w>mailto:foo.bar@baz.comnak</w><c>.</c></s>
+
+windows halozati szolgaltatas (vagy mi)
+IN : Az ADMIN$ van.
+OUT: <s><w>Az</w><ws> </ws><w>ADMIN$</w><ws> </ws><w>van</w><c>.</c></s>
+IN : Az ADMIN$.
+OUT: <s><w>Az</w><ws> </ws><w>ADMIN$</w><c>.</c></s>
+IN : Az ADMIN$-nak van.
+OUT: <s><w>Az</w><ws> </ws><w>ADMIN$-nak</w><ws> </ws><w>van</w><c>.</c></s>
+IN : Az ADMIN$-nak.
+OUT: <s><w>Az</w><ws> </ws><w>ADMIN$-nak</w><c>.</c></s>
+
+legfelso tartomanynevek
+IN : A .edu, .hu vannak.
+OUT: <s><w>A</w><ws> </ws><w>.edu</w><c>,</c><ws> </ws><w>.hu</w><ws> </ws><w>vannak</w><c>.</c></s>
+IN : A .edu, .hu.
+OUT: <s><w>A</w><ws> </ws><w>.edu</w><c>,</c><ws> </ws><w>.hu</w><c>.</c></s>
+
+slash, backslash
+IN : A / van.
+OUT: <s><w>A</w><ws> </ws><c>/</c><ws> </ws><w>van</w><c>.</c></s>
+IN : A \ van.
+OUT: <s><w>A</w><ws> </ws><c>\</c><ws> </ws><w>van</w><c>.</c></s>
+
+windows path
+IN : C:, \ es c:\ van.
+OUT: <s><w>C:</w><c>,</c><ws> </ws><c>\</c><ws> </ws><w>es</w><ws> </ws><w>c:\</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A \\test\test$\TEST.xls van.
+OUT: <s><w>A</w><ws> </ws><w>\\test\test$\TEST.xls</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A \\server\share\folder\myfile.txt van.
+OUT: <s><w>A</w><ws> </ws><w>\\server\share\folder\myfile.txt</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A \\server\share\myfile.txt van.
+OUT: <s><w>A</w><ws> </ws><w>\\server\share\myfile.txt</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A \\123.123.123.123\share\folder\myfile.txt van.
+OUT: <s><w>A</w><ws> </ws><w>\\123.123.123.123\share\folder\myfile.txt</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A c:\folder\myfile.txt van.
+OUT: <s><w>A</w><ws> </ws><w>c:\folder\myfile.txt</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A c:\folder\myfile.txt-ben van.
+OUT: <s><w>A</w><ws> </ws><w>c:\folder\myfile.txt-ben</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A c:\folder\myfileWithoutExtension van.
+OUT: <s><w>A</w><ws> </ws><w>c:\folder\myfileWithoutExtension</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A c:\folder\myfile.txt.
+OUT: <s><w>A</w><ws> </ws><w>c:\folder\myfile.txt</w><c>.</c></s>
+IN : A c:\folder\myfile.txt-ben.
+OUT: <s><w>A</w><ws> </ws><w>c:\folder\myfile.txt-ben</w><c>.</c></s>
+
+unix path
+IN : A ./ van.
+OUT: <s><w>A</w><ws> </ws><w>./</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A ../ van.
+OUT: <s><w>A</w><ws> </ws><w>../</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A ./valami1/valami2/ van.
+OUT: <s><w>A</w><ws> </ws><w>./valami1/valami2/</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A /valami/valami.txt van.
+OUT: <s><w>A</w><ws> </ws><w>/valami/valami.txt</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A /valami/valami.txt-ben van.
+OUT: <s><w>A</w><ws> </ws><w>/valami/valami.txt-ben</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A /valami/valami.txt.
+OUT: <s><w>A</w><ws> </ws><w>/valami/valami.txt</w><c>.</c></s>
+IN : A /valami/valami.txt-ben.
+OUT: <s><w>A</w><ws> </ws><w>/valami/valami.txt-ben</w><c>.</c></s>
+
+Huntokenes tesztek vegyesen
+1.
+IN : A C:\DINNYE\MACSKA.JPG van.
+OUT: <s><w>A</w><ws> </ws><w>C:\DINNYE\MACSKA.JPG</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A C:\DINNYE\MACSKA.JPG-ben van.
+OUT: <s><w>A</w><ws> </ws><w>C:\DINNYE\MACSKA.JPG-ben</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A C:\DINNYE\MACSKA.JPG.
+OUT: <s><w>A</w><ws> </ws><w>C:\DINNYE\MACSKA.JPG</w><c>.</c></s>
+IN : A C:\DINNYE\MACSKA.JPG-ben.
+OUT: <s><w>A</w><ws> </ws><w>C:\DINNYE\MACSKA.JPG-ben</w><c>.</c></s>
+2.
+IN : A \\SZERVER\SZOLGALTATAS$ van.
+OUT: <s><w>A</w><ws> </ws><w>\\SZERVER\SZOLGALTATAS$</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A \\SZERVER\SZOLGALTATAS$-ben van.
+OUT: <s><w>A</w><ws> </ws><w>\\SZERVER\SZOLGALTATAS$-ben</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A \\SZERVER\SZOLGALTATAS$.
+OUT: <s><w>A</w><ws> </ws><w>\\SZERVER\SZOLGALTATAS$</w><c>.</c></s>
+IN : A \\SZERVER\SZOLGALTATAS$-ben.
+OUT: <s><w>A</w><ws> </ws><w>\\SZERVER\SZOLGALTATAS$-ben</w><c>.</c></s>
+3.
+IN : A /etc/.././home/ van.
+OUT: <s><w>A</w><ws> </ws><w>/etc/.././home/</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A /etc/.././home/-ben van.
+OUT: <s><w>A</w><ws> </ws><w>/etc/.././home/-ben</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A /etc/.././home/.
+OUT: <s><w>A</w><ws> </ws><w>/etc/.././home/</w><c>.</c></s>
+IN : A /etc/.././home/-ben.
+OUT: <s><w>A</w><ws> </ws><w>/etc/.././home/-ben</w><c>.</c></s>
+4.
+IN : A *.doc van.
+OUT: <s><w>A</w><ws> </ws><w>*.doc</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A *.doc-ben van.
+OUT: <s><w>A</w><ws> </ws><w>*.doc-ben</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A *.doc.
+OUT: <s><w>A</w><ws> </ws><w>*.doc</w><c>.</c></s>
+IN : A *.doc-ben.
+OUT: <s><w>A</w><ws> </ws><w>*.doc-ben</w><c>.</c></s>
+5.
+IN : A *.* van.
+OUT: <s><w>A</w><ws> </ws><w>*.*</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A *.*-ben van.
+OUT: <s><w>A</w><ws> </ws><w>*.*-ben</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A *.*.
+OUT: <s><w>A</w><ws> </ws><w>*.*</w><c>.</c></s>
+IN : A *.*-ben.
+OUT: <s><w>A</w><ws> </ws><w>*.*-ben</w><c>.</c></s>
+6.
+IN : A .bmp.zip van.
+OUT: <s><w>A</w><ws> </ws><w>.bmp.zip</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A .bmp.zip-ben van.
+OUT: <s><w>A</w><ws> </ws><w>.bmp.zip-ben</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A .bmp.zip.
+OUT: <s><w>A</w><ws> </ws><w>.bmp.zip</w><c>.</c></s>
+IN : A .bmp.zip-ben.
+OUT: <s><w>A</w><ws> </ws><w>.bmp.zip-ben</w><c>.</c></s>
+
+
+fajlnevek
+IN : A teszt.txt van.
+OUT: <s><w>A</w><ws> </ws><w>teszt.txt</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A teszt.txt-ben van.
+OUT: <s><w>A</w><ws> </ws><w>teszt.txt-ben</w><ws> </ws><w>van</w><c>.</c></s>
+IN : A teszt.txt.
+OUT: <s><w>A</w><ws> </ws><w>teszt.txt</w><c>.</c></s>
+IN : A teszt.txt-ben.
+OUT: <s><w>A</w><ws> </ws><w>teszt.txt-ben</w><c>.</c></s>
+IN : A .txt van.
+OUT: <s><w>A</w><ws> </ws><w>.txt</w><ws> </ws><w>van</w><c>.</c></s>
+
+
+    MINTA
+    IN : A  van.
+    OUT: <s><w>A</w><ws> </ws><w></w><ws> </ws><w>van</w><c>.</c></s>
+    IN : A -ben van.
+    OUT: <s><w>A</w><ws> </ws><w>-ben</w><ws> </ws><w>van</w><c>.</c></s>
+    IN : A .
+    OUT: <s><w>A</w><ws> </ws><w></w><c>.</c></s>
+    IN : A -ben.
+    OUT: <s><w>A</w><ws> </ws><w>-ben</w><c>.</c></s>
+
+
+
+
+
+
+
+
+
+
+
+
+
--- a/spacy/tests/hu/tokenizer/test_default_token_misc.txt
+++ b/spacy/tests/hu/tokenizer/test_default_token_misc.txt
@ -0,0 +1,22 @@
+# TOKEN misc
+
+    TODO: html entities
+    IN : Moli&egrave;re-rol van.
+    OUT: <s><w>Moli&egrave;re-rol</w><ws> </ws><w>van</w><c>.</c></s>
+
+
+&-t tartalmazo nagybatus szavak
+IN : AT&T van.
+OUT: <s><w>AT&T</w><ws> </ws><w>van</w><c>.</c></s>
+
+
+    zarojeles mondatkozi valtozatok
+    IN : (La)kik-e van?
+    OUT: <s><w>(La)kik</w><w>-e</w><ws> </ws><w>van</w><c>?</c></s>
+    IN : L(a)kik-e van?
+    OUT: <s><w>L(a)kik</w><w>-e</w><ws> </ws><w>van</w><c>?</c></s>
+    IN : Lak(ik)-e van?
+    OUT: <s><w>Lak(ik)</w><w>-e</w><ws> </ws><w>van</w><c>?</c></s>
+
+
+
--- a/spacy/tests/hu/tokenizer/test_tokenizer.py
+++ b/spacy/tests/hu/tokenizer/test_tokenizer.py
@ -58,9 +58,11 @@ _DOTS_CASES = list(TokenizerTestCase.read_from_file(_MODULE_PATH + "/test_defaul
 _HYPHEN_CASES = list(TokenizerTestCase.read_from_file(_MODULE_PATH + "/test_default_token_hyphen.txt"))
 _QUOTE_CASES = list(TokenizerTestCase.read_from_file(_MODULE_PATH + "/test_default_token_quote.txt"))
 _NUMBER_CASES = list(TokenizerTestCase.read_from_file(_MODULE_PATH + "/test_default_token_numbers.txt"))
-ALL_TESTCASES = _DOTS_CASES + _HYPHEN_CASES + _QUOTE_CASES  + _NUMBER_CASES
-ALL_TESTCASES = _NUMBER_CASES
-# ALL_TESTCASES = [TokenizerTestCase("A 1:20:36.7.", "A 1:20:36.7.".split())]
+_MISC_CASES = list(TokenizerTestCase.read_from_file(_MODULE_PATH + "/test_default_token_misc.txt"))
+_IT_CASES = list(TokenizerTestCase.read_from_file(_MODULE_PATH + "/test_default_token_it.txt"))
+
+# TODO: Until this get fixed we cannot really test the urls: https://github.com/explosion/spaCy/issues/344
+ALL_TESTCASES = _DOTS_CASES + _HYPHEN_CASES + _QUOTE_CASES + _NUMBER_CASES + _MISC_CASES  # + _IT_CASES


@pytest.fixture(scope="session")