spaCy/spacy/tests/vocab_vectors/test_stringstore.py

98 lines
3.3 KiB
Python
Raw Normal View History

2017-01-12 17:07:31 +03:00
import pytest
💫 Refactor test suite (#2568) ## Description Related issues: #2379 (should be fixed by separating model tests) * **total execution time down from > 300 seconds to under 60 seconds** 🎉 * removed all model-specific tests that could only really be run manually anyway – those will now live in a separate test suite in the [`spacy-models`](https://github.com/explosion/spacy-models) repository and are already integrated into our new model training infrastructure * changed all relative imports to absolute imports to prepare for moving the test suite from `/spacy/tests` to `/tests` (it'll now always test against the installed version) * merged old regression tests into collections, e.g. `test_issue1001-1500.py` (about 90% of the regression tests are very short anyways) * tidied up and rewrote existing tests wherever possible ### Todo - [ ] move tests to `/tests` and adjust CI commands accordingly - [x] move model test suite from internal repo to `spacy-models` - [x] ~~investigate why `pipeline/test_textcat.py` is flakey~~ - [x] review old regression tests (leftover files) and see if they can be merged, simplified or deleted - [ ] update documentation on how to run tests ### Types of change enhancement, tests ## Checklist <!--- Before you submit the PR, go over this checklist and make sure you can tick off all the boxes. [] -> [x] --> - [x] I have submitted the spaCy Contributor Agreement. - [x] I ran the tests, and all new and existing tests passed. - [ ] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-07-25 00:38:44 +03:00
from spacy.strings import StringStore
@pytest.fixture
def stringstore():
return StringStore()
2017-01-12 17:07:31 +03:00
2017-09-16 12:21:35 +03:00
def test_string_hash(stringstore):
💫 Refactor test suite (#2568) ## Description Related issues: #2379 (should be fixed by separating model tests) * **total execution time down from > 300 seconds to under 60 seconds** 🎉 * removed all model-specific tests that could only really be run manually anyway – those will now live in a separate test suite in the [`spacy-models`](https://github.com/explosion/spacy-models) repository and are already integrated into our new model training infrastructure * changed all relative imports to absolute imports to prepare for moving the test suite from `/spacy/tests` to `/tests` (it'll now always test against the installed version) * merged old regression tests into collections, e.g. `test_issue1001-1500.py` (about 90% of the regression tests are very short anyways) * tidied up and rewrote existing tests wherever possible ### Todo - [ ] move tests to `/tests` and adjust CI commands accordingly - [x] move model test suite from internal repo to `spacy-models` - [x] ~~investigate why `pipeline/test_textcat.py` is flakey~~ - [x] review old regression tests (leftover files) and see if they can be merged, simplified or deleted - [ ] update documentation on how to run tests ### Types of change enhancement, tests ## Checklist <!--- Before you submit the PR, go over this checklist and make sure you can tick off all the boxes. [] -> [x] --> - [x] I have submitted the spaCy Contributor Agreement. - [x] I ran the tests, and all new and existing tests passed. - [ ] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-07-25 00:38:44 +03:00
"""Test that string hashing is stable across platforms"""
assert stringstore.add("apple") == 8566208034543834098
heart = "\U0001f499"
💫 Refactor test suite (#2568) ## Description Related issues: #2379 (should be fixed by separating model tests) * **total execution time down from > 300 seconds to under 60 seconds** 🎉 * removed all model-specific tests that could only really be run manually anyway – those will now live in a separate test suite in the [`spacy-models`](https://github.com/explosion/spacy-models) repository and are already integrated into our new model training infrastructure * changed all relative imports to absolute imports to prepare for moving the test suite from `/spacy/tests` to `/tests` (it'll now always test against the installed version) * merged old regression tests into collections, e.g. `test_issue1001-1500.py` (about 90% of the regression tests are very short anyways) * tidied up and rewrote existing tests wherever possible ### Todo - [ ] move tests to `/tests` and adjust CI commands accordingly - [x] move model test suite from internal repo to `spacy-models` - [x] ~~investigate why `pipeline/test_textcat.py` is flakey~~ - [x] review old regression tests (leftover files) and see if they can be merged, simplified or deleted - [ ] update documentation on how to run tests ### Types of change enhancement, tests ## Checklist <!--- Before you submit the PR, go over this checklist and make sure you can tick off all the boxes. [] -> [x] --> - [x] I have submitted the spaCy Contributor Agreement. - [x] I ran the tests, and all new and existing tests passed. - [ ] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-07-25 00:38:44 +03:00
h = stringstore.add(heart)
2017-09-16 17:22:38 +03:00
assert h == 11841826740069053588
💫 Refactor test suite (#2568) ## Description Related issues: #2379 (should be fixed by separating model tests) * **total execution time down from > 300 seconds to under 60 seconds** 🎉 * removed all model-specific tests that could only really be run manually anyway – those will now live in a separate test suite in the [`spacy-models`](https://github.com/explosion/spacy-models) repository and are already integrated into our new model training infrastructure * changed all relative imports to absolute imports to prepare for moving the test suite from `/spacy/tests` to `/tests` (it'll now always test against the installed version) * merged old regression tests into collections, e.g. `test_issue1001-1500.py` (about 90% of the regression tests are very short anyways) * tidied up and rewrote existing tests wherever possible ### Todo - [ ] move tests to `/tests` and adjust CI commands accordingly - [x] move model test suite from internal repo to `spacy-models` - [x] ~~investigate why `pipeline/test_textcat.py` is flakey~~ - [x] review old regression tests (leftover files) and see if they can be merged, simplified or deleted - [ ] update documentation on how to run tests ### Types of change enhancement, tests ## Checklist <!--- Before you submit the PR, go over this checklist and make sure you can tick off all the boxes. [] -> [x] --> - [x] I have submitted the spaCy Contributor Agreement. - [x] I ran the tests, and all new and existing tests passed. - [ ] My changes don't require a change to the documentation, or if they do, I've added all required information.
2018-07-25 00:38:44 +03:00
2017-09-16 12:21:35 +03:00
2017-05-29 02:05:53 +03:00
def test_stringstore_from_api_docs(stringstore):
apple_hash = stringstore.add("apple")
2017-05-29 02:05:53 +03:00
assert apple_hash == 8566208034543834098
assert stringstore[apple_hash] == "apple"
assert "apple" in stringstore
assert "cherry" not in stringstore
2018-11-30 19:43:08 +03:00
stringstore.add("orange")
2017-05-29 02:05:53 +03:00
all_strings = [s for s in stringstore]
assert all_strings == ["apple", "orange"]
banana_hash = stringstore.add("banana")
2017-05-29 02:05:53 +03:00
assert len(stringstore) == 3
assert banana_hash == 2525716904149915114
assert stringstore[banana_hash] == "banana"
assert stringstore["banana"] == banana_hash
2017-05-29 02:05:53 +03:00
@pytest.mark.parametrize("text1,text2,text3", [(b"Hello", b"goodbye", b"hello")])
2017-01-12 17:07:31 +03:00
def test_stringstore_save_bytes(stringstore, text1, text2, text3):
key = stringstore.add(text1)
assert stringstore[text1] == key
assert stringstore[text2] != key
assert stringstore[text3] != key
2017-01-12 17:07:31 +03:00
@pytest.mark.parametrize("text1,text2,text3", [("Hello", "goodbye", "hello")])
2017-01-12 17:07:31 +03:00
def test_stringstore_save_unicode(stringstore, text1, text2, text3):
key = stringstore.add(text1)
assert stringstore[text1] == key
assert stringstore[text2] != key
assert stringstore[text3] != key
2017-01-12 17:07:31 +03:00
@pytest.mark.parametrize("text", [b"A"])
2017-01-12 17:07:31 +03:00
def test_stringstore_retrieve_id(stringstore, text):
key = stringstore.add(text)
assert len(stringstore) == 1
assert stringstore[key] == text.decode("utf8")
with pytest.raises(KeyError):
2017-05-28 15:08:09 +03:00
stringstore[20000]
2017-01-12 17:07:31 +03:00
@pytest.mark.parametrize("text1,text2", [(b"0123456789", b"A")])
2017-01-12 17:07:31 +03:00
def test_stringstore_med_string(stringstore, text1, text2):
store = stringstore.add(text1)
assert stringstore[store] == text1.decode("utf8")
2018-11-30 19:43:08 +03:00
stringstore.add(text2)
2017-01-12 17:07:31 +03:00
assert stringstore[text1] == store
def test_stringstore_long_string(stringstore):
text = "INFORMATIVE](http://www.google.com/search?as_q=RedditMonkey&amp;hl=en&amp;num=50&amp;btnG=Google+Search&amp;as_epq=&amp;as_oq=&amp;as_eq=&amp;lr=&amp;as_ft=i&amp;as_filetype=&amp;as_qdr=all&amp;as_nlo=&amp;as_nhi=&amp;as_occt=any&amp;as_dt=i&amp;as_sitesearch=&amp;as_rights=&amp;safe=off"
store = stringstore.add(text)
2017-01-12 17:07:31 +03:00
assert stringstore[store] == text
@pytest.mark.parametrize("factor", [254, 255, 256])
2017-01-12 17:07:31 +03:00
def test_stringstore_multiply(stringstore, factor):
text = "a" * factor
store = stringstore.add(text)
2017-01-12 17:07:31 +03:00
assert stringstore[store] == text
def test_stringstore_massive_strings(stringstore):
text = "a" * 511
store = stringstore.add(text)
2017-01-12 17:07:31 +03:00
assert stringstore[store] == text
text2 = "z" * 512
store = stringstore.add(text2)
2017-01-12 17:07:31 +03:00
assert stringstore[store] == text2
text3 = "1" * 513
store = stringstore.add(text3)
2017-01-12 17:07:31 +03:00
assert stringstore[store] == text3
@pytest.mark.parametrize("text", ["qqqqq"])
def test_stringstore_to_bytes(stringstore, text):
store = stringstore.add(text)
serialized = stringstore.to_bytes()
new_stringstore = StringStore().from_bytes(serialized)
2017-01-12 17:07:31 +03:00
assert new_stringstore[store] == text