explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-01-11 17:56:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Adjust names and formatting

Browse Source
This commit is contained in:
Ines Montani 2017-01-10 15:56:35 +01:00
parent e10d4ca964
commit 2185d31907
1 changed files with 20 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
spacy/tests/tokenizer/test_urls.py
View File

@ -13,62 +13,68 @@ URLS = [
"mailto:foo-bar@baz-co.com" "mailto:foo-bar@baz-co.com"
] ]
# Punctuation we want to check is split away before the URL # Punctuation we want to check is split away before the URL
PREFIXES = [ PREFIXES = [
"(", '"', ">" "(", '"', ">"
] ]
# Punctuation we want to check is split away after the URL # Punctuation we want to check is split away after the URL
SUFFIXES = [ SUFFIXES = [
'"', ":", ">"] '"', ":", ">"]
@pytest.mark.parametrize("text", URLS)
def test_simple_url(tokenizer, text): @pytest.mark.parametrize("url", URLS)
tokens = tokenizer(text) def test_tokenizer_handles_simple_url(tokenizer, url):
assert tokens[0].orth_ == text tokens = tokenizer(url)
assert len(tokens) == 1 assert len(tokens) == 1
assert tokens[0].text == url
@pytest.mark.parametrize("prefix", PREFIXES) @pytest.mark.parametrize("prefix", PREFIXES)
@pytest.mark.parametrize("url", URLS) @pytest.mark.parametrize("url", URLS)
def test_prefixed_url(tokenizer, prefix, url): def test_tokenizer_handles_prefixed_url(tokenizer, prefix, url):
tokens = tokenizer(prefix + url) tokens = tokenizer(prefix + url)
assert tokens[0].text == prefix assert tokens[0].text == prefix
assert tokens[1].text == url assert tokens[1].text == url
assert len(tokens) == 2 assert len(tokens) == 2
@pytest.mark.parametrize("suffix", SUFFIXES) @pytest.mark.parametrize("suffix", SUFFIXES)
@pytest.mark.parametrize("url", URLS) @pytest.mark.parametrize("url", URLS)
def test_suffixed_url(tokenizer, url, suffix): def test_tokenizer_handles_suffixed_url(tokenizer, url, suffix):
tokens = tokenizer(url + suffix) tokens = tokenizer(url + suffix)
assert tokens[0].text == url assert tokens[0].text == url
assert tokens[1].text == suffix assert tokens[1].text == suffix
assert len(tokens) == 2 assert len(tokens) == 2
@pytest.mark.parametrize("prefix", PREFIXES) @pytest.mark.parametrize("prefix", PREFIXES)
@pytest.mark.parametrize("suffix", SUFFIXES) @pytest.mark.parametrize("suffix", SUFFIXES)
@pytest.mark.parametrize("url", URLS) @pytest.mark.parametrize("url", URLS)
def test_surround_url(tokenizer, prefix, suffix, url): def test_tokenizer_handles_surround_url(tokenizer, prefix, suffix, url):
tokens = tokenizer(prefix + url + suffix) tokens = tokenizer(prefix + url + suffix)
assert tokens[0].text == prefix assert tokens[0].text == prefix
assert tokens[1].text == url assert tokens[1].text == url
assert tokens[2].text == suffix assert tokens[2].text == suffix
assert len(tokens) == 3
@pytest.mark.parametrize("prefix1", PREFIXES) @pytest.mark.parametrize("prefix1", PREFIXES)
@pytest.mark.parametrize("prefix2", PREFIXES) @pytest.mark.parametrize("prefix2", PREFIXES)
@pytest.mark.parametrize("url", URLS) @pytest.mark.parametrize("url", URLS)
def test_two_prefix_url(tokenizer, prefix1, prefix2, url): def test_tokenizer_handles_two_prefix_url(tokenizer, prefix1, prefix2, url):
tokens = tokenizer(prefix1 + prefix2 + url) tokens = tokenizer(prefix1 + prefix2 + url)
assert tokens[0].text == prefix1 assert tokens[0].text == prefix1
assert tokens[1].text == prefix2 assert tokens[1].text == prefix2
assert tokens[2].text == url assert tokens[2].text == url
assert len(tokens) == 3 assert len(tokens) == 3
@pytest.mark.parametrize("suffix1", SUFFIXES) @pytest.mark.parametrize("suffix1", SUFFIXES)
@pytest.mark.parametrize("suffix2", SUFFIXES) @pytest.mark.parametrize("suffix2", SUFFIXES)
@pytest.mark.parametrize("url", URLS) @pytest.mark.parametrize("url", URLS)
def test_two_prefix_url(tokenizer, suffix1, suffix2, url): def test_tokenizer_handles_two_prefix_url(tokenizer, suffix1, suffix2, url):
tokens = tokenizer(url + suffix1 + suffix2) tokens = tokenizer(url + suffix1 + suffix2)
assert tokens[0].text == url assert tokens[0].text == url
assert tokens[1].text == suffix1 assert tokens[1].text == suffix1