explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-11-11 12:18:04 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into spacy.io

Browse Source
This commit is contained in:
Ines Montani 2019-10-21 12:26:19 +02:00
commit 2f223a5dd8
5 changed files with 26 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
spacy/cli/debug_data.py
View File

@ -206,6 +206,9 @@ def debug_data(
missing_values, "value" if missing_values == 1 else "values"
)
)
for label in new_labels:
if len(label) == 0:
msg.fail("Empty label found in new labels")
if new_labels:
labels_with_counts = [
(label, count)

1
spacy/errors.py
View File

@ -503,6 +503,7 @@ class Errors(object):
"names: {names}")
E175 = ("Can't remove rule for unknown match pattern ID: {key}")
E176 = ("Alias '{alias}' is not defined in the Knowledge Base.")
E177 = ("Ill-formed IOB input detected: {tag}")
@add_codes

2
spacy/gold.pyx
View File

@ -522,6 +522,8 @@ def _consume_ent(tags):
tags.pop(0)
label = tag[2:]
if length == 1:
if len(label) == 0:
raise ValueError(Errors.E177.format(tag=tag))
return ["U-" + label]
else:
start = "B-" + label

12
spacy/tests/test_gold.py
View File

@ -2,7 +2,7 @@
from __future__ import unicode_literals
from spacy.gold import biluo_tags_from_offsets, offsets_from_biluo_tags
from spacy.gold import spans_from_biluo_tags, GoldParse
from spacy.gold import spans_from_biluo_tags, GoldParse, iob_to_biluo
from spacy.gold import GoldCorpus, docs_to_json
from spacy.lang.en import English
from spacy.tokens import Doc
@ -87,6 +87,16 @@ def test_gold_ner_missing_tags(en_tokenizer):
gold = GoldParse(doc, entities=biluo_tags) # noqa: F841
def test_iob_to_biluo():
good_iob = ["O", "O", "B-LOC", "I-LOC", "O", "B-PERSON"]
good_biluo = ["O", "O", "B-LOC", "L-LOC", "O", "U-PERSON"]
bad_iob = ["O", "O", "\"", "B-LOC", "I-LOC"]
converted_biluo = iob_to_biluo(good_iob)
assert good_biluo == converted_biluo
with pytest.raises(ValueError):
iob_to_biluo(bad_iob)
def test_roundtrip_docs_to_json():
text = "I flew to Silicon Valley via London."
cats = {"TRAVEL": 1.0, "BAKING": 0.0}

19
website/docs/api/vocab.md
View File

@ -166,10 +166,9 @@ cosines are calculated in minibatches, to reduce memory usage.
## Vocab.get_vector {#get_vector tag="method" new="2"}
Retrieve a vector for a word in the vocabulary. Words can be looked up by string
or hash value. If no vectors data is loaded, a `ValueError` is raised.
If `minn` is defined, then the resulting vector uses Fasttext's
subword features by average over ngrams of `orth`. (Introduced in spaCy `v2.1`)
or hash value. If no vectors data is loaded, a `ValueError` is raised. If `minn`
is defined, then the resulting vector uses [FastText](https://fasttext.cc/)'s
subword features by average over ngrams of `orth` (introduced in spaCy `v2.1`).
> #### Example
>
@ -178,12 +177,12 @@ subword features by average over ngrams of `orth`. (Introduced in spaCy `v2.1`)
> nlp.vocab.get_vector("apple", minn=1, maxn=5)
> ```
| Name | Type | Description |
| ----------- | ---------------------------------------- | ---------------------------------------------------------------------------------------------- |
| `orth` | int / unicode | The hash value of a word, or its unicode string. |
| `minn` | int | Minimum n-gram length used for Fasttext's ngram computation. Defaults to the length of `orth`. |
| `maxn` | int | Maximum n-gram length used for Fasttext's ngram computation. Defaults to the length of `orth`. |
| **RETURNS** | `numpy.ndarray[ndim=1, dtype='float32']` | A word vector. Size and shape are determined by the `Vocab.vectors` instance. |
| Name | Type | Description |
| ----------------------------------- | ---------------------------------------- | ---------------------------------------------------------------------------------------------- |
| `orth` | int / unicode | The hash value of a word, or its unicode string. |
| `minn` <Tag variant="new">2.1</Tag> | int | Minimum n-gram length used for FastText's ngram computation. Defaults to the length of `orth`. |
| `maxn` <Tag variant="new">2.1</Tag> | int | Maximum n-gram length used for FastText's ngram computation. Defaults to the length of `orth`. |
| **RETURNS** | `numpy.ndarray[ndim=1, dtype='float32']` | A word vector. Size and shape are determined by the `Vocab.vectors` instance. |
## Vocab.set_vector {#set_vector tag="method" new="2"}