diff --git a/spacy/syntax/nn_parser.pyx b/spacy/syntax/nn_parser.pyx
index c4edef137..85f7b5bb9 100644
--- a/spacy/syntax/nn_parser.pyx
+++ b/spacy/syntax/nn_parser.pyx
@@ -130,10 +130,6 @@ cdef class Parser:
def __reduce__(self):
return (Parser, (self.vocab, self.moves, self.model), None, None)
- @property
- def tok2vec(self):
- return self.model.tok2vec
-
@property
def move_names(self):
names = []
diff --git a/spacy/tests/pipeline/test_entity_linker.py b/spacy/tests/pipeline/test_entity_linker.py
index b10d55267..0c89a2e14 100644
--- a/spacy/tests/pipeline/test_entity_linker.py
+++ b/spacy/tests/pipeline/test_entity_linker.py
@@ -6,6 +6,7 @@ import pytest
from spacy.kb import KnowledgeBase
from spacy.lang.en import English
from spacy.pipeline import EntityRuler
+from spacy.tokens import Span
@pytest.fixture
@@ -171,3 +172,31 @@ def test_preserving_links_asdoc(nlp):
for s_ent in sent_doc.ents:
if s_ent.text == orig_text:
assert s_ent.kb_id_ == orig_kb_id
+
+
+def test_preserving_links_ents(nlp):
+ """Test that doc.ents preserves KB annotations"""
+ text = "She lives in Boston. He lives in Denver."
+ doc = nlp(text)
+ assert len(list(doc.ents)) == 0
+
+ boston_ent = Span(doc, 3, 4, label="LOC", kb_id="Q1")
+ doc.ents = [boston_ent]
+ assert len(list(doc.ents)) == 1
+ assert list(doc.ents)[0].label_ == "LOC"
+ assert list(doc.ents)[0].kb_id_ == "Q1"
+
+
+def test_preserving_links_ents_2(nlp):
+ """Test that doc.ents preserves KB annotations"""
+ text = "She lives in Boston. He lives in Denver."
+ doc = nlp(text)
+ assert len(list(doc.ents)) == 0
+
+ loc = doc.vocab.strings.add("LOC")
+ q1 = doc.vocab.strings.add("Q1")
+
+ doc.ents = [(loc, q1, 3, 4)]
+ assert len(list(doc.ents)) == 1
+ assert list(doc.ents)[0].label_ == "LOC"
+ assert list(doc.ents)[0].kb_id_ == "Q1"
diff --git a/spacy/tokens/_retokenize.pyx b/spacy/tokens/_retokenize.pyx
index 55ad909bb..741be7e6a 100644
--- a/spacy/tokens/_retokenize.pyx
+++ b/spacy/tokens/_retokenize.pyx
@@ -146,6 +146,7 @@ def _merge(Doc doc, merges):
syntactic root of the span.
RETURNS (Token): The first newly merged token.
"""
+ cdef int i, merge_index, start, end, token_index
cdef Span span
cdef const LexemeC* lex
cdef TokenC* token
diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx
index e5c213383..e863b0807 100644
--- a/spacy/tokens/doc.pyx
+++ b/spacy/tokens/doc.pyx
@@ -534,7 +534,7 @@ cdef class Doc:
cdef attr_t entity_type
cdef int ent_start, ent_end
for ent_info in ents:
- entity_type, ent_start, ent_end = get_entity_info(ent_info)
+ entity_type, kb_id, ent_start, ent_end = get_entity_info(ent_info)
for token_index in range(ent_start, ent_end):
if token_index in tokens_in_ents.keys():
raise ValueError(Errors.E103.format(
@@ -542,7 +542,7 @@ cdef class Doc:
tokens_in_ents[token_index][1],
self.vocab.strings[tokens_in_ents[token_index][2]]),
span2=(ent_start, ent_end, self.vocab.strings[entity_type])))
- tokens_in_ents[token_index] = (ent_start, ent_end, entity_type)
+ tokens_in_ents[token_index] = (ent_start, ent_end, entity_type, kb_id)
cdef int i
for i in range(self.length):
self.c[i].ent_type = 0
@@ -551,16 +551,18 @@ cdef class Doc:
cdef attr_t ent_type
cdef int start, end
for ent_info in ents:
- ent_type, start, end = get_entity_info(ent_info)
+ ent_type, ent_kb_id, start, end = get_entity_info(ent_info)
if ent_type is None or ent_type < 0:
# Mark as O
for i in range(start, end):
self.c[i].ent_type = 0
+ self.c[i].ent_kb_id = 0
self.c[i].ent_iob = 2
else:
# Mark (inside) as I
for i in range(start, end):
self.c[i].ent_type = ent_type
+ self.c[i].ent_kb_id = ent_kb_id
self.c[i].ent_iob = 1
# Set start as B
self.c[start].ent_iob = 3
@@ -1251,10 +1253,14 @@ def fix_attributes(doc, attributes):
def get_entity_info(ent_info):
if isinstance(ent_info, Span):
ent_type = ent_info.label
+ ent_kb_id = ent_info.kb_id
start = ent_info.start
end = ent_info.end
elif len(ent_info) == 3:
ent_type, start, end = ent_info
+ ent_kb_id = 0
+ elif len(ent_info) == 4:
+ ent_type, ent_kb_id, start, end = ent_info
else:
- ent_id, ent_type, start, end = ent_info
- return ent_type, start, end
+ ent_id, ent_kb_id, ent_type, start, end = ent_info
+ return ent_type, ent_kb_id, start, end
diff --git a/website/docs/api/annotation.md b/website/docs/api/annotation.md
index ac888cec9..7f7b46260 100644
--- a/website/docs/api/annotation.md
+++ b/website/docs/api/annotation.md
@@ -186,63 +186,63 @@ The German part-of-speech tagger uses the
annotation scheme. We also map the tags to the simpler Google Universal POS tag
set.
-| Tag | POS | Morphology | Description |
-| --------- | ------- | ------------------------------------------- | ------------------------------------------------- |
-| `$(` | `PUNCT` | `PunctType=brck` | other sentence-internal punctuation mark |
-| `$,` | `PUNCT` | `PunctType=comm` | comma |
-| `$.` | `PUNCT` | `PunctType=peri` | sentence-final punctuation mark |
-| `ADJA` | `ADJ` | | adjective, attributive |
-| `ADJD` | `ADJ` | `Variant=short` | adjective, adverbial or predicative |
-| `ADV` | `ADV` | | adverb |
-| `APPO` | `ADP` | `AdpType=post` | postposition |
-| `APPR` | `ADP` | `AdpType=prep` | preposition; circumposition left |
-| `APPRART` | `ADP` | `AdpType=prep PronType=art` | preposition with article |
-| `APZR` | `ADP` | `AdpType=circ` | circumposition right |
-| `ART` | `DET` | `PronType=art` | definite or indefinite article |
-| `CARD` | `NUM` | `NumType=card` | cardinal number |
-| `FM` | `X` | `Foreign=yes` | foreign language material |
-| `ITJ` | `INTJ` | | interjection |
-| `KOKOM` | `CONJ` | `ConjType=comp` | comparative conjunction |
-| `KON` | `CONJ` | | coordinate conjunction |
-| `KOUI` | `SCONJ` | | subordinate conjunction with "zu" and infinitive |
-| `KOUS` | `SCONJ` | | subordinate conjunction with sentence |
-| `NE` | `PROPN` | | proper noun |
-| `NNE` | `PROPN` | | proper noun |
-| `NN` | `NOUN` | | noun, singular or mass |
-| `PROAV` | `ADV` | `PronType=dem` | pronominal adverb |
-| `PDAT` | `DET` | `PronType=dem` | attributive demonstrative pronoun |
-| `PDS` | `PRON` | `PronType=dem` | substituting demonstrative pronoun |
-| `PIAT` | `DET` | `PronType=ind\|neg\|tot` | attributive indefinite pronoun without determiner |
-| `PIS` | `PRON` | `PronType=ind\|neg\|tot` | substituting indefinite pronoun |
-| `PPER` | `PRON` | `PronType=prs` | non-reflexive personal pronoun |
-| `PPOSAT` | `DET` | `Poss=yes PronType=prs` | attributive possessive pronoun |
-| `PPOSS` | `PRON` | `PronType=rel` | substituting possessive pronoun |
-| `PRELAT` | `DET` | `PronType=rel` | attributive relative pronoun |
-| `PRELS` | `PRON` | `PronType=rel` | substituting relative pronoun |
-| `PRF` | `PRON` | `PronType=prs Reflex=yes` | reflexive personal pronoun |
-| `PTKA` | `PART` | | particle with adjective or adverb |
-| `PTKANT` | `PART` | `PartType=res` | answer particle |
-| `PTKNEG` | `PART` | `Negative=yes` | negative particle |
-| `PTKVZ` | `PART` | `PartType=vbp` | separable verbal particle |
-| `PTKZU` | `PART` | `PartType=inf` | "zu" before infinitive |
-| `PWAT` | `DET` | `PronType=int` | attributive interrogative pronoun |
-| `PWAV` | `ADV` | `PronType=int` | adverbial interrogative or relative pronoun |
-| `PWS` | `PRON` | `PronType=int` | substituting interrogative pronoun |
-| `TRUNC` | `X` | `Hyph=yes` | word remnant |
-| `VAFIN` | `AUX` | `Mood=ind VerbForm=fin` | finite verb, auxiliary |
-| `VAIMP` | `AUX` | `Mood=imp VerbForm=fin` | imperative, auxiliary |
-| `VAINF` | `AUX` | `VerbForm=inf` | infinitive, auxiliary |
-| `VAPP` | `AUX` | `Aspect=perf VerbForm=fin` | perfect participle, auxiliary |
-| `VMFIN` | `VERB` | `Mood=ind VerbForm=fin VerbType=mod` | finite verb, modal |
-| `VMINF` | `VERB` | `VerbForm=fin VerbType=mod` | infinitive, modal |
-| `VMPP` | `VERB` | `Aspect=perf VerbForm=part VerbType=mod` | perfect participle, modal |
-| `VVFIN` | `VERB` | `Mood=ind VerbForm=fin` | finite verb, full |
-| `VVIMP` | `VERB` | `Mood=imp VerbForm=fin` | imperative, full |
-| `VVINF` | `VERB` | `VerbForm=inf` | infinitive, full |
-| `VVIZU` | `VERB` | `VerbForm=inf` | infinitive with "zu", full |
-| `VVPP` | `VERB` | `Aspect=perf VerbForm=part` | perfect participle, full |
-| `XY` | `X` | | non-word containing non-letter |
-| `SP` | `SPACE` | | space |
+| Tag | POS | Morphology | Description |
+| --------- | ------- | ---------------------------------------- | ------------------------------------------------- |
+| `$(` | `PUNCT` | `PunctType=brck` | other sentence-internal punctuation mark |
+| `$,` | `PUNCT` | `PunctType=comm` | comma |
+| `$.` | `PUNCT` | `PunctType=peri` | sentence-final punctuation mark |
+| `ADJA` | `ADJ` | | adjective, attributive |
+| `ADJD` | `ADJ` | `Variant=short` | adjective, adverbial or predicative |
+| `ADV` | `ADV` | | adverb |
+| `APPO` | `ADP` | `AdpType=post` | postposition |
+| `APPR` | `ADP` | `AdpType=prep` | preposition; circumposition left |
+| `APPRART` | `ADP` | `AdpType=prep PronType=art` | preposition with article |
+| `APZR` | `ADP` | `AdpType=circ` | circumposition right |
+| `ART` | `DET` | `PronType=art` | definite or indefinite article |
+| `CARD` | `NUM` | `NumType=card` | cardinal number |
+| `FM` | `X` | `Foreign=yes` | foreign language material |
+| `ITJ` | `INTJ` | | interjection |
+| `KOKOM` | `CONJ` | `ConjType=comp` | comparative conjunction |
+| `KON` | `CONJ` | | coordinate conjunction |
+| `KOUI` | `SCONJ` | | subordinate conjunction with "zu" and infinitive |
+| `KOUS` | `SCONJ` | | subordinate conjunction with sentence |
+| `NE` | `PROPN` | | proper noun |
+| `NNE` | `PROPN` | | proper noun |
+| `NN` | `NOUN` | | noun, singular or mass |
+| `PROAV` | `ADV` | `PronType=dem` | pronominal adverb |
+| `PDAT` | `DET` | `PronType=dem` | attributive demonstrative pronoun |
+| `PDS` | `PRON` | `PronType=dem` | substituting demonstrative pronoun |
+| `PIAT` | `DET` | `PronType=ind\|neg\|tot` | attributive indefinite pronoun without determiner |
+| `PIS` | `PRON` | `PronType=ind\|neg\|tot` | substituting indefinite pronoun |
+| `PPER` | `PRON` | `PronType=prs` | non-reflexive personal pronoun |
+| `PPOSAT` | `DET` | `Poss=yes PronType=prs` | attributive possessive pronoun |
+| `PPOSS` | `PRON` | `PronType=rel` | substituting possessive pronoun |
+| `PRELAT` | `DET` | `PronType=rel` | attributive relative pronoun |
+| `PRELS` | `PRON` | `PronType=rel` | substituting relative pronoun |
+| `PRF` | `PRON` | `PronType=prs Reflex=yes` | reflexive personal pronoun |
+| `PTKA` | `PART` | | particle with adjective or adverb |
+| `PTKANT` | `PART` | `PartType=res` | answer particle |
+| `PTKNEG` | `PART` | `Negative=yes` | negative particle |
+| `PTKVZ` | `PART` | `PartType=vbp` | separable verbal particle |
+| `PTKZU` | `PART` | `PartType=inf` | "zu" before infinitive |
+| `PWAT` | `DET` | `PronType=int` | attributive interrogative pronoun |
+| `PWAV` | `ADV` | `PronType=int` | adverbial interrogative or relative pronoun |
+| `PWS` | `PRON` | `PronType=int` | substituting interrogative pronoun |
+| `TRUNC` | `X` | `Hyph=yes` | word remnant |
+| `VAFIN` | `AUX` | `Mood=ind VerbForm=fin` | finite verb, auxiliary |
+| `VAIMP` | `AUX` | `Mood=imp VerbForm=fin` | imperative, auxiliary |
+| `VAINF` | `AUX` | `VerbForm=inf` | infinitive, auxiliary |
+| `VAPP` | `AUX` | `Aspect=perf VerbForm=fin` | perfect participle, auxiliary |
+| `VMFIN` | `VERB` | `Mood=ind VerbForm=fin VerbType=mod` | finite verb, modal |
+| `VMINF` | `VERB` | `VerbForm=fin VerbType=mod` | infinitive, modal |
+| `VMPP` | `VERB` | `Aspect=perf VerbForm=part VerbType=mod` | perfect participle, modal |
+| `VVFIN` | `VERB` | `Mood=ind VerbForm=fin` | finite verb, full |
+| `VVIMP` | `VERB` | `Mood=imp VerbForm=fin` | imperative, full |
+| `VVINF` | `VERB` | `VerbForm=inf` | infinitive, full |
+| `VVIZU` | `VERB` | `VerbForm=inf` | infinitive with "zu", full |
+| `VVPP` | `VERB` | `Aspect=perf VerbForm=part` | perfect participle, full |
+| `XY` | `X` | | non-word containing non-letter |
+| `SP` | `SPACE` | | space |
@@ -379,51 +379,51 @@ The German dependency labels use the
[TIGER Treebank](http://www.ims.uni-stuttgart.de/forschung/ressourcen/korpora/TIGERCorpus/annotation/index.html)
annotation scheme.
-| Label | Description |
-| ------ | ------------------------------- |
-| `ac` | adpositional case marker |
-| `adc` | adjective component |
-| `ag` | genitive attribute |
-| `ams` | measure argument of adjective |
-| `app` | apposition |
-| `avc` | adverbial phrase component |
-| `cc` | comparative complement |
-| `cd` | coordinating conjunction |
-| `cj` | conjunct |
-| `cm` | comparative conjunction |
-| `cp` | complementizer |
-| `cvc` | collocational verb construction |
-| `da` | dative |
-| `dm` | discourse marker |
-| `ep` | expletive es |
-| `ju` | junctor |
-| `mnr` | postnominal modifier |
-| `mo` | modifier |
-| `ng` | negation |
-| `nk` | noun kernel element |
-| `nmc` | numerical component |
-| `oa` | accusative object |
-| `oa2` | second accusative object |
-| `oc` | clausal object |
-| `og` | genitive object |
-| `op` | prepositional object |
-| `par` | parenthetical element |
-| `pd` | predicate |
-| `pg` | phrasal genitive |
-| `ph` | placeholder |
-| `pm` | morphological particle |
-| `pnc` | proper noun component |
-| `punct` | punctuation |
-| `rc` | relative clause |
-| `re` | repeated element |
-| `rs` | reported speech |
-| `sb` | subject |
-| `sbp` | passivized subject (PP) |
-| `sp` | subject or predicate |
-| `svp` | separable verb prefix |
-| `uc` | unit component |
-| `vo` | vocative |
-| `ROOT` | root |
+| Label | Description |
+| ------- | ------------------------------- |
+| `ac` | adpositional case marker |
+| `adc` | adjective component |
+| `ag` | genitive attribute |
+| `ams` | measure argument of adjective |
+| `app` | apposition |
+| `avc` | adverbial phrase component |
+| `cc` | comparative complement |
+| `cd` | coordinating conjunction |
+| `cj` | conjunct |
+| `cm` | comparative conjunction |
+| `cp` | complementizer |
+| `cvc` | collocational verb construction |
+| `da` | dative |
+| `dm` | discourse marker |
+| `ep` | expletive es |
+| `ju` | junctor |
+| `mnr` | postnominal modifier |
+| `mo` | modifier |
+| `ng` | negation |
+| `nk` | noun kernel element |
+| `nmc` | numerical component |
+| `oa` | accusative object |
+| `oa2` | second accusative object |
+| `oc` | clausal object |
+| `og` | genitive object |
+| `op` | prepositional object |
+| `par` | parenthetical element |
+| `pd` | predicate |
+| `pg` | phrasal genitive |
+| `ph` | placeholder |
+| `pm` | morphological particle |
+| `pnc` | proper noun component |
+| `punct` | punctuation |
+| `rc` | relative clause |
+| `re` | repeated element |
+| `rs` | reported speech |
+| `sb` | subject |
+| `sbp` | passivized subject (PP) |
+| `sp` | subject or predicate |
+| `svp` | separable verb prefix |
+| `uc` | unit component |
+| `vo` | vocative |
+| `ROOT` | root |
@@ -584,8 +584,8 @@ data.
```python
### Entry structure
{
- "orth": string,
- "id": int,
+ "orth": string, # the word text
+ "id": int, # can correspond to row in vectors table
"lower": string,
"norm": string,
"shape": string
diff --git a/website/docs/api/cli.md b/website/docs/api/cli.md
index 32e3623b0..0920e7e07 100644
--- a/website/docs/api/cli.md
+++ b/website/docs/api/cli.md
@@ -174,12 +174,12 @@ All output files generated by this command are compatible with
-| ID | Description |
-| ------------------------------ | --------------------------------------------------------------- |
-| `auto` | Automatically pick converter based on file extension and file content (default). |
-| `conll`, `conllu`, `conllubio` | Universal Dependencies `.conllu` or `.conll` format. |
+| ID | Description |
+| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `auto` | Automatically pick converter based on file extension and file content (default). |
+| `conll`, `conllu`, `conllubio` | Universal Dependencies `.conllu` or `.conll` format. |
| `ner` | NER with IOB/IOB2 tags, one token per line with columns separated by whitespace. The first column is the token and the final column is the IOB tag. Sentences are separated by blank lines and documents are separated by the line `-DOCSTART- -X- O O`. Supports CoNLL 2003 NER format. See [sample data](https://github.com/explosion/spaCy/tree/master/examples/training/ner_example_data). |
-| `iob` | NER with IOB/IOB2 tags, one sentence per line with tokens separated by whitespace and annotation separated by `|`, either `word|B-ENT` or `word|POS|B-ENT`. See [sample data](https://github.com/explosion/spaCy/tree/master/examples/training/ner_example_data). |
+| `iob` | NER with IOB/IOB2 tags, one sentence per line with tokens separated by whitespace and annotation separated by `|`, either `word|B-ENT` or `word|POS|B-ENT`. See [sample data](https://github.com/explosion/spaCy/tree/master/examples/training/ner_example_data). |
## Train {#train}
@@ -291,26 +291,26 @@ $ python -m spacy pretrain [texts_loc] [vectors_model] [output_dir]
[--seed] [--n-iter] [--use-vectors] [--n-save_every] [--init-tok2vec] [--epoch-start]
```
-| Argument | Type | Description |
-| ----------------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------- |
-| `texts_loc` | positional | Path to JSONL file with raw texts to learn from, with text provided as the key `"text"` or tokens as the key `"tokens"`. [See here](#pretrain-jsonl) for details. |
-| `vectors_model` | positional | Name or path to spaCy model with vectors to learn from. |
-| `output_dir` | positional | Directory to write models to on each epoch. |
-| `--width`, `-cw` | option | Width of CNN layers. |
-| `--depth`, `-cd` | option | Depth of CNN layers. |
-| `--embed-rows`, `-er` | option | Number of embedding rows. |
-| `--loss-func`, `-L` | option | Loss function to use for the objective. Either `"L2"` or `"cosine"`. |
-| `--dropout`, `-d` | option | Dropout rate. |
-| `--batch-size`, `-bs` | option | Number of words per training batch. |
-| `--max-length`, `-xw` | option | Maximum words per example. Longer examples are discarded. |
-| `--min-length`, `-nw` | option | Minimum words per example. Shorter examples are discarded. |
-| `--seed`, `-s` | option | Seed for random number generators. |
-| `--n-iter`, `-i` | option | Number of iterations to pretrain. |
-| `--use-vectors`, `-uv` | flag | Whether to use the static vectors as input features. |
-| `--n-save-every`, `-se` | option | Save model every X batches. |
-| `--init-tok2vec`, `-t2v` 2.1 | option | Path to pretrained weights for the token-to-vector parts of the models. See `spacy pretrain`. Experimental.|
-| `--epoch-start`, `-es` 2.1.5 | option | The epoch to start counting at. Only relevant when using `--init-tok2vec` and the given weight file has been renamed. Prevents unintended overwriting of existing weight files.|
-| **CREATES** | weights | The pre-trained weights that can be used to initialize `spacy train`. |
+| Argument | Type | Description |
+| ----------------------------------------------------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `texts_loc` | positional | Path to JSONL file with raw texts to learn from, with text provided as the key `"text"` or tokens as the key `"tokens"`. [See here](#pretrain-jsonl) for details. |
+| `vectors_model` | positional | Name or path to spaCy model with vectors to learn from. |
+| `output_dir` | positional | Directory to write models to on each epoch. |
+| `--width`, `-cw` | option | Width of CNN layers. |
+| `--depth`, `-cd` | option | Depth of CNN layers. |
+| `--embed-rows`, `-er` | option | Number of embedding rows. |
+| `--loss-func`, `-L` | option | Loss function to use for the objective. Either `"L2"` or `"cosine"`. |
+| `--dropout`, `-d` | option | Dropout rate. |
+| `--batch-size`, `-bs` | option | Number of words per training batch. |
+| `--max-length`, `-xw` | option | Maximum words per example. Longer examples are discarded. |
+| `--min-length`, `-nw` | option | Minimum words per example. Shorter examples are discarded. |
+| `--seed`, `-s` | option | Seed for random number generators. |
+| `--n-iter`, `-i` | option | Number of iterations to pretrain. |
+| `--use-vectors`, `-uv` | flag | Whether to use the static vectors as input features. |
+| `--n-save-every`, `-se` | option | Save model every X batches. |
+| `--init-tok2vec`, `-t2v` 2.1 | option | Path to pretrained weights for the token-to-vector parts of the models. See `spacy pretrain`. Experimental. |
+| `--epoch-start`, `-es` 2.1.5 | option | The epoch to start counting at. Only relevant when using `--init-tok2vec` and the given weight file has been renamed. Prevents unintended overwriting of existing weight files. |
+| **CREATES** | weights | The pre-trained weights that can be used to initialize `spacy train`. |
### JSONL format for raw text {#pretrain-jsonl}
@@ -330,10 +330,10 @@ tokenization can be provided.
> srsly.write_jsonl("/path/to/text.jsonl", data)
> ```
-| Key | Type | Description |
-| -------- | ------- | -------------------------------------------- |
+| Key | Type | Description |
+| -------- | ------- | ---------------------------------------------------------- |
| `text` | unicode | The raw input text. Is not required if `tokens` available. |
-| `tokens` | list | Optional tokenization, one string per token. |
+| `tokens` | list | Optional tokenization, one string per token. |
```json
### Example
@@ -347,14 +347,17 @@ tokenization can be provided.
Create a new model directory from raw data, like word frequencies, Brown
clusters and word vectors. This command is similar to the `spacy model` command
-in v1.x.
+in v1.x. Note that in order to populate the model's vocab, you need to pass in a
+JSONL-formatted [vocabulary file](<(/api/annotation#vocab-jsonl)>) as
+`--jsonl-loc` with optional `id` values that correspond to the vectors table.
+Just loading in vectors will not automatically populate the vocab.
As of v2.1.0, the `--freqs-loc` and `--clusters-loc` are deprecated and have
been replaced with the `--jsonl-loc` argument, which lets you pass in a a
-[newline-delimited JSON](http://jsonlines.org/) (JSONL) file containing one
-lexical entry per line. For more details on the format, see the
+[JSONL](http://jsonlines.org/) file containing one lexical entry per line. For
+more details on the format, see the
[annotation specs](/api/annotation#vocab-jsonl).
@@ -368,7 +371,7 @@ $ python -m spacy init-model [lang] [output_dir] [--jsonl-loc] [--vectors-loc]
| ----------------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `lang` | positional | Model language [ISO code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes), e.g. `en`. |
| `output_dir` | positional | Model output directory. Will be created if it doesn't exist. |
-| `--jsonl-loc`, `-j` | option | Optional location of JSONL-formatted vocabulary file with lexical attributes. |
+| `--jsonl-loc`, `-j` | option | Optional location of JSONL-formatted [vocabulary file](/api/annotation#vocab-jsonl) with lexical attributes. |
| `--vectors-loc`, `-v` | option | Optional location of vectors file. Should be a tab-separated file in Word2Vec format where the first column contains the word and the remaining columns the values. File can be provided in `.txt` format or as a zipped text file in `.zip` or `.tar.gz` format. |
| `--prune-vectors`, `-V` | flag | Number of vectors to prune the vocabulary to. Defaults to `-1` for no pruning. |
| **CREATES** | model | A spaCy model containing the vocab and vectors. |
@@ -424,7 +427,7 @@ pip install dist/en_model-0.0.0.tar.gz
| `input_dir` | positional | Path to directory containing model data. |
| `output_dir` | positional | Directory to create package folder in. |
| `--meta-path`, `-m` 2 | option | Path to `meta.json` file (optional). |
-| `--create-meta`, `-c` 2 | flag | Create a `meta.json` file on the command line, even if one already exists in the directory. If an existing file is found, its entries will be shown as the defaults in the command line prompt.
-| `--force`, `-f` | flag | Force overwriting of existing folder in output directory. |
+| `--create-meta`, `-c` 2 | flag | Create a `meta.json` file on the command line, even if one already exists in the directory. If an existing file is found, its entries will be shown as the defaults in the command line prompt. |
+| `--force`, `-f` | flag | Force overwriting of existing folder in output directory. |
| `--help`, `-h` | flag | Show help message and available arguments. |
| **CREATES** | directory | A Python package containing the spaCy model. |