mirror of
https://github.com/explosion/spaCy.git
synced 2024-09-21 03:19:13 +03:00
392c4880d9
* Switch to train_dataset() function in train CLI * Fixes for pipe() methods in pipeline components * Don't clobber `examples` variable with `as_example` in pipe() methods * Remove unnecessary traversals of `examples` * Update Parser.pipe() for Examples * Add `as_examples` kwarg to `pipe()` with implementation to return `Example`s * Accept `Doc` or `Example` in `pipe()` with `_get_doc()` (copied from `Pipe`) * Fixes to Example implementation in spacy.gold * Move `make_projective` from an attribute of Example to an argument of `Example.get_gold_parses()` * Head of 0 are not treated as unset * Unset heads are set to self rather than `None` (which causes problems while projectivizing) * Check for `Doc` (not just not `None`) when creating GoldParses for pre-merged example * Don't clobber `examples` variable in `iter_gold_docs()` * Add/modify gold tests for handling projectivity * In JSON roundtrip compare results from `dev_dataset` rather than `train_dataset` to avoid projectivization (and other potential modifications) * Add test for projective train vs. nonprojective dev versions of the same `Doc` * Handle ignore_misaligned as arg rather than attr Move `ignore_misaligned` from an attribute of `Example` to an argument to `Example.get_gold_parses()`, which makes it parallel to `make_projective`. Add test with old and new align that checks whether `ignore_misaligned` errors are raised as expected (only for new align). * Remove unused attrs from gold.pxd Remove `ignore_misaligned` and `make_projective` from `gold.pxd` * Restructure Example with merged sents as default An `Example` now includes a single `TokenAnnotation` that includes all the information from one `Doc` (=JSON `paragraph`). If required, the individual sentences can be returned as a list of examples with `Example.split_sents()` with no raw text available. * Input/output a single `Example.token_annotation` * Add `sent_starts` to `TokenAnnotation` to handle sentence boundaries * Replace `Example.merge_sents()` with `Example.split_sents()` * Modify components to use a single `Example.token_annotation` * Pipeline components * conllu2json converter * Rework/rename `add_token_annotation()` and `add_doc_annotation()` to `set_token_annotation()` and `set_doc_annotation()`, functions that set rather then appending/extending. * Rename `morphology` to `morphs` in `TokenAnnotation` and `GoldParse` * Add getters to `TokenAnnotation` to supply default values when a given attribute is not available * `Example.get_gold_parses()` in `spacy.gold._make_golds()` is only applied on single examples, so the `GoldParse` is returned saved in the provided `Example` rather than creating a new `Example` with no other internal annotation * Update tests for API changes and `merge_sents()` vs. `split_sents()` * Refer to Example.goldparse in iter_gold_docs() Use `Example.goldparse` in `iter_gold_docs()` instead of `Example.gold` because a `None` `GoldParse` is generated with ignore_misaligned and generating it on-the-fly can raise an unwanted AlignmentError * Fix make_orth_variants() Fix bug in make_orth_variants() related to conversion from multiple to one TokenAnnotation per Example. * Add basic test for make_orth_variants() * Replace try/except with conditionals * Replace default morph value with set
65 lines
1.3 KiB
Cython
65 lines
1.3 KiB
Cython
from cymem.cymem cimport Pool
|
|
|
|
from spacy.tokens import Doc
|
|
from .typedefs cimport attr_t
|
|
from .syntax.transition_system cimport Transition
|
|
|
|
|
|
cdef struct GoldParseC:
|
|
int* tags
|
|
int* heads
|
|
int* has_dep
|
|
int* sent_start
|
|
attr_t* labels
|
|
int** brackets
|
|
Transition* ner
|
|
|
|
|
|
cdef class GoldParse:
|
|
cdef Pool mem
|
|
|
|
cdef GoldParseC c
|
|
cdef readonly TokenAnnotation orig
|
|
|
|
cdef int length
|
|
cdef public int loss
|
|
cdef public list words
|
|
cdef public list tags
|
|
cdef public list morphs
|
|
cdef public list heads
|
|
cdef public list labels
|
|
cdef public dict orths
|
|
cdef public list ner
|
|
cdef public dict brackets
|
|
cdef public dict cats
|
|
cdef public dict links
|
|
|
|
cdef readonly list cand_to_gold
|
|
cdef readonly list gold_to_cand
|
|
|
|
|
|
cdef class TokenAnnotation:
|
|
cdef public list ids
|
|
cdef public list words
|
|
cdef public list tags
|
|
cdef public list heads
|
|
cdef public list deps
|
|
cdef public list entities
|
|
cdef public list morphs
|
|
cdef public list sent_starts
|
|
cdef public list brackets
|
|
|
|
|
|
cdef class DocAnnotation:
|
|
cdef public object cats
|
|
cdef public object links
|
|
|
|
|
|
cdef class Example:
|
|
cdef public object doc
|
|
cdef public TokenAnnotation token_annotation
|
|
cdef public DocAnnotation doc_annotation
|
|
cdef public object goldparse
|
|
|
|
|