mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-11 12:18:04 +03:00
b5d999e510
* Add doc.cats to spacy.gold at the paragraph level
Support `doc.cats` as `"cats": [{"label": string, "value": number}]` in
the spacy JSON training format at the paragraph level.
* `spacy.gold.docs_to_json()` writes `docs.cats`
* `GoldCorpus` reads in cats in each `GoldParse`
* Update instances of gold_tuples to handle cats
Update iteration over gold_tuples / gold_parses to handle addition of
cats at the paragraph level.
* Add textcat to train CLI
* Add textcat options to train CLI
* Add textcat labels in `TextCategorizer.begin_training()`
* Add textcat evaluation to `Scorer`:
* For binary exclusive classes with provided label: F1 for label
* For 2+ exclusive classes: F1 macro average
* For multilabel (not exclusive): ROC AUC macro average (currently
relying on sklearn)
* Provide user info on textcat evaluation settings, potential
incompatibilities
* Provide pipeline to Scorer in `Language.evaluate` for textcat config
* Customize train CLI output to include only metrics relevant to current
pipeline
* Add textcat evaluation to evaluate CLI
* Fix handling of unset arguments and config params
Fix handling of unset arguments and model confiug parameters in Scorer
initialization.
* Temporarily add sklearn requirement
* Remove sklearn version number
* Improve Scorer handling of models without textcats
* Fixing Scorer handling of models without textcats
* Update Scorer output for python 2.7
* Modify inf in Scorer for python 2.7
* Auto-format
Also make small adjustments to make auto-formatting with black easier and produce nicer results
* Move error message to Errors
* Update documentation
* Add cats to annotation JSON format [ci skip]
* Fix tpl flag and docs [ci skip]
* Switch to internal roc_auc_score
Switch to internal `roc_auc_score()` adapted from scikit-learn.
* Add AUCROCScore tests and improve errors/warnings
* Add tests for AUCROCScore and roc_auc_score
* Add missing error for only positive/negative values
* Remove unnecessary warnings and errors
* Make reduced roc_auc_score functions private
Because most of the checks and warnings have been stripped for the
internal functions and access is only intended through `ROCAUCScore`,
make the functions for roc_auc_score adapted from scikit-learn private.
* Check that data corresponds with multilabel flag
Check that the training instances correspond with the multilabel flag,
adding the multilabel flag if required.
* Add textcat score to early stopping check
* Add more checks to debug-data for textcat
* Add example training data for textcat
* Add more checks to textcat train CLI
* Check configuration when extending base model
* Fix typos
* Update textcat example data
* Provide licensing details and licenses for data
* Remove two labels with no positive instances from jigsaw-toxic-comment
data.
Co-authored-by: Ines Montani <ines@ines.io>
97 lines
3.3 KiB
Python
97 lines
3.3 KiB
Python
# coding: utf8
|
|
from __future__ import unicode_literals, division, print_function
|
|
|
|
import plac
|
|
from timeit import default_timer as timer
|
|
from wasabi import Printer
|
|
|
|
from ..gold import GoldCorpus
|
|
from .. import util
|
|
from .. import displacy
|
|
|
|
|
|
@plac.annotations(
|
|
model=("Model name or path", "positional", None, str),
|
|
data_path=("Location of JSON-formatted evaluation data", "positional", None, str),
|
|
gold_preproc=("Use gold preprocessing", "flag", "G", bool),
|
|
gpu_id=("Use GPU", "option", "g", int),
|
|
displacy_path=("Directory to output rendered parses as HTML", "option", "dp", str),
|
|
displacy_limit=("Limit of parses to render as HTML", "option", "dl", int),
|
|
return_scores=("Return dict containing model scores", "flag", "R", bool),
|
|
)
|
|
def evaluate(
|
|
model,
|
|
data_path,
|
|
gpu_id=-1,
|
|
gold_preproc=False,
|
|
displacy_path=None,
|
|
displacy_limit=25,
|
|
return_scores=False,
|
|
):
|
|
"""
|
|
Evaluate a model. To render a sample of parses in a HTML file, set an
|
|
output directory as the displacy_path argument.
|
|
"""
|
|
msg = Printer()
|
|
util.fix_random_seed()
|
|
if gpu_id >= 0:
|
|
util.use_gpu(gpu_id)
|
|
util.set_env_log(False)
|
|
data_path = util.ensure_path(data_path)
|
|
displacy_path = util.ensure_path(displacy_path)
|
|
if not data_path.exists():
|
|
msg.fail("Evaluation data not found", data_path, exits=1)
|
|
if displacy_path and not displacy_path.exists():
|
|
msg.fail("Visualization output directory not found", displacy_path, exits=1)
|
|
corpus = GoldCorpus(data_path, data_path)
|
|
nlp = util.load_model(model)
|
|
dev_docs = list(corpus.dev_docs(nlp, gold_preproc=gold_preproc))
|
|
begin = timer()
|
|
scorer = nlp.evaluate(dev_docs, verbose=False)
|
|
end = timer()
|
|
nwords = sum(len(doc_gold[0]) for doc_gold in dev_docs)
|
|
results = {
|
|
"Time": "%.2f s" % (end - begin),
|
|
"Words": nwords,
|
|
"Words/s": "%.0f" % (nwords / (end - begin)),
|
|
"TOK": "%.2f" % scorer.token_acc,
|
|
"POS": "%.2f" % scorer.tags_acc,
|
|
"UAS": "%.2f" % scorer.uas,
|
|
"LAS": "%.2f" % scorer.las,
|
|
"NER P": "%.2f" % scorer.ents_p,
|
|
"NER R": "%.2f" % scorer.ents_r,
|
|
"NER F": "%.2f" % scorer.ents_f,
|
|
"Textcat": "%.2f" % scorer.textcat_score,
|
|
}
|
|
msg.table(results, title="Results")
|
|
|
|
if displacy_path:
|
|
docs, golds = zip(*dev_docs)
|
|
render_deps = "parser" in nlp.meta.get("pipeline", [])
|
|
render_ents = "ner" in nlp.meta.get("pipeline", [])
|
|
render_parses(
|
|
docs,
|
|
displacy_path,
|
|
model_name=model,
|
|
limit=displacy_limit,
|
|
deps=render_deps,
|
|
ents=render_ents,
|
|
)
|
|
msg.good("Generated {} parses as HTML".format(displacy_limit), displacy_path)
|
|
if return_scores:
|
|
return scorer.scores
|
|
|
|
|
|
def render_parses(docs, output_path, model_name="", limit=250, deps=True, ents=True):
|
|
docs[0].user_data["title"] = model_name
|
|
if ents:
|
|
html = displacy.render(docs[:limit], style="ent", page=True)
|
|
with (output_path / "entities.html").open("w", encoding="utf8") as file_:
|
|
file_.write(html)
|
|
if deps:
|
|
html = displacy.render(
|
|
docs[:limit], style="dep", page=True, options={"compact": True}
|
|
)
|
|
with (output_path / "parses.html").open("w", encoding="utf8") as file_:
|
|
file_.write(html)
|