From b64243ed557a1cc591a2f436449072b27f432de7 Mon Sep 17 00:00:00 2001 From: Luka Dragar Date: Fri, 5 Aug 2022 10:10:18 +0200 Subject: [PATCH 001/179] Updates to Slovenian language (#11162) * Added examples for Slovene * Update spacy/lang/sl/examples.py Co-authored-by: Adriane Boyd * Corrected a typo in one of the sentences * Updated support for Slovenian * Some minor changes to corrections * Added forint currency * Corrected HYPHENS_PERMITTED regex and some formatting * Minor changes * Un-xfail tokenizer test * Format Co-authored-by: Luka Dragar Co-authored-by: Adriane Boyd --- spacy/lang/sl/__init__.py | 8 + spacy/lang/sl/lex_attrs.py | 145 ++++++++++ spacy/lang/sl/punctuation.py | 84 ++++++ spacy/lang/sl/stop_words.py | 394 +++++--------------------- spacy/lang/sl/tokenizer_exceptions.py | 272 ++++++++++++++++++ spacy/tests/lang/sl/test_text.py | 1 - 6 files changed, 585 insertions(+), 319 deletions(-) create mode 100644 spacy/lang/sl/lex_attrs.py create mode 100644 spacy/lang/sl/punctuation.py create mode 100644 spacy/lang/sl/tokenizer_exceptions.py diff --git a/spacy/lang/sl/__init__.py b/spacy/lang/sl/__init__.py index 9ddd676bf..0070e9fa1 100644 --- a/spacy/lang/sl/__init__.py +++ b/spacy/lang/sl/__init__.py @@ -1,9 +1,17 @@ +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES, TOKENIZER_PREFIXES from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from ...language import Language, BaseDefaults class SlovenianDefaults(BaseDefaults): stop_words = STOP_WORDS + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + lex_attr_getters = LEX_ATTRS class Slovenian(Language): diff --git a/spacy/lang/sl/lex_attrs.py b/spacy/lang/sl/lex_attrs.py new file mode 100644 index 000000000..958152e37 --- /dev/null +++ b/spacy/lang/sl/lex_attrs.py @@ -0,0 +1,145 @@ +from ...attrs import LIKE_NUM +from ...attrs import IS_CURRENCY +import unicodedata + + +_num_words = set( + """ + nula ničla nič ena dva tri štiri pet šest sedem osem + devet deset enajst dvanajst trinajst štirinajst petnajst + šestnajst sedemnajst osemnajst devetnajst dvajset trideset štirideset + petdeset šestdest sedemdeset osemdeset devedeset sto tisoč + milijon bilijon trilijon kvadrilijon nešteto + + en eden enega enemu ennem enim enih enima enimi ene eni eno + dveh dvema dvem dvoje trije treh trem tremi troje štirje štirih štirim štirimi + petih petim petimi šestih šestim šestimi sedmih sedmim sedmimi osmih osmim osmimi + devetih devetim devetimi desetih desetim desetimi enajstih enajstim enajstimi + dvanajstih dvanajstim dvanajstimi trinajstih trinajstim trinajstimi + šestnajstih šestnajstim šestnajstimi petnajstih petnajstim petnajstimi + sedemnajstih sedemnajstim sedemnajstimi osemnajstih osemnajstim osemnajstimi + devetnajstih devetnajstim devetnajstimi dvajsetih dvajsetim dvajsetimi + """.split() +) + +_ordinal_words = set( + """ + prvi drugi tretji četrti peti šesti sedmi osmi + deveti deseti enajsti dvanajsti trinajsti štirinajsti + petnajsti šestnajsti sedemnajsti osemnajsti devetnajsti + dvajseti trideseti štirideseti petdeseti šestdeseti sedemdeseti + osemdeseti devetdeseti stoti tisoči milijonti bilijonti + trilijonti kvadrilijonti nešteti + + prva druga tretja četrta peta šesta sedma osma + deveta deseta enajsta dvanajsta trinajsta štirnajsta + petnajsta šestnajsta sedemnajsta osemnajsta devetnajsta + dvajseta trideseta štirideseta petdeseta šestdeseta sedemdeseta + osemdeseta devetdeseta stota tisoča milijonta bilijonta + trilijonta kvadrilijonta nešteta + + prvo drugo tretje četrto peto šestro sedmo osmo + deveto deseto enajsto dvanajsto trinajsto štirnajsto + petnajsto šestnajsto sedemnajsto osemnajsto devetnajsto + dvajseto trideseto štirideseto petdeseto šestdeseto sedemdeseto + osemdeseto devetdeseto stoto tisočo milijonto bilijonto + trilijonto kvadrilijonto nešteto + + prvega drugega tretjega četrtega petega šestega sedmega osmega + devega desetega enajstega dvanajstega trinajstega štirnajstega + petnajstega šestnajstega sedemnajstega osemnajstega devetnajstega + dvajsetega tridesetega štiridesetega petdesetega šestdesetega sedemdesetega + osemdesetega devetdesetega stotega tisočega milijontega bilijontega + trilijontega kvadrilijontega neštetega + + prvemu drugemu tretjemu četrtemu petemu šestemu sedmemu osmemu devetemu desetemu + enajstemu dvanajstemu trinajstemu štirnajstemu petnajstemu šestnajstemu sedemnajstemu + osemnajstemu devetnajstemu dvajsetemu tridesetemu štiridesetemu petdesetemu šestdesetemu + sedemdesetemu osemdesetemu devetdesetemu stotemu tisočemu milijontemu bilijontemu + trilijontemu kvadrilijontemu neštetemu + + prvem drugem tretjem četrtem petem šestem sedmem osmem devetem desetem + enajstem dvanajstem trinajstem štirnajstem petnajstem šestnajstem sedemnajstem + osemnajstem devetnajstem dvajsetem tridesetem štiridesetem petdesetem šestdesetem + sedemdesetem osemdesetem devetdesetem stotem tisočem milijontem bilijontem + trilijontem kvadrilijontem neštetem + + prvim drugim tretjim četrtim petim šestim sedtim osmim devetim desetim + enajstim dvanajstim trinajstim štirnajstim petnajstim šestnajstim sedemnajstim + osemnajstim devetnajstim dvajsetim tridesetim štiridesetim petdesetim šestdesetim + sedemdesetim osemdesetim devetdesetim stotim tisočim milijontim bilijontim + trilijontim kvadrilijontim neštetim + + prvih drugih tretjih četrthih petih šestih sedmih osmih deveth desetih + enajstih dvanajstih trinajstih štirnajstih petnajstih šestnajstih sedemnajstih + osemnajstih devetnajstih dvajsetih tridesetih štiridesetih petdesetih šestdesetih + sedemdesetih osemdesetih devetdesetih stotih tisočih milijontih bilijontih + trilijontih kvadrilijontih nešteth + + prvima drugima tretjima četrtima petima šestima sedmima osmima devetima desetima + enajstima dvanajstima trinajstima štirnajstima petnajstima šestnajstima sedemnajstima + osemnajstima devetnajstima dvajsetima tridesetima štiridesetima petdesetima šestdesetima + sedemdesetima osemdesetima devetdesetima stotima tisočima milijontima bilijontima + trilijontima kvadrilijontima neštetima + + prve druge četrte pete šeste sedme osme devete desete + enajste dvanajste trinajste štirnajste petnajste šestnajste sedemnajste + osemnajste devetnajste dvajsete tridesete štiridesete petdesete šestdesete + sedemdesete osemdesete devetdesete stote tisoče milijonte bilijonte + trilijonte kvadrilijonte neštete + + prvimi drugimi tretjimi četrtimi petimi šestimi sedtimi osmimi devetimi desetimi + enajstimi dvanajstimi trinajstimi štirnajstimi petnajstimi šestnajstimi sedemnajstimi + osemnajstimi devetnajstimi dvajsetimi tridesetimi štiridesetimi petdesetimi šestdesetimi + sedemdesetimi osemdesetimi devetdesetimi stotimi tisočimi milijontimi bilijontimi + trilijontimi kvadrilijontimi neštetimi + """.split() +) + +_currency_words = set( + """ + evro evra evru evrom evrov evroma evrih evrom evre evri evr eur + cent centa centu cenom centov centoma centih centom cente centi + dolar dolarja dolarji dolarju dolarjem dolarjev dolarjema dolarjih dolarje usd + tolar tolarja tolarji tolarju tolarjem tolarjev tolarjema tolarjih tolarje tol + dinar dinarja dinarji dinarju dinarjem dinarjev dinarjema dinarjih dinarje din + funt funta funti funtu funtom funtov funtoma funtih funte gpb + forint forinta forinti forintu forintom forintov forintoma forintih forinte + zlot zlota zloti zlotu zlotom zlotov zlotoma zlotih zlote + rupij rupija rupiji rupiju rupijem rupijev rupijema rupijih rupije + jen jena jeni jenu jenom jenov jenoma jenih jene + kuna kuni kune kuno kun kunama kunah kunam kunami + marka marki marke markama markah markami + """.split() +) + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + text_lower = text.lower() + if text_lower in _num_words: + return True + if text_lower in _ordinal_words: + return True + return False + + +def is_currency(text): + text_lower = text.lower() + if text in _currency_words: + return True + for char in text: + if unicodedata.category(char) != "Sc": + return False + return True + + +LEX_ATTRS = {LIKE_NUM: like_num, IS_CURRENCY: is_currency} diff --git a/spacy/lang/sl/punctuation.py b/spacy/lang/sl/punctuation.py new file mode 100644 index 000000000..b6ca1830e --- /dev/null +++ b/spacy/lang/sl/punctuation.py @@ -0,0 +1,84 @@ +from ..char_classes import ( + LIST_ELLIPSES, + LIST_ICONS, + HYPHENS, + LIST_PUNCT, + LIST_QUOTES, + CURRENCY, + UNITS, + PUNCT, + LIST_CURRENCY, + CONCAT_QUOTES, +) +from ..char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA +from ..char_classes import merge_chars +from ..punctuation import TOKENIZER_PREFIXES as BASE_TOKENIZER_PREFIXES + + +INCLUDE_SPECIAL = ["\\+", "\\/", "\\•", "\\¯", "\\=", "\\×"] + HYPHENS.split("|") + +_prefixes = INCLUDE_SPECIAL + BASE_TOKENIZER_PREFIXES + +_suffixes = ( + INCLUDE_SPECIAL + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + [ + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[{al}{e}{p}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT + ), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + # split initials like J.K. Rowling + r"(?<=[A-Z]\.)(?:[A-Z].)", + ] +) + +# a list of all suffixes following a hyphen that are shouldn't split (eg. BTC-jev) +# source: Obeliks tokenizer - https://github.com/clarinsi/obeliks/blob/master/obeliks/res/TokRulesPart1.txt +CONCAT_QUOTES = CONCAT_QUOTES.replace("'", "") +HYPHENS_PERMITTED = ( + "((a)|(evemu)|(evskega)|(i)|(jevega)|(jevska)|(jevskimi)|(jinemu)|(oma)|(ovim)|" + "(ovski)|(e)|(evi)|(evskem)|(ih)|(jevem)|(jevske)|(jevsko)|(jini)|(ov)|(ovima)|" + "(ovskih)|(em)|(evih)|(evskemu)|(ja)|(jevemu)|(jevskega)|(ji)|(jinih)|(ova)|" + "(ovimi)|(ovskim)|(ema)|(evim)|(evski)|(je)|(jevi)|(jevskem)|(jih)|(jinim)|" + "(ove)|(ovo)|(ovskima)|(ev)|(evima)|(evskih)|(jem)|(jevih)|(jevskemu)|(jin)|" + "(jinima)|(ovega)|(ovska)|(ovskimi)|(eva)|(evimi)|(evskim)|(jema)|(jevim)|" + "(jevski)|(jina)|(jinimi)|(ovem)|(ovske)|(ovsko)|(eve)|(evo)|(evskima)|(jev)|" + "(jevima)|(jevskih)|(jine)|(jino)|(ovemu)|(ovskega)|(u)|(evega)|(evska)|" + "(evskimi)|(jeva)|(jevimi)|(jevskim)|(jinega)|(ju)|(ovi)|(ovskem)|(evem)|" + "(evske)|(evsko)|(jeve)|(jevo)|(jevskima)|(jinem)|(om)|(ovih)|(ovskemu)|" + "(ovec)|(ovca)|(ovcu)|(ovcem)|(ovcev)|(ovcema)|(ovcih)|(ovci)|(ovce)|(ovcimi)|" + "(evec)|(evca)|(evcu)|(evcem)|(evcev)|(evcema)|(evcih)|(evci)|(evce)|(evcimi)|" + "(jevec)|(jevca)|(jevcu)|(jevcem)|(jevcev)|(jevcema)|(jevcih)|(jevci)|(jevce)|" + "(jevcimi)|(ovka)|(ovke)|(ovki)|(ovko)|(ovk)|(ovkama)|(ovkah)|(ovkam)|(ovkami)|" + "(evka)|(evke)|(evki)|(evko)|(evk)|(evkama)|(evkah)|(evkam)|(evkami)|(jevka)|" + "(jevke)|(jevki)|(jevko)|(jevk)|(jevkama)|(jevkah)|(jevkam)|(jevkami)|(timi)|" + "(im)|(ima)|(a)|(imi)|(e)|(o)|(ega)|(ti)|(em)|(tih)|(emu)|(tim)|(i)|(tima)|" + "(ih)|(ta)|(te)|(to)|(tega)|(tem)|(temu))" +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])(?:{h})(?!{hp}$)(?=[{a}])".format( + a=ALPHA, h=HYPHENS, hp=HYPHENS_PERMITTED + ), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/sl/stop_words.py b/spacy/lang/sl/stop_words.py index c9004ed5d..8491efcb5 100644 --- a/spacy/lang/sl/stop_words.py +++ b/spacy/lang/sl/stop_words.py @@ -1,326 +1,84 @@ # Source: https://github.com/stopwords-iso/stopwords-sl -# Removed various words that are not normally considered stop words, such as months. STOP_WORDS = set( """ -a -ali -b -bi -bil -bila -bile -bili -bilo -biti -blizu -bo -bodo -bolj -bom -bomo -boste -bova -boš -brez -c -cel -cela -celi -celo -d -da -daleč -dan -danes -do -dober -dobra -dobri -dobro -dokler -dol -dovolj -e -eden -en -ena -ene -eni -enkrat -eno -etc. +a ali + +b bi bil bila bile bili bilo biti blizu bo bodo bojo bolj bom bomo +boste bova boš brez + +c cel cela celi celo + +č če često četrta četrtek četrti četrto čez čigav + +d da daleč dan danes datum deset deseta deseti deseto devet +deveta deveti deveto do dober dobra dobri dobro dokler dol dolg +dolga dolgi dovolj drug druga drugi drugo dva dve + +e eden en ena ene eni enkrat eno etc. + f -g -g. -ga -ga. -gor -gospa -gospod -h -halo -i -idr. -ii -iii -in -iv -ix -iz -j -jaz -je -ji -jih -jim -jo -k -kadarkoli -kaj -kajti -kako -kakor -kamor -kamorkoli -kar -karkoli -katerikoli -kdaj -kdo -kdorkoli -ker -ki -kje -kjer -kjerkoli -ko -koderkoli -koga -komu -kot -l -le -lep -lepa -lepe -lepi -lepo -m -manj -me -med -medtem -mene -mi -midva -midve -mnogo -moj -moja -moje -mora -morajo -moram -moramo -morate -moraš -morem -mu -n -na -nad -naj -najina -najino -najmanj -naju -največ -nam -nas -nato -nazaj -naš -naša -naše -ne -nedavno -nek -neka -nekaj -nekatere -nekateri -nekatero -nekdo -neke -nekega -neki -nekje -neko -nekoga -nekoč -ni -nikamor -nikdar -nikjer -nikoli -nič -nje -njega -njegov -njegova -njegovo -njej -njemu -njen -njena -njeno -nji -njih -njihov -njihova -njihovo -njiju -njim -njo -njun -njuna -njuno -no -nocoj -npr. -o -ob -oba -obe -oboje -od -okoli -on -onadva -one -oni -onidve -oz. -p -pa -po -pod -pogosto -poleg -ponavadi -ponovno -potem -povsod -prbl. -precej -pred -prej -preko -pri -pribl. -približno -proti -r -redko -res -s -saj -sam -sama -same -sami -samo -se -sebe -sebi -sedaj -sem -seveda -si -sicer -skoraj -skozi -smo -so -spet -sta -ste -sva -t -ta -tak -taka -take -taki -tako -takoj -tam -te -tebe -tebi -tega -ti -tista -tiste -tisti -tisto -tj. -tja -to -toda -tu -tudi -tukaj -tvoj -tvoja -tvoje + +g g. ga ga. gor gospa gospod + +h halo + +i idr. ii iii in iv ix iz + +j jaz je ji jih jim jo jutri + +k kadarkoli kaj kajti kako kakor kamor kamorkoli kar karkoli +katerikoli kdaj kdo kdorkoli ker ki kje kjer kjerkoli +ko koder koderkoli koga komu kot kratek kratka kratke kratki + +l lahka lahke lahki lahko le lep lepa lepe lepi lepo leto + +m majhen majhna majhni malce malo manj me med medtem mene +mesec mi midva midve mnogo moj moja moje mora morajo moram +moramo morate moraš morem mu + +n na nad naj najina najino najmanj naju največ nam narobe +nas nato nazaj naš naša naše ne nedavno nedelja nek neka +nekaj nekatere nekateri nekatero nekdo neke nekega neki +nekje neko nekoga nekoč ni nikamor nikdar nikjer nikoli +nič nje njega njegov njegova njegovo njej njemu njen +njena njeno nji njih njihov njihova njihovo njiju njim +njo njun njuna njuno no nocoj npr. + +o ob oba obe oboje od odprt odprta odprti okoli on +onadva one oni onidve osem osma osmi osmo oz. + +p pa pet peta petek peti peto po pod pogosto poleg poln +polna polni polno ponavadi ponedeljek ponovno potem +povsod pozdravljen pozdravljeni prav prava prave pravi +pravo prazen prazna prazno prbl. precej pred prej preko +pri pribl. približno primer pripravljen pripravljena +pripravljeni proti prva prvi prvo + +r ravno redko res reč + +s saj sam sama same sami samo se sebe sebi sedaj sedem +sedma sedmi sedmo sem seveda si sicer skoraj skozi slab sm +so sobota spet sreda srednja srednji sta ste stran stvar sva + +š šest šesta šesti šesto štiri + +t ta tak taka take taki tako takoj tam te tebe tebi tega +težak težka težki težko ti tista tiste tisti tisto tj. +tja to toda torek tretja tretje tretji tri tu tudi tukaj +tvoj tvoja tvoje + u -v -vaju -vam -vas -vaš -vaša -vaše -ve -vedno -vendar -ves -več -vi -vidva -vii -viii -vsa -vsaj -vsak -vsaka -vsakdo -vsake -vsaki -vsakomur -vse -vsega -vsi -vso -včasih -x -z -za -zadaj -zadnji -zakaj -zdaj -zelo -zunaj -č -če -često -čez -čigav -š -ž -že + +v vaju vam vas vaš vaša vaše ve vedno velik velika veliki +veliko vendar ves več vi vidva vii viii visok visoka visoke +visoki vsa vsaj vsak vsaka vsakdo vsake vsaki vsakomur vse +vsega vsi vso včasih včeraj + +x + +z za zadaj zadnji zakaj zaprta zaprti zaprto zdaj zelo zunaj + +ž že """.split() ) diff --git a/spacy/lang/sl/tokenizer_exceptions.py b/spacy/lang/sl/tokenizer_exceptions.py new file mode 100644 index 000000000..3d4109228 --- /dev/null +++ b/spacy/lang/sl/tokenizer_exceptions.py @@ -0,0 +1,272 @@ +from typing import Dict, List +from ..tokenizer_exceptions import BASE_EXCEPTIONS +from ...symbols import ORTH, NORM +from ...util import update_exc + +_exc: Dict[str, List[Dict]] = {} + +_other_exc = { + "t.i.": [{ORTH: "t.", NORM: "tako"}, {ORTH: "i.", NORM: "imenovano"}], + "t.j.": [{ORTH: "t.", NORM: "to"}, {ORTH: "j.", NORM: "je"}], + "T.j.": [{ORTH: "T.", NORM: "to"}, {ORTH: "j.", NORM: "je"}], + "d.o.o.": [ + {ORTH: "d.", NORM: "družba"}, + {ORTH: "o.", NORM: "omejeno"}, + {ORTH: "o.", NORM: "odgovornostjo"}, + ], + "D.O.O.": [ + {ORTH: "D.", NORM: "družba"}, + {ORTH: "O.", NORM: "omejeno"}, + {ORTH: "O.", NORM: "odgovornostjo"}, + ], + "d.n.o.": [ + {ORTH: "d.", NORM: "družba"}, + {ORTH: "n.", NORM: "neomejeno"}, + {ORTH: "o.", NORM: "odgovornostjo"}, + ], + "D.N.O.": [ + {ORTH: "D.", NORM: "družba"}, + {ORTH: "N.", NORM: "neomejeno"}, + {ORTH: "O.", NORM: "odgovornostjo"}, + ], + "d.d.": [{ORTH: "d.", NORM: "delniška"}, {ORTH: "d.", NORM: "družba"}], + "D.D.": [{ORTH: "D.", NORM: "delniška"}, {ORTH: "D.", NORM: "družba"}], + "s.p.": [{ORTH: "s.", NORM: "samostojni"}, {ORTH: "p.", NORM: "podjetnik"}], + "S.P.": [{ORTH: "S.", NORM: "samostojni"}, {ORTH: "P.", NORM: "podjetnik"}], + "l.r.": [{ORTH: "l.", NORM: "lastno"}, {ORTH: "r.", NORM: "ročno"}], + "le-te": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "te"}], + "Le-te": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "te"}], + "le-ti": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "ti"}], + "Le-ti": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "ti"}], + "le-to": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "to"}], + "Le-to": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "to"}], + "le-ta": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "ta"}], + "Le-ta": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "ta"}], + "le-tega": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "tega"}], + "Le-tega": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "tega"}], +} + +_exc.update(_other_exc) + + +for exc_data in [ + {ORTH: "adm.", NORM: "administracija"}, + {ORTH: "aer.", NORM: "aeronavtika"}, + {ORTH: "agr.", NORM: "agronomija"}, + {ORTH: "amer.", NORM: "ameriško"}, + {ORTH: "anat.", NORM: "anatomija"}, + {ORTH: "angl.", NORM: "angleški"}, + {ORTH: "ant.", NORM: "antonim"}, + {ORTH: "antr.", NORM: "antropologija"}, + {ORTH: "apr.", NORM: "april"}, + {ORTH: "arab.", NORM: "arabsko"}, + {ORTH: "arheol.", NORM: "arheologija"}, + {ORTH: "arhit.", NORM: "arhitektura"}, + {ORTH: "avg.", NORM: "avgust"}, + {ORTH: "avstr.", NORM: "avstrijsko"}, + {ORTH: "avt.", NORM: "avtomobilizem"}, + {ORTH: "bibl.", NORM: "biblijsko"}, + {ORTH: "biokem.", NORM: "biokemija"}, + {ORTH: "biol.", NORM: "biologija"}, + {ORTH: "bolg.", NORM: "bolgarski"}, + {ORTH: "bot.", NORM: "botanika"}, + {ORTH: "cit.", NORM: "citat"}, + {ORTH: "daj.", NORM: "dajalnik"}, + {ORTH: "del.", NORM: "deležnik"}, + {ORTH: "ed.", NORM: "ednina"}, + {ORTH: "etn.", NORM: "etnografija"}, + {ORTH: "farm.", NORM: "farmacija"}, + {ORTH: "filat.", NORM: "filatelija"}, + {ORTH: "filoz.", NORM: "filozofija"}, + {ORTH: "fin.", NORM: "finančništvo"}, + {ORTH: "fiz.", NORM: "fizika"}, + {ORTH: "fot.", NORM: "fotografija"}, + {ORTH: "fr.", NORM: "francoski"}, + {ORTH: "friz.", NORM: "frizerstvo"}, + {ORTH: "gastr.", NORM: "gastronomija"}, + {ORTH: "geogr.", NORM: "geografija"}, + {ORTH: "geol.", NORM: "geologija"}, + {ORTH: "geom.", NORM: "geometrija"}, + {ORTH: "germ.", NORM: "germanski"}, + {ORTH: "gl.", NORM: "glej"}, + {ORTH: "glag.", NORM: "glagolski"}, + {ORTH: "glasb.", NORM: "glasba"}, + {ORTH: "gled.", NORM: "gledališče"}, + {ORTH: "gost.", NORM: "gostinstvo"}, + {ORTH: "gozd.", NORM: "gozdarstvo"}, + {ORTH: "gr.", NORM: "grški"}, + {ORTH: "grad.", NORM: "gradbeništvo"}, + {ORTH: "hebr.", NORM: "hebrejsko"}, + {ORTH: "hrv.", NORM: "hrvaško"}, + {ORTH: "ide.", NORM: "indoevropsko"}, + {ORTH: "igr.", NORM: "igre"}, + {ORTH: "im.", NORM: "imenovalnik"}, + {ORTH: "iron.", NORM: "ironično"}, + {ORTH: "it.", NORM: "italijanski"}, + {ORTH: "itd.", NORM: "in tako dalje"}, + {ORTH: "itn.", NORM: "in tako naprej"}, + {ORTH: "ipd.", NORM: "in podobno"}, + {ORTH: "jap.", NORM: "japonsko"}, + {ORTH: "jul.", NORM: "julij"}, + {ORTH: "jun.", NORM: "junij"}, + {ORTH: "kit.", NORM: "kitajsko"}, + {ORTH: "knj.", NORM: "knjižno"}, + {ORTH: "knjiž.", NORM: "knjižno"}, + {ORTH: "kor.", NORM: "koreografija"}, + {ORTH: "lat.", NORM: "latinski"}, + {ORTH: "les.", NORM: "lesna stroka"}, + {ORTH: "lingv.", NORM: "lingvistika"}, + {ORTH: "lit.", NORM: "literarni"}, + {ORTH: "ljubk.", NORM: "ljubkovalno"}, + {ORTH: "lov.", NORM: "lovstvo"}, + {ORTH: "m.", NORM: "moški"}, + {ORTH: "mak.", NORM: "makedonski"}, + {ORTH: "mar.", NORM: "marec"}, + {ORTH: "mat.", NORM: "matematika"}, + {ORTH: "med.", NORM: "medicina"}, + {ORTH: "meh.", NORM: "mehiško"}, + {ORTH: "mest.", NORM: "mestnik"}, + {ORTH: "mdr.", NORM: "med drugim"}, + {ORTH: "min.", NORM: "mineralogija"}, + {ORTH: "mitol.", NORM: "mitologija"}, + {ORTH: "mn.", NORM: "množina"}, + {ORTH: "mont.", NORM: "montanistika"}, + {ORTH: "muz.", NORM: "muzikologija"}, + {ORTH: "nam.", NORM: "namenilnik"}, + {ORTH: "nar.", NORM: "narečno"}, + {ORTH: "nav.", NORM: "navadno"}, + {ORTH: "nedol.", NORM: "nedoločnik"}, + {ORTH: "nedov.", NORM: "nedovršni"}, + {ORTH: "neprav.", NORM: "nepravilno"}, + {ORTH: "nepreh.", NORM: "neprehodno"}, + {ORTH: "neskl.", NORM: "nesklonljiv(o)"}, + {ORTH: "nestrok.", NORM: "nestrokovno"}, + {ORTH: "num.", NORM: "numizmatika"}, + {ORTH: "npr.", NORM: "na primer"}, + {ORTH: "obrt.", NORM: "obrtništvo"}, + {ORTH: "okt.", NORM: "oktober"}, + {ORTH: "or.", NORM: "orodnik"}, + {ORTH: "os.", NORM: "oseba"}, + {ORTH: "otr.", NORM: "otroško"}, + {ORTH: "oz.", NORM: "oziroma"}, + {ORTH: "pal.", NORM: "paleontologija"}, + {ORTH: "papir.", NORM: "papirništvo"}, + {ORTH: "ped.", NORM: "pedagogika"}, + {ORTH: "pisar.", NORM: "pisarniško"}, + {ORTH: "pog.", NORM: "pogovorno"}, + {ORTH: "polit.", NORM: "politika"}, + {ORTH: "polj.", NORM: "poljsko"}, + {ORTH: "poljud.", NORM: "poljudno"}, + {ORTH: "preg.", NORM: "pregovor"}, + {ORTH: "preh.", NORM: "prehodno"}, + {ORTH: "pren.", NORM: "preneseno"}, + {ORTH: "prid.", NORM: "pridevnik"}, + {ORTH: "prim.", NORM: "primerjaj"}, + {ORTH: "prisl.", NORM: "prislov"}, + {ORTH: "psih.", NORM: "psihologija"}, + {ORTH: "psiht.", NORM: "psihiatrija"}, + {ORTH: "rad.", NORM: "radiotehnika"}, + {ORTH: "rač.", NORM: "računalništvo"}, + {ORTH: "rib.", NORM: "ribištvo"}, + {ORTH: "rod.", NORM: "rodilnik"}, + {ORTH: "rus.", NORM: "rusko"}, + {ORTH: "s.", NORM: "srednji"}, + {ORTH: "sam.", NORM: "samostalniški"}, + {ORTH: "sed.", NORM: "sedanjik"}, + {ORTH: "sep.", NORM: "september"}, + {ORTH: "slabš.", NORM: "slabšalno"}, + {ORTH: "slovan.", NORM: "slovansko"}, + {ORTH: "slovaš.", NORM: "slovaško"}, + {ORTH: "srb.", NORM: "srbsko"}, + {ORTH: "star.", NORM: "starinsko"}, + {ORTH: "stil.", NORM: "stilno"}, + {ORTH: "sv.", NORM: "svet(i)"}, + {ORTH: "teh.", NORM: "tehnika"}, + {ORTH: "tisk.", NORM: "tiskarstvo"}, + {ORTH: "tj.", NORM: "to je"}, + {ORTH: "tož.", NORM: "tožilnik"}, + {ORTH: "trg.", NORM: "trgovina"}, + {ORTH: "ukr.", NORM: "ukrajinski"}, + {ORTH: "um.", NORM: "umetnost"}, + {ORTH: "vel.", NORM: "velelnik"}, + {ORTH: "vet.", NORM: "veterina"}, + {ORTH: "vez.", NORM: "veznik"}, + {ORTH: "vn.", NORM: "visokonemško"}, + {ORTH: "voj.", NORM: "vojska"}, + {ORTH: "vrtn.", NORM: "vrtnarstvo"}, + {ORTH: "vulg.", NORM: "vulgarno"}, + {ORTH: "vznes.", NORM: "vzneseno"}, + {ORTH: "zal.", NORM: "založništvo"}, + {ORTH: "zastar.", NORM: "zastarelo"}, + {ORTH: "zgod.", NORM: "zgodovina"}, + {ORTH: "zool.", NORM: "zoologija"}, + {ORTH: "čeb.", NORM: "čebelarstvo"}, + {ORTH: "češ.", NORM: "češki"}, + {ORTH: "člov.", NORM: "človeškost"}, + {ORTH: "šah.", NORM: "šahovski"}, + {ORTH: "šalj.", NORM: "šaljivo"}, + {ORTH: "šp.", NORM: "španski"}, + {ORTH: "špan.", NORM: "špansko"}, + {ORTH: "šport.", NORM: "športni"}, + {ORTH: "štev.", NORM: "števnik"}, + {ORTH: "šved.", NORM: "švedsko"}, + {ORTH: "švic.", NORM: "švicarsko"}, + {ORTH: "ž.", NORM: "ženski"}, + {ORTH: "žarg.", NORM: "žargonsko"}, + {ORTH: "žel.", NORM: "železnica"}, + {ORTH: "živ.", NORM: "živost"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +abbrv = """ +Co. Ch. DIPL. DR. Dr. Ev. Inc. Jr. Kr. Mag. M. MR. Mr. Mt. Murr. Npr. OZ. +Opr. Osn. Prim. Roj. ST. Sim. Sp. Sred. St. Sv. Škofl. Tel. UR. Zb. +a. aa. ab. abc. abit. abl. abs. abt. acc. accel. add. adj. adv. aet. afr. akad. al. alban. all. alleg. +alp. alt. alter. alžir. am. an. andr. ang. anh. anon. ans. antrop. apoc. app. approx. apt. ar. arc. arch. +arh. arr. as. asist. assist. assoc. asst. astr. attn. aug. avstral. az. b. bab. bal. bbl. bd. belg. bioinf. +biomed. bk. bl. bn. borg. bp. br. braz. brit. bros. broš. bt. bu. c. ca. cal. can. cand. cantab. cap. capt. +cat. cath. cc. cca. cd. cdr. cdre. cent. cerkv. cert. cf. cfr. ch. chap. chem. chr. chs. cic. circ. civ. cl. +cm. cmd. cnr. co. cod. col. coll. colo. com. comp. con. conc. cond. conn. cons. cont. coop. corr. cost. cp. +cpl. cr. crd. cres. cresc. ct. cu. d. dan. dat. davč. ddr. dec. ded. def. dem. dent. dept. dia. dip. dipl. +dir. disp. diss. div. do. doc. dok. dol. doo. dop. dott. dr. dram. druž. družb. drž. dt. duh. dur. dvr. dwt. e. +ea. ecc. eccl. eccles. econ. edn. egipt. egr. ekon. eksp. el. em. enc. eng. eo. ep. err. esp. esq. est. +et. etc. etnogr. etnol. ev. evfem. evr. ex. exc. excl. exp. expl. ext. exx. f. fa. facs. fak. faks. fas. +fasc. fco. fcp. feb. febr. fec. fed. fem. ff. fff. fid. fig. fil. film. fiziol. fiziot. flam. fm. fo. fol. folk. +frag. fran. franc. fsc. g. ga. gal. gdč. ge. gen. geod. geog. geotehnol. gg. gimn. glas. glav. gnr. go. gor. +gosp. gp. graf. gram. gren. grš. gs. h. hab. hf. hist. ho. hort. i. ia. ib. ibid. id. idr. idridr. ill. imen. +imp. impf. impr. in. inc. incl. ind. indus. inf. inform. ing. init. ins. int. inv. inšp. inštr. inž. is. islam. +ist. ital. iur. iz. izbr. izd. izg. izgr. izr. izv. j. jak. jam. jan. jav. je. jez. jr. jsl. jud. jug. +jugoslovan. jur. juž. jv. jz. k. kal. kan. kand. kat. kdo. kem. kip. kmet. kol. kom. komp. konf. kont. kost. kov. +kp. kpfw. kr. kraj. krat. kub. kult. kv. kval. l. la. lab. lb. ld. let. lib. lik. litt. lj. ljud. ll. loc. log. +loč. lt. ma. madž. mag. manag. manjš. masc. mass. mater. max. maxmax. mb. md. mech. medic. medij. medn. +mehč. mem. menedž. mes. mess. metal. meteor. meteorol. mex. mi. mikr. mil. minn. mio. misc. miss. mit. mk. +mkt. ml. mlad. mlle. mlr. mm. mme. množ. mo. moj. moš. možn. mr. mrd. mrs. ms. msc. msgr. mt. murr. mus. mut. +n. na. nad. nadalj. nadom. nagl. nakl. namer. nan. naniz. nasl. nat. navt. nač. ned. nem. nik. nizoz. nm. nn. +no. nom. norv. notr. nov. novogr. ns. o. ob. obd. obj. oblač. obl. oblik. obr. obraz. obs. obst. obt. obč. oc. +oct. od. odd. odg. odn. odst. odv. oec. off. ok. okla. okr. ont. oo. op. opis. opp. opr. orch. ord. ore. oreg. +org. orient. orig. ork. ort. oseb. osn. ot. ozir. ošk. p. pag. par. para. parc. parl. part. past. pat. pdk. +pen. perf. pert. perz. pesn. pet. pev. pf. pfc. ph. pharm. phil. pis. pl. po. pod. podr. podaljš. pogl. pogoj. pojm. +pok. pokr. pol. poljed. poljub. polu. pom. pomen. pon. ponov. pop. por. port. pos. posl. posn. pov. pp. ppl. pr. +praet. prav. pravopis. pravosl. preb. pred. predl. predm. predp. preds. pref. pregib. prel. prem. premen. prep. +pres. pret. prev. pribl. prih. pril. primerj. primor. prip. pripor. prir. prist. priv. proc. prof. prog. proiz. +prom. pron. prop. prot. protest. prov. ps. pss. pt. publ. pz. q. qld. qu. quad. que. r. racc. rastl. razgl. +razl. razv. rd. red. ref. reg. rel. relig. rep. repr. rer. resp. rest. ret. rev. revol. rež. rim. rist. rkp. rm. +roj. rom. romun. rp. rr. rt. rud. ruš. ry. sal. samogl. san. sc. scen. sci. scr. sdv. seg. sek. sen. sept. ser. +sev. sg. sgt. sh. sig. sigg. sign. sim. sin. sing. sinh. skand. skl. sklad. sklanj. sklep. skr. sl. slik. slov. +slovak. slovn. sn. so. sob. soc. sociol. sod. sopomen. sopr. sor. sov. sovj. sp. spec. spl. spr. spreg. sq. sr. +sre. sred. sredoz. srh. ss. ssp. st. sta. stan. stanstar. stcsl. ste. stim. stol. stom. str. stroj. strok. stsl. +stud. sup. supl. suppl. svet. sz. t. tab. tech. ted. tehn. tehnol. tek. teks. tekst. tel. temp. ten. teol. ter. +term. test. th. theol. tim. tip. tisočl. tit. tl. tol. tolmač. tom. tor. tov. tr. trad. traj. trans. tren. +trib. tril. trop. trp. trž. ts. tt. tu. tur. turiz. tvor. tvorb. tč. u. ul. umet. un. univ. up. upr. ur. urad. +us. ust. utr. v. va. val. var. varn. ven. ver. verb. vest. vezal. vic. vis. viv. viz. viš. vod. vok. vol. vpr. +vrst. vrstil. vs. vv. vzd. vzg. vzh. vzor. w. wed. wg. wk. x. y. z. zah. zaim. zak. zap. zasl. zavar. zač. zb. +združ. zg. zn. znan. znanstv. zoot. zun. zv. zvd. á. é. ć. č. čas. čet. čl. člen. čustv. đ. ľ. ł. ş. ŠT. š. šir. +škofl. škot. šol. št. števil. štud. ů. ű. žen. žival. +""".split() + +for orth in abbrv: + _exc[orth] = [{ORTH: orth}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/tests/lang/sl/test_text.py b/spacy/tests/lang/sl/test_text.py index ddc5b6b5d..a2a932077 100644 --- a/spacy/tests/lang/sl/test_text.py +++ b/spacy/tests/lang/sl/test_text.py @@ -20,7 +20,6 @@ od katerih so te svoboščine odvisne, assert len(tokens) == 116 -@pytest.mark.xfail def test_ordinal_number(sl_tokenizer): text = "10. decembra 1948" tokens = sl_tokenizer(text) From 5fa8f4faca966fe58c5c8de861900724c7659f25 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Mon, 22 Aug 2022 11:27:14 +0200 Subject: [PATCH 002/179] Switch ru and uk lemmatizers to pymorphy3 (#11345) * Switch ru and uk lemmatizers to pymorphy3 * Switch to pymorphy3 in tests --- spacy/lang/ru/__init__.py | 2 +- spacy/lang/ru/lemmatizer.py | 15 ++++++++++++++- spacy/lang/uk/__init__.py | 2 +- spacy/lang/uk/lemmatizer.py | 13 ++++++++++++- spacy/tests/conftest.py | 10 +++++----- website/docs/api/lemmatizer.md | 6 +++--- website/meta/languages.json | 8 ++++---- 7 files changed, 40 insertions(+), 16 deletions(-) diff --git a/spacy/lang/ru/__init__.py b/spacy/lang/ru/__init__.py index c118c26ff..7d17628c4 100644 --- a/spacy/lang/ru/__init__.py +++ b/spacy/lang/ru/__init__.py @@ -28,7 +28,7 @@ class Russian(Language): assigns=["token.lemma"], default_config={ "model": None, - "mode": "pymorphy2", + "mode": "pymorphy3", "overwrite": False, "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, }, diff --git a/spacy/lang/ru/lemmatizer.py b/spacy/lang/ru/lemmatizer.py index 85180b1e4..720d3a8cb 100644 --- a/spacy/lang/ru/lemmatizer.py +++ b/spacy/lang/ru/lemmatizer.py @@ -19,7 +19,7 @@ class RussianLemmatizer(Lemmatizer): model: Optional[Model], name: str = "lemmatizer", *, - mode: str = "pymorphy2", + mode: str = "pymorphy3", overwrite: bool = False, scorer: Optional[Callable] = lemmatizer_score, ) -> None: @@ -33,6 +33,16 @@ class RussianLemmatizer(Lemmatizer): ) from None if getattr(self, "_morph", None) is None: self._morph = MorphAnalyzer() + elif mode == "pymorphy3": + try: + from pymorphy3 import MorphAnalyzer + except ImportError: + raise ImportError( + "The Russian lemmatizer mode 'pymorphy3' requires the " + "pymorphy3 library. Install it with: pip install pymorphy3" + ) from None + if getattr(self, "_morph", None) is None: + self._morph = MorphAnalyzer() super().__init__( vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer ) @@ -104,6 +114,9 @@ class RussianLemmatizer(Lemmatizer): return [analyses[0].normal_form] return [string] + def pymorphy3_lemmatize(self, token: Token) -> List[str]: + return self.pymorphy2_lemmatize(token) + def oc2ud(oc_tag: str) -> Tuple[str, Dict[str, str]]: gram_map = { diff --git a/spacy/lang/uk/__init__.py b/spacy/lang/uk/__init__.py index 737243b66..bfea9ff69 100644 --- a/spacy/lang/uk/__init__.py +++ b/spacy/lang/uk/__init__.py @@ -29,7 +29,7 @@ class Ukrainian(Language): assigns=["token.lemma"], default_config={ "model": None, - "mode": "pymorphy2", + "mode": "pymorphy3", "overwrite": False, "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, }, diff --git a/spacy/lang/uk/lemmatizer.py b/spacy/lang/uk/lemmatizer.py index a8bc56057..97ee80479 100644 --- a/spacy/lang/uk/lemmatizer.py +++ b/spacy/lang/uk/lemmatizer.py @@ -14,7 +14,7 @@ class UkrainianLemmatizer(RussianLemmatizer): model: Optional[Model], name: str = "lemmatizer", *, - mode: str = "pymorphy2", + mode: str = "pymorphy3", overwrite: bool = False, scorer: Optional[Callable] = lemmatizer_score, ) -> None: @@ -29,6 +29,17 @@ class UkrainianLemmatizer(RussianLemmatizer): ) from None if getattr(self, "_morph", None) is None: self._morph = MorphAnalyzer(lang="uk") + elif mode == "pymorphy3": + try: + from pymorphy3 import MorphAnalyzer + except ImportError: + raise ImportError( + "The Ukrainian lemmatizer mode 'pymorphy3' requires the " + "pymorphy3 library and dictionaries. Install them with: " + "pip install pymorphy3 pymorphy3-dicts-uk" + ) from None + if getattr(self, "_morph", None) is None: + self._morph = MorphAnalyzer(lang="uk") super().__init__( vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer ) diff --git a/spacy/tests/conftest.py b/spacy/tests/conftest.py index eb643ec2f..76de8f373 100644 --- a/spacy/tests/conftest.py +++ b/spacy/tests/conftest.py @@ -323,13 +323,13 @@ def ro_tokenizer(): @pytest.fixture(scope="session") def ru_tokenizer(): - pytest.importorskip("pymorphy2") + pytest.importorskip("pymorphy3") return get_lang_class("ru")().tokenizer @pytest.fixture def ru_lemmatizer(): - pytest.importorskip("pymorphy2") + pytest.importorskip("pymorphy3") return get_lang_class("ru")().add_pipe("lemmatizer") @@ -401,14 +401,14 @@ def ky_tokenizer(): @pytest.fixture(scope="session") def uk_tokenizer(): - pytest.importorskip("pymorphy2") + pytest.importorskip("pymorphy3") return get_lang_class("uk")().tokenizer @pytest.fixture def uk_lemmatizer(): - pytest.importorskip("pymorphy2") - pytest.importorskip("pymorphy2_dicts_uk") + pytest.importorskip("pymorphy3") + pytest.importorskip("pymorphy3_dicts_uk") return get_lang_class("uk")().add_pipe("lemmatizer") diff --git a/website/docs/api/lemmatizer.md b/website/docs/api/lemmatizer.md index 422f34040..905096338 100644 --- a/website/docs/api/lemmatizer.md +++ b/website/docs/api/lemmatizer.md @@ -70,7 +70,7 @@ lemmatizer is available. The lemmatizer modes `rule` and `pos_lookup` require [`token.pos`](/api/token) from a previous pipeline component (see example pipeline configurations in the [pretrained pipeline design details](/models#design-cnn)) or rely on third-party -libraries (`pymorphy2`). +libraries (`pymorphy3`). | Language | Default Mode | | -------- | ------------ | @@ -86,9 +86,9 @@ libraries (`pymorphy2`). | `nb` | `rule` | | `nl` | `rule` | | `pl` | `pos_lookup` | -| `ru` | `pymorphy2` | +| `ru` | `pymorphy3` | | `sv` | `rule` | -| `uk` | `pymorphy2` | +| `uk` | `pymorphy3` | ```python %%GITHUB_SPACY/spacy/pipeline/lemmatizer.py diff --git a/website/meta/languages.json b/website/meta/languages.json index 6bc2309ed..5305ceffc 100644 --- a/website/meta/languages.json +++ b/website/meta/languages.json @@ -369,8 +369,8 @@ "has_examples": true, "dependencies": [ { - "name": "pymorphy2", - "url": "https://github.com/kmike/pymorphy2" + "name": "pymorphy3", + "url": "https://github.com/no-plagiarism/pymorphy3" } ], "models": [ @@ -469,8 +469,8 @@ "has_examples": true, "dependencies": [ { - "name": "pymorphy2", - "url": "https://github.com/kmike/pymorphy2" + "name": "pymorphy3", + "url": "https://github.com/no-plagiarism/pymorphy3" } ] }, From aafee5e1b7c8d13d9ac14c438063621a18bec743 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Mon, 29 Aug 2022 17:32:38 +0900 Subject: [PATCH 003/179] Fix lookup usage in French/Catalan (fix #11347) (#11382) * Fix lookup usage (fix #11347) Before using the lookups table in the French (and Catalan) lemmatizers, there's a check to see if the current term is in the table. But it's checking a string against hashes, so it's always false. Also the table lookup function is designed so you don't have to do that anyway. * Use the lookup table directly * Use string, not token --- spacy/lang/ca/lemmatizer.py | 6 +++--- spacy/lang/fr/lemmatizer.py | 13 ++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/spacy/lang/ca/lemmatizer.py b/spacy/lang/ca/lemmatizer.py index 2fd012912..0f15e6e65 100644 --- a/spacy/lang/ca/lemmatizer.py +++ b/spacy/lang/ca/lemmatizer.py @@ -72,10 +72,10 @@ class CatalanLemmatizer(Lemmatizer): oov_forms.append(form) if not forms: forms.extend(oov_forms) - if not forms and string in lookup_table.keys(): - forms.append(self.lookup_lemmatize(token)[0]) + + # use lookups, and fall back to the token itself if not forms: - forms.append(string) + forms.append(lookup_table.get(string, [string])[0]) forms = list(dict.fromkeys(forms)) self.cache[cache_key] = forms return forms diff --git a/spacy/lang/fr/lemmatizer.py b/spacy/lang/fr/lemmatizer.py index c6422cf96..a7cbe0bcf 100644 --- a/spacy/lang/fr/lemmatizer.py +++ b/spacy/lang/fr/lemmatizer.py @@ -53,11 +53,16 @@ class FrenchLemmatizer(Lemmatizer): rules = rules_table.get(univ_pos, []) string = string.lower() forms = [] + # first try lookup in table based on upos if string in index: forms.append(string) self.cache[cache_key] = forms return forms + + # then add anything in the exceptions table forms.extend(exceptions.get(string, [])) + + # if nothing found yet, use the rules oov_forms = [] if not forms: for old, new in rules: @@ -69,12 +74,14 @@ class FrenchLemmatizer(Lemmatizer): forms.append(form) else: oov_forms.append(form) + + # if still nothing, add the oov forms from rules if not forms: forms.extend(oov_forms) - if not forms and string in lookup_table.keys(): - forms.append(self.lookup_lemmatize(token)[0]) + + # use lookups, which fall back to the token itself if not forms: - forms.append(string) + forms.append(lookup_table.get(string, [string])[0]) forms = list(dict.fromkeys(forms)) self.cache[cache_key] = forms return forms From 5ae63b1fbd549fdfc0f7399c0b9656d4a6681544 Mon Sep 17 00:00:00 2001 From: "Patrick J. Burns" Date: Tue, 30 Aug 2022 08:04:54 -0400 Subject: [PATCH 004/179] Add Latin language support (#11349) * Add lang folder for la (Latin) * Add Latin lang classes * Add minimal tokenizer exceptions * Add minimal stopwords * Add minimal lex_attrs * Update stopwords, tokenizer exceptions * Add la tests; register la_tokenizer in conftest.py * Update spacy/lang/la/lex_attrs.py Remove duplicate form in Latin lex_attrs Co-authored-by: Sofie Van Landeghem * Update natto-py version spec (#11222) * Update natto-py version spec * Update setup.cfg Co-authored-by: Adriane Boyd Co-authored-by: Adriane Boyd * Add scorer to textcat API docs config settings (#11263) * Update docs for pipeline initialize() methods (#11221) * Update documentation for dependency parser * Update documentation for trainable_lemmatizer * Update documentation for entity_linker * Update documentation for ner * Update documentation for morphologizer * Update documentation for senter * Update documentation for spancat * Update documentation for tagger * Update documentation for textcat * Update documentation for tok2vec * Run prettier on edited files * Apply similar changes in transformer docs * Remove need to say annotated example explicitly I removed the need to say "Must contain at least one annotated Example" because it's often a given that Examples will contain some gold-standard annotation. * Run prettier on transformer docs * chore: add 'concepCy' to spacy universe (#11255) * chore: add 'concepCy' to spacy universe * docs: add 'slogan' to concepCy * Support full prerelease versions in the compat table (#11228) * Support full prerelease versions in the compat table * Fix types * adding spans to doc_annotation in Example.to_dict (#11261) * adding spans to doc_annotation in Example.to_dict * to_dict compatible with from_dict: tuples instead of spans * use strings for label and kb_id * Simplify test * Update data formats docs Co-authored-by: Stefanie Wolf Co-authored-by: Adriane Boyd * Fix regex invalid escape sequences (#11276) * Add W605 to the errors raised by flake8 in the CI (#11283) * Clean up automated label-based issue handling (#11284) * Clean up automated label-based issue handline 1. upgrade tiangolo/issue-manager to latest 2. move needs-more-info to tiangolo 3. change needs-more-info close time to 7 days 4. delete old needs-more-info config * Use old, longer message * Fix label name * Fix Dutch noun chunks to skip overlapping spans (#11275) * Add test for overlapping noun chunks * Skip overlapping noun chunks * Update spacy/tests/lang/nl/test_noun_chunks.py Co-authored-by: Sofie Van Landeghem Co-authored-by: Sofie Van Landeghem * Docs: displaCy documentation - data types, `parse_{deps,ents,spans}`, spans example (#10950) * add in spans example and parse references * rm autoformatter * rm extra ents copy * TypedDict draft * type fixes * restore non-documentation files * docs update * fix spans example * fix hyperlinks * add parse example * example fix + argument fix * fix api arg in docs * fix bad variable replacement * fix spacing in style Co-authored-by: Sofie Van Landeghem * fix spacing on table * fix spacing on table * rm temp files Co-authored-by: Sofie Van Landeghem * include span_ruler for default warning filter (#11333) * Add uk pipelines to website (#11332) * Check for . in factory names (#11336) * Make fixes for PR #11349 * Fix roman numeral coverage in #11349 Co-authored-by: Patrick J. Burns Co-authored-by: Sofie Van Landeghem Co-authored-by: Paul O'Leary McCann Co-authored-by: Adriane Boyd Co-authored-by: Lj Miranda <12949683+ljvmiranda921@users.noreply.github.com> Co-authored-by: Jules Belveze <32683010+JulesBelveze@users.noreply.github.com> Co-authored-by: stefawolf Co-authored-by: Stefanie Wolf Co-authored-by: Peter Baumgartner <5107405+pmbaumgartner@users.noreply.github.com> --- spacy/lang/la/__init__.py | 18 +++++++++++++ spacy/lang/la/lex_attrs.py | 32 +++++++++++++++++++++++ spacy/lang/la/stop_words.py | 37 +++++++++++++++++++++++++++ spacy/lang/la/tokenizer_exceptions.py | 30 ++++++++++++++++++++++ spacy/tests/conftest.py | 5 ++++ spacy/tests/lang/la/__init__.py | 0 spacy/tests/lang/la/test_exception.py | 7 +++++ spacy/tests/lang/la/test_text.py | 33 ++++++++++++++++++++++++ website/docs/api/top-level.md | 2 +- 9 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 spacy/lang/la/__init__.py create mode 100644 spacy/lang/la/lex_attrs.py create mode 100644 spacy/lang/la/stop_words.py create mode 100644 spacy/lang/la/tokenizer_exceptions.py create mode 100644 spacy/tests/lang/la/__init__.py create mode 100644 spacy/tests/lang/la/test_exception.py create mode 100644 spacy/tests/lang/la/test_text.py diff --git a/spacy/lang/la/__init__.py b/spacy/lang/la/__init__.py new file mode 100644 index 000000000..5f2cccee3 --- /dev/null +++ b/spacy/lang/la/__init__.py @@ -0,0 +1,18 @@ +from ...language import Language, BaseDefaults +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS +from .stop_words import STOP_WORDS +from .lex_attrs import LEX_ATTRS + + +class LatinDefaults(BaseDefaults): + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + stop_words = STOP_WORDS + lex_attr_getters = LEX_ATTRS + + +class Latin(Language): + lang = "la" + Defaults = LatinDefaults + + +__all__ = ["Latin"] diff --git a/spacy/lang/la/lex_attrs.py b/spacy/lang/la/lex_attrs.py new file mode 100644 index 000000000..9348a811a --- /dev/null +++ b/spacy/lang/la/lex_attrs.py @@ -0,0 +1,32 @@ +from ...attrs import LIKE_NUM +import re + +# cf. Goyvaerts/Levithan 2009; case-insensitive, allow 4 +roman_numerals_compile = re.compile(r'(?i)^(?=[MDCLXVI])M*(C[MD]|D?C{0,4})(X[CL]|L?X{0,4})(I[XV]|V?I{0,4})$') + +_num_words = set( + """ +unus una unum duo duae tres tria quattuor quinque sex septem octo novem decem +""".split() +) + +_ordinal_words = set( + """ +primus prima primum secundus secunda secundum tertius tertia tertium +""".split() +) + + +def like_num(text): + if text.isdigit(): + return True + if roman_numerals_compile.match(text): + return True + if text.lower() in _num_words: + return True + if text.lower() in _ordinal_words: + return True + return False + + +LEX_ATTRS = {LIKE_NUM: like_num} diff --git a/spacy/lang/la/stop_words.py b/spacy/lang/la/stop_words.py new file mode 100644 index 000000000..8b590bb67 --- /dev/null +++ b/spacy/lang/la/stop_words.py @@ -0,0 +1,37 @@ +# Corrected Perseus list, cf. https://wiki.digitalclassicist.org/Stopwords_for_Greek_and_Latin + +STOP_WORDS = set( + """ +ab ac ad adhuc aliqui aliquis an ante apud at atque aut autem + +cum cur + +de deinde dum + +ego enim ergo es est et etiam etsi ex + +fio + +haud hic + +iam idem igitur ille in infra inter interim ipse is ita + +magis modo mox + +nam ne nec necque neque nisi non nos + +o ob + +per possum post pro + +quae quam quare qui quia quicumque quidem quilibet quis quisnam quisquam quisque quisquis quo quoniam + +sed si sic sive sub sui sum super suus + +tam tamen trans tu tum + +ubi uel uero + +vel vero +""".split() +) diff --git a/spacy/lang/la/tokenizer_exceptions.py b/spacy/lang/la/tokenizer_exceptions.py new file mode 100644 index 000000000..905304188 --- /dev/null +++ b/spacy/lang/la/tokenizer_exceptions.py @@ -0,0 +1,30 @@ +from ..tokenizer_exceptions import BASE_EXCEPTIONS +from ...symbols import ORTH +from ...util import update_exc + + +## TODO: Look into systematically handling u/v +_exc = { + "mecum": [{ORTH: "me"}, {ORTH: "cum"}], + "tecum": [{ORTH: "te"}, {ORTH: "cum"}], + "nobiscum": [{ORTH: "nobis"}, {ORTH: "cum"}], + "vobiscum": [{ORTH: "vobis"}, {ORTH: "cum"}], + "uobiscum": [{ORTH: "uobis"}, {ORTH: "cum"}], +} + +for orth in [ + + 'A.', 'Agr.', 'Ap.', 'C.', 'Cn.', 'D.', 'F.', 'K.', 'L.', "M'.", 'M.', 'Mam.', 'N.', 'Oct.', + 'Opet.', 'P.', 'Paul.', 'Post.', 'Pro.', 'Q.', 'S.', 'Ser.', 'Sert.', 'Sex.', 'St.', 'Sta.', + 'T.', 'Ti.', 'V.', 'Vol.', 'Vop.', 'U.', 'Uol.', 'Uop.', + + 'Ian.', 'Febr.', 'Mart.', 'Apr.', 'Mai.', 'Iun.', 'Iul.', 'Aug.', 'Sept.', 'Oct.', 'Nov.', 'Nou.', + 'Dec.', + + 'Non.', 'Id.', 'A.D.', + + 'Coll.', 'Cos.', 'Ord.', 'Pl.', 'S.C.', 'Suff.', 'Trib.', +]: + _exc[orth] = [{ORTH: orth}] + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/tests/conftest.py b/spacy/tests/conftest.py index 5193bd301..0395ba7ca 100644 --- a/spacy/tests/conftest.py +++ b/spacy/tests/conftest.py @@ -256,6 +256,11 @@ def ko_tokenizer_tokenizer(): return nlp.tokenizer +@pytest.fixture(scope="module") +def la_tokenizer(): + return get_lang_class("la")().tokenizer + + @pytest.fixture(scope="session") def lb_tokenizer(): return get_lang_class("lb")().tokenizer diff --git a/spacy/tests/lang/la/__init__.py b/spacy/tests/lang/la/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/spacy/tests/lang/la/test_exception.py b/spacy/tests/lang/la/test_exception.py new file mode 100644 index 000000000..04bc1d489 --- /dev/null +++ b/spacy/tests/lang/la/test_exception.py @@ -0,0 +1,7 @@ +import pytest + +def test_la_tokenizer_handles_exc_in_text(la_tokenizer): + text = "scio te omnia facturum, ut nobiscum quam primum sis" + tokens = la_tokenizer(text) + assert len(tokens) == 11 + assert tokens[6].text == "nobis" diff --git a/spacy/tests/lang/la/test_text.py b/spacy/tests/lang/la/test_text.py new file mode 100644 index 000000000..11676b92b --- /dev/null +++ b/spacy/tests/lang/la/test_text.py @@ -0,0 +1,33 @@ +import pytest +from spacy.lang.la.lex_attrs import like_num + +@pytest.mark.parametrize( + "text,match", + [ + ("IIII", True), + ("VI", True), + ("vi", True), + ("IV", True), + ("iv", True), + ("IX", True), + ("ix", True), + ("MMXXII", True), + ("0", True), + ("1", True), + ("quattuor", True), + ("decem", True), + ("tertius", True), + ("canis", False), + ("MMXX11", False), + (",", False), + ], +) +def test_lex_attrs_like_number(la_tokenizer, text, match): + tokens = la_tokenizer(text) + assert len(tokens) == 1 + assert tokens[0].like_num == match + +@pytest.mark.parametrize("word", ["quinque"]) +def test_la_lex_attrs_capitals(word): + assert like_num(word) + assert like_num(word.upper()) diff --git a/website/docs/api/top-level.md b/website/docs/api/top-level.md index c3dc42f1a..724f2775e 100644 --- a/website/docs/api/top-level.md +++ b/website/docs/api/top-level.md @@ -451,7 +451,7 @@ factories. | Registry name | Description | | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `architectures` | Registry for functions that create [model architectures](/api/architectures). Can be used to register custom model architectures and reference them in the `config.cfg`. | -| `augmenters` | Registry for functions that create [data augmentation](#augmenters) callbacks for corpora and other training data iterators. | +| `augmenters` | Registry for functions that create [data augmentation](#augmenters) callbacks for corpora and other training data iterators. | | `batchers` | Registry for training and evaluation [data batchers](#batchers). | | `callbacks` | Registry for custom callbacks to [modify the `nlp` object](/usage/training#custom-code-nlp-callbacks) before training. | | `displacy_colors` | Registry for custom color scheme for the [`displacy` NER visualizer](/usage/visualizers). Automatically reads from [entry points](/usage/saving-loading#entry-points). | From 3f4b4b7b4fa2df6c5d888cdc97efb71093d3fb6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Tue, 30 Aug 2022 14:21:02 +0200 Subject: [PATCH 005/179] Fix `test_{prefer,require}_gpu` (#11390) * Fix `test_{prefer,require}_gpu` These tests assumed that GPUs are only supported with CuPy, but since Thinc 8.1 we also support Metal Performance Shaders. * test_misc: arrange thinc imports to be together --- spacy/tests/test_misc.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/spacy/tests/test_misc.py b/spacy/tests/test_misc.py index d8743d322..1c9b045ac 100644 --- a/spacy/tests/test_misc.py +++ b/spacy/tests/test_misc.py @@ -10,7 +10,8 @@ from spacy.ml._precomputable_affine import _backprop_precomputable_affine_paddin from spacy.util import dot_to_object, SimpleFrozenList, import_file from spacy.util import to_ternary_int from thinc.api import Config, Optimizer, ConfigValidationError -from thinc.api import set_current_ops +from thinc.api import get_current_ops, set_current_ops, NumpyOps, CupyOps, MPSOps +from thinc.compat import has_cupy_gpu, has_torch_mps_gpu from spacy.training.batchers import minibatch_by_words from spacy.lang.en import English from spacy.lang.nl import Dutch @@ -18,7 +19,6 @@ from spacy.language import DEFAULT_CONFIG_PATH from spacy.schemas import ConfigSchemaTraining, TokenPattern, TokenPatternSchema from pydantic import ValidationError -from thinc.api import get_current_ops, NumpyOps, CupyOps from .util import get_random_doc, make_tempdir @@ -111,26 +111,25 @@ def test_PrecomputableAffine(nO=4, nI=5, nF=3, nP=2): def test_prefer_gpu(): current_ops = get_current_ops() - try: - import cupy # noqa: F401 - - prefer_gpu() + if has_cupy_gpu: + assert prefer_gpu() assert isinstance(get_current_ops(), CupyOps) - except ImportError: + elif has_torch_mps_gpu: + assert prefer_gpu() + assert isinstance(get_current_ops(), MPSOps) + else: assert not prefer_gpu() set_current_ops(current_ops) def test_require_gpu(): current_ops = get_current_ops() - try: - import cupy # noqa: F401 - + if has_cupy_gpu: require_gpu() assert isinstance(get_current_ops(), CupyOps) - except ImportError: - with pytest.raises(ValueError): - require_gpu() + elif has_torch_mps_gpu: + require_gpu() + assert isinstance(get_current_ops(), MPSOps) set_current_ops(current_ops) From 8fc0efc502da2f02076575e0887cb585d0e0f391 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Wed, 31 Aug 2022 09:02:34 +0200 Subject: [PATCH 006/179] Allow string argument for disable/enable/exclude (#11406) * adding unit test for spacy.load with disable/exclude string arg * allow pure strings in from_config * update docs * upstream type adjustements * docs update * make docstring more consistent * Update spacy/language.py Co-authored-by: Adriane Boyd * two more cleanups * fix type in internal method Co-authored-by: Adriane Boyd --- spacy/__init__.py | 12 ++--- spacy/language.py | 32 +++++++----- spacy/tests/pipeline/test_pipe_methods.py | 11 +++++ spacy/util.py | 60 +++++++++++------------ website/docs/api/language.md | 27 +++++----- website/docs/api/top-level.md | 58 +++++++++++----------- 6 files changed, 112 insertions(+), 88 deletions(-) diff --git a/spacy/__init__.py b/spacy/__init__.py index 069215fda..d60f46b96 100644 --- a/spacy/__init__.py +++ b/spacy/__init__.py @@ -31,21 +31,21 @@ def load( name: Union[str, Path], *, vocab: Union[Vocab, bool] = True, - disable: Iterable[str] = util.SimpleFrozenList(), - enable: Iterable[str] = util.SimpleFrozenList(), - exclude: Iterable[str] = util.SimpleFrozenList(), + disable: Union[str, Iterable[str]] = util.SimpleFrozenList(), + enable: Union[str, Iterable[str]] = util.SimpleFrozenList(), + exclude: Union[str, Iterable[str]] = util.SimpleFrozenList(), config: Union[Dict[str, Any], Config] = util.SimpleFrozenDict(), ) -> Language: """Load a spaCy model from an installed package or a local path. name (str): Package name or model path. vocab (Vocab): A Vocab object. If True, a vocab is created. - disable (Iterable[str]): Names of pipeline components to disable. Disabled + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. - enable (Iterable[str]): Names of pipeline components to enable. All other + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other pipes will be disabled (but can be enabled later using nlp.enable_pipe). - exclude (Iterable[str]): Names of pipeline components to exclude. Excluded + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded components won't be loaded. config (Dict[str, Any] / Config): Config overrides as nested dict or dict keyed by section values in dot notation. diff --git a/spacy/language.py b/spacy/language.py index e89ae142b..ec330753c 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -1063,7 +1063,7 @@ class Language: """ if enable is None and disable is None: raise ValueError(Errors.E991) - if disable is not None and isinstance(disable, str): + if isinstance(disable, str): disable = [disable] if enable is not None: if isinstance(enable, str): @@ -1698,9 +1698,9 @@ class Language: config: Union[Dict[str, Any], Config] = {}, *, vocab: Union[Vocab, bool] = True, - disable: Iterable[str] = SimpleFrozenList(), - enable: Iterable[str] = SimpleFrozenList(), - exclude: Iterable[str] = SimpleFrozenList(), + disable: Union[str, Iterable[str]] = SimpleFrozenList(), + enable: Union[str, Iterable[str]] = SimpleFrozenList(), + exclude: Union[str, Iterable[str]] = SimpleFrozenList(), meta: Dict[str, Any] = SimpleFrozenDict(), auto_fill: bool = True, validate: bool = True, @@ -1711,12 +1711,12 @@ class Language: config (Dict[str, Any] / Config): The loaded config. vocab (Vocab): A Vocab object. If True, a vocab is created. - disable (Iterable[str]): Names of pipeline components to disable. + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. - enable (Iterable[str]): Names of pipeline components to enable. All other + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other pipes will be disabled (and can be enabled using `nlp.enable_pipe`). - exclude (Iterable[str]): Names of pipeline components to exclude. + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded components won't be loaded. meta (Dict[str, Any]): Meta overrides for nlp.meta. auto_fill (bool): Automatically fill in missing values in config based @@ -1727,6 +1727,12 @@ class Language: DOCS: https://spacy.io/api/language#from_config """ + if isinstance(disable, str): + disable = [disable] + if isinstance(enable, str): + enable = [enable] + if isinstance(exclude, str): + exclude = [exclude] if auto_fill: config = Config( cls.default_config, section_order=CONFIG_SECTION_ORDER @@ -2031,25 +2037,29 @@ class Language: @staticmethod def _resolve_component_status( - disable: Iterable[str], enable: Iterable[str], pipe_names: Collection[str] + disable: Union[str, Iterable[str]], + enable: Union[str, Iterable[str]], + pipe_names: Iterable[str], ) -> Tuple[str, ...]: """Derives whether (1) `disable` and `enable` values are consistent and (2) resolves those to a single set of disabled components. Raises an error in case of inconsistency. - disable (Iterable[str]): Names of components or serialization fields to disable. - enable (Iterable[str]): Names of pipeline components to enable. + disable (Union[str, Iterable[str]]): Name(s) of component(s) or serialization fields to disable. + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. pipe_names (Iterable[str]): Names of all pipeline components. RETURNS (Tuple[str, ...]): Names of components to exclude from pipeline w.r.t. specified includes and excludes. """ - if disable is not None and isinstance(disable, str): + if isinstance(disable, str): disable = [disable] to_disable = disable if enable: + if isinstance(enable, str): + enable = [enable] to_disable = [ pipe_name for pipe_name in pipe_names if pipe_name not in enable ] diff --git a/spacy/tests/pipeline/test_pipe_methods.py b/spacy/tests/pipeline/test_pipe_methods.py index 6f00a1cd9..b946061f6 100644 --- a/spacy/tests/pipeline/test_pipe_methods.py +++ b/spacy/tests/pipeline/test_pipe_methods.py @@ -618,6 +618,7 @@ def test_load_disable_enable() -> None: base_nlp.to_disk(tmp_dir) to_disable = ["parser", "tagger"] to_enable = ["tagger", "parser"] + single_str = "tagger" # Setting only `disable`. nlp = spacy.load(tmp_dir, disable=to_disable) @@ -632,6 +633,16 @@ def test_load_disable_enable() -> None: ] ) + # Loading with a string representing one component + nlp = spacy.load(tmp_dir, exclude=single_str) + assert single_str not in nlp.component_names + + nlp = spacy.load(tmp_dir, disable=single_str) + assert single_str in nlp.component_names + assert single_str not in nlp.pipe_names + assert nlp._disabled == {single_str} + assert nlp.disabled == [single_str] + # Testing consistent enable/disable combination. nlp = spacy.load( tmp_dir, diff --git a/spacy/util.py b/spacy/util.py index d170fc15b..4e1a62d05 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -398,9 +398,9 @@ def load_model( name: Union[str, Path], *, vocab: Union["Vocab", bool] = True, - disable: Iterable[str] = SimpleFrozenList(), - enable: Iterable[str] = SimpleFrozenList(), - exclude: Iterable[str] = SimpleFrozenList(), + disable: Union[str, Iterable[str]] = SimpleFrozenList(), + enable: Union[str, Iterable[str]] = SimpleFrozenList(), + exclude: Union[str, Iterable[str]] = SimpleFrozenList(), config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), ) -> "Language": """Load a model from a package or data path. @@ -408,9 +408,9 @@ def load_model( name (str): Package name or model path. vocab (Vocab / True): Optional vocab to pass in on initialization. If True, a new Vocab object will be created. - disable (Iterable[str]): Names of pipeline components to disable. - enable (Iterable[str]): Names of pipeline components to enable. All others will be disabled. - exclude (Iterable[str]): Names of pipeline components to exclude. + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All others will be disabled. + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. config (Dict[str, Any] / Config): Config overrides as nested dict or dict keyed by section values in dot notation. RETURNS (Language): The loaded nlp object. @@ -440,9 +440,9 @@ def load_model_from_package( name: str, *, vocab: Union["Vocab", bool] = True, - disable: Iterable[str] = SimpleFrozenList(), - enable: Iterable[str] = SimpleFrozenList(), - exclude: Iterable[str] = SimpleFrozenList(), + disable: Union[str, Iterable[str]] = SimpleFrozenList(), + enable: Union[str, Iterable[str]] = SimpleFrozenList(), + exclude: Union[str, Iterable[str]] = SimpleFrozenList(), config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), ) -> "Language": """Load a model from an installed package. @@ -450,12 +450,12 @@ def load_model_from_package( name (str): The package name. vocab (Vocab / True): Optional vocab to pass in on initialization. If True, a new Vocab object will be created. - disable (Iterable[str]): Names of pipeline components to disable. Disabled + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. - enable (Iterable[str]): Names of pipeline components to enable. All other + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other pipes will be disabled (and can be enabled using `nlp.enable_pipe`). - exclude (Iterable[str]): Names of pipeline components to exclude. Excluded + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded components won't be loaded. config (Dict[str, Any] / Config): Config overrides as nested dict or dict keyed by section values in dot notation. @@ -470,9 +470,9 @@ def load_model_from_path( *, meta: Optional[Dict[str, Any]] = None, vocab: Union["Vocab", bool] = True, - disable: Iterable[str] = SimpleFrozenList(), - enable: Iterable[str] = SimpleFrozenList(), - exclude: Iterable[str] = SimpleFrozenList(), + disable: Union[str, Iterable[str]] = SimpleFrozenList(), + enable: Union[str, Iterable[str]] = SimpleFrozenList(), + exclude: Union[str, Iterable[str]] = SimpleFrozenList(), config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), ) -> "Language": """Load a model from a data directory path. Creates Language class with @@ -482,12 +482,12 @@ def load_model_from_path( meta (Dict[str, Any]): Optional model meta. vocab (Vocab / True): Optional vocab to pass in on initialization. If True, a new Vocab object will be created. - disable (Iterable[str]): Names of pipeline components to disable. Disabled + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. - enable (Iterable[str]): Names of pipeline components to enable. All other + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other pipes will be disabled (and can be enabled using `nlp.enable_pipe`). - exclude (Iterable[str]): Names of pipeline components to exclude. Excluded + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded components won't be loaded. config (Dict[str, Any] / Config): Config overrides as nested dict or dict keyed by section values in dot notation. @@ -516,9 +516,9 @@ def load_model_from_config( *, meta: Dict[str, Any] = SimpleFrozenDict(), vocab: Union["Vocab", bool] = True, - disable: Iterable[str] = SimpleFrozenList(), - enable: Iterable[str] = SimpleFrozenList(), - exclude: Iterable[str] = SimpleFrozenList(), + disable: Union[str, Iterable[str]] = SimpleFrozenList(), + enable: Union[str, Iterable[str]] = SimpleFrozenList(), + exclude: Union[str, Iterable[str]] = SimpleFrozenList(), auto_fill: bool = False, validate: bool = True, ) -> "Language": @@ -529,12 +529,12 @@ def load_model_from_config( meta (Dict[str, Any]): Optional model meta. vocab (Vocab / True): Optional vocab to pass in on initialization. If True, a new Vocab object will be created. - disable (Iterable[str]): Names of pipeline components to disable. Disabled + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. - enable (Iterable[str]): Names of pipeline components to enable. All other + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other pipes will be disabled (and can be enabled using `nlp.enable_pipe`). - exclude (Iterable[str]): Names of pipeline components to exclude. Excluded + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded components won't be loaded. auto_fill (bool): Whether to auto-fill config with missing defaults. validate (bool): Whether to show config validation errors. @@ -616,9 +616,9 @@ def load_model_from_init_py( init_file: Union[Path, str], *, vocab: Union["Vocab", bool] = True, - disable: Iterable[str] = SimpleFrozenList(), - enable: Iterable[str] = SimpleFrozenList(), - exclude: Iterable[str] = SimpleFrozenList(), + disable: Union[str, Iterable[str]] = SimpleFrozenList(), + enable: Union[str, Iterable[str]] = SimpleFrozenList(), + exclude: Union[str, Iterable[str]] = SimpleFrozenList(), config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), ) -> "Language": """Helper function to use in the `load()` method of a model package's @@ -626,12 +626,12 @@ def load_model_from_init_py( vocab (Vocab / True): Optional vocab to pass in on initialization. If True, a new Vocab object will be created. - disable (Iterable[str]): Names of pipeline components to disable. Disabled + disable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to disable. Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling nlp.enable_pipe. - enable (Iterable[str]): Names of pipeline components to enable. All other + enable (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to enable. All other pipes will be disabled (and can be enabled using `nlp.enable_pipe`). - exclude (Iterable[str]): Names of pipeline components to exclude. Excluded + exclude (Union[str, Iterable[str]]): Name(s) of pipeline component(s) to exclude. Excluded components won't be loaded. config (Dict[str, Any] / Config): Config overrides as nested dict or dict keyed by section values in dot notation. diff --git a/website/docs/api/language.md b/website/docs/api/language.md index 9a413efaf..ed763e36a 100644 --- a/website/docs/api/language.md +++ b/website/docs/api/language.md @@ -63,17 +63,18 @@ spaCy loads a model under the hood based on its > nlp = Language.from_config(config) > ``` -| Name | Description | -| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `config` | The loaded config. ~~Union[Dict[str, Any], Config]~~ | -| _keyword-only_ | | -| `vocab` | A `Vocab` object. If `True`, a vocab is created using the default language data settings. ~~Vocab~~ | -| `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [`nlp.enable_pipe`](/api/language#enable_pipe). ~~List[str]~~ | -| `exclude` | Names of pipeline components to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~List[str]~~ | -| `meta` | [Meta data](/api/data-formats#meta) overrides. ~~Dict[str, Any]~~ | -| `auto_fill` | Whether to automatically fill in missing values in the config, based on defaults and function argument annotations. Defaults to `True`. ~~bool~~ | -| `validate` | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | -| **RETURNS** | The initialized object. ~~Language~~ | +| Name | Description | +| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `config` | The loaded config. ~~Union[Dict[str, Any], Config]~~ | +| _keyword-only_ | | +| `vocab` | A `Vocab` object. If `True`, a vocab is created using the default language data settings. ~~Vocab~~ | +| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled, but can be enabled again using [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `exclude` | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | +| `meta` | [Meta data](/api/data-formats#meta) overrides. ~~Dict[str, Any]~~ | +| `auto_fill` | Whether to automatically fill in missing values in the config, based on defaults and function argument annotations. Defaults to `True`. ~~bool~~ | +| `validate` | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | +| **RETURNS** | The initialized object. ~~Language~~ | ## Language.component {#component tag="classmethod" new="3"} @@ -695,8 +696,8 @@ As of spaCy v3.0, the `disable_pipes` method has been renamed to `select_pipes`: | Name | Description | | -------------- | ------------------------------------------------------------------------------------------------------ | | _keyword-only_ | | -| `disable` | Name(s) of pipeline components to disable. ~~Optional[Union[str, Iterable[str]]]~~ | -| `enable` | Name(s) of pipeline components that will not be disabled. ~~Optional[Union[str, Iterable[str]]]~~ | +| `disable` | Name(s) of pipeline component(s) to disable. ~~Optional[Union[str, Iterable[str]]]~~ | +| `enable` | Name(s) of pipeline component(s) that will not be disabled. ~~Optional[Union[str, Iterable[str]]]~~ | | **RETURNS** | The disabled pipes that can be restored by calling the object's `.restore()` method. ~~DisabledPipes~~ | ## Language.get_factory_meta {#get_factory_meta tag="classmethod" new="3"} diff --git a/website/docs/api/top-level.md b/website/docs/api/top-level.md index 724f2775e..220b2d6e9 100644 --- a/website/docs/api/top-level.md +++ b/website/docs/api/top-level.md @@ -45,16 +45,16 @@ specified separately using the new `exclude` keyword argument. > nlp = spacy.load("en_core_web_sm", exclude=["parser", "tagger"]) > ``` -| Name | Description | -| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `name` | Pipeline to load, i.e. package name or path. ~~Union[str, Path]~~ | -| _keyword-only_ | | -| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | -| `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). ~~List[str]~~ | -| `enable` | Names of pipeline components to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled. ~~List[str]~~ | -| `exclude` 3 | Names of pipeline components to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~List[str]~~ | -| `config` 3 | Optional config overrides, either as nested dict or dict keyed by section value in dot notation, e.g. `"components.name.value"`. ~~Union[Dict[str, Any], Config]~~ | -| **RETURNS** | A `Language` object with the loaded pipeline. ~~Language~~ | +| Name | Description | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `name` | Pipeline to load, i.e. package name or path. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | +| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled. ~~Union[str, Iterable[str]]~~ | +| `exclude` 3 | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | +| `config` 3 | Optional config overrides, either as nested dict or dict keyed by section value in dot notation, e.g. `"components.name.value"`. ~~Union[Dict[str, Any], Config]~~ | +| **RETURNS** | A `Language` object with the loaded pipeline. ~~Language~~ | Essentially, `spacy.load()` is a convenience wrapper that reads the pipeline's [`config.cfg`](/api/data-formats#config), uses the language and pipeline @@ -1049,15 +1049,16 @@ and create a `Language` object. The model data will then be loaded in via > nlp = util.load_model("/path/to/data") > ``` -| Name | Description | -| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `name` | Package name or path. ~~str~~ | -| _keyword-only_ | | -| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | -| `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [`nlp.enable_pipe`](/api/language#enable_pipe). ~~List[str]~~ | -| `exclude` 3 | Names of pipeline components to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~List[str]~~ | -| `config` 3 | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ | -| **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ | +| Name | Description | +| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Package name or path. ~~str~~ | +| _keyword-only_ | | +| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | +| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled, but can be enabled again using [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `exclude` | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | +| `config` 3 | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ | +| **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ | ### util.load_model_from_init_py {#util.load_model_from_init_py tag="function" new="2"} @@ -1073,15 +1074,16 @@ A helper function to use in the `load()` method of a pipeline package's > return load_model_from_init_py(__file__, **overrides) > ``` -| Name | Description | -| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `init_file` | Path to package's `__init__.py`, i.e. `__file__`. ~~Union[str, Path]~~ | -| _keyword-only_ | | -| `vocab` 3 | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | -| `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). ~~List[str]~~ | -| `exclude` 3 | Names of pipeline components to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~List[str]~~ | -| `config` 3 | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ | -| **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ | +| Name | Description | +| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `init_file` | Path to package's `__init__.py`, i.e. `__file__`. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `vocab` 3 | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | +| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled, but can be enabled again using [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `exclude` 3 | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | +| `config` 3 | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ | +| **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ | ### util.load_config {#util.load_config tag="function" new="3"} From 604a7c3c26bcc6737a9676c3ba1b16c9ac705be3 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Wed, 31 Aug 2022 09:03:20 +0200 Subject: [PATCH 007/179] `SpanGroup(s)`-related optimizations (#11380) * `SpanGroup`: Add support for binding copies to a new reference document * `SpanGroups`: Replace superfluous serialize-deserialize roundtrip in `copy` Instead, directly copy the in-memory representations of the constituent `SpanGroup`s. * Update `SpanGroup.copy()` signature * Rename `new_doc` param to `doc` * Fix kwdarg * Update `.pyi` file and docstrings * `mypy` fix * Update spacy/tokens/span_group.pyx * Update docs Co-authored-by: Adriane Boyd --- spacy/tokens/_dict_proxies.py | 3 ++- spacy/tokens/span_group.pyi | 4 ++-- spacy/tokens/span_group.pyx | 7 +++++-- website/docs/api/spangroup.md | 7 ++++--- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/spacy/tokens/_dict_proxies.py b/spacy/tokens/_dict_proxies.py index 9630da261..6edcce13d 100644 --- a/spacy/tokens/_dict_proxies.py +++ b/spacy/tokens/_dict_proxies.py @@ -42,7 +42,8 @@ class SpanGroups(UserDict): def copy(self, doc: Optional["Doc"] = None) -> "SpanGroups": if doc is None: doc = self._ensure_doc() - return SpanGroups(doc).from_bytes(self.to_bytes()) + data_copy = ((k, v.copy(doc=doc)) for k, v in self.items()) + return SpanGroups(doc, items=data_copy) def setdefault(self, key, default=None): if not isinstance(default, SpanGroup): diff --git a/spacy/tokens/span_group.pyi b/spacy/tokens/span_group.pyi index 245eb4dbe..21cd124ab 100644 --- a/spacy/tokens/span_group.pyi +++ b/spacy/tokens/span_group.pyi @@ -1,4 +1,4 @@ -from typing import Any, Dict, Iterable +from typing import Any, Dict, Iterable, Optional from .doc import Doc from .span import Span @@ -24,4 +24,4 @@ class SpanGroup: def __getitem__(self, i: int) -> Span: ... def to_bytes(self) -> bytes: ... def from_bytes(self, bytes_data: bytes) -> SpanGroup: ... - def copy(self) -> SpanGroup: ... + def copy(self, doc: Optional[Doc] = ...) -> SpanGroup: ... diff --git a/spacy/tokens/span_group.pyx b/spacy/tokens/span_group.pyx index bb0fab24f..1aa3c0bc8 100644 --- a/spacy/tokens/span_group.pyx +++ b/spacy/tokens/span_group.pyx @@ -241,15 +241,18 @@ cdef class SpanGroup: cdef void push_back(self, SpanC span) nogil: self.c.push_back(span) - def copy(self) -> SpanGroup: + def copy(self, doc: Optional["Doc"] = None) -> SpanGroup: """Clones the span group. + doc (Doc): New reference document to which the copy is bound. RETURNS (SpanGroup): A copy of the span group. DOCS: https://spacy.io/api/spangroup#copy """ + if doc is None: + doc = self.doc return SpanGroup( - self.doc, + doc, name=self.name, attrs=deepcopy(self.attrs), spans=list(self), diff --git a/website/docs/api/spangroup.md b/website/docs/api/spangroup.md index 8dbdefc01..2d1cf73c4 100644 --- a/website/docs/api/spangroup.md +++ b/website/docs/api/spangroup.md @@ -255,9 +255,10 @@ Return a copy of the span group. > new_group = doc.spans["errors"].copy() > ``` -| Name | Description | -| ----------- | ----------------------------------------------- | -| **RETURNS** | A copy of the `SpanGroup` object. ~~SpanGroup~~ | +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------- | +| `doc` | The document to which the copy is bound. Defaults to `None` for the current doc. ~~Optional[Doc]~~ | +| **RETURNS** | A copy of the `SpanGroup` object. ~~SpanGroup~~ | ## SpanGroup.to_bytes {#to_bytes tag="method"} From 78f5503a29b3ab27b860220499346b79d26e666b Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Thu, 1 Sep 2022 19:37:23 +0200 Subject: [PATCH 008/179] Check for any non-Doc returned value for components (#11424) --- spacy/errors.py | 5 +++-- spacy/language.py | 4 ++-- spacy/tests/test_language.py | 22 ++++++++++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/spacy/errors.py b/spacy/errors.py index 608305a06..5ee1476c2 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -230,8 +230,9 @@ class Errors(metaclass=ErrorsWithCodes): "initialized component.") E004 = ("Can't set up pipeline component: a factory for '{name}' already " "exists. Existing factory: {func}. New factory: {new_func}") - E005 = ("Pipeline component '{name}' returned None. If you're using a " - "custom component, maybe you forgot to return the processed Doc?") + E005 = ("Pipeline component '{name}' returned {returned_type} instead of a " + "Doc. If you're using a custom component, maybe you forgot to " + "return the processed Doc?") E006 = ("Invalid constraints for adding pipeline component. You can only " "set one of the following: before (component name or index), " "after (component name or index), first (True) or last (True). " diff --git a/spacy/language.py b/spacy/language.py index ec330753c..34a06e576 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -1028,8 +1028,8 @@ class Language: raise ValueError(Errors.E109.format(name=name)) from e except Exception as e: error_handler(name, proc, [doc], e) - if doc is None: - raise ValueError(Errors.E005.format(name=name)) + if not isinstance(doc, Doc): + raise ValueError(Errors.E005.format(name=name, returned_type=type(doc))) return doc def disable_pipes(self, *names) -> "DisabledPipes": diff --git a/spacy/tests/test_language.py b/spacy/tests/test_language.py index 6f3ba8acc..03a98d32f 100644 --- a/spacy/tests/test_language.py +++ b/spacy/tests/test_language.py @@ -670,3 +670,25 @@ def test_dot_in_factory_names(nlp): with pytest.raises(ValueError, match="not permitted"): Language.factory("my.evil.component.v1", func=evil_component) + + +def test_component_return(): + """Test that an error is raised if components return a type other than a + doc.""" + nlp = English() + + @Language.component("test_component_good_pipe") + def good_pipe(doc): + return doc + + nlp.add_pipe("test_component_good_pipe") + nlp("text") + nlp.remove_pipe("test_component_good_pipe") + + @Language.component("test_component_bad_pipe") + def bad_pipe(doc): + return doc.text + + nlp.add_pipe("test_component_bad_pipe") + with pytest.raises(ValueError, match="instead of a Doc"): + nlp("text") From d1760ebe027852a10b3ba7c5c7a187859bdae76b Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Fri, 2 Sep 2022 09:09:48 +0200 Subject: [PATCH 009/179] Better handling of unexpected types in `SetPredicate` (#11312) * `Matcher`: Better type checking of values in `SetPredicate` `SetPredicate`: Emit warning and return `False` on unexpected value types * Rename `value_type_mismatch` variable * Inline warning * Remove unexpected type warning from `_SetPredicate` * Ensure that `str` values are not interpreted as sequences Check elements of sequence values for convertibility to `str` or `int` * Add more `INTERSECT` and `IN` test cases * Test for inputs with multiple characters * Return `False` early instead of using a boolean flag * Remove superfluous `int` check, parentheses * Apply suggestions from code review Co-authored-by: Adriane Boyd * Appy suggestions from code review * Clarify test comment Co-authored-by: Adriane Boyd --- spacy/matcher/matcher.pyx | 23 +++++++++++++++-------- spacy/tests/matcher/test_matcher_api.py | 20 +++++++++++++++++++- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/spacy/matcher/matcher.pyx b/spacy/matcher/matcher.pyx index 5105f69ed..e1dba01a2 100644 --- a/spacy/matcher/matcher.pyx +++ b/spacy/matcher/matcher.pyx @@ -1,5 +1,5 @@ # cython: infer_types=True, cython: profile=True -from typing import List +from typing import List, Iterable from libcpp.vector cimport vector from libc.stdint cimport int32_t, int8_t @@ -867,20 +867,27 @@ class _SetPredicate: def __call__(self, Token token): if self.is_extension: - value = get_string_id(token._.get(self.attr)) + value = token._.get(self.attr) else: value = get_token_attr_for_matcher(token.c, self.attr) - if self.predicate in ("IS_SUBSET", "IS_SUPERSET", "INTERSECTS"): + if self.predicate in ("IN", "NOT_IN"): + if isinstance(value, (str, int)): + value = get_string_id(value) + else: + return False + elif self.predicate in ("IS_SUBSET", "IS_SUPERSET", "INTERSECTS"): + # ensure that all values are enclosed in a set if self.attr == MORPH: # break up MORPH into individual Feat=Val values value = set(get_string_id(v) for v in MorphAnalysis.from_id(self.vocab, value)) + elif isinstance(value, (str, int)): + value = set((get_string_id(value),)) + elif isinstance(value, Iterable) and all(isinstance(v, (str, int)) for v in value): + value = set(get_string_id(v) for v in value) else: - # treat a single value as a list - if isinstance(value, (str, int)): - value = set([get_string_id(value)]) - else: - value = set(get_string_id(v) for v in value) + return False + if self.predicate == "IN": return value in self.value elif self.predicate == "NOT_IN": diff --git a/spacy/tests/matcher/test_matcher_api.py b/spacy/tests/matcher/test_matcher_api.py index 7c16da9f8..ac905eeb4 100644 --- a/spacy/tests/matcher/test_matcher_api.py +++ b/spacy/tests/matcher/test_matcher_api.py @@ -368,6 +368,16 @@ def test_matcher_intersect_value_operator(en_vocab): doc[0]._.ext = ["A", "B"] assert len(matcher(doc)) == 1 + # INTERSECTS matches nothing for iterables that aren't all str or int + matcher = Matcher(en_vocab) + pattern = [{"_": {"ext": {"INTERSECTS": ["Abx", "C"]}}}] + matcher.add("M", [pattern]) + doc = Doc(en_vocab, words=["a", "b", "c"]) + doc[0]._.ext = [["Abx"], "B"] + assert len(matcher(doc)) == 0 + doc[0]._.ext = ["Abx", "B"] + assert len(matcher(doc)) == 1 + # INTERSECTS with an empty pattern list matches nothing matcher = Matcher(en_vocab) pattern = [{"_": {"ext": {"INTERSECTS": []}}}] @@ -476,14 +486,22 @@ def test_matcher_extension_set_membership(en_vocab): assert len(matches) == 0 -@pytest.mark.xfail(reason="IN predicate must handle sequence values in extensions") def test_matcher_extension_in_set_predicate(en_vocab): matcher = Matcher(en_vocab) Token.set_extension("ext", default=[]) pattern = [{"_": {"ext": {"IN": ["A", "C"]}}}] matcher.add("M", [pattern]) doc = Doc(en_vocab, words=["a", "b", "c"]) + + # The IN predicate expects an exact match between the + # extension value and one of the pattern's values. doc[0]._.ext = ["A", "B"] + assert len(matcher(doc)) == 0 + + doc[0]._.ext = ["A"] + assert len(matcher(doc)) == 0 + + doc[0]._.ext = "A" assert len(matcher(doc)) == 1 From 71884d0942c9b45f0ce5408496aec1aff2f0a4b7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 2 Sep 2022 11:43:20 +0200 Subject: [PATCH 010/179] Auto-format code with black (#11427) Co-authored-by: explosion-bot --- spacy/lang/la/__init__.py | 2 +- spacy/lang/la/lex_attrs.py | 4 +- spacy/lang/la/tokenizer_exceptions.py | 70 ++++++++++++++++++++++----- spacy/tests/conftest.py | 2 +- spacy/tests/lang/la/test_exception.py | 1 + spacy/tests/lang/la/test_text.py | 4 +- 6 files changed, 67 insertions(+), 16 deletions(-) diff --git a/spacy/lang/la/__init__.py b/spacy/lang/la/__init__.py index 5f2cccee3..15b87c5b9 100644 --- a/spacy/lang/la/__init__.py +++ b/spacy/lang/la/__init__.py @@ -6,7 +6,7 @@ from .lex_attrs import LEX_ATTRS class LatinDefaults(BaseDefaults): tokenizer_exceptions = TOKENIZER_EXCEPTIONS - stop_words = STOP_WORDS + stop_words = STOP_WORDS lex_attr_getters = LEX_ATTRS diff --git a/spacy/lang/la/lex_attrs.py b/spacy/lang/la/lex_attrs.py index 9348a811a..9efb4dd3c 100644 --- a/spacy/lang/la/lex_attrs.py +++ b/spacy/lang/la/lex_attrs.py @@ -2,7 +2,9 @@ from ...attrs import LIKE_NUM import re # cf. Goyvaerts/Levithan 2009; case-insensitive, allow 4 -roman_numerals_compile = re.compile(r'(?i)^(?=[MDCLXVI])M*(C[MD]|D?C{0,4})(X[CL]|L?X{0,4})(I[XV]|V?I{0,4})$') +roman_numerals_compile = re.compile( + r"(?i)^(?=[MDCLXVI])M*(C[MD]|D?C{0,4})(X[CL]|L?X{0,4})(I[XV]|V?I{0,4})$" +) _num_words = set( """ diff --git a/spacy/lang/la/tokenizer_exceptions.py b/spacy/lang/la/tokenizer_exceptions.py index 905304188..060f6e085 100644 --- a/spacy/lang/la/tokenizer_exceptions.py +++ b/spacy/lang/la/tokenizer_exceptions.py @@ -9,21 +9,67 @@ _exc = { "tecum": [{ORTH: "te"}, {ORTH: "cum"}], "nobiscum": [{ORTH: "nobis"}, {ORTH: "cum"}], "vobiscum": [{ORTH: "vobis"}, {ORTH: "cum"}], - "uobiscum": [{ORTH: "uobis"}, {ORTH: "cum"}], + "uobiscum": [{ORTH: "uobis"}, {ORTH: "cum"}], } for orth in [ - - 'A.', 'Agr.', 'Ap.', 'C.', 'Cn.', 'D.', 'F.', 'K.', 'L.', "M'.", 'M.', 'Mam.', 'N.', 'Oct.', - 'Opet.', 'P.', 'Paul.', 'Post.', 'Pro.', 'Q.', 'S.', 'Ser.', 'Sert.', 'Sex.', 'St.', 'Sta.', - 'T.', 'Ti.', 'V.', 'Vol.', 'Vop.', 'U.', 'Uol.', 'Uop.', - - 'Ian.', 'Febr.', 'Mart.', 'Apr.', 'Mai.', 'Iun.', 'Iul.', 'Aug.', 'Sept.', 'Oct.', 'Nov.', 'Nou.', - 'Dec.', - - 'Non.', 'Id.', 'A.D.', - - 'Coll.', 'Cos.', 'Ord.', 'Pl.', 'S.C.', 'Suff.', 'Trib.', + "A.", + "Agr.", + "Ap.", + "C.", + "Cn.", + "D.", + "F.", + "K.", + "L.", + "M'.", + "M.", + "Mam.", + "N.", + "Oct.", + "Opet.", + "P.", + "Paul.", + "Post.", + "Pro.", + "Q.", + "S.", + "Ser.", + "Sert.", + "Sex.", + "St.", + "Sta.", + "T.", + "Ti.", + "V.", + "Vol.", + "Vop.", + "U.", + "Uol.", + "Uop.", + "Ian.", + "Febr.", + "Mart.", + "Apr.", + "Mai.", + "Iun.", + "Iul.", + "Aug.", + "Sept.", + "Oct.", + "Nov.", + "Nou.", + "Dec.", + "Non.", + "Id.", + "A.D.", + "Coll.", + "Cos.", + "Ord.", + "Pl.", + "S.C.", + "Suff.", + "Trib.", ]: _exc[orth] = [{ORTH: orth}] diff --git a/spacy/tests/conftest.py b/spacy/tests/conftest.py index 0395ba7ca..742bfcc6a 100644 --- a/spacy/tests/conftest.py +++ b/spacy/tests/conftest.py @@ -258,7 +258,7 @@ def ko_tokenizer_tokenizer(): @pytest.fixture(scope="module") def la_tokenizer(): - return get_lang_class("la")().tokenizer + return get_lang_class("la")().tokenizer @pytest.fixture(scope="session") diff --git a/spacy/tests/lang/la/test_exception.py b/spacy/tests/lang/la/test_exception.py index 04bc1d489..966ae22cf 100644 --- a/spacy/tests/lang/la/test_exception.py +++ b/spacy/tests/lang/la/test_exception.py @@ -1,5 +1,6 @@ import pytest + def test_la_tokenizer_handles_exc_in_text(la_tokenizer): text = "scio te omnia facturum, ut nobiscum quam primum sis" tokens = la_tokenizer(text) diff --git a/spacy/tests/lang/la/test_text.py b/spacy/tests/lang/la/test_text.py index 11676b92b..48e7359a4 100644 --- a/spacy/tests/lang/la/test_text.py +++ b/spacy/tests/lang/la/test_text.py @@ -1,6 +1,7 @@ import pytest from spacy.lang.la.lex_attrs import like_num + @pytest.mark.parametrize( "text,match", [ @@ -13,7 +14,7 @@ from spacy.lang.la.lex_attrs import like_num ("ix", True), ("MMXXII", True), ("0", True), - ("1", True), + ("1", True), ("quattuor", True), ("decem", True), ("tertius", True), @@ -27,6 +28,7 @@ def test_lex_attrs_like_number(la_tokenizer, text, match): assert len(tokens) == 1 assert tokens[0].like_num == match + @pytest.mark.parametrize("word", ["quinque"]) def test_la_lex_attrs_capitals(word): assert like_num(word) From 977dc33312dd189b5b4ae1d791031d090c169c24 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Fri, 2 Sep 2022 18:58:21 +0900 Subject: [PATCH 011/179] Add a way to get the URL to download a pipeline to the CLI (#11175) * Add a dry run flag to download * Remove --dry-run, add --url option to `spacy info` instead * Make mypy happy * Print only the URL, so it's easier to use in scripts * Don't add the egg hash unless downloading an sdist * Update spacy/cli/info.py Co-authored-by: Sofie Van Landeghem * Add two implementations of requirements * Clean up requirements sample slightly This should make mypy happy * Update URL help string * Remove requirements option * Add url option to docs * Add URL to spacy info model output, when available * Add types-setuptools to testing reqs * Add types-setuptools to requirements * Add "compatible", expand docstring * Update spacy/cli/info.py Co-authored-by: Adriane Boyd * Run prettier on CLI docs * Update docs Add a sidebar about finding download URLs, with some examples of the new command. * Add download URLs to table on model page * Apply suggestions from code review Co-authored-by: Adriane Boyd * Updates from review * download url -> download link * Update docs Co-authored-by: Sofie Van Landeghem Co-authored-by: Adriane Boyd --- requirements.txt | 1 + spacy/cli/download.py | 32 ++++++++++--- spacy/cli/info.py | 58 +++++++++++++++++++++++- spacy/tests/package/test_requirements.py | 1 + website/docs/api/cli.md | 17 +++---- website/docs/usage/models.md | 36 ++++++++++----- website/src/templates/models.js | 10 ++++ 7 files changed, 127 insertions(+), 28 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3b8d66e0e..3e8501b2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -34,4 +34,5 @@ mypy>=0.910,<0.970; platform_machine!='aarch64' types-dataclasses>=0.1.3; python_version < "3.7" types-mock>=0.1.1 types-requests +types-setuptools>=57.0.0 black>=22.0,<23.0 diff --git a/spacy/cli/download.py b/spacy/cli/download.py index b7de88729..0c9a32b93 100644 --- a/spacy/cli/download.py +++ b/spacy/cli/download.py @@ -20,7 +20,7 @@ def download_cli( ctx: typer.Context, model: str = Arg(..., help="Name of pipeline package to download"), direct: bool = Opt(False, "--direct", "-d", "-D", help="Force direct download of name + version"), - sdist: bool = Opt(False, "--sdist", "-S", help="Download sdist (.tar.gz) archive instead of pre-built binary wheel") + sdist: bool = Opt(False, "--sdist", "-S", help="Download sdist (.tar.gz) archive instead of pre-built binary wheel"), # fmt: on ): """ @@ -36,7 +36,12 @@ def download_cli( download(model, direct, sdist, *ctx.args) -def download(model: str, direct: bool = False, sdist: bool = False, *pip_args) -> None: +def download( + model: str, + direct: bool = False, + sdist: bool = False, + *pip_args, +) -> None: if ( not (is_package("spacy") or is_package("spacy-nightly")) and "--no-deps" not in pip_args @@ -50,13 +55,10 @@ def download(model: str, direct: bool = False, sdist: bool = False, *pip_args) - "dependencies, you'll have to install them manually." ) pip_args = pip_args + ("--no-deps",) - suffix = SDIST_SUFFIX if sdist else WHEEL_SUFFIX - dl_tpl = "{m}-{v}/{m}-{v}{s}#egg={m}=={v}" if direct: components = model.split("-") model_name = "".join(components[:-1]) version = components[-1] - download_model(dl_tpl.format(m=model_name, v=version, s=suffix), pip_args) else: model_name = model if model in OLD_MODEL_SHORTCUTS: @@ -67,13 +69,26 @@ def download(model: str, direct: bool = False, sdist: bool = False, *pip_args) - model_name = OLD_MODEL_SHORTCUTS[model] compatibility = get_compatibility() version = get_version(model_name, compatibility) - download_model(dl_tpl.format(m=model_name, v=version, s=suffix), pip_args) + + filename = get_model_filename(model_name, version, sdist) + + download_model(filename, pip_args) msg.good( "Download and installation successful", f"You can now load the package via spacy.load('{model_name}')", ) +def get_model_filename(model_name: str, version: str, sdist: bool = False) -> str: + dl_tpl = "{m}-{v}/{m}-{v}{s}" + egg_tpl = "#egg={m}=={v}" + suffix = SDIST_SUFFIX if sdist else WHEEL_SUFFIX + filename = dl_tpl.format(m=model_name, v=version, s=suffix) + if sdist: + filename += egg_tpl.format(m=model_name, v=version) + return filename + + def get_compatibility() -> dict: if is_prerelease_version(about.__version__): version: Optional[str] = about.__version__ @@ -105,6 +120,11 @@ def get_version(model: str, comp: dict) -> str: return comp[model][0] +def get_latest_version(model: str) -> str: + comp = get_compatibility() + return get_version(model, comp) + + def download_model( filename: str, user_pip_args: Optional[Sequence[str]] = None ) -> None: diff --git a/spacy/cli/info.py b/spacy/cli/info.py index e6a1cb616..e6ac4270f 100644 --- a/spacy/cli/info.py +++ b/spacy/cli/info.py @@ -1,10 +1,13 @@ from typing import Optional, Dict, Any, Union, List import platform +import pkg_resources +import json from pathlib import Path from wasabi import Printer, MarkdownRenderer import srsly from ._util import app, Arg, Opt, string_to_list +from .download import get_model_filename, get_latest_version from .. import util from .. import about @@ -16,6 +19,7 @@ def info_cli( markdown: bool = Opt(False, "--markdown", "-md", help="Generate Markdown for GitHub issues"), silent: bool = Opt(False, "--silent", "-s", "-S", help="Don't print anything (just return)"), exclude: str = Opt("labels", "--exclude", "-e", help="Comma-separated keys to exclude from the print-out"), + url: bool = Opt(False, "--url", "-u", help="Print the URL to download the most recent compatible version of the pipeline"), # fmt: on ): """ @@ -23,10 +27,19 @@ def info_cli( print its meta information. Flag --markdown prints details in Markdown for easy copy-pasting to GitHub issues. + Flag --url prints only the download URL of the most recent compatible + version of the pipeline. + DOCS: https://spacy.io/api/cli#info """ exclude = string_to_list(exclude) - info(model, markdown=markdown, silent=silent, exclude=exclude) + info( + model, + markdown=markdown, + silent=silent, + exclude=exclude, + url=url, + ) def info( @@ -35,11 +48,20 @@ def info( markdown: bool = False, silent: bool = True, exclude: Optional[List[str]] = None, + url: bool = False, ) -> Union[str, dict]: msg = Printer(no_print=silent, pretty=not silent) if not exclude: exclude = [] - if model: + if url: + if model is not None: + title = f"Download info for pipeline '{model}'" + data = info_model_url(model) + print(data["download_url"]) + return data + else: + msg.fail("--url option requires a pipeline name", exits=1) + elif model: title = f"Info about pipeline '{model}'" data = info_model(model, silent=silent) else: @@ -99,11 +121,43 @@ def info_model(model: str, *, silent: bool = True) -> Dict[str, Any]: meta["source"] = str(model_path.resolve()) else: meta["source"] = str(model_path) + download_url = info_installed_model_url(model) + if download_url: + meta["download_url"] = download_url return { k: v for k, v in meta.items() if k not in ("accuracy", "performance", "speed") } +def info_installed_model_url(model: str) -> Optional[str]: + """Given a pipeline name, get the download URL if available, otherwise + return None. + + This is only available for pipelines installed as modules that have + dist-info available. + """ + try: + dist = pkg_resources.get_distribution(model) + data = json.loads(dist.get_metadata("direct_url.json")) + return data["url"] + except pkg_resources.DistributionNotFound: + # no such package + return None + except Exception: + # something else, like no file or invalid JSON + return None + +def info_model_url(model: str) -> Dict[str, Any]: + """Return the download URL for the latest version of a pipeline.""" + version = get_latest_version(model) + + filename = get_model_filename(model, version) + download_url = about.__download_url__ + "/" + filename + release_tpl = "https://github.com/explosion/spacy-models/releases/tag/{m}-{v}" + release_url = release_tpl.format(m=model, v=version) + return {"download_url": download_url, "release_url": release_url} + + def get_markdown( data: Dict[str, Any], title: Optional[str] = None, diff --git a/spacy/tests/package/test_requirements.py b/spacy/tests/package/test_requirements.py index e20227455..b403f274f 100644 --- a/spacy/tests/package/test_requirements.py +++ b/spacy/tests/package/test_requirements.py @@ -17,6 +17,7 @@ def test_build_dependencies(): "types-dataclasses", "types-mock", "types-requests", + "types-setuptools", ] # ignore language-specific packages that shouldn't be installed by all libs_ignore_setup = [ diff --git a/website/docs/api/cli.md b/website/docs/api/cli.md index cbd1f794a..e5cd3089b 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.md @@ -77,14 +77,15 @@ $ python -m spacy info [--markdown] [--silent] [--exclude] $ python -m spacy info [model] [--markdown] [--silent] [--exclude] ``` -| Name | Description | -| ------------------------------------------------ | --------------------------------------------------------------------------------------------- | -| `model` | A trained pipeline, i.e. package name or path (optional). ~~Optional[str] \(option)~~ | -| `--markdown`, `-md` | Print information as Markdown. ~~bool (flag)~~ | -| `--silent`, `-s` 2.0.12 | Don't print anything, just return the values. ~~bool (flag)~~ | -| `--exclude`, `-e` | Comma-separated keys to exclude from the print-out. Defaults to `"labels"`. ~~Optional[str]~~ | -| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | -| **PRINTS** | Information about your spaCy installation. | +| Name | Description | +| ------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------- | +| `model` | A trained pipeline, i.e. package name or path (optional). ~~Optional[str] \(option)~~ | +| `--markdown`, `-md` | Print information as Markdown. ~~bool (flag)~~ | +| `--silent`, `-s` 2.0.12 | Don't print anything, just return the values. ~~bool (flag)~~ | +| `--exclude`, `-e` | Comma-separated keys to exclude from the print-out. Defaults to `"labels"`. ~~Optional[str]~~ | +| `--url`, `-u` 3.5.0 | Print the URL to download the most recent compatible version of the pipeline. Requires a pipeline name. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **PRINTS** | Information about your spaCy installation. | ## validate {#validate new="2" tag="command"} diff --git a/website/docs/usage/models.md b/website/docs/usage/models.md index 56992e7e3..6971ac8b4 100644 --- a/website/docs/usage/models.md +++ b/website/docs/usage/models.md @@ -365,15 +365,32 @@ pipeline package can be found. To download a trained pipeline directly using [pip](https://pypi.python.org/pypi/pip), point `pip install` to the URL or local path of the wheel file or archive. Installing the wheel is usually more -efficient. To find the direct link to a package, head over to the -[releases](https://github.com/explosion/spacy-models/releases), right click on -the archive link and copy it to your clipboard. +efficient. + +> #### Pipeline Package URLs {#pipeline-urls} +> +> Pretrained pipeline distributions are hosted on +> [Github Releases](https://github.com/explosion/spacy-models/releases), and you +> can find download links there, as well as on the model page. You can also get +> URLs directly from the command line by using `spacy info` with the `--url` +> flag, which may be useful for automation. +> +> ```bash +> spacy info en_core_web_sm --url +> ``` +> +> This command will print the URL for the latest version of a pipeline +> compatible with the version of spaCy you're using. Note that in order to look +> up the compatibility information an internet connection is required. ```bash # With external URL $ pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0-py3-none-any.whl $ pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0.tar.gz +# Using spacy info to get the external URL +$ pip install $(spacy info en_core_web_sm --url) + # With local file $ pip install /Users/you/en_core_web_sm-3.0.0-py3-none-any.whl $ pip install /Users/you/en_core_web_sm-3.0.0.tar.gz @@ -514,21 +531,16 @@ should be specifying them directly. Because pipeline packages are valid Python packages, you can add them to your application's `requirements.txt`. If you're running your own internal PyPi installation, you can upload the pipeline packages there. pip's -[requirements file format](https://pip.pypa.io/en/latest/reference/pip_install/#requirements-file-format) -supports both package names to download via a PyPi server, as well as direct -URLs. +[requirements file format](https://pip.pypa.io/en/latest/reference/requirements-file-format/) +supports both package names to download via a PyPi server, as well as +[direct URLs](#pipeline-urls). ```text ### requirements.txt spacy>=3.0.0,<4.0.0 -https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.0.0/en_core_web_sm-3.0.0.tar.gz#egg=en_core_web_sm +en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.4.0/en_core_web_sm-3.4.0-py3-none-any.whl ``` -Specifying `#egg=` with the package name tells pip which package to expect from -the download URL. This way, the package won't be re-downloaded and overwritten -if it's already installed - just like when you're downloading a package from -PyPi. - All pipeline packages are versioned and specify their spaCy dependency. This ensures cross-compatibility and lets you specify exact version requirements for each pipeline. If you've [trained](/usage/training) your own pipeline, you can diff --git a/website/src/templates/models.js b/website/src/templates/models.js index df53f8c3c..16a2360d5 100644 --- a/website/src/templates/models.js +++ b/website/src/templates/models.js @@ -76,6 +76,7 @@ const MODEL_META = { benchmark_ner: 'NER accuracy', benchmark_speed: 'Speed', compat: 'Latest compatible package version for your spaCy installation', + download_link: 'Download link for the pipeline', } const LABEL_SCHEME_META = { @@ -138,6 +139,13 @@ function formatAccuracy(data, lang) { .filter(item => item) } +function formatDownloadLink(lang, name, version) { + const fullName = `${lang}_${name}-${version}` + const filename = `${fullName}-py3-none-any.whl` + const url = `https://github.com/explosion/spacy-models/releases/download/${fullName}/${filename}` + return {filename} +} + function formatModelMeta(data) { return { fullName: `${data.lang}_${data.name}-${data.version}`, @@ -154,6 +162,7 @@ function formatModelMeta(data) { labels: isEmptyObj(data.labels) ? null : data.labels, vectors: formatVectors(data.vectors), accuracy: formatAccuracy(data.performance, data.lang), + download_link: formatDownloadLink(data.lang, data.name, data.version), } } @@ -244,6 +253,7 @@ const Model = ({ { label: 'Components', content: components, help: MODEL_META.components }, { label: 'Pipeline', content: pipeline, help: MODEL_META.pipeline }, { label: 'Vectors', content: meta.vectors, help: MODEL_META.vecs }, + { label: 'Download Link', content: meta.download_link, help: MODEL_META.download_link }, { label: 'Sources', content: sources, help: MODEL_META.sources }, { label: 'Author', content: author }, { label: 'License', content: license }, From ff0522f8daac603e4dfb2773e1a73da61acc621d Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Thu, 1 Sep 2022 11:35:52 +0900 Subject: [PATCH 012/179] Fix asent pip package name --- website/meta/universe.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/meta/universe.json b/website/meta/universe.json index 6c8caa6a6..9145855c6 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -1192,7 +1192,7 @@ "slogan": "Fast, flexible and transparent sentiment analysis", "description": "Asent is a rule-based sentiment analysis library for Python made using spaCy. It is inspired by VADER, but uses a more modular ruleset, that allows the user to change e.g. the method for finding negations. Furthermore it includes visualisers to visualize the model predictions, making the model easily interpretable.", "github": "kennethenevoldsen/asent", - "pip": "aseny", + "pip": "asent", "code_example": [ "import spacy", "import asent", From 515d5c65d5f5d05eb8d2777e59cb5680dfcb4bd9 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Wed, 7 Sep 2022 22:24:22 +0900 Subject: [PATCH 013/179] Add dev docs on satellite packages (#11435) * Add dev docs on satellite packages * Apply suggestions from code review Co-authored-by: Sofie Van Landeghem * Add displacy link Co-authored-by: Sofie Van Landeghem --- extra/DEVELOPER_DOCS/Satellite Packages.md | 82 ++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 extra/DEVELOPER_DOCS/Satellite Packages.md diff --git a/extra/DEVELOPER_DOCS/Satellite Packages.md b/extra/DEVELOPER_DOCS/Satellite Packages.md new file mode 100644 index 000000000..02b06a90e --- /dev/null +++ b/extra/DEVELOPER_DOCS/Satellite Packages.md @@ -0,0 +1,82 @@ +# spaCy Satellite Packages + +This is a list of all the active repos relevant to spaCy besides the main one, with short descriptions, history, and current status. Archived repos will not be covered. + +## Always Included in spaCy + +These packages are always pulled in when you install spaCy. Most of them are direct dependencies, but some are transitive dependencies through other packages. + +- [spacy-legacy](https://github.com/explosion/spacy-legacy): When an architecture in spaCy changes enough to get a new version, the old version is frozen and moved to spacy-legacy. This allows us to keep the core library slim while also preserving backwards compatability. +- [thinc](https://github.com/explosion/thinc): Thinc is the machine learning library that powers trainable components in spaCy. It wraps backends like Numpy, PyTorch, and Tensorflow to provide a functional interface for specifying architectures. +- [catalogue](https://github.com/explosion/catalogue): Small library for adding function registries, like those used for model architectures in spaCy. +- [confection](https://github.com/explosion/confection): This library contains the functionality for config parsing that was formerly contained directly in Thinc. +- [spacy-loggers](https://github.com/explosion/spacy-loggers): Contains loggers beyond the default logger available in spaCy's core code base. This includes loggers integrated with third-party services, which may differ in release cadence from spaCy itself. +- [wasabi](https://github.com/explosion/wasabi): A command line formatting library, used for terminal output in spaCy. +- [srsly](https://github.com/explosion/srsly): A wrapper that vendors several serialization libraries for spaCy. Includes parsers for JSON, JSONL, MessagePack, (extended) Pickle, and YAML. +- [preshed](https://github.com/explosion/preshed): A Cython library for low-level data structures like hash maps, used for memory efficient data storage. +- [cython-blis](https://github.com/explosion/cython-blis): Fast matrix multiplication using BLIS without depending on system libraries. Required by Thinc, rather than spaCy directly. +- [murmurhash](https://github.com/explosion/murmurhash): A wrapper library for a C++ murmurhash implementation, used for string IDs in spaCy and preshed. +- [cymem](https://github.com/explosion/cymem): A small library for RAII-style memory management in Cython. + +## Optional Extensions for spaCy + +These are repos that can be used by spaCy but aren't part of a default installation. Many of these are wrappers to integrate various kinds of third-party libraries. + +- [spacy-transformers](https://github.com/explosion/spacy-transformers): A wrapper for the [HuggingFace Transformers](https://huggingface.co/docs/transformers/index) library, this handles the extensive conversion necessary to coordinate spaCy's powerful `Doc` representation, training pipeline, and the Transformer embeddings. When released, this was known as `spacy-pytorch-transformers`, but it changed to the current name when HuggingFace update the name of their library as well. +- [spacy-huggingface-hub](https://github.com/explosion/spacy-huggingface-hub): This package has a CLI script for uploading a packaged spaCy pipeline (created with `spacy package`) to the [Hugging Face Hub](https://huggingface.co/models). +- [spacy-alignments](https://github.com/explosion/spacy-alignments): A wrapper for the tokenizations library (mentioned below) with a modified build system to simplify cross-platform wheel creation. Used in spacy-transformers for aligning spaCy and HuggingFace tokenizations. +- [spacy-experimental](https://github.com/explosion/spacy-experimental): Experimental components that are not quite ready for inclusion in the main spaCy library. Usually there are unresolved questions around their APIs, so the experimental library allows us to expose them to the community for feedback before fully integrating them. +- [spacy-lookups-data](https://github.com/explosion/spacy-lookups-data): A repository of linguistic data, such as lemmas, that takes up a lot of disk space. Originally created to reduce the size of the spaCy core library. This is mainly useful if you want the data included but aren't using a pretrained pipeline; for the affected languages, the relevant data is included in pretrained pipelines directly. +- [coreferee](https://github.com/explosion/coreferee): Coreference resolution for English, French, German and Polish, optimised for limited training data and easily extensible for further languages. Used as a spaCy pipeline component. +- [spacy-stanza](https://github.com/explosion/spacy-stanza): This is a wrapper that allows the use of Stanford's Stanza library in spaCy. +- [spacy-streamlit](https://github.com/explosion/spacy-streamlit): A wrapper for the Streamlit dashboard building library to help with integrating [displaCy](https://spacy.io/api/top-level/#displacy). +- [spacymoji](https://github.com/explosion/spacymoji): A library to add extra support for emoji to spaCy, such as including character names. +- [thinc-apple-ops](https://github.com/explosion/thinc-apple-ops): A special backend for OSX that uses Apple's native libraries for improved performance. +- [os-signpost](https://github.com/explosion/os-signpost): A Python package that allows you to use the `OSSignposter` API in OSX for performance analysis. +- [spacy-ray](https://github.com/explosion/spacy-ray): A wrapper to integrate spaCy with Ray, a distributed training framework. Currently a work in progress. + +## Prodigy + +[Prodigy](https://prodi.gy) is Explosion's easy to use and highly customizable tool for annotating data. Prodigy itself requires a license, but the repos below contain documentation, examples, and editor or notebook integrations. + +- [prodigy-recipes](https://github.com/explosion/prodigy-recipes): Sample recipes for Prodigy, along with notebooks and other examples of usage. +- [vscode-prodigy](https://github.com/explosion/vscode-prodigy): A VS Code extension that lets you run Prodigy inside VS Code. +- [jupyterlab-prodigy](https://github.com/explosion/jupyterlab-prodigy): An extension for JupyterLab that lets you run Prodigy inside JupyterLab. + +## Independent Tools or Projects + +These are tools that may be related to or use spaCy, but are functional independent projects in their own right as well. + +- [floret](https://github.com/explosion/floret): A modification of fastText to use Bloom Embeddings. Can be used to add vectors with subword features to spaCy, and also works independently in the same manner as fastText. +- [sense2vec](https://github.com/explosion/sense2vec): A library to make embeddings of noun phrases or words coupled with their part of speech. This library uses spaCy. +- [spacy-vectors-builder](https://github.com/explosion/spacy-vectors-builder): This is a spaCy project that builds vectors using floret and a lot of input text. It handles downloading the input data as well as the actual building of vectors. +- [holmes-extractor](https://github.com/explosion/holmes-extractor): Information extraction from English and German texts based on predicate logic. Uses spaCy. +- [healthsea](https://github.com/explosion/healthsea): Healthsea is a project to extract information from comments about health supplements. Structurally, it's a self-contained, large spaCy project. +- [spacy-pkuseg](https://github.com/explosion/spacy-pkuseg): A fork of the pkuseg Chinese tokenizer. Used for Chinese support in spaCy, but also works independently. +- [ml-datasets](https://github.com/explosion/ml-datasets): This repo includes loaders for several standard machine learning datasets, like MNIST or WikiNER, and has historically been used in spaCy example code and documentation. + +## Documentation and Informational Repos + +These repos are used to support the spaCy docs or otherwise present information about spaCy or other Explosion projects. + +- [projects](https://github.com/explosion/projects): The projects repo is used to show detailed examples of spaCy usage. Individual projects can be checked out using the spaCy command line tool, rather than checking out the projects repo directly. +- [spacy-course](https://github.com/explosion/spacy-course): Home to the interactive spaCy course for learning about how to use the library and some basic NLP principles. +- [spacy-io-binder](https://github.com/explosion/spacy-io-binder): Home to the notebooks used for interactive examples in the documentation. + +## Organizational / Meta + +These repos are used for organizing data around spaCy, but are not something an end user would need to install as part of using the library. + +- [spacy-models](https://github.com/explosion/spacy-models): This repo contains metadata (but not training data) for all the spaCy models. This includes information about where their training data came from, version compatability, and performance information. It also includes tests for the model packages, and the built models are hosted as releases of this repo. +- [wheelwright](https://github.com/explosion/wheelwright): A tool for automating our PyPI builds and releases. +- [ec2buildwheel](https://github.com/explosion/ec2buildwheel): A small project that allows you to build Python packages in the manner of cibuildwheel, but on any EC2 image. Used by wheelwright. + +## Other + +Repos that don't fit in any of the above categories. + +- [blis](https://github.com/explosion/blis): A fork of the official BLIS library. The main branch is not updated, but work continues in various branches. This is used for cython-blis. +- [tokenizations](https://github.com/explosion/tokenizations): A library originally by Yohei Tamura to align strings with tolerance to some variations in features like case and diacritics, used for aligning tokens and wordpieces. Adopted and maintained by Explosion, but usually spacy-alignments is used instead. +- [conll-2012](https://github.com/explosion/conll-2012): A repo to hold some slightly cleaned up versions of the official scripts for the CoNLL 2012 shared task involving coreference resolution. Used in the coref project. +- [fastapi-explosion-extras](https://github.com/explosion/fastapi-explosion-extras): Some small tweaks to FastAPI used at Explosion. + From 1f23c615d7a7326ca5a38a7d768b8b70caaa0e17 Mon Sep 17 00:00:00 2001 From: Raphael Mitsch Date: Thu, 8 Sep 2022 10:38:07 +0200 Subject: [PATCH 014/179] Refactor KB for easier customization (#11268) * Add implementation of batching + backwards compatibility fixes. Tests indicate issue with batch disambiguation for custom singular entity lookups. * Fix tests. Add distinction w.r.t. batch size. * Remove redundant and add new comments. * Adjust comments. Fix variable naming in EL prediction. * Fix mypy errors. * Remove KB entity type config option. Change return types of candidate retrieval functions to Iterable from Iterator. Fix various other issues. * Update spacy/pipeline/entity_linker.py Co-authored-by: Paul O'Leary McCann * Update spacy/pipeline/entity_linker.py Co-authored-by: Paul O'Leary McCann * Update spacy/kb_base.pyx Co-authored-by: Paul O'Leary McCann * Update spacy/kb_base.pyx Co-authored-by: Paul O'Leary McCann * Update spacy/pipeline/entity_linker.py Co-authored-by: Paul O'Leary McCann * Add error messages to NotImplementedErrors. Remove redundant comment. * Fix imports. * Remove redundant comments. * Rename KnowledgeBase to InMemoryLookupKB and BaseKnowledgeBase to KnowledgeBase. * Fix tests. * Update spacy/errors.py Co-authored-by: Sofie Van Landeghem * Move KB into subdirectory. * Adjust imports after KB move to dedicated subdirectory. * Fix config imports. * Move Candidate + retrieval functions to separate module. Fix other, small issues. * Fix docstrings and error message w.r.t. class names. Fix typing for candidate retrieval functions. * Update spacy/kb/kb_in_memory.pyx Co-authored-by: Sofie Van Landeghem * Update spacy/ml/models/entity_linker.py Co-authored-by: Sofie Van Landeghem * Fix typing. * Change typing of mentions to be Span instead of Union[Span, str]. * Update docs. * Update EntityLinker and _architecture docs. * Update website/docs/api/entitylinker.md Co-authored-by: Paul O'Leary McCann * Adjust message for E1046. * Re-add section for Candidate in kb.md, add reference to dedicated page. * Update docs and docstrings. * Re-add section + reference for KnowledgeBase.get_alias_candidates() in docs. * Update spacy/kb/candidate.pyx * Update spacy/kb/kb_in_memory.pyx * Update spacy/pipeline/legacy/entity_linker.py * Remove canididate.md. Remove mistakenly added config snippet in entity_linker.py. Co-authored-by: Paul O'Leary McCann Co-authored-by: Sofie Van Landeghem --- setup.py | 4 +- spacy/errors.py | 10 +- spacy/kb/__init__.py | 3 + spacy/kb/candidate.pxd | 12 + spacy/kb/candidate.pyx | 74 +++++ spacy/kb/kb.pxd | 10 + spacy/kb/kb.pyx | 108 +++++++ spacy/{kb.pxd => kb/kb_in_memory.pxd} | 24 +- spacy/{kb.pyx => kb/kb_in_memory.pyx} | 96 ++---- spacy/ml/models/entity_linker.py | 30 +- spacy/pipeline/entity_linker.py | 184 +++++++---- spacy/pipeline/legacy/entity_linker.py | 5 +- spacy/tests/pipeline/test_entity_linker.py | 98 +++--- .../tests/serialize/test_resource_warning.py | 8 +- spacy/tests/serialize/test_serialize_kb.py | 16 +- website/docs/api/architectures.md | 14 +- website/docs/api/entitylinker.md | 5 +- website/docs/api/kb.md | 219 +++++-------- website/docs/api/kb_in_memory.md | 302 ++++++++++++++++++ website/docs/usage/101/_architecture.md | 4 +- 20 files changed, 854 insertions(+), 372 deletions(-) create mode 100644 spacy/kb/__init__.py create mode 100644 spacy/kb/candidate.pxd create mode 100644 spacy/kb/candidate.pyx create mode 100644 spacy/kb/kb.pxd create mode 100644 spacy/kb/kb.pyx rename spacy/{kb.pxd => kb/kb_in_memory.pxd} (92%) rename spacy/{kb.pyx => kb/kb_in_memory.pyx} (90%) create mode 100644 website/docs/api/kb_in_memory.md diff --git a/setup.py b/setup.py index ec1bd35fa..899d940ed 100755 --- a/setup.py +++ b/setup.py @@ -30,7 +30,9 @@ MOD_NAMES = [ "spacy.lexeme", "spacy.vocab", "spacy.attrs", - "spacy.kb", + "spacy.kb.candidate", + "spacy.kb.kb", + "spacy.kb.kb_in_memory", "spacy.ml.parser_model", "spacy.morphology", "spacy.pipeline.dep_parser", diff --git a/spacy/errors.py b/spacy/errors.py index 5ee1476c2..e2201284f 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -709,9 +709,9 @@ class Errors(metaclass=ErrorsWithCodes): "`nlp.enable_pipe` instead.") E927 = ("Can't write to frozen list Maybe you're trying to modify a computed " "property or default function argument?") - E928 = ("A KnowledgeBase can only be serialized to/from from a directory, " + E928 = ("An InMemoryLookupKB can only be serialized to/from from a directory, " "but the provided argument {loc} points to a file.") - E929 = ("Couldn't read KnowledgeBase from {loc}. The path does not seem to exist.") + E929 = ("Couldn't read InMemoryLookupKB from {loc}. The path does not seem to exist.") E930 = ("Received invalid get_examples callback in `{method}`. " "Expected function that returns an iterable of Example objects but " "got: {obj}") @@ -941,6 +941,12 @@ class Errors(metaclass=ErrorsWithCodes): "`{arg2}`={arg2_values} but these arguments are conflicting.") E1043 = ("Expected None or a value in range [{range_start}, {range_end}] for entity linker threshold, but got " "{value}.") + E1044 = ("Expected `candidates_batch_size` to be >= 1, but got: {value}") + E1045 = ("Encountered {parent} subclass without `{parent}.{method}` " + "method in '{name}'. If you want to use this method, make " + "sure it's overwritten on the subclass.") + E1046 = ("{cls_name} is an abstract class and cannot be instantiated. If you are looking for spaCy's default " + "knowledge base, use `InMemoryLookupKB`.") # Deprecated model shortcuts, only used in errors and warnings diff --git a/spacy/kb/__init__.py b/spacy/kb/__init__.py new file mode 100644 index 000000000..1d70a9b34 --- /dev/null +++ b/spacy/kb/__init__.py @@ -0,0 +1,3 @@ +from .kb import KnowledgeBase +from .kb_in_memory import InMemoryLookupKB +from .candidate import Candidate, get_candidates, get_candidates_batch diff --git a/spacy/kb/candidate.pxd b/spacy/kb/candidate.pxd new file mode 100644 index 000000000..942ce9dd0 --- /dev/null +++ b/spacy/kb/candidate.pxd @@ -0,0 +1,12 @@ +from .kb cimport KnowledgeBase +from libcpp.vector cimport vector +from ..typedefs cimport hash_t + +# Object used by the Entity Linker that summarizes one entity-alias candidate combination. +cdef class Candidate: + cdef readonly KnowledgeBase kb + cdef hash_t entity_hash + cdef float entity_freq + cdef vector[float] entity_vector + cdef hash_t alias_hash + cdef float prior_prob diff --git a/spacy/kb/candidate.pyx b/spacy/kb/candidate.pyx new file mode 100644 index 000000000..c89efeb03 --- /dev/null +++ b/spacy/kb/candidate.pyx @@ -0,0 +1,74 @@ +# cython: infer_types=True, profile=True + +from typing import Iterable +from .kb cimport KnowledgeBase +from ..tokens import Span + +cdef class Candidate: + """A `Candidate` object refers to a textual mention (`alias`) that may or may not be resolved + to a specific `entity` from a Knowledge Base. This will be used as input for the entity linking + algorithm which will disambiguate the various candidates to the correct one. + Each candidate (alias, entity) pair is assigned a certain prior probability. + + DOCS: https://spacy.io/api/kb/#candidate-init + """ + + def __init__(self, KnowledgeBase kb, entity_hash, entity_freq, entity_vector, alias_hash, prior_prob): + self.kb = kb + self.entity_hash = entity_hash + self.entity_freq = entity_freq + self.entity_vector = entity_vector + self.alias_hash = alias_hash + self.prior_prob = prior_prob + + @property + def entity(self) -> int: + """RETURNS (uint64): hash of the entity's KB ID/name""" + return self.entity_hash + + @property + def entity_(self) -> str: + """RETURNS (str): ID/name of this entity in the KB""" + return self.kb.vocab.strings[self.entity_hash] + + @property + def alias(self) -> int: + """RETURNS (uint64): hash of the alias""" + return self.alias_hash + + @property + def alias_(self) -> str: + """RETURNS (str): ID of the original alias""" + return self.kb.vocab.strings[self.alias_hash] + + @property + def entity_freq(self) -> float: + return self.entity_freq + + @property + def entity_vector(self) -> Iterable[float]: + return self.entity_vector + + @property + def prior_prob(self) -> float: + return self.prior_prob + + +def get_candidates(kb: KnowledgeBase, mention: Span) -> Iterable[Candidate]: + """ + Return candidate entities for a given mention and fetching appropriate entries from the index. + kb (KnowledgeBase): Knowledge base to query. + mention (Span): Entity mention for which to identify candidates. + RETURNS (Iterable[Candidate]): Identified candidates. + """ + return kb.get_candidates(mention) + + +def get_candidates_batch(kb: KnowledgeBase, mentions: Iterable[Span]) -> Iterable[Iterable[Candidate]]: + """ + Return candidate entities for the given mentions and fetching appropriate entries from the index. + kb (KnowledgeBase): Knowledge base to query. + mention (Iterable[Span]): Entity mentions for which to identify candidates. + RETURNS (Iterable[Iterable[Candidate]]): Identified candidates. + """ + return kb.get_candidates_batch(mentions) diff --git a/spacy/kb/kb.pxd b/spacy/kb/kb.pxd new file mode 100644 index 000000000..1adeef8ae --- /dev/null +++ b/spacy/kb/kb.pxd @@ -0,0 +1,10 @@ +"""Knowledge-base for entity or concept linking.""" + +from cymem.cymem cimport Pool +from libc.stdint cimport int64_t +from ..vocab cimport Vocab + +cdef class KnowledgeBase: + cdef Pool mem + cdef readonly Vocab vocab + cdef readonly int64_t entity_vector_length diff --git a/spacy/kb/kb.pyx b/spacy/kb/kb.pyx new file mode 100644 index 000000000..ce4bc0138 --- /dev/null +++ b/spacy/kb/kb.pyx @@ -0,0 +1,108 @@ +# cython: infer_types=True, profile=True + +from pathlib import Path +from typing import Iterable, Tuple, Union +from cymem.cymem cimport Pool + +from .candidate import Candidate +from ..tokens import Span +from ..util import SimpleFrozenList +from ..errors import Errors + + +cdef class KnowledgeBase: + """A `KnowledgeBase` instance stores unique identifiers for entities and their textual aliases, + to support entity linking of named entities to real-world concepts. + This is an abstract class and requires its operations to be implemented. + + DOCS: https://spacy.io/api/kb + """ + + def __init__(self, vocab: Vocab, entity_vector_length: int): + """Create a KnowledgeBase.""" + # Make sure abstract KB is not instantiated. + if self.__class__ == KnowledgeBase: + raise TypeError( + Errors.E1046.format(cls_name=self.__class__.__name__) + ) + + self.vocab = vocab + self.entity_vector_length = entity_vector_length + self.mem = Pool() + + def get_candidates_batch(self, mentions: Iterable[Span]) -> Iterable[Iterable[Candidate]]: + """ + Return candidate entities for specified texts. Each candidate defines the entity, the original alias, + and the prior probability of that alias resolving to that entity. + If no candidate is found for a given text, an empty list is returned. + mentions (Iterable[Span]): Mentions for which to get candidates. + RETURNS (Iterable[Iterable[Candidate]]): Identified candidates. + """ + return [self.get_candidates(span) for span in mentions] + + def get_candidates(self, mention: Span) -> Iterable[Candidate]: + """ + Return candidate entities for specified text. Each candidate defines the entity, the original alias, + and the prior probability of that alias resolving to that entity. + If the no candidate is found for a given text, an empty list is returned. + mention (Span): Mention for which to get candidates. + RETURNS (Iterable[Candidate]): Identified candidates. + """ + raise NotImplementedError( + Errors.E1045.format(parent="KnowledgeBase", method="get_candidates", name=self.__name__) + ) + + def get_vectors(self, entities: Iterable[str]) -> Iterable[Iterable[float]]: + """ + Return vectors for entities. + entity (str): Entity name/ID. + RETURNS (Iterable[Iterable[float]]): Vectors for specified entities. + """ + return [self.get_vector(entity) for entity in entities] + + def get_vector(self, str entity) -> Iterable[float]: + """ + Return vector for entity. + entity (str): Entity name/ID. + RETURNS (Iterable[float]): Vector for specified entity. + """ + raise NotImplementedError( + Errors.E1045.format(parent="KnowledgeBase", method="get_vector", name=self.__name__) + ) + + def to_bytes(self, **kwargs) -> bytes: + """Serialize the current state to a binary string. + RETURNS (bytes): Current state as binary string. + """ + raise NotImplementedError( + Errors.E1045.format(parent="KnowledgeBase", method="to_bytes", name=self.__name__) + ) + + def from_bytes(self, bytes_data: bytes, *, exclude: Tuple[str] = tuple()): + """Load state from a binary string. + bytes_data (bytes): KB state. + exclude (Tuple[str]): Properties to exclude when restoring KB. + """ + raise NotImplementedError( + Errors.E1045.format(parent="KnowledgeBase", method="from_bytes", name=self.__name__) + ) + + def to_disk(self, path: Union[str, Path], exclude: Iterable[str] = SimpleFrozenList()) -> None: + """ + Write KnowledgeBase content to disk. + path (Union[str, Path]): Target file path. + exclude (Iterable[str]): List of components to exclude. + """ + raise NotImplementedError( + Errors.E1045.format(parent="KnowledgeBase", method="to_disk", name=self.__name__) + ) + + def from_disk(self, path: Union[str, Path], exclude: Iterable[str] = SimpleFrozenList()) -> None: + """ + Load KnowledgeBase content from disk. + path (Union[str, Path]): Target file path. + exclude (Iterable[str]): List of components to exclude. + """ + raise NotImplementedError( + Errors.E1045.format(parent="KnowledgeBase", method="from_disk", name=self.__name__) + ) diff --git a/spacy/kb.pxd b/spacy/kb/kb_in_memory.pxd similarity index 92% rename from spacy/kb.pxd rename to spacy/kb/kb_in_memory.pxd index a823dbe1e..825a6bde9 100644 --- a/spacy/kb.pxd +++ b/spacy/kb/kb_in_memory.pxd @@ -1,14 +1,12 @@ """Knowledge-base for entity or concept linking.""" -from cymem.cymem cimport Pool from preshed.maps cimport PreshMap from libcpp.vector cimport vector from libc.stdint cimport int32_t, int64_t from libc.stdio cimport FILE -from .vocab cimport Vocab -from .typedefs cimport hash_t -from .structs cimport KBEntryC, AliasC - +from ..typedefs cimport hash_t +from ..structs cimport KBEntryC, AliasC +from .kb cimport KnowledgeBase ctypedef vector[KBEntryC] entry_vec ctypedef vector[AliasC] alias_vec @@ -16,21 +14,7 @@ ctypedef vector[float] float_vec ctypedef vector[float_vec] float_matrix -# Object used by the Entity Linker that summarizes one entity-alias candidate combination. -cdef class Candidate: - cdef readonly KnowledgeBase kb - cdef hash_t entity_hash - cdef float entity_freq - cdef vector[float] entity_vector - cdef hash_t alias_hash - cdef float prior_prob - - -cdef class KnowledgeBase: - cdef Pool mem - cdef readonly Vocab vocab - cdef int64_t entity_vector_length - +cdef class InMemoryLookupKB(KnowledgeBase): # This maps 64bit keys (hash of unique entity string) # to 64bit values (position of the _KBEntryC struct in the _entries vector). # The PreshMap is pretty space efficient, as it uses open addressing. So diff --git a/spacy/kb.pyx b/spacy/kb/kb_in_memory.pyx similarity index 90% rename from spacy/kb.pyx rename to spacy/kb/kb_in_memory.pyx index ae1983a8d..485e52c2f 100644 --- a/spacy/kb.pyx +++ b/spacy/kb/kb_in_memory.pyx @@ -1,8 +1,7 @@ # cython: infer_types=True, profile=True -from typing import Iterator, Iterable, Callable, Dict, Any +from typing import Iterable, Callable, Dict, Any, Union import srsly -from cymem.cymem cimport Pool from preshed.maps cimport PreshMap from cpython.exc cimport PyErr_SetFromErrno from libc.stdio cimport fopen, fclose, fread, fwrite, feof, fseek @@ -12,85 +11,28 @@ from libcpp.vector cimport vector from pathlib import Path import warnings -from .typedefs cimport hash_t -from .errors import Errors, Warnings -from . import util -from .util import SimpleFrozenList, ensure_path - -cdef class Candidate: - """A `Candidate` object refers to a textual mention (`alias`) that may or may not be resolved - to a specific `entity` from a Knowledge Base. This will be used as input for the entity linking - algorithm which will disambiguate the various candidates to the correct one. - Each candidate (alias, entity) pair is assigned to a certain prior probability. - - DOCS: https://spacy.io/api/kb/#candidate_init - """ - - def __init__(self, KnowledgeBase kb, entity_hash, entity_freq, entity_vector, alias_hash, prior_prob): - self.kb = kb - self.entity_hash = entity_hash - self.entity_freq = entity_freq - self.entity_vector = entity_vector - self.alias_hash = alias_hash - self.prior_prob = prior_prob - - @property - def entity(self): - """RETURNS (uint64): hash of the entity's KB ID/name""" - return self.entity_hash - - @property - def entity_(self): - """RETURNS (str): ID/name of this entity in the KB""" - return self.kb.vocab.strings[self.entity_hash] - - @property - def alias(self): - """RETURNS (uint64): hash of the alias""" - return self.alias_hash - - @property - def alias_(self): - """RETURNS (str): ID of the original alias""" - return self.kb.vocab.strings[self.alias_hash] - - @property - def entity_freq(self): - return self.entity_freq - - @property - def entity_vector(self): - return self.entity_vector - - @property - def prior_prob(self): - return self.prior_prob +from ..tokens import Span +from ..typedefs cimport hash_t +from ..errors import Errors, Warnings +from .. import util +from ..util import SimpleFrozenList, ensure_path +from ..vocab cimport Vocab +from .kb cimport KnowledgeBase +from .candidate import Candidate as Candidate -def get_candidates(KnowledgeBase kb, span) -> Iterator[Candidate]: - """ - Return candidate entities for a given span by using the text of the span as the alias - and fetching appropriate entries from the index. - This particular function is optimized to work with the built-in KB functionality, - but any other custom candidate generation method can be used in combination with the KB as well. - """ - return kb.get_alias_candidates(span.text) - - -cdef class KnowledgeBase: - """A `KnowledgeBase` instance stores unique identifiers for entities and their textual aliases, +cdef class InMemoryLookupKB(KnowledgeBase): + """An `InMemoryLookupKB` instance stores unique identifiers for entities and their textual aliases, to support entity linking of named entities to real-world concepts. - DOCS: https://spacy.io/api/kb + DOCS: https://spacy.io/api/kb_in_memory """ def __init__(self, Vocab vocab, entity_vector_length): - """Create a KnowledgeBase.""" - self.mem = Pool() - self.entity_vector_length = entity_vector_length + """Create an InMemoryLookupKB.""" + super().__init__(vocab, entity_vector_length) self._entry_index = PreshMap() self._alias_index = PreshMap() - self.vocab = vocab self._create_empty_vectors(dummy_hash=self.vocab.strings[""]) def _initialize_entities(self, int64_t nr_entities): @@ -104,11 +46,6 @@ cdef class KnowledgeBase: self._alias_index = PreshMap(nr_aliases + 1) self._aliases_table = alias_vec(nr_aliases + 1) - @property - def entity_vector_length(self): - """RETURNS (uint64): length of the entity vectors""" - return self.entity_vector_length - def __len__(self): return self.get_size_entities() @@ -286,7 +223,10 @@ cdef class KnowledgeBase: alias_entry.probs = probs self._aliases_table[alias_index] = alias_entry - def get_alias_candidates(self, str alias) -> Iterator[Candidate]: + def get_candidates(self, mention: Span) -> Iterable[Candidate]: + return self.get_alias_candidates(mention.text) # type: ignore + + def get_alias_candidates(self, str alias) -> Iterable[Candidate]: """ Return candidate entities for an alias. Each candidate defines the entity, the original alias, and the prior probability of that alias resolving to that entity. diff --git a/spacy/ml/models/entity_linker.py b/spacy/ml/models/entity_linker.py index d847342a3..4d18d216a 100644 --- a/spacy/ml/models/entity_linker.py +++ b/spacy/ml/models/entity_linker.py @@ -1,11 +1,12 @@ from pathlib import Path from typing import Optional, Callable, Iterable, List, Tuple from thinc.types import Floats2d -from thinc.api import chain, clone, list2ragged, reduce_mean, residual -from thinc.api import Model, Maxout, Linear, noop, tuplify, Ragged +from thinc.api import chain, list2ragged, reduce_mean, residual +from thinc.api import Model, Maxout, Linear, tuplify, Ragged from ...util import registry -from ...kb import KnowledgeBase, Candidate, get_candidates +from ...kb import KnowledgeBase, InMemoryLookupKB +from ...kb import Candidate, get_candidates, get_candidates_batch from ...vocab import Vocab from ...tokens import Span, Doc from ..extract_spans import extract_spans @@ -78,9 +79,11 @@ def span_maker_forward(model, docs: List[Doc], is_train) -> Tuple[Ragged, Callab @registry.misc("spacy.KBFromFile.v1") -def load_kb(kb_path: Path) -> Callable[[Vocab], KnowledgeBase]: - def kb_from_file(vocab): - kb = KnowledgeBase(vocab, entity_vector_length=1) +def load_kb( + kb_path: Path, +) -> Callable[[Vocab], KnowledgeBase]: + def kb_from_file(vocab: Vocab): + kb = InMemoryLookupKB(vocab, entity_vector_length=1) kb.from_disk(kb_path) return kb @@ -88,9 +91,11 @@ def load_kb(kb_path: Path) -> Callable[[Vocab], KnowledgeBase]: @registry.misc("spacy.EmptyKB.v1") -def empty_kb(entity_vector_length: int) -> Callable[[Vocab], KnowledgeBase]: - def empty_kb_factory(vocab): - return KnowledgeBase(vocab=vocab, entity_vector_length=entity_vector_length) +def empty_kb( + entity_vector_length: int, +) -> Callable[[Vocab], KnowledgeBase]: + def empty_kb_factory(vocab: Vocab): + return InMemoryLookupKB(vocab=vocab, entity_vector_length=entity_vector_length) return empty_kb_factory @@ -98,3 +103,10 @@ def empty_kb(entity_vector_length: int) -> Callable[[Vocab], KnowledgeBase]: @registry.misc("spacy.CandidateGenerator.v1") def create_candidates() -> Callable[[KnowledgeBase, Span], Iterable[Candidate]]: return get_candidates + + +@registry.misc("spacy.CandidateBatchGenerator.v1") +def create_candidates_batch() -> Callable[ + [KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]] +]: + return get_candidates_batch diff --git a/spacy/pipeline/entity_linker.py b/spacy/pipeline/entity_linker.py index 73a90b268..62845287b 100644 --- a/spacy/pipeline/entity_linker.py +++ b/spacy/pipeline/entity_linker.py @@ -53,9 +53,11 @@ DEFAULT_NEL_MODEL = Config().from_str(default_model_config)["model"] "incl_context": True, "entity_vector_length": 64, "get_candidates": {"@misc": "spacy.CandidateGenerator.v1"}, + "get_candidates_batch": {"@misc": "spacy.CandidateBatchGenerator.v1"}, "overwrite": True, "scorer": {"@scorers": "spacy.entity_linker_scorer.v1"}, "use_gold_ents": True, + "candidates_batch_size": 1, "threshold": None, }, default_score_weights={ @@ -75,9 +77,13 @@ def make_entity_linker( incl_context: bool, entity_vector_length: int, get_candidates: Callable[[KnowledgeBase, Span], Iterable[Candidate]], + get_candidates_batch: Callable[ + [KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]] + ], overwrite: bool, scorer: Optional[Callable], use_gold_ents: bool, + candidates_batch_size: int, threshold: Optional[float] = None, ): """Construct an EntityLinker component. @@ -90,17 +96,21 @@ def make_entity_linker( incl_prior (bool): Whether or not to include prior probabilities from the KB in the model. incl_context (bool): Whether or not to include the local context in the model. entity_vector_length (int): Size of encoding vectors in the KB. - get_candidates (Callable[[KnowledgeBase, "Span"], Iterable[Candidate]]): Function that + get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that produces a list of candidates, given a certain knowledge base and a textual mention. + get_candidates_batch ( + Callable[[KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]]], Iterable[Candidate]] + ): Function that produces a list of candidates, given a certain knowledge base and several textual mentions. scorer (Optional[Callable]): The scoring method. use_gold_ents (bool): Whether to copy entities from gold docs or not. If false, another component must provide entity annotations. + candidates_batch_size (int): Size of batches for entity candidate generation. threshold (Optional[float]): Confidence threshold for entity predictions. If confidence is below the threshold, prediction is discarded. If None, predictions are not filtered by any threshold. """ if not model.attrs.get("include_span_maker", False): - # The only difference in arguments here is that use_gold_ents is not available + # The only difference in arguments here is that use_gold_ents and threshold aren't available. return EntityLinker_v1( nlp.vocab, model, @@ -124,9 +134,11 @@ def make_entity_linker( incl_context=incl_context, entity_vector_length=entity_vector_length, get_candidates=get_candidates, + get_candidates_batch=get_candidates_batch, overwrite=overwrite, scorer=scorer, use_gold_ents=use_gold_ents, + candidates_batch_size=candidates_batch_size, threshold=threshold, ) @@ -160,9 +172,13 @@ class EntityLinker(TrainablePipe): incl_context: bool, entity_vector_length: int, get_candidates: Callable[[KnowledgeBase, Span], Iterable[Candidate]], + get_candidates_batch: Callable[ + [KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]] + ], overwrite: bool = BACKWARD_OVERWRITE, scorer: Optional[Callable] = entity_linker_score, use_gold_ents: bool, + candidates_batch_size: int, threshold: Optional[float] = None, ) -> None: """Initialize an entity linker. @@ -178,10 +194,14 @@ class EntityLinker(TrainablePipe): entity_vector_length (int): Size of encoding vectors in the KB. get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that produces a list of candidates, given a certain knowledge base and a textual mention. - scorer (Optional[Callable]): The scoring method. Defaults to - Scorer.score_links. + get_candidates_batch ( + Callable[[KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]]], + Iterable[Candidate]] + ): Function that produces a list of candidates, given a certain knowledge base and several textual mentions. + scorer (Optional[Callable]): The scoring method. Defaults to Scorer.score_links. use_gold_ents (bool): Whether to copy entities from gold docs or not. If false, another component must provide entity annotations. + candidates_batch_size (int): Size of batches for entity candidate generation. threshold (Optional[float]): Confidence threshold for entity predictions. If confidence is below the threshold, prediction is discarded. If None, predictions are not filtered by any threshold. DOCS: https://spacy.io/api/entitylinker#init @@ -204,22 +224,27 @@ class EntityLinker(TrainablePipe): self.incl_prior = incl_prior self.incl_context = incl_context self.get_candidates = get_candidates + self.get_candidates_batch = get_candidates_batch self.cfg: Dict[str, Any] = {"overwrite": overwrite} self.distance = CosineDistance(normalize=False) # how many neighbour sentences to take into account - # create an empty KB by default. If you want to load a predefined one, specify it in 'initialize'. + # create an empty KB by default self.kb = empty_kb(entity_vector_length)(self.vocab) self.scorer = scorer self.use_gold_ents = use_gold_ents + self.candidates_batch_size = candidates_batch_size self.threshold = threshold + if candidates_batch_size < 1: + raise ValueError(Errors.E1044) + def set_kb(self, kb_loader: Callable[[Vocab], KnowledgeBase]): """Define the KB of this pipe by providing a function that will create it using this object's vocab.""" if not callable(kb_loader): raise ValueError(Errors.E885.format(arg_type=type(kb_loader))) - self.kb = kb_loader(self.vocab) + self.kb = kb_loader(self.vocab) # type: ignore def validate_kb(self) -> None: # Raise an error if the knowledge base is not initialized. @@ -241,8 +266,8 @@ class EntityLinker(TrainablePipe): get_examples (Callable[[], Iterable[Example]]): Function that returns a representative sample of gold-standard Example objects. nlp (Language): The current nlp object the component is part of. - kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates a KnowledgeBase from a Vocab instance. - Note that providing this argument, will overwrite all data accumulated in the current KB. + kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates a KnowledgeBase from a Vocab + instance. Note that providing this argument will overwrite all data accumulated in the current KB. Use this only when loading a KB as-such from file. DOCS: https://spacy.io/api/entitylinker#initialize @@ -419,66 +444,93 @@ class EntityLinker(TrainablePipe): if len(doc) == 0: continue sentences = [s for s in doc.sents] - # Looping through each entity (TODO: rewrite) - for ent in doc.ents: - sent_index = sentences.index(ent.sent) - assert sent_index >= 0 - if self.incl_context: - # get n_neighbour sentences, clipped to the length of the document - start_sentence = max(0, sent_index - self.n_sents) - end_sentence = min(len(sentences) - 1, sent_index + self.n_sents) - start_token = sentences[start_sentence].start - end_token = sentences[end_sentence].end - sent_doc = doc[start_token:end_token].as_doc() - # currently, the context is the same for each entity in a sentence (should be refined) - sentence_encoding = self.model.predict([sent_doc])[0] - sentence_encoding_t = sentence_encoding.T - sentence_norm = xp.linalg.norm(sentence_encoding_t) - entity_count += 1 - if ent.label_ in self.labels_discard: - # ignoring this entity - setting to NIL - final_kb_ids.append(self.NIL) - else: - candidates = list(self.get_candidates(self.kb, ent)) - if not candidates: - # no prediction possible for this entity - setting to NIL - final_kb_ids.append(self.NIL) - elif len(candidates) == 1 and self.threshold is None: - # shortcut for efficiency reasons: take the 1 candidate - final_kb_ids.append(candidates[0].entity_) - else: - random.shuffle(candidates) - # set all prior probabilities to 0 if incl_prior=False - prior_probs = xp.asarray([c.prior_prob for c in candidates]) - if not self.incl_prior: - prior_probs = xp.asarray([0.0 for _ in candidates]) - scores = prior_probs - # add in similarity from the context - if self.incl_context: - entity_encodings = xp.asarray( - [c.entity_vector for c in candidates] - ) - entity_norm = xp.linalg.norm(entity_encodings, axis=1) - if len(entity_encodings) != len(prior_probs): - raise RuntimeError( - Errors.E147.format( - method="predict", - msg="vectors not of equal length", - ) - ) - # cosine similarity - sims = xp.dot(entity_encodings, sentence_encoding_t) / ( - sentence_norm * entity_norm - ) - if sims.shape != prior_probs.shape: - raise ValueError(Errors.E161) - scores = prior_probs + sims - (prior_probs * sims) - final_kb_ids.append( - candidates[scores.argmax().item()].entity_ - if self.threshold is None or scores.max() >= self.threshold - else EntityLinker.NIL + # Loop over entities in batches. + for ent_idx in range(0, len(doc.ents), self.candidates_batch_size): + ent_batch = doc.ents[ent_idx : ent_idx + self.candidates_batch_size] + + # Look up candidate entities. + valid_ent_idx = [ + idx + for idx in range(len(ent_batch)) + if ent_batch[idx].label_ not in self.labels_discard + ] + + batch_candidates = list( + self.get_candidates_batch( + self.kb, [ent_batch[idx] for idx in valid_ent_idx] + ) + if self.candidates_batch_size > 1 + else [ + self.get_candidates(self.kb, ent_batch[idx]) + for idx in valid_ent_idx + ] + ) + + # Looping through each entity in batch (TODO: rewrite) + for j, ent in enumerate(ent_batch): + sent_index = sentences.index(ent.sent) + assert sent_index >= 0 + + if self.incl_context: + # get n_neighbour sentences, clipped to the length of the document + start_sentence = max(0, sent_index - self.n_sents) + end_sentence = min( + len(sentences) - 1, sent_index + self.n_sents ) + start_token = sentences[start_sentence].start + end_token = sentences[end_sentence].end + sent_doc = doc[start_token:end_token].as_doc() + # currently, the context is the same for each entity in a sentence (should be refined) + sentence_encoding = self.model.predict([sent_doc])[0] + sentence_encoding_t = sentence_encoding.T + sentence_norm = xp.linalg.norm(sentence_encoding_t) + entity_count += 1 + if ent.label_ in self.labels_discard: + # ignoring this entity - setting to NIL + final_kb_ids.append(self.NIL) + else: + candidates = list(batch_candidates[j]) + if not candidates: + # no prediction possible for this entity - setting to NIL + final_kb_ids.append(self.NIL) + elif len(candidates) == 1 and self.threshold is None: + # shortcut for efficiency reasons: take the 1 candidate + final_kb_ids.append(candidates[0].entity_) + else: + random.shuffle(candidates) + # set all prior probabilities to 0 if incl_prior=False + prior_probs = xp.asarray([c.prior_prob for c in candidates]) + if not self.incl_prior: + prior_probs = xp.asarray([0.0 for _ in candidates]) + scores = prior_probs + # add in similarity from the context + if self.incl_context: + entity_encodings = xp.asarray( + [c.entity_vector for c in candidates] + ) + entity_norm = xp.linalg.norm(entity_encodings, axis=1) + if len(entity_encodings) != len(prior_probs): + raise RuntimeError( + Errors.E147.format( + method="predict", + msg="vectors not of equal length", + ) + ) + # cosine similarity + sims = xp.dot(entity_encodings, sentence_encoding_t) / ( + sentence_norm * entity_norm + ) + if sims.shape != prior_probs.shape: + raise ValueError(Errors.E161) + scores = prior_probs + sims - (prior_probs * sims) + final_kb_ids.append( + candidates[scores.argmax().item()].entity_ + if self.threshold is None + or scores.max() >= self.threshold + else EntityLinker.NIL + ) + if not (len(final_kb_ids) == entity_count): err = Errors.E147.format( method="predict", msg="result variables not of equal length" diff --git a/spacy/pipeline/legacy/entity_linker.py b/spacy/pipeline/legacy/entity_linker.py index 2f8a1f8ea..c14dfa1db 100644 --- a/spacy/pipeline/legacy/entity_linker.py +++ b/spacy/pipeline/legacy/entity_linker.py @@ -68,8 +68,7 @@ class EntityLinker_v1(TrainablePipe): entity_vector_length (int): Size of encoding vectors in the KB. get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that produces a list of candidates, given a certain knowledge base and a textual mention. - scorer (Optional[Callable]): The scoring method. Defaults to - Scorer.score_links. + scorer (Optional[Callable]): The scoring method. Defaults to Scorer.score_links. DOCS: https://spacy.io/api/entitylinker#init """ self.vocab = vocab @@ -115,7 +114,7 @@ class EntityLinker_v1(TrainablePipe): get_examples (Callable[[], Iterable[Example]]): Function that returns a representative sample of gold-standard Example objects. nlp (Language): The current nlp object the component is part of. - kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates a KnowledgeBase from a Vocab instance. + kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates an InMemoryLookupKB from a Vocab instance. Note that providing this argument, will overwrite all data accumulated in the current KB. Use this only when loading a KB as-such from file. diff --git a/spacy/tests/pipeline/test_entity_linker.py b/spacy/tests/pipeline/test_entity_linker.py index 82bc976bb..4d683acc5 100644 --- a/spacy/tests/pipeline/test_entity_linker.py +++ b/spacy/tests/pipeline/test_entity_linker.py @@ -6,7 +6,7 @@ from numpy.testing import assert_equal from spacy import registry, util from spacy.attrs import ENT_KB_ID from spacy.compat import pickle -from spacy.kb import Candidate, KnowledgeBase, get_candidates +from spacy.kb import Candidate, InMemoryLookupKB, get_candidates, KnowledgeBase from spacy.lang.en import English from spacy.ml import load_kb from spacy.pipeline import EntityLinker @@ -34,7 +34,7 @@ def assert_almost_equal(a, b): def test_issue4674(): """Test that setting entities with overlapping identifiers does not mess up IO""" nlp = English() - kb = KnowledgeBase(nlp.vocab, entity_vector_length=3) + kb = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) vector1 = [0.9, 1.1, 1.01] vector2 = [1.8, 2.25, 2.01] with pytest.warns(UserWarning): @@ -51,7 +51,7 @@ def test_issue4674(): dir_path.mkdir() file_path = dir_path / "kb" kb.to_disk(str(file_path)) - kb2 = KnowledgeBase(nlp.vocab, entity_vector_length=3) + kb2 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) kb2.from_disk(str(file_path)) assert kb2.get_size_entities() == 1 @@ -59,9 +59,9 @@ def test_issue4674(): @pytest.mark.issue(6730) def test_issue6730(en_vocab): """Ensure that the KB does not accept empty strings, but otherwise IO works fine.""" - from spacy.kb import KnowledgeBase + from spacy.kb.kb_in_memory import InMemoryLookupKB - kb = KnowledgeBase(en_vocab, entity_vector_length=3) + kb = InMemoryLookupKB(en_vocab, entity_vector_length=3) kb.add_entity(entity="1", freq=148, entity_vector=[1, 2, 3]) with pytest.raises(ValueError): @@ -127,7 +127,7 @@ def test_issue7065_b(): def create_kb(vocab): # create artificial KB - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q270853", freq=12, entity_vector=[9, 1, -7]) mykb.add_alias( alias="No. 8", @@ -190,7 +190,7 @@ def test_no_entities(): def create_kb(vocab): # create artificial KB - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) mykb.add_alias("Russ Cochran", ["Q2146908"], [0.9]) return mykb @@ -231,7 +231,7 @@ def test_partial_links(): def create_kb(vocab): # create artificial KB - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) mykb.add_alias("Russ Cochran", ["Q2146908"], [0.9]) return mykb @@ -263,7 +263,7 @@ def test_partial_links(): def test_kb_valid_entities(nlp): """Test the valid construction of a KB with 3 entities and two aliases""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=3) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) # adding entities mykb.add_entity(entity="Q1", freq=19, entity_vector=[8, 4, 3]) @@ -292,7 +292,7 @@ def test_kb_valid_entities(nlp): def test_kb_invalid_entities(nlp): """Test the invalid construction of a KB with an alias linked to a non-existing entity""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) # adding entities mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) @@ -308,7 +308,7 @@ def test_kb_invalid_entities(nlp): def test_kb_invalid_probabilities(nlp): """Test the invalid construction of a KB with wrong prior probabilities""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) # adding entities mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) @@ -322,7 +322,7 @@ def test_kb_invalid_probabilities(nlp): def test_kb_invalid_combination(nlp): """Test the invalid construction of a KB with non-matching entity and probability lists""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) # adding entities mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) @@ -338,7 +338,7 @@ def test_kb_invalid_combination(nlp): def test_kb_invalid_entity_vector(nlp): """Test the invalid construction of a KB with non-matching entity vector lengths""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=3) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) # adding entities mykb.add_entity(entity="Q1", freq=19, entity_vector=[1, 2, 3]) @@ -376,7 +376,7 @@ def test_kb_initialize_empty(nlp): def test_kb_serialize(nlp): """Test serialization of the KB""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) with make_tempdir() as d: # normal read-write behaviour mykb.to_disk(d / "kb") @@ -393,12 +393,12 @@ def test_kb_serialize(nlp): @pytest.mark.issue(9137) def test_kb_serialize_2(nlp): v = [5, 6, 7, 8] - kb1 = KnowledgeBase(vocab=nlp.vocab, entity_vector_length=4) + kb1 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4) kb1.set_entities(["E1"], [1], [v]) assert kb1.get_vector("E1") == v with make_tempdir() as d: kb1.to_disk(d / "kb") - kb2 = KnowledgeBase(vocab=nlp.vocab, entity_vector_length=4) + kb2 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4) kb2.from_disk(d / "kb") assert kb2.get_vector("E1") == v @@ -408,7 +408,7 @@ def test_kb_set_entities(nlp): v = [5, 6, 7, 8] v1 = [1, 1, 1, 0] v2 = [2, 2, 2, 3] - kb1 = KnowledgeBase(vocab=nlp.vocab, entity_vector_length=4) + kb1 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4) kb1.set_entities(["E0"], [1], [v]) assert kb1.get_entity_strings() == ["E0"] kb1.set_entities(["E1", "E2"], [1, 9], [v1, v2]) @@ -417,7 +417,7 @@ def test_kb_set_entities(nlp): assert kb1.get_vector("E2") == v2 with make_tempdir() as d: kb1.to_disk(d / "kb") - kb2 = KnowledgeBase(vocab=nlp.vocab, entity_vector_length=4) + kb2 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4) kb2.from_disk(d / "kb") assert set(kb2.get_entity_strings()) == {"E1", "E2"} assert kb2.get_vector("E1") == v1 @@ -428,7 +428,7 @@ def test_kb_serialize_vocab(nlp): """Test serialization of the KB and custom strings""" entity = "MyFunnyID" assert entity not in nlp.vocab.strings - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) assert not mykb.contains_entity(entity) mykb.add_entity(entity, freq=342, entity_vector=[3]) assert mykb.contains_entity(entity) @@ -436,14 +436,14 @@ def test_kb_serialize_vocab(nlp): with make_tempdir() as d: # normal read-write behaviour mykb.to_disk(d / "kb") - mykb_new = KnowledgeBase(Vocab(), entity_vector_length=1) + mykb_new = InMemoryLookupKB(Vocab(), entity_vector_length=1) mykb_new.from_disk(d / "kb") assert entity in mykb_new.vocab.strings def test_candidate_generation(nlp): """Test correct candidate generation""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) doc = nlp("douglas adam Adam shrubbery") douglas_ent = doc[0:1] @@ -481,7 +481,7 @@ def test_el_pipe_configuration(nlp): ruler.add_patterns([pattern]) def create_kb(vocab): - kb = KnowledgeBase(vocab, entity_vector_length=1) + kb = InMemoryLookupKB(vocab, entity_vector_length=1) kb.add_entity(entity="Q2", freq=12, entity_vector=[2]) kb.add_entity(entity="Q3", freq=5, entity_vector=[3]) kb.add_alias(alias="douglas", entities=["Q2", "Q3"], probabilities=[0.8, 0.1]) @@ -500,10 +500,21 @@ def test_el_pipe_configuration(nlp): def get_lowercased_candidates(kb, span): return kb.get_alias_candidates(span.text.lower()) + def get_lowercased_candidates_batch(kb, spans): + return [get_lowercased_candidates(kb, span) for span in spans] + @registry.misc("spacy.LowercaseCandidateGenerator.v1") - def create_candidates() -> Callable[[KnowledgeBase, "Span"], Iterable[Candidate]]: + def create_candidates() -> Callable[ + [InMemoryLookupKB, "Span"], Iterable[Candidate] + ]: return get_lowercased_candidates + @registry.misc("spacy.LowercaseCandidateBatchGenerator.v1") + def create_candidates_batch() -> Callable[ + [InMemoryLookupKB, Iterable["Span"]], Iterable[Iterable[Candidate]] + ]: + return get_lowercased_candidates_batch + # replace the pipe with a new one with with a different candidate generator entity_linker = nlp.replace_pipe( "entity_linker", @@ -511,6 +522,9 @@ def test_el_pipe_configuration(nlp): config={ "incl_context": False, "get_candidates": {"@misc": "spacy.LowercaseCandidateGenerator.v1"}, + "get_candidates_batch": { + "@misc": "spacy.LowercaseCandidateBatchGenerator.v1" + }, }, ) entity_linker.set_kb(create_kb) @@ -532,7 +546,7 @@ def test_nel_nsents(nlp): def test_vocab_serialization(nlp): """Test that string information is retained across storage""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) # adding entities mykb.add_entity(entity="Q1", freq=27, entity_vector=[1]) @@ -552,7 +566,7 @@ def test_vocab_serialization(nlp): with make_tempdir() as d: mykb.to_disk(d / "kb") - kb_new_vocab = KnowledgeBase(Vocab(), entity_vector_length=1) + kb_new_vocab = InMemoryLookupKB(Vocab(), entity_vector_length=1) kb_new_vocab.from_disk(d / "kb") candidates = kb_new_vocab.get_alias_candidates("adam") @@ -568,7 +582,7 @@ def test_vocab_serialization(nlp): def test_append_alias(nlp): """Test that we can append additional alias-entity pairs""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) # adding entities mykb.add_entity(entity="Q1", freq=27, entity_vector=[1]) @@ -599,7 +613,7 @@ def test_append_alias(nlp): @pytest.mark.filterwarnings("ignore:\\[W036") def test_append_invalid_alias(nlp): """Test that append an alias will throw an error if prior probs are exceeding 1""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) # adding entities mykb.add_entity(entity="Q1", freq=27, entity_vector=[1]) @@ -621,7 +635,7 @@ def test_preserving_links_asdoc(nlp): vector_length = 1 def create_kb(vocab): - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) # adding entities mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) mykb.add_entity(entity="Q2", freq=8, entity_vector=[1]) @@ -723,7 +737,7 @@ def test_overfitting_IO(): # create artificial KB - assign same prior weight to the two russ cochran's # Q2146908 (Russ Cochran): American golfer # Q7381115 (Russ Cochran): publisher - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) mykb.add_entity(entity="Q7381115", freq=12, entity_vector=[9, 1, -7]) mykb.add_alias( @@ -805,7 +819,7 @@ def test_kb_serialization(): kb_dir = tmp_dir / "kb" nlp1 = English() assert "Q2146908" not in nlp1.vocab.strings - mykb = KnowledgeBase(nlp1.vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(nlp1.vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) mykb.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) assert "Q2146908" in nlp1.vocab.strings @@ -828,7 +842,7 @@ def test_kb_serialization(): def test_kb_pickle(): # Test that the KB can be pickled nlp = English() - kb_1 = KnowledgeBase(nlp.vocab, entity_vector_length=3) + kb_1 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) kb_1.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) assert not kb_1.contains_alias("Russ Cochran") kb_1.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) @@ -842,7 +856,7 @@ def test_kb_pickle(): def test_nel_pickle(): # Test that a pipeline with an EL component can be pickled def create_kb(vocab): - kb = KnowledgeBase(vocab, entity_vector_length=3) + kb = InMemoryLookupKB(vocab, entity_vector_length=3) kb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) kb.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) return kb @@ -864,7 +878,7 @@ def test_nel_pickle(): def test_kb_to_bytes(): # Test that the KB's to_bytes method works correctly nlp = English() - kb_1 = KnowledgeBase(nlp.vocab, entity_vector_length=3) + kb_1 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) kb_1.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) kb_1.add_entity(entity="Q66", freq=9, entity_vector=[1, 2, 3]) kb_1.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) @@ -874,7 +888,7 @@ def test_kb_to_bytes(): ) assert kb_1.contains_alias("Russ Cochran") kb_bytes = kb_1.to_bytes() - kb_2 = KnowledgeBase(nlp.vocab, entity_vector_length=3) + kb_2 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) assert not kb_2.contains_alias("Russ Cochran") kb_2 = kb_2.from_bytes(kb_bytes) # check that both KBs are exactly the same @@ -897,7 +911,7 @@ def test_kb_to_bytes(): def test_nel_to_bytes(): # Test that a pipeline with an EL component can be converted to bytes def create_kb(vocab): - kb = KnowledgeBase(vocab, entity_vector_length=3) + kb = InMemoryLookupKB(vocab, entity_vector_length=3) kb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) kb.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) return kb @@ -987,7 +1001,7 @@ def test_legacy_architectures(name, config): train_examples.append(Example.from_dict(doc, annotation)) def create_kb(vocab): - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) mykb.add_entity(entity="Q7381115", freq=12, entity_vector=[9, 1, -7]) mykb.add_alias( @@ -1054,7 +1068,7 @@ def test_no_gold_ents(patterns): def create_kb(vocab): # create artificial KB - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q613241", freq=12, entity_vector=[6, -4, 3]) mykb.add_alias("Kirby", ["Q613241"], [0.9]) # Placeholder @@ -1104,7 +1118,7 @@ def test_tokenization_mismatch(): def create_kb(vocab): # create placeholder KB - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q613241", freq=12, entity_vector=[6, -4, 3]) mykb.add_alias("Kirby", ["Q613241"], [0.9]) return mykb @@ -1121,6 +1135,12 @@ def test_tokenization_mismatch(): nlp.evaluate(train_examples) +def test_abstract_kb_instantiation(): + """Test whether instantiation of abstract KB base class fails.""" + with pytest.raises(TypeError): + KnowledgeBase(None, 3) + + # fmt: off @pytest.mark.parametrize( "meet_threshold,config", @@ -1151,7 +1171,7 @@ def test_threshold(meet_threshold: bool, config: Dict[str, Any]): def create_kb(vocab): # create artificial KB - mykb = KnowledgeBase(vocab, entity_vector_length=3) + mykb = InMemoryLookupKB(vocab, entity_vector_length=3) mykb.add_entity(entity=entity_id, freq=12, entity_vector=[6, -4, 3]) mykb.add_alias( alias="Mahler", diff --git a/spacy/tests/serialize/test_resource_warning.py b/spacy/tests/serialize/test_resource_warning.py index a00b2a688..38701c6d9 100644 --- a/spacy/tests/serialize/test_resource_warning.py +++ b/spacy/tests/serialize/test_resource_warning.py @@ -3,7 +3,7 @@ from unittest import TestCase import pytest import srsly from numpy import zeros -from spacy.kb import KnowledgeBase, Writer +from spacy.kb.kb_in_memory import InMemoryLookupKB, Writer from spacy.vectors import Vectors from spacy.language import Language from spacy.pipeline import TrainablePipe @@ -71,7 +71,7 @@ def entity_linker(): nlp = Language() def create_kb(vocab): - kb = KnowledgeBase(vocab, entity_vector_length=1) + kb = InMemoryLookupKB(vocab, entity_vector_length=1) kb.add_entity("test", 0.0, zeros((1, 1), dtype="f")) return kb @@ -120,7 +120,7 @@ def test_writer_with_path_py35(): def test_save_and_load_knowledge_base(): nlp = Language() - kb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + kb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) with make_tempdir() as d: path = d / "kb" try: @@ -129,7 +129,7 @@ def test_save_and_load_knowledge_base(): pytest.fail(str(e)) try: - kb_loaded = KnowledgeBase(nlp.vocab, entity_vector_length=1) + kb_loaded = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) kb_loaded.from_disk(path) except Exception as e: pytest.fail(str(e)) diff --git a/spacy/tests/serialize/test_serialize_kb.py b/spacy/tests/serialize/test_serialize_kb.py index 1e0ae3c76..8d3653ab1 100644 --- a/spacy/tests/serialize/test_serialize_kb.py +++ b/spacy/tests/serialize/test_serialize_kb.py @@ -2,7 +2,7 @@ from typing import Callable from spacy import util from spacy.util import ensure_path, registry, load_model_from_config -from spacy.kb import KnowledgeBase +from spacy.kb.kb_in_memory import InMemoryLookupKB from spacy.vocab import Vocab from thinc.api import Config @@ -22,7 +22,7 @@ def test_serialize_kb_disk(en_vocab): dir_path.mkdir() file_path = dir_path / "kb" kb1.to_disk(str(file_path)) - kb2 = KnowledgeBase(vocab=en_vocab, entity_vector_length=3) + kb2 = InMemoryLookupKB(vocab=en_vocab, entity_vector_length=3) kb2.from_disk(str(file_path)) # final assertions @@ -30,7 +30,7 @@ def test_serialize_kb_disk(en_vocab): def _get_dummy_kb(vocab): - kb = KnowledgeBase(vocab, entity_vector_length=3) + kb = InMemoryLookupKB(vocab, entity_vector_length=3) kb.add_entity(entity="Q53", freq=33, entity_vector=[0, 5, 3]) kb.add_entity(entity="Q17", freq=2, entity_vector=[7, 1, 0]) kb.add_entity(entity="Q007", freq=7, entity_vector=[0, 0, 7]) @@ -104,7 +104,7 @@ def test_serialize_subclassed_kb(): custom_field = 666 """ - class SubKnowledgeBase(KnowledgeBase): + class SubInMemoryLookupKB(InMemoryLookupKB): def __init__(self, vocab, entity_vector_length, custom_field): super().__init__(vocab, entity_vector_length) self.custom_field = custom_field @@ -112,9 +112,9 @@ def test_serialize_subclassed_kb(): @registry.misc("spacy.CustomKB.v1") def custom_kb( entity_vector_length: int, custom_field: int - ) -> Callable[[Vocab], KnowledgeBase]: + ) -> Callable[[Vocab], InMemoryLookupKB]: def custom_kb_factory(vocab): - kb = SubKnowledgeBase( + kb = SubInMemoryLookupKB( vocab=vocab, entity_vector_length=entity_vector_length, custom_field=custom_field, @@ -129,7 +129,7 @@ def test_serialize_subclassed_kb(): nlp.initialize() entity_linker = nlp.get_pipe("entity_linker") - assert type(entity_linker.kb) == SubKnowledgeBase + assert type(entity_linker.kb) == SubInMemoryLookupKB assert entity_linker.kb.entity_vector_length == 342 assert entity_linker.kb.custom_field == 666 @@ -139,6 +139,6 @@ def test_serialize_subclassed_kb(): nlp2 = util.load_model_from_path(tmp_dir) entity_linker2 = nlp2.get_pipe("entity_linker") # After IO, the KB is the standard one - assert type(entity_linker2.kb) == KnowledgeBase + assert type(entity_linker2.kb) == InMemoryLookupKB assert entity_linker2.kb.entity_vector_length == 342 assert not hasattr(entity_linker2.kb, "custom_field") diff --git a/website/docs/api/architectures.md b/website/docs/api/architectures.md index 2537faff6..a3cb07b44 100644 --- a/website/docs/api/architectures.md +++ b/website/docs/api/architectures.md @@ -587,8 +587,8 @@ consists of either two or three subnetworks: run once for each batch. - **lower**: Construct a feature-specific vector for each `(token, feature)` pair. This is also run once for each batch. Constructing the state - representation is then a matter of summing the component features and - applying the non-linearity. + representation is then a matter of summing the component features and applying + the non-linearity. - **upper** (optional): A feed-forward network that predicts scores from the state representation. If not present, the output from the lower model is used as action scores directly. @@ -628,8 +628,8 @@ same signature, but the `use_upper` argument was `True` by default. > ``` Build a tagger model, using a provided token-to-vector component. The tagger -model adds a linear layer with softmax activation to predict scores given -the token vectors. +model adds a linear layer with softmax activation to predict scores given the +token vectors. | Name | Description | | ----------- | ------------------------------------------------------------------------------------------ | @@ -919,6 +919,6 @@ A function that reads an existing `KnowledgeBase` from file. A function that takes as input a [`KnowledgeBase`](/api/kb) and a [`Span`](/api/span) object denoting a named entity, and returns a list of -plausible [`Candidate`](/api/kb/#candidate) objects. The default -`CandidateGenerator` uses the text of a mention to find its potential -aliases in the `KnowledgeBase`. Note that this function is case-dependent. +plausible [`Candidate`](/api/kb#candidate) objects. The default +`CandidateGenerator` uses the text of a mention to find its potential aliases in +the `KnowledgeBase`. Note that this function is case-dependent. diff --git a/website/docs/api/entitylinker.md b/website/docs/api/entitylinker.md index 43e08a39c..40ec8afb5 100644 --- a/website/docs/api/entitylinker.md +++ b/website/docs/api/entitylinker.md @@ -14,7 +14,8 @@ entities) to unique identifiers, grounding the named entities into the "real world". It requires a `KnowledgeBase`, as well as a function to generate plausible candidates from that `KnowledgeBase` given a certain textual mention, and a machine learning model to pick the right candidate, given the local -context of the mention. +context of the mention. `EntityLinker` defaults to using the +[`InMemoryLookupKB`](/api/kb_in_memory) implementation. ## Assigned Attributes {#assigned-attributes} @@ -170,7 +171,7 @@ with the current vocab. > > ```python > def create_kb(vocab): -> kb = KnowledgeBase(vocab, entity_vector_length=128) +> kb = InMemoryLookupKB(vocab, entity_vector_length=128) > kb.add_entity(...) > kb.add_alias(...) > return kb diff --git a/website/docs/api/kb.md b/website/docs/api/kb.md index e7a8fcd6f..b217a1678 100644 --- a/website/docs/api/kb.md +++ b/website/docs/api/kb.md @@ -4,27 +4,45 @@ teaser: A storage class for entities and aliases of a specific knowledge base (ontology) tag: class -source: spacy/kb.pyx +source: spacy/kb/kb.pyx new: 2.2 --- -The `KnowledgeBase` object provides a method to generate -[`Candidate`](/api/kb/#candidate) objects, which are plausible external +The `KnowledgeBase` object is an abstract class providing a method to generate +[`Candidate`](/api/kb#candidate) objects, which are plausible external identifiers given a certain textual mention. Each such `Candidate` holds information from the relevant KB entities, such as its frequency in text and possible aliases. Each entity in the knowledge base also has a pretrained entity vector of a fixed size. +Beyond that, `KnowledgeBase` classes have to implement a number of utility +functions called by the [`EntityLinker`](/api/entitylinker) component. + + + +This class was not abstract up to spaCy version 3.5. The `KnowledgeBase` +implementation up to that point is available as `InMemoryLookupKB` from 3.5 +onwards. + + + ## KnowledgeBase.\_\_init\_\_ {#init tag="method"} -Create the knowledge base. +`KnowledgeBase` is an abstract class and cannot be instantiated. Its child +classes should call `__init__()` to set up some necessary attributes. > #### Example > > ```python > from spacy.kb import KnowledgeBase +> from spacy.vocab import Vocab +> +> class FullyImplementedKB(KnowledgeBase): +> def __init__(self, vocab: Vocab, entity_vector_length: int): +> super().__init__(vocab, entity_vector_length) +> ... > vocab = nlp.vocab -> kb = KnowledgeBase(vocab=vocab, entity_vector_length=64) +> kb = FullyImplementedKB(vocab=vocab, entity_vector_length=64) > ``` | Name | Description | @@ -40,133 +58,66 @@ The length of the fixed-size entity vectors in the knowledge base. | ----------- | ------------------------------------------------ | | **RETURNS** | Length of the fixed-size entity vectors. ~~int~~ | -## KnowledgeBase.add_entity {#add_entity tag="method"} +## KnowledgeBase.get_candidates {#get_candidates tag="method"} -Add an entity to the knowledge base, specifying its corpus frequency and entity -vector, which should be of length -[`entity_vector_length`](/api/kb#entity_vector_length). +Given a certain textual mention as input, retrieve a list of candidate entities +of type [`Candidate`](/api/kb#candidate). > #### Example > > ```python -> kb.add_entity(entity="Q42", freq=32, entity_vector=vector1) -> kb.add_entity(entity="Q463035", freq=111, entity_vector=vector2) +> from spacy.lang.en import English +> nlp = English() +> doc = nlp("Douglas Adams wrote 'The Hitchhiker's Guide to the Galaxy'.") +> candidates = kb.get_candidates(doc[0:2]) > ``` -| Name | Description | -| --------------- | ---------------------------------------------------------- | -| `entity` | The unique entity identifier. ~~str~~ | -| `freq` | The frequency of the entity in a typical corpus. ~~float~~ | -| `entity_vector` | The pretrained vector of the entity. ~~numpy.ndarray~~ | +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `mention` | The textual mention or alias. ~~Span~~ | +| **RETURNS** | An iterable of relevant `Candidate` objects. ~~Iterable[Candidate]~~ | -## KnowledgeBase.set_entities {#set_entities tag="method"} +## KnowledgeBase.get_candidates_batch {#get_candidates_batch tag="method"} -Define the full list of entities in the knowledge base, specifying the corpus -frequency and entity vector for each entity. +Same as [`get_candidates()`](/api/kb#get_candidates), but for an arbitrary +number of mentions. The [`EntityLinker`](/api/entitylinker) component will call +`get_candidates_batch()` instead of `get_candidates()`, if the config parameter +`candidates_batch_size` is greater or equal than 1. + +The default implementation of `get_candidates_batch()` executes +`get_candidates()` in a loop. We recommend implementing a more efficient way to +retrieve candidates for multiple mentions at once, if performance is of concern +to you. > #### Example > > ```python -> kb.set_entities(entity_list=["Q42", "Q463035"], freq_list=[32, 111], vector_list=[vector1, vector2]) +> from spacy.lang.en import English +> nlp = English() +> doc = nlp("Douglas Adams wrote 'The Hitchhiker's Guide to the Galaxy'.") +> candidates = kb.get_candidates((doc[0:2], doc[3:])) > ``` -| Name | Description | -| ------------- | ---------------------------------------------------------------- | -| `entity_list` | List of unique entity identifiers. ~~Iterable[Union[str, int]]~~ | -| `freq_list` | List of entity frequencies. ~~Iterable[int]~~ | -| `vector_list` | List of entity vectors. ~~Iterable[numpy.ndarray]~~ | - -## KnowledgeBase.add_alias {#add_alias tag="method"} - -Add an alias or mention to the knowledge base, specifying its potential KB -identifiers and their prior probabilities. The entity identifiers should refer -to entities previously added with [`add_entity`](/api/kb#add_entity) or -[`set_entities`](/api/kb#set_entities). The sum of the prior probabilities -should not exceed 1. Note that an empty string can not be used as alias. - -> #### Example -> -> ```python -> kb.add_alias(alias="Douglas", entities=["Q42", "Q463035"], probabilities=[0.6, 0.3]) -> ``` - -| Name | Description | -| --------------- | --------------------------------------------------------------------------------- | -| `alias` | The textual mention or alias. Can not be the empty string. ~~str~~ | -| `entities` | The potential entities that the alias may refer to. ~~Iterable[Union[str, int]]~~ | -| `probabilities` | The prior probabilities of each entity. ~~Iterable[float]~~ | - -## KnowledgeBase.\_\_len\_\_ {#len tag="method"} - -Get the total number of entities in the knowledge base. - -> #### Example -> -> ```python -> total_entities = len(kb) -> ``` - -| Name | Description | -| ----------- | ----------------------------------------------------- | -| **RETURNS** | The number of entities in the knowledge base. ~~int~~ | - -## KnowledgeBase.get_entity_strings {#get_entity_strings tag="method"} - -Get a list of all entity IDs in the knowledge base. - -> #### Example -> -> ```python -> all_entities = kb.get_entity_strings() -> ``` - -| Name | Description | -| ----------- | --------------------------------------------------------- | -| **RETURNS** | The list of entities in the knowledge base. ~~List[str]~~ | - -## KnowledgeBase.get_size_aliases {#get_size_aliases tag="method"} - -Get the total number of aliases in the knowledge base. - -> #### Example -> -> ```python -> total_aliases = kb.get_size_aliases() -> ``` - -| Name | Description | -| ----------- | ---------------------------------------------------- | -| **RETURNS** | The number of aliases in the knowledge base. ~~int~~ | - -## KnowledgeBase.get_alias_strings {#get_alias_strings tag="method"} - -Get a list of all aliases in the knowledge base. - -> #### Example -> -> ```python -> all_aliases = kb.get_alias_strings() -> ``` - -| Name | Description | -| ----------- | -------------------------------------------------------- | -| **RETURNS** | The list of aliases in the knowledge base. ~~List[str]~~ | +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------- | +| `mentions` | The textual mention or alias. ~~Iterable[Span]~~ | +| **RETURNS** | An iterable of iterable with relevant `Candidate` objects. ~~Iterable[Iterable[Candidate]]~~ | ## KnowledgeBase.get_alias_candidates {#get_alias_candidates tag="method"} -Given a certain textual mention as input, retrieve a list of candidate entities -of type [`Candidate`](/api/kb/#candidate). + +This method is _not_ available from spaCy 3.5 onwards. + -> #### Example -> -> ```python -> candidates = kb.get_alias_candidates("Douglas") -> ``` - -| Name | Description | -| ----------- | ------------------------------------------------------------- | -| `alias` | The textual mention or alias. ~~str~~ | -| **RETURNS** | The list of relevant `Candidate` objects. ~~List[Candidate]~~ | +From spaCy 3.5 on `KnowledgeBase` is an abstract class (with +[`InMemoryLookupKB`](/api/kb_in_memory) being a drop-in replacement) to allow +more flexibility in customizing knowledge bases. Some of its methods were moved +to [`InMemoryLookupKB`](/api/kb_in_memory) during this refactoring, one of those +being `get_alias_candidates()`. This method is now available as +[`InMemoryLookupKB.get_alias_candidates()`](/api/kb_in_memory#get_alias_candidates). +Note: [`InMemoryLookupKB.get_candidates()`](/api/kb_in_memory#get_candidates) +defaults to +[`InMemoryLookupKB.get_alias_candidates()`](/api/kb_in_memory#get_alias_candidates). ## KnowledgeBase.get_vector {#get_vector tag="method"} @@ -178,27 +129,30 @@ Given a certain entity ID, retrieve its pretrained entity vector. > vector = kb.get_vector("Q42") > ``` -| Name | Description | -| ----------- | ------------------------------------ | -| `entity` | The entity ID. ~~str~~ | -| **RETURNS** | The entity vector. ~~numpy.ndarray~~ | +| Name | Description | +| ----------- | -------------------------------------- | +| `entity` | The entity ID. ~~str~~ | +| **RETURNS** | The entity vector. ~~Iterable[float]~~ | -## KnowledgeBase.get_prior_prob {#get_prior_prob tag="method"} +## KnowledgeBase.get_vectors {#get_vectors tag="method"} -Given a certain entity ID and a certain textual mention, retrieve the prior -probability of the fact that the mention links to the entity ID. +Same as [`get_vector()`](/api/kb#get_vector), but for an arbitrary number of +entity IDs. + +The default implementation of `get_vectors()` executes `get_vector()` in a loop. +We recommend implementing a more efficient way to retrieve vectors for multiple +entities at once, if performance is of concern to you. > #### Example > > ```python -> probability = kb.get_prior_prob("Q42", "Douglas") +> vectors = kb.get_vectors(("Q42", "Q3107329")) > ``` -| Name | Description | -| ----------- | ------------------------------------------------------------------------- | -| `entity` | The entity ID. ~~str~~ | -| `alias` | The textual mention or alias. ~~str~~ | -| **RETURNS** | The prior probability of the `alias` referring to the `entity`. ~~float~~ | +| Name | Description | +| ----------- | --------------------------------------------------------- | +| `entities` | The entity IDs. ~~Iterable[str]~~ | +| **RETURNS** | The entity vectors. ~~Iterable[Iterable[numpy.ndarray]]~~ | ## KnowledgeBase.to_disk {#to_disk tag="method"} @@ -207,12 +161,13 @@ Save the current state of the knowledge base to a directory. > #### Example > > ```python -> kb.to_disk(loc) +> kb.to_disk(path) > ``` -| Name | Description | -| ----- | ------------------------------------------------------------------------------------------------------------------------------------------ | -| `loc` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| Name | Description | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| `exclude` | List of components to exclude. ~~Iterable[str]~~ | ## KnowledgeBase.from_disk {#from_disk tag="method"} @@ -222,16 +177,16 @@ Restore the state of the knowledge base from a given directory. Note that the > #### Example > > ```python -> from spacy.kb import KnowledgeBase > from spacy.vocab import Vocab > vocab = Vocab().from_disk("/path/to/vocab") -> kb = KnowledgeBase(vocab=vocab, entity_vector_length=64) +> kb = FullyImplementedKB(vocab=vocab, entity_vector_length=64) > kb.from_disk("/path/to/kb") > ``` | Name | Description | | ----------- | ----------------------------------------------------------------------------------------------- | | `loc` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| `exclude` | List of components to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `KnowledgeBase` object. ~~KnowledgeBase~~ | ## Candidate {#candidate tag="class"} diff --git a/website/docs/api/kb_in_memory.md b/website/docs/api/kb_in_memory.md new file mode 100644 index 000000000..c9ce624f0 --- /dev/null +++ b/website/docs/api/kb_in_memory.md @@ -0,0 +1,302 @@ +--- +title: InMemoryLookupKB +teaser: + The default implementation of the KnowledgeBase interface. Stores all + information in-memory. +tag: class +source: spacy/kb/kb_in_memory.pyx +new: 3.5 +--- + +The `InMemoryLookupKB` class inherits from [`KnowledgeBase`](/api/kb) and +implements all of its methods. It stores all KB data in-memory and generates +[`Candidate`](/api/kb#candidate) objects by exactly matching mentions with +entity names. It's highly optimized for both a low memory footprint and speed of +retrieval. + +## InMemoryLookupKB.\_\_init\_\_ {#init tag="method"} + +Create the knowledge base. + +> #### Example +> +> ```python +> from spacy.kb import KnowledgeBase +> vocab = nlp.vocab +> kb = KnowledgeBase(vocab=vocab, entity_vector_length=64) +> ``` + +| Name | Description | +| ---------------------- | ------------------------------------------------ | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `entity_vector_length` | Length of the fixed-size entity vectors. ~~int~~ | + +## InMemoryLookupKB.entity_vector_length {#entity_vector_length tag="property"} + +The length of the fixed-size entity vectors in the knowledge base. + +| Name | Description | +| ----------- | ------------------------------------------------ | +| **RETURNS** | Length of the fixed-size entity vectors. ~~int~~ | + +## InMemoryLookupKB.add_entity {#add_entity tag="method"} + +Add an entity to the knowledge base, specifying its corpus frequency and entity +vector, which should be of length +[`entity_vector_length`](/api/kb_in_memory#entity_vector_length). + +> #### Example +> +> ```python +> kb.add_entity(entity="Q42", freq=32, entity_vector=vector1) +> kb.add_entity(entity="Q463035", freq=111, entity_vector=vector2) +> ``` + +| Name | Description | +| --------------- | ---------------------------------------------------------- | +| `entity` | The unique entity identifier. ~~str~~ | +| `freq` | The frequency of the entity in a typical corpus. ~~float~~ | +| `entity_vector` | The pretrained vector of the entity. ~~numpy.ndarray~~ | + +## InMemoryLookupKB.set_entities {#set_entities tag="method"} + +Define the full list of entities in the knowledge base, specifying the corpus +frequency and entity vector for each entity. + +> #### Example +> +> ```python +> kb.set_entities(entity_list=["Q42", "Q463035"], freq_list=[32, 111], vector_list=[vector1, vector2]) +> ``` + +| Name | Description | +| ------------- | ---------------------------------------------------------------- | +| `entity_list` | List of unique entity identifiers. ~~Iterable[Union[str, int]]~~ | +| `freq_list` | List of entity frequencies. ~~Iterable[int]~~ | +| `vector_list` | List of entity vectors. ~~Iterable[numpy.ndarray]~~ | + +## InMemoryLookupKB.add_alias {#add_alias tag="method"} + +Add an alias or mention to the knowledge base, specifying its potential KB +identifiers and their prior probabilities. The entity identifiers should refer +to entities previously added with [`add_entity`](/api/kb_in_memory#add_entity) +or [`set_entities`](/api/kb_in_memory#set_entities). The sum of the prior +probabilities should not exceed 1. Note that an empty string can not be used as +alias. + +> #### Example +> +> ```python +> kb.add_alias(alias="Douglas", entities=["Q42", "Q463035"], probabilities=[0.6, 0.3]) +> ``` + +| Name | Description | +| --------------- | --------------------------------------------------------------------------------- | +| `alias` | The textual mention or alias. Can not be the empty string. ~~str~~ | +| `entities` | The potential entities that the alias may refer to. ~~Iterable[Union[str, int]]~~ | +| `probabilities` | The prior probabilities of each entity. ~~Iterable[float]~~ | + +## InMemoryLookupKB.\_\_len\_\_ {#len tag="method"} + +Get the total number of entities in the knowledge base. + +> #### Example +> +> ```python +> total_entities = len(kb) +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------- | +| **RETURNS** | The number of entities in the knowledge base. ~~int~~ | + +## InMemoryLookupKB.get_entity_strings {#get_entity_strings tag="method"} + +Get a list of all entity IDs in the knowledge base. + +> #### Example +> +> ```python +> all_entities = kb.get_entity_strings() +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------- | +| **RETURNS** | The list of entities in the knowledge base. ~~List[str]~~ | + +## InMemoryLookupKB.get_size_aliases {#get_size_aliases tag="method"} + +Get the total number of aliases in the knowledge base. + +> #### Example +> +> ```python +> total_aliases = kb.get_size_aliases() +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------- | +| **RETURNS** | The number of aliases in the knowledge base. ~~int~~ | + +## InMemoryLookupKB.get_alias_strings {#get_alias_strings tag="method"} + +Get a list of all aliases in the knowledge base. + +> #### Example +> +> ```python +> all_aliases = kb.get_alias_strings() +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------- | +| **RETURNS** | The list of aliases in the knowledge base. ~~List[str]~~ | + +## InMemoryLookupKB.get_candidates {#get_candidates tag="method"} + +Given a certain textual mention as input, retrieve a list of candidate entities +of type [`Candidate`](/api/kb#candidate). Wraps +[`get_alias_candidates()`](/api/kb_in_memory#get_alias_candidates). + +> #### Example +> +> ```python +> from spacy.lang.en import English +> nlp = English() +> doc = nlp("Douglas Adams wrote 'The Hitchhiker's Guide to the Galaxy'.") +> candidates = kb.get_candidates(doc[0:2]) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `mention` | The textual mention or alias. ~~Span~~ | +| **RETURNS** | An iterable of relevant `Candidate` objects. ~~Iterable[Candidate]~~ | + +## InMemoryLookupKB.get_candidates_batch {#get_candidates_batch tag="method"} + +Same as [`get_candidates()`](/api/kb_in_memory#get_candidates), but for an +arbitrary number of mentions. The [`EntityLinker`](/api/entitylinker) component +will call `get_candidates_batch()` instead of `get_candidates()`, if the config +parameter `candidates_batch_size` is greater or equal than 1. + +The default implementation of `get_candidates_batch()` executes +`get_candidates()` in a loop. We recommend implementing a more efficient way to +retrieve candidates for multiple mentions at once, if performance is of concern +to you. + +> #### Example +> +> ```python +> from spacy.lang.en import English +> nlp = English() +> doc = nlp("Douglas Adams wrote 'The Hitchhiker's Guide to the Galaxy'.") +> candidates = kb.get_candidates((doc[0:2], doc[3:])) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------- | +| `mentions` | The textual mention or alias. ~~Iterable[Span]~~ | +| **RETURNS** | An iterable of iterable with relevant `Candidate` objects. ~~Iterable[Iterable[Candidate]]~~ | + +## InMemoryLookupKB.get_alias_candidates {#get_alias_candidates tag="method"} + +Given a certain textual mention as input, retrieve a list of candidate entities +of type [`Candidate`](/api/kb#candidate). + +> #### Example +> +> ```python +> candidates = kb.get_alias_candidates("Douglas") +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------- | +| `alias` | The textual mention or alias. ~~str~~ | +| **RETURNS** | The list of relevant `Candidate` objects. ~~List[Candidate]~~ | + +## InMemoryLookupKB.get_vector {#get_vector tag="method"} + +Given a certain entity ID, retrieve its pretrained entity vector. + +> #### Example +> +> ```python +> vector = kb.get_vector("Q42") +> ``` + +| Name | Description | +| ----------- | ------------------------------------ | +| `entity` | The entity ID. ~~str~~ | +| **RETURNS** | The entity vector. ~~numpy.ndarray~~ | + +## InMemoryLookupKB.get_vectors {#get_vectors tag="method"} + +Same as [`get_vector()`](/api/kb_in_memory#get_vector), but for an arbitrary +number of entity IDs. + +The default implementation of `get_vectors()` executes `get_vector()` in a loop. +We recommend implementing a more efficient way to retrieve vectors for multiple +entities at once, if performance is of concern to you. + +> #### Example +> +> ```python +> vectors = kb.get_vectors(("Q42", "Q3107329")) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------- | +| `entities` | The entity IDs. ~~Iterable[str]~~ | +| **RETURNS** | The entity vectors. ~~Iterable[Iterable[numpy.ndarray]]~~ | + +## InMemoryLookupKB.get_prior_prob {#get_prior_prob tag="method"} + +Given a certain entity ID and a certain textual mention, retrieve the prior +probability of the fact that the mention links to the entity ID. + +> #### Example +> +> ```python +> probability = kb.get_prior_prob("Q42", "Douglas") +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------- | +| `entity` | The entity ID. ~~str~~ | +| `alias` | The textual mention or alias. ~~str~~ | +| **RETURNS** | The prior probability of the `alias` referring to the `entity`. ~~float~~ | + +## InMemoryLookupKB.to_disk {#to_disk tag="method"} + +Save the current state of the knowledge base to a directory. + +> #### Example +> +> ```python +> kb.to_disk(path) +> ``` + +| Name | Description | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| `exclude` | List of components to exclude. ~~Iterable[str]~~ | + +## InMemoryLookupKB.from_disk {#from_disk tag="method"} + +Restore the state of the knowledge base from a given directory. Note that the +[`Vocab`](/api/vocab) should also be the same as the one used to create the KB. + +> #### Example +> +> ```python +> from spacy.vocab import Vocab +> vocab = Vocab().from_disk("/path/to/vocab") +> kb = FullyImplementedKB(vocab=vocab, entity_vector_length=64) +> kb.from_disk("/path/to/kb") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| `loc` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| `exclude` | List of components to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `KnowledgeBase` object. ~~KnowledgeBase~~ | diff --git a/website/docs/usage/101/_architecture.md b/website/docs/usage/101/_architecture.md index 22e2b961e..4ebca2756 100644 --- a/website/docs/usage/101/_architecture.md +++ b/website/docs/usage/101/_architecture.md @@ -78,7 +78,9 @@ operates on a `Doc` and gives you access to the matched tokens **in context**. | Name | Description | | ------------------------------------------------ | -------------------------------------------------------------------------------------------------- | | [`Corpus`](/api/corpus) | Class for managing annotated corpora for training and evaluation data. | -| [`KnowledgeBase`](/api/kb) | Storage for entities and aliases of a knowledge base for entity linking. | +| [`KnowledgeBase`](/api/kb) | Abstract base class for storage and retrieval of data for entity linking. | +| [`InMemoryLookupKB`](/api/kb_in_memory) | Implementation of `KnowledgeBase` storing all data in memory. | +| [`Candidate`](/api/kb#candidate) | Object associating a textual mention with a specific entity contained in a `KnowledgeBase`. | | [`Lookups`](/api/lookups) | Container for convenient access to large lookup tables and dictionaries. | | [`MorphAnalysis`](/api/morphology#morphanalysis) | A morphological analysis. | | [`Morphology`](/api/morphology) | Store morphological analyses and map them to and from hash values. | From 2602a30d326e561776fecce95ac03cc5df55652b Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Thu, 8 Sep 2022 20:42:47 +0900 Subject: [PATCH 015/179] Fix DVC command example (#11457) This command doesn't have the project dir, but it's required. --- website/docs/usage/projects.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/usage/projects.md b/website/docs/usage/projects.md index 566ae561b..35150035a 100644 --- a/website/docs/usage/projects.md +++ b/website/docs/usage/projects.md @@ -758,7 +758,7 @@ and [`dvc repro`](https://dvc.org/doc/command-reference/repro) to reproduce the workflow or individual commands. ```cli -$ python -m spacy project dvc [workflow_name] +$ python -m spacy project dvc [project_dir] [workflow_name] ``` From aac9a58c2935768c7751b8db7043e7c073362c90 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Fri, 9 Sep 2022 10:46:01 +0200 Subject: [PATCH 016/179] Add docs for the `spacy.models_and_pipes_with_nvtx_range.v1` callback (#11463) * Add docs for the `spacy.models_and_pipes_with_nvtx_range.v1` callback * Add `new` tag --- website/docs/api/top-level.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/website/docs/api/top-level.md b/website/docs/api/top-level.md index 220b2d6e9..bc53fc868 100644 --- a/website/docs/api/top-level.md +++ b/website/docs/api/top-level.md @@ -887,6 +887,27 @@ backprop passes. | `backprop_color` | Color identifier for backpropagation passes. Defaults to `-1`. ~~int~~ | | **CREATES** | A function that takes the current `nlp` and wraps forward/backprop passes in NVTX ranges. ~~Callable[[Language], Language]~~ | +### spacy.models_and_pipes_with_nvtx_range.v1 {#models_and_pipes_with_nvtx_range tag="registered function" new="3.4"} + +> #### Example config +> +> ```ini +> [nlp] +> after_pipeline_creation = {"@callbacks":"spacy.models_and_pipes_with_nvtx_range.v1"} +> ``` + +Recursively wrap both the models and methods of each pipe using +[NVTX](https://nvidia.github.io/NVTX/) range markers. By default, the following +methods are wrapped: `pipe`, `predict`, `set_annotations`, `update`, `rehearse`, +`get_loss`, `initialize`, `begin_update`, `finish_update`, `update`. + +| Name | Description | +| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `forward_color` | Color identifier for model forward passes. Defaults to `-1`. ~~int~~ | +| `backprop_color` | Color identifier for model backpropagation passes. Defaults to `-1`. ~~int~~ | +| `additional_pipe_functions` | Additional pipeline methods to wrap. Keys are pipeline names and values are lists of method identifiers. Defaults to `None`. ~~Optional[Dict[str, List[str]]]~~ | +| **CREATES** | A function that takes the current `nlp` and wraps pipe models and methods in NVTX ranges. ~~Callable[[Language], Language]~~ | + ## Training data and alignment {#gold source="spacy/training"} ### training.offsets_to_biluo_tags {#offsets_to_biluo_tags tag="function"} From 0c72c6bb2c04677654ffeda2a706e3df3a58b3cc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 9 Sep 2022 11:21:17 +0200 Subject: [PATCH 017/179] Auto-format code with black (#11468) Co-authored-by: explosion-bot --- spacy/cli/info.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spacy/cli/info.py b/spacy/cli/info.py index e6ac4270f..974bc0f4e 100644 --- a/spacy/cli/info.py +++ b/spacy/cli/info.py @@ -147,6 +147,7 @@ def info_installed_model_url(model: str) -> Optional[str]: # something else, like no file or invalid JSON return None + def info_model_url(model: str) -> Dict[str, Any]: """Return the download URL for the latest version of a pipeline.""" version = get_latest_version(model) From 8a86a35eab45a69d795c2950da61058047d1a516 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 9 Sep 2022 15:10:04 +0200 Subject: [PATCH 018/179] Remove has_letters in config template (#11465) Due to problems with the javascript conversion in the website quickstart, remove the `has_letters` setting to simplify generating `attrs` for the default `tok2vec`. Additionally reduce `PREFIX` as in the trained pipelines. --- spacy/cli/templates/quickstart_training.jinja | 7 +------ .../cli/templates/quickstart_training_recommendations.yml | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/spacy/cli/templates/quickstart_training.jinja b/spacy/cli/templates/quickstart_training.jinja index ae11dcafc..58864883a 100644 --- a/spacy/cli/templates/quickstart_training.jinja +++ b/spacy/cli/templates/quickstart_training.jinja @@ -271,13 +271,8 @@ factory = "tok2vec" [components.tok2vec.model.embed] @architectures = "spacy.MultiHashEmbed.v2" width = ${components.tok2vec.model.encode.width} -{% if has_letters -%} attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"] -rows = [5000, 2500, 2500, 2500] -{% else -%} -attrs = ["ORTH", "SHAPE"] -rows = [5000, 2500] -{% endif -%} +rows = [5000, 1000, 2500, 2500] include_static_vectors = {{ "true" if optimize == "accuracy" else "false" }} [components.tok2vec.model.encode] diff --git a/spacy/cli/templates/quickstart_training_recommendations.yml b/spacy/cli/templates/quickstart_training_recommendations.yml index a7bf9b74a..27945e27a 100644 --- a/spacy/cli/templates/quickstart_training_recommendations.yml +++ b/spacy/cli/templates/quickstart_training_recommendations.yml @@ -271,4 +271,3 @@ zh: accuracy: name: bert-base-chinese size_factor: 3 - has_letters: false From 6b83fee58db27cee70ef8d893cbbf7470db4e242 Mon Sep 17 00:00:00 2001 From: kadarakos Date: Fri, 9 Sep 2022 17:17:10 +0200 Subject: [PATCH 019/179] Assets message (#11458) * new error message when 'project run assets' * new error message when 'project run assets' * Update spacy/cli/project/run.py Co-authored-by: Sofie Van Landeghem Co-authored-by: Sofie Van Landeghem --- spacy/cli/project/run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spacy/cli/project/run.py b/spacy/cli/project/run.py index 734803bc4..d42d95465 100644 --- a/spacy/cli/project/run.py +++ b/spacy/cli/project/run.py @@ -195,6 +195,8 @@ def validate_subcommand( msg.fail(f"No commands or workflows defined in {PROJECT_FILE}", exits=1) if subcommand not in commands and subcommand not in workflows: help_msg = [] + if subcommand in ["assets", "asset"]: + help_msg.append("Did you mean to run: python -m spacy project assets?") if commands: help_msg.append(f"Available commands: {', '.join(commands)}") if workflows: From 0ec9a696e60933807c189c7be22623a81a840289 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Mon, 12 Sep 2022 14:55:41 +0200 Subject: [PATCH 020/179] Fix config validation failures caused by NVTX pipeline wrappers (#11460) * Enable Cython<->Python bindings for `Pipe` and `TrainablePipe` methods * `pipes_with_nvtx_range`: Skip hooking methods whose signature cannot be ascertained When loading pipelines from a config file, the arguments passed to individual pipeline components is validated by `pydantic` during init. For this, the validation model attempts to parse the function signature of the component's c'tor/entry point so that it can check if all mandatory parameters are present in the config file. When using the `models_and_pipes_with_nvtx_range` as a `after_pipeline_creation` callback, the methods of all pipeline components get replaced by a NVTX range wrapper **before** the above-mentioned validation takes place. This can be problematic for components that are implemented as Cython extension types - if the extension type is not compiled with Python bindings for its methods, they will have no signatures at runtime. This resulted in `pydantic` matching the *wrapper's* parameters with the those in the config and raising errors. To avoid this, we now skip applying the wrapper to any (Cython) methods that do not have signatures. --- spacy/ml/callbacks.py | 7 +++++-- spacy/pipeline/pipe.pyx | 2 +- spacy/pipeline/trainable_pipe.pyx | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/spacy/ml/callbacks.py b/spacy/ml/callbacks.py index 18290b947..3b60ec2ab 100644 --- a/spacy/ml/callbacks.py +++ b/spacy/ml/callbacks.py @@ -89,11 +89,14 @@ def pipes_with_nvtx_range( types.MethodType(nvtx_range_wrapper_for_pipe_method, pipe), func ) - # Try to preserve the original function signature. + # We need to preserve the original function signature so that + # the original parameters are passed to pydantic for validation downstream. try: wrapped_func.__signature__ = inspect.signature(func) # type: ignore except: - pass + # Can fail for Cython methods that do not have bindings. + warnings.warn(Warnings.W122.format(method=name, pipe=pipe.name)) + continue try: setattr( diff --git a/spacy/pipeline/pipe.pyx b/spacy/pipeline/pipe.pyx index 4e3ae1cf0..8407acc45 100644 --- a/spacy/pipeline/pipe.pyx +++ b/spacy/pipeline/pipe.pyx @@ -1,4 +1,4 @@ -# cython: infer_types=True, profile=True +# cython: infer_types=True, profile=True, binding=True from typing import Optional, Tuple, Iterable, Iterator, Callable, Union, Dict import srsly import warnings diff --git a/spacy/pipeline/trainable_pipe.pyx b/spacy/pipeline/trainable_pipe.pyx index 76b0733cf..3f0507d4b 100644 --- a/spacy/pipeline/trainable_pipe.pyx +++ b/spacy/pipeline/trainable_pipe.pyx @@ -1,4 +1,4 @@ -# cython: infer_types=True, profile=True +# cython: infer_types=True, profile=True, binding=True from typing import Iterable, Iterator, Optional, Dict, Tuple, Callable import srsly from thinc.api import set_dropout_rate, Model, Optimizer From cc10a27c59a3e5fe3c2d08667534fcbf22908f06 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Mon, 12 Sep 2022 15:36:48 +0200 Subject: [PATCH 021/179] Prevent tok2vec to broadcast to listeners when predicting (#11385) * replicate bug with tok2vec in annotating components * add overfitting test with a frozen tok2vec * remove broadcast from predict and check doc.tensor instead * remove broadcast * proper error * slight rephrase of documentation --- spacy/errors.py | 2 + spacy/pipeline/tok2vec.py | 20 ++++--- spacy/tests/pipeline/test_tok2vec.py | 81 ++++++++++++++++++++++++++++ website/docs/usage/training.md | 2 +- 4 files changed, 98 insertions(+), 7 deletions(-) diff --git a/spacy/errors.py b/spacy/errors.py index e2201284f..7e63dc76c 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -538,6 +538,8 @@ class Errors(metaclass=ErrorsWithCodes): E199 = ("Unable to merge 0-length span at `doc[{start}:{end}]`.") E200 = ("Can't set {attr} from Span.") E202 = ("Unsupported {name} mode '{mode}'. Supported modes: {modes}.") + E203 = ("If the {name} embedding layer is not updated " + "during training, make sure to include it in 'annotating components'") # New errors added in v3.x E853 = ("Unsupported component factory name '{name}'. The character '.' is " diff --git a/spacy/pipeline/tok2vec.py b/spacy/pipeline/tok2vec.py index 2e3dde3cb..c742aaeaa 100644 --- a/spacy/pipeline/tok2vec.py +++ b/spacy/pipeline/tok2vec.py @@ -123,9 +123,6 @@ class Tok2Vec(TrainablePipe): width = self.model.get_dim("nO") return [self.model.ops.alloc((0, width)) for doc in docs] tokvecs = self.model.predict(docs) - batch_id = Tok2VecListener.get_batch_id(docs) - for listener in self.listeners: - listener.receive(batch_id, tokvecs, _empty_backprop) return tokvecs def set_annotations(self, docs: Sequence[Doc], tokvecses) -> None: @@ -286,8 +283,19 @@ class Tok2VecListener(Model): def forward(model: Tok2VecListener, inputs, is_train: bool): """Supply the outputs from the upstream Tok2Vec component.""" if is_train: - model.verify_inputs(inputs) - return model._outputs, model._backprop + # This might occur during training when the tok2vec layer is frozen / hasn't been updated. + # In that case, it should be set to "annotating" so we can retrieve the embeddings from the doc. + if model._batch_id is None: + outputs = [] + for doc in inputs: + if doc.tensor.size == 0: + raise ValueError(Errors.E203.format(name="tok2vec")) + else: + outputs.append(doc.tensor) + return outputs, _empty_backprop + else: + model.verify_inputs(inputs) + return model._outputs, model._backprop else: # This is pretty grim, but it's hard to do better :(. # It's hard to avoid relying on the doc.tensor attribute, because the @@ -306,7 +314,7 @@ def forward(model: Tok2VecListener, inputs, is_train: bool): outputs.append(model.ops.alloc2f(len(doc), width)) else: outputs.append(doc.tensor) - return outputs, lambda dX: [] + return outputs, _empty_backprop def _empty_backprop(dX): # for pickling diff --git a/spacy/tests/pipeline/test_tok2vec.py b/spacy/tests/pipeline/test_tok2vec.py index 64faf133d..659274db9 100644 --- a/spacy/tests/pipeline/test_tok2vec.py +++ b/spacy/tests/pipeline/test_tok2vec.py @@ -230,6 +230,87 @@ def test_tok2vec_listener_callback(): assert get_dX(Y) is not None +def test_tok2vec_listener_overfitting(): + """ Test that a pipeline with a listener properly overfits, even if 'tok2vec' is in the annotating components """ + orig_config = Config().from_str(cfg_string) + nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses, annotates=["tok2vec"]) + assert losses["tagger"] < 0.00001 + + # test the trained model + test_text = "I like blue eggs" + doc = nlp(test_text) + assert doc[0].tag_ == "N" + assert doc[1].tag_ == "V" + assert doc[2].tag_ == "J" + assert doc[3].tag_ == "N" + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + assert doc2[0].tag_ == "N" + assert doc2[1].tag_ == "V" + assert doc2[2].tag_ == "J" + assert doc2[3].tag_ == "N" + + +def test_tok2vec_frozen_not_annotating(): + """ Test that a pipeline with a frozen tok2vec raises an error when the tok2vec is not annotating """ + orig_config = Config().from_str(cfg_string) + nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + + for i in range(2): + losses = {} + with pytest.raises(ValueError, match=r"the tok2vec embedding layer is not updated"): + nlp.update(train_examples, sgd=optimizer, losses=losses, exclude=["tok2vec"]) + + +def test_tok2vec_frozen_overfitting(): + """ Test that a pipeline with a frozen & annotating tok2vec can still overfit """ + orig_config = Config().from_str(cfg_string) + nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + + for i in range(100): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses, exclude=["tok2vec"], annotates=["tok2vec"]) + assert losses["tagger"] < 0.0001 + + # test the trained model + test_text = "I like blue eggs" + doc = nlp(test_text) + assert doc[0].tag_ == "N" + assert doc[1].tag_ == "V" + assert doc[2].tag_ == "J" + assert doc[3].tag_ == "N" + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + assert doc2[0].tag_ == "N" + assert doc2[1].tag_ == "V" + assert doc2[2].tag_ == "J" + assert doc2[3].tag_ == "N" + + def test_replace_listeners(): orig_config = Config().from_str(cfg_string) nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) diff --git a/website/docs/usage/training.md b/website/docs/usage/training.md index 5e064b269..27a8bbca7 100644 --- a/website/docs/usage/training.md +++ b/website/docs/usage/training.md @@ -480,7 +480,7 @@ as-is. They are also excluded when calling > parse. So the evaluation results should always reflect what your pipeline will > produce at runtime. If you want a frozen component to run (without updating) > during training as well, so that downstream components can use its -> **predictions**, you can add it to the list of +> **predictions**, you should add it to the list of > [`annotating_components`](/usage/training#annotating-components). ```ini From 6be6913ba5aaa7aa35deb1a9fcd4418d93824b24 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 13 Sep 2022 09:04:53 +0200 Subject: [PATCH 022/179] Update cupy extras (#11279) * Update cupy extras: * Extend to v11 * Add `cupy-cuda11x` and `cupy-wheel` * Update quickstart to use `cupy-wheel` for CUDA 10.2+ * Rename cuda-wheel to cuda-autodetect, remove repeated CUDA in menu --- setup.cfg | 36 +++++++++++++---------- website/src/widgets/quickstart-install.js | 12 ++------ 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/setup.cfg b/setup.cfg index 5fd820a96..2dc5e7042 100644 --- a/setup.cfg +++ b/setup.cfg @@ -76,37 +76,41 @@ transformers = ray = spacy_ray>=0.1.0,<1.0.0 cuda = - cupy>=5.0.0b4,<11.0.0 + cupy>=5.0.0b4,<12.0.0 cuda80 = - cupy-cuda80>=5.0.0b4,<11.0.0 + cupy-cuda80>=5.0.0b4,<12.0.0 cuda90 = - cupy-cuda90>=5.0.0b4,<11.0.0 + cupy-cuda90>=5.0.0b4,<12.0.0 cuda91 = - cupy-cuda91>=5.0.0b4,<11.0.0 + cupy-cuda91>=5.0.0b4,<12.0.0 cuda92 = - cupy-cuda92>=5.0.0b4,<11.0.0 + cupy-cuda92>=5.0.0b4,<12.0.0 cuda100 = - cupy-cuda100>=5.0.0b4,<11.0.0 + cupy-cuda100>=5.0.0b4,<12.0.0 cuda101 = - cupy-cuda101>=5.0.0b4,<11.0.0 + cupy-cuda101>=5.0.0b4,<12.0.0 cuda102 = - cupy-cuda102>=5.0.0b4,<11.0.0 + cupy-cuda102>=5.0.0b4,<12.0.0 cuda110 = - cupy-cuda110>=5.0.0b4,<11.0.0 + cupy-cuda110>=5.0.0b4,<12.0.0 cuda111 = - cupy-cuda111>=5.0.0b4,<11.0.0 + cupy-cuda111>=5.0.0b4,<12.0.0 cuda112 = - cupy-cuda112>=5.0.0b4,<11.0.0 + cupy-cuda112>=5.0.0b4,<12.0.0 cuda113 = - cupy-cuda113>=5.0.0b4,<11.0.0 + cupy-cuda113>=5.0.0b4,<12.0.0 cuda114 = - cupy-cuda114>=5.0.0b4,<11.0.0 + cupy-cuda114>=5.0.0b4,<12.0.0 cuda115 = - cupy-cuda115>=5.0.0b4,<11.0.0 + cupy-cuda115>=5.0.0b4,<12.0.0 cuda116 = - cupy-cuda116>=5.0.0b4,<11.0.0 + cupy-cuda116>=5.0.0b4,<12.0.0 cuda117 = - cupy-cuda117>=5.0.0b4,<11.0.0 + cupy-cuda117>=5.0.0b4,<12.0.0 +cuda11x = + cupy-cuda11x>=11.0.0,<12.0.0 +cuda-autodetect = + cupy-wheel>=11.0.0,<12.0.0 apple = thinc-apple-ops>=0.1.0.dev0,<1.0.0 # Language tokenizers with external dependencies diff --git a/website/src/widgets/quickstart-install.js b/website/src/widgets/quickstart-install.js index 61c0678dd..0d2186acb 100644 --- a/website/src/widgets/quickstart-install.js +++ b/website/src/widgets/quickstart-install.js @@ -9,7 +9,7 @@ const DEFAULT_PLATFORM = 'x86' const DEFAULT_MODELS = ['en'] const DEFAULT_OPT = 'efficiency' const DEFAULT_HARDWARE = 'cpu' -const DEFAULT_CUDA = 'cuda113' +const DEFAULT_CUDA = 'cuda-autodetect' const CUDA = { '8.0': 'cuda80', '9.0': 'cuda90', @@ -17,15 +17,7 @@ const CUDA = { '9.2': 'cuda92', '10.0': 'cuda100', '10.1': 'cuda101', - '10.2': 'cuda102', - '11.0': 'cuda110', - '11.1': 'cuda111', - '11.2': 'cuda112', - '11.3': 'cuda113', - '11.4': 'cuda114', - '11.5': 'cuda115', - '11.6': 'cuda116', - '11.7': 'cuda117', + '10.2, 11.0+': 'cuda-autodetect', } const LANG_EXTRAS = ['ja'] // only for languages with models From 3f0c3ad7d30d493cd017b6bb41b174d991bbcdc1 Mon Sep 17 00:00:00 2001 From: Richard Hudson Date: Wed, 14 Sep 2022 09:36:55 +0200 Subject: [PATCH 023/179] Correct alignment example and documentation (#11491) * Correct example and documentation * Added altered example.md * Changes based on review + apply prettier * Remote unnecessary 'the' Co-authored-by: Madeesh Kannan Co-authored-by: Madeesh Kannan --- website/docs/api/example.md | 16 ++++++++++------ website/docs/usage/linguistic-features.md | 10 +++++----- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/website/docs/api/example.md b/website/docs/api/example.md index ca9d3c056..0228e8935 100644 --- a/website/docs/api/example.md +++ b/website/docs/api/example.md @@ -286,10 +286,14 @@ Calculate alignment tables between two tokenizations. ### Alignment attributes {#alignment-attributes"} -| Name | Description | -| ----- | --------------------------------------------------------------------- | -| `x2y` | The `Ragged` object holding the alignment from `x` to `y`. ~~Ragged~~ | -| `y2x` | The `Ragged` object holding the alignment from `y` to `x`. ~~Ragged~~ | +Alignment attributes are managed using `AlignmentArray`, which is a +simplified version of Thinc's [Ragged](https://thinc.ai/docs/api-types#ragged) +type that only supports the `data` and `length` attributes. + +| Name | Description | +| ----- | ------------------------------------------------------------------------------------- | +| `x2y` | The `AlignmentArray` object holding the alignment from `x` to `y`. ~~AlignmentArray~~ | +| `y2x` | The `AlignmentArray` object holding the alignment from `y` to `x`. ~~AlignmentArray~~ | @@ -309,10 +313,10 @@ tokenizations add up to the same string. For example, you'll be able to align > spacy_tokens = ["obama", "'s", "podcast"] > alignment = Alignment.from_strings(bert_tokens, spacy_tokens) > a2b = alignment.x2y -> assert list(a2b.dataXd) == [0, 1, 1, 2] +> assert list(a2b.data) == [0, 1, 1, 2] > ``` > -> If `a2b.dataXd[1] == a2b.dataXd[2] == 1`, that means that `A[1]` (`"'"`) and +> If `a2b.data[1] == a2b.data[2] == 1`, that means that `A[1]` (`"'"`) and > `A[2]` (`"s"`) both align to `B[1]` (`"'s"`). ### Alignment.from_strings {#classmethod tag="function"} diff --git a/website/docs/usage/linguistic-features.md b/website/docs/usage/linguistic-features.md index 82472c67e..099678c40 100644 --- a/website/docs/usage/linguistic-features.md +++ b/website/docs/usage/linguistic-features.md @@ -1422,9 +1422,9 @@ other_tokens = ["i", "listened", "to", "obama", "'", "s", "podcasts", "."] spacy_tokens = ["i", "listened", "to", "obama", "'s", "podcasts", "."] align = Alignment.from_strings(other_tokens, spacy_tokens) print(f"a -> b, lengths: {align.x2y.lengths}") # array([1, 1, 1, 1, 1, 1, 1, 1]) -print(f"a -> b, mapping: {align.x2y.dataXd}") # array([0, 1, 2, 3, 4, 4, 5, 6]) : two tokens both refer to "'s" +print(f"a -> b, mapping: {align.x2y.data}") # array([0, 1, 2, 3, 4, 4, 5, 6]) : two tokens both refer to "'s" print(f"b -> a, lengths: {align.y2x.lengths}") # array([1, 1, 1, 1, 2, 1, 1]) : the token "'s" refers to two tokens -print(f"b -> a, mappings: {align.y2x.dataXd}") # array([0, 1, 2, 3, 4, 5, 6, 7]) +print(f"b -> a, mappings: {align.y2x.data}") # array([0, 1, 2, 3, 4, 5, 6, 7]) ``` Here are some insights from the alignment information generated in the example @@ -1433,10 +1433,10 @@ above: - The one-to-one mappings for the first four tokens are identical, which means they map to each other. This makes sense because they're also identical in the input: `"i"`, `"listened"`, `"to"` and `"obama"`. -- The value of `x2y.dataXd[6]` is `5`, which means that `other_tokens[6]` +- The value of `x2y.data[6]` is `5`, which means that `other_tokens[6]` (`"podcasts"`) aligns to `spacy_tokens[5]` (also `"podcasts"`). -- `x2y.dataXd[4]` and `x2y.dataXd[5]` are both `4`, which means that both tokens - 4 and 5 of `other_tokens` (`"'"` and `"s"`) align to token 4 of `spacy_tokens` +- `x2y.data[4]` and `x2y.data[5]` are both `4`, which means that both tokens 4 + and 5 of `other_tokens` (`"'"` and `"s"`) align to token 4 of `spacy_tokens` (`"'s"`). From 7c98245c0c0f9c6c0c4a523c0bf1a75690e58620 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 14 Sep 2022 17:05:22 +0200 Subject: [PATCH 024/179] Add levenshtein from polyleven (#11418) Add a simple levenshtein distance function using the implementation from the polyleven library as `spacy.matcher.levenshtein`. --- .gitignore | 1 + licenses/3rd_party_licenses.txt | 31 ++ setup.py | 11 + spacy/matcher/__init__.py | 3 +- spacy/matcher/levenshtein.pyx | 15 + spacy/matcher/polyleven.c | 384 ++++++++++++++++++++++++ spacy/tests/matcher/test_levenshtein.py | 36 +++ 7 files changed, 480 insertions(+), 1 deletion(-) create mode 100644 spacy/matcher/levenshtein.pyx create mode 100644 spacy/matcher/polyleven.c create mode 100644 spacy/tests/matcher/test_levenshtein.py diff --git a/.gitignore b/.gitignore index ac72f2bbf..ac333f958 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ quickstart-training-generator.js cythonize.json spacy/*.html *.cpp +*.c *.so # Vim / VSCode / editors diff --git a/licenses/3rd_party_licenses.txt b/licenses/3rd_party_licenses.txt index d58da9c4a..851e09585 100644 --- a/licenses/3rd_party_licenses.txt +++ b/licenses/3rd_party_licenses.txt @@ -127,3 +127,34 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + + +polyleven +--------- + +* Files: spacy/matcher/polyleven.c + +MIT License + +Copyright (c) 2021 Fujimoto Seiji +Copyright (c) 2021 Max Bachmann +Copyright (c) 2022 Nick Mazuk +Copyright (c) 2022 Michael Weiss + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/setup.py b/setup.py index ec1bd35fa..c4138aa93 100755 --- a/setup.py +++ b/setup.py @@ -205,6 +205,17 @@ def setup_package(): get_python_inc(plat_specific=True), ] ext_modules = [] + ext_modules.append( + Extension( + "spacy.matcher.levenshtein", + [ + "spacy/matcher/levenshtein.pyx", + "spacy/matcher/polyleven.c", + ], + language="c", + include_dirs=include_dirs, + ) + ) for name in MOD_NAMES: mod_path = name.replace(".", "/") + ".pyx" ext = Extension( diff --git a/spacy/matcher/__init__.py b/spacy/matcher/__init__.py index 286844787..a4f164847 100644 --- a/spacy/matcher/__init__.py +++ b/spacy/matcher/__init__.py @@ -1,5 +1,6 @@ from .matcher import Matcher from .phrasematcher import PhraseMatcher from .dependencymatcher import DependencyMatcher +from .levenshtein import levenshtein -__all__ = ["Matcher", "PhraseMatcher", "DependencyMatcher"] +__all__ = ["Matcher", "PhraseMatcher", "DependencyMatcher", "levenshtein"] diff --git a/spacy/matcher/levenshtein.pyx b/spacy/matcher/levenshtein.pyx new file mode 100644 index 000000000..8463d913d --- /dev/null +++ b/spacy/matcher/levenshtein.pyx @@ -0,0 +1,15 @@ +# cython: profile=True, binding=True, infer_types=True +from cpython.object cimport PyObject +from libc.stdint cimport int64_t + +from typing import Optional + + +cdef extern from "polyleven.c": + int64_t polyleven(PyObject *o1, PyObject *o2, int64_t k) + + +cpdef int64_t levenshtein(a: str, b: str, k: Optional[int] = None): + if k is None: + k = -1 + return polyleven(a, b, k) diff --git a/spacy/matcher/polyleven.c b/spacy/matcher/polyleven.c new file mode 100644 index 000000000..2f2b8826c --- /dev/null +++ b/spacy/matcher/polyleven.c @@ -0,0 +1,384 @@ +/* + * Adapted from Polyleven (https://ceptord.net/) + * + * Source: https://github.com/fujimotos/polyleven/blob/c3f95a080626c5652f0151a2e449963288ccae84/polyleven.c + * + * Copyright (c) 2021 Fujimoto Seiji + * Copyright (c) 2021 Max Bachmann + * Copyright (c) 2022 Nick Mazuk + * Copyright (c) 2022 Michael Weiss + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include +#include + +#define MIN(a,b) ((a) < (b) ? (a) : (b)) +#define MAX(a,b) ((a) > (b) ? (a) : (b)) +#define CDIV(a,b) ((a) / (b) + ((a) % (b) > 0)) +#define BIT(i,n) (((i) >> (n)) & 1) +#define FLIP(i,n) ((i) ^ ((uint64_t) 1 << (n))) +#define ISASCII(kd) ((kd) == PyUnicode_1BYTE_KIND) + +/* + * Bare bone of PyUnicode + */ +struct strbuf { + void *ptr; + int kind; + int64_t len; +}; + +static void strbuf_init(struct strbuf *s, PyObject *o) +{ + s->ptr = PyUnicode_DATA(o); + s->kind = PyUnicode_KIND(o); + s->len = PyUnicode_GET_LENGTH(o); +} + +#define strbuf_read(s, i) PyUnicode_READ((s)->kind, (s)->ptr, (i)) + +/* + * An encoded mbleven model table. + * + * Each 8-bit integer represents an edit sequence, with using two + * bits for a single operation. + * + * 01 = DELETE, 10 = INSERT, 11 = REPLACE + * + * For example, 13 is '1101' in binary notation, so it means + * DELETE + REPLACE. + */ +static const uint8_t MBLEVEN_MATRIX[] = { + 3, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, + 15, 9, 6, 0, 0, 0, 0, 0, + 13, 7, 0, 0, 0, 0, 0, 0, + 5, 0, 0, 0, 0, 0, 0, 0, + 63, 39, 45, 57, 54, 30, 27, 0, + 61, 55, 31, 37, 25, 22, 0, 0, + 53, 29, 23, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 0, 0, 0, 0, +}; + +#define MBLEVEN_MATRIX_GET(k, d) ((((k) + (k) * (k)) / 2 - 1) + (d)) * 8 + +static int64_t mbleven_ascii(char *s1, int64_t len1, + char *s2, int64_t len2, int k) +{ + int pos; + uint8_t m; + int64_t i, j, c, r; + + pos = MBLEVEN_MATRIX_GET(k, len1 - len2); + r = k + 1; + + while (MBLEVEN_MATRIX[pos]) { + m = MBLEVEN_MATRIX[pos++]; + i = j = c = 0; + while (i < len1 && j < len2) { + if (s1[i] != s2[j]) { + c++; + if (!m) break; + if (m & 1) i++; + if (m & 2) j++; + m >>= 2; + } else { + i++; + j++; + } + } + c += (len1 - i) + (len2 - j); + r = MIN(r, c); + if (r < 2) { + return r; + } + } + return r; +} + +static int64_t mbleven(PyObject *o1, PyObject *o2, int64_t k) +{ + int pos; + uint8_t m; + int64_t i, j, c, r; + struct strbuf s1, s2; + + strbuf_init(&s1, o1); + strbuf_init(&s2, o2); + + if (s1.len < s2.len) + return mbleven(o2, o1, k); + + if (k > 3) + return -1; + + if (k < s1.len - s2.len) + return k + 1; + + if (ISASCII(s1.kind) && ISASCII(s2.kind)) + return mbleven_ascii(s1.ptr, s1.len, s2.ptr, s2.len, k); + + pos = MBLEVEN_MATRIX_GET(k, s1.len - s2.len); + r = k + 1; + + while (MBLEVEN_MATRIX[pos]) { + m = MBLEVEN_MATRIX[pos++]; + i = j = c = 0; + while (i < s1.len && j < s2.len) { + if (strbuf_read(&s1, i) != strbuf_read(&s2, j)) { + c++; + if (!m) break; + if (m & 1) i++; + if (m & 2) j++; + m >>= 2; + } else { + i++; + j++; + } + } + c += (s1.len - i) + (s2.len - j); + r = MIN(r, c); + if (r < 2) { + return r; + } + } + return r; +} + +/* + * Data structure to store Peq (equality bit-vector). + */ +struct blockmap_entry { + uint32_t key[128]; + uint64_t val[128]; +}; + +struct blockmap { + int64_t nr; + struct blockmap_entry *list; +}; + +#define blockmap_key(c) ((c) | 0x80000000U) +#define blockmap_hash(c) ((c) % 128) + +static int blockmap_init(struct blockmap *map, struct strbuf *s) +{ + int64_t i; + struct blockmap_entry *be; + uint32_t c, k; + uint8_t h; + + map->nr = CDIV(s->len, 64); + map->list = calloc(1, map->nr * sizeof(struct blockmap_entry)); + if (map->list == NULL) { + PyErr_NoMemory(); + return -1; + } + + for (i = 0; i < s->len; i++) { + be = &(map->list[i / 64]); + c = strbuf_read(s, i); + h = blockmap_hash(c); + k = blockmap_key(c); + + while (be->key[h] && be->key[h] != k) + h = blockmap_hash(h + 1); + be->key[h] = k; + be->val[h] |= (uint64_t) 1 << (i % 64); + } + return 0; +} + +static void blockmap_clear(struct blockmap *map) +{ + if (map->list) + free(map->list); + map->list = NULL; + map->nr = 0; +} + +static uint64_t blockmap_get(struct blockmap *map, int block, uint32_t c) +{ + struct blockmap_entry *be; + uint8_t h; + uint32_t k; + + h = blockmap_hash(c); + k = blockmap_key(c); + + be = &(map->list[block]); + while (be->key[h] && be->key[h] != k) + h = blockmap_hash(h + 1); + return be->key[h] == k ? be->val[h] : 0; +} + +/* + * Myers' bit-parallel algorithm + * + * See: G. Myers. "A fast bit-vector algorithm for approximate string + * matching based on dynamic programming." Journal of the ACM, 1999. + */ +static int64_t myers1999_block(struct strbuf *s1, struct strbuf *s2, + struct blockmap *map) +{ + uint64_t Eq, Xv, Xh, Ph, Mh, Pv, Mv, Last; + uint64_t *Mhc, *Phc; + int64_t i, b, hsize, vsize, Score; + uint8_t Pb, Mb; + + hsize = CDIV(s1->len, 64); + vsize = CDIV(s2->len, 64); + Score = s2->len; + + Phc = malloc(hsize * 2 * sizeof(uint64_t)); + if (Phc == NULL) { + PyErr_NoMemory(); + return -1; + } + Mhc = Phc + hsize; + memset(Phc, -1, hsize * sizeof(uint64_t)); + memset(Mhc, 0, hsize * sizeof(uint64_t)); + Last = (uint64_t)1 << ((s2->len - 1) % 64); + + for (b = 0; b < vsize; b++) { + Mv = 0; + Pv = (uint64_t) -1; + Score = s2->len; + + for (i = 0; i < s1->len; i++) { + Eq = blockmap_get(map, b, strbuf_read(s1, i)); + + Pb = BIT(Phc[i / 64], i % 64); + Mb = BIT(Mhc[i / 64], i % 64); + + Xv = Eq | Mv; + Xh = ((((Eq | Mb) & Pv) + Pv) ^ Pv) | Eq | Mb; + + Ph = Mv | ~ (Xh | Pv); + Mh = Pv & Xh; + + if (Ph & Last) Score++; + if (Mh & Last) Score--; + + if ((Ph >> 63) ^ Pb) + Phc[i / 64] = FLIP(Phc[i / 64], i % 64); + + if ((Mh >> 63) ^ Mb) + Mhc[i / 64] = FLIP(Mhc[i / 64], i % 64); + + Ph = (Ph << 1) | Pb; + Mh = (Mh << 1) | Mb; + + Pv = Mh | ~ (Xv | Ph); + Mv = Ph & Xv; + } + } + free(Phc); + return Score; +} + +static int64_t myers1999_simple(uint8_t *s1, int64_t len1, uint8_t *s2, int64_t len2) +{ + uint64_t Peq[256]; + uint64_t Eq, Xv, Xh, Ph, Mh, Pv, Mv, Last; + int64_t i; + int64_t Score = len2; + + memset(Peq, 0, sizeof(Peq)); + + for (i = 0; i < len2; i++) + Peq[s2[i]] |= (uint64_t) 1 << i; + + Mv = 0; + Pv = (uint64_t) -1; + Last = (uint64_t) 1 << (len2 - 1); + + for (i = 0; i < len1; i++) { + Eq = Peq[s1[i]]; + + Xv = Eq | Mv; + Xh = (((Eq & Pv) + Pv) ^ Pv) | Eq; + + Ph = Mv | ~ (Xh | Pv); + Mh = Pv & Xh; + + if (Ph & Last) Score++; + if (Mh & Last) Score--; + + Ph = (Ph << 1) | 1; + Mh = (Mh << 1); + + Pv = Mh | ~ (Xv | Ph); + Mv = Ph & Xv; + } + return Score; +} + +static int64_t myers1999(PyObject *o1, PyObject *o2) +{ + struct strbuf s1, s2; + struct blockmap map; + int64_t ret; + + strbuf_init(&s1, o1); + strbuf_init(&s2, o2); + + if (s1.len < s2.len) + return myers1999(o2, o1); + + if (ISASCII(s1.kind) && ISASCII(s2.kind) && s2.len < 65) + return myers1999_simple(s1.ptr, s1.len, s2.ptr, s2.len); + + if (blockmap_init(&map, &s2)) + return -1; + + ret = myers1999_block(&s1, &s2, &map); + blockmap_clear(&map); + return ret; +} + +/* + * Interface functions + */ +static int64_t polyleven(PyObject *o1, PyObject *o2, int64_t k) +{ + int64_t len1, len2; + + len1 = PyUnicode_GET_LENGTH(o1); + len2 = PyUnicode_GET_LENGTH(o2); + + if (len1 < len2) + return polyleven(o2, o1, k); + + if (k == 0) + return PyUnicode_Compare(o1, o2) ? 1 : 0; + + if (0 < k && k < len1 - len2) + return k + 1; + + if (len2 == 0) + return len1; + + if (0 < k && k < 4) + return mbleven(o1, o2, k); + + return myers1999(o1, o2); +} diff --git a/spacy/tests/matcher/test_levenshtein.py b/spacy/tests/matcher/test_levenshtein.py new file mode 100644 index 000000000..6c7793f63 --- /dev/null +++ b/spacy/tests/matcher/test_levenshtein.py @@ -0,0 +1,36 @@ +import pytest +from spacy.matcher import levenshtein + + +# empty string plus 10 random ASCII, 10 random unicode, and 2 random long tests +# from polyleven +@pytest.mark.parametrize( + "dist,a,b", + [ + (0, "", ""), + (4, "bbcb", "caba"), + (3, "abcb", "cacc"), + (3, "aa", "ccc"), + (1, "cca", "ccac"), + (1, "aba", "aa"), + (4, "bcbb", "abac"), + (3, "acbc", "bba"), + (3, "cbba", "a"), + (2, "bcc", "ba"), + (4, "aaa", "ccbb"), + (3, "うあい", "いいうい"), + (2, "あううい", "うあい"), + (3, "いういい", "うううあ"), + (2, "うい", "あいあ"), + (2, "いあい", "いう"), + (1, "いい", "あいい"), + (3, "あうあ", "いいああ"), + (4, "いあうう", "ううああ"), + (3, "いあいい", "ういああ"), + (3, "いいああ", "ううあう"), + (166,"TCTGGGCACGGATTCGTCAGATTCCATGTCCATATTTGAGGCTCTTGCAGGCAAAATTTGGGCATGTGAACTCCTTATAGTCCCCGTGC","ATATGGATTGGGGGCATTCAAAGATACGGTTTCCCTTTCTTCAGTTTCGCGCGGCGCACGTCCGGGTGCGAGCCAGTTCGTCTTACTCACATTGTCGACTTCACGAATCGCGCATGATGTGCTTAGCCTGTACTTACGAACGAACTTTCGGTCCAAATACATTCTATCAACACCGAGGTATCCGTGCCACACGCCGAAGCTCGACCGTGTTCGTTGAGAGGTGGAAATGGTAAAAGATGAACATAGTC"), + (111,"GGTTCGGCCGAATTCATAGAGCGTGGTAGTCGACGGTATCCCGCCTGGTAGGGGCCCCTTCTACCTAGCGGAAGTTTGTCAGTACTCTATAACACGAGGGCCTCTCACACCCTAGATCGTCCAGCCACTCGAAGATCGCAGCACCCTTACAGAAAGGCATTAATGTTTCTCCTAGCACTTGTGCAATGGTGAAGGAGTGATG","CGTAACACTTCGCGCTACTGGGCTGCAACGTCTTGGGCATACATGCAAGATTATCTAATGCAAGCTTGAGCCCCGCTTGCGGAATTTCCCTAATCGGGGTCCCTTCCTGTTACGATAAGGACGCGTGCACT"), + ], +) +def test_levenshtein(dist, a, b): + assert levenshtein(a, b) == dist From ca1ad67458d96562ab28d03892e926908cb583e1 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Thu, 15 Sep 2022 15:51:19 +0200 Subject: [PATCH 025/179] disable mypy run for Python 3.10 (#11508) --- .github/azure-steps.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml index 18224ba8c..c7722391f 100644 --- a/.github/azure-steps.yml +++ b/.github/azure-steps.yml @@ -27,6 +27,7 @@ steps: - script: python -m mypy spacy displayName: 'Run mypy' + condition: ne(variables['python_version'], '3.10') - task: DeleteFiles@1 inputs: From 0509f908743afc86a185346ca6cb2e4789041732 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Thu, 15 Sep 2022 17:29:42 +0200 Subject: [PATCH 026/179] add dot (#11500) --- spacy/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/errors.py b/spacy/errors.py index 5ee1476c2..f55b378e9 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -707,7 +707,7 @@ class Errors(metaclass=ErrorsWithCodes): "need to modify the pipeline, use the built-in methods like " "`nlp.add_pipe`, `nlp.remove_pipe`, `nlp.disable_pipe` or " "`nlp.enable_pipe` instead.") - E927 = ("Can't write to frozen list Maybe you're trying to modify a computed " + E927 = ("Can't write to frozen list. Maybe you're trying to modify a computed " "property or default function argument?") E928 = ("A KnowledgeBase can only be serialized to/from from a directory, " "but the provided argument {loc} points to a file.") From d5c8498f2f8c26fdfb1f18aafeeebbe94e6126bb Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Thu, 15 Sep 2022 17:41:25 +0200 Subject: [PATCH 027/179] disable mypy run for Python 3.10 (#11508) (#11511) --- .github/azure-steps.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml index 18224ba8c..c7722391f 100644 --- a/.github/azure-steps.yml +++ b/.github/azure-steps.yml @@ -27,6 +27,7 @@ steps: - script: python -m mypy spacy displayName: 'Run mypy' + condition: ne(variables['python_version'], '3.10') - task: DeleteFiles@1 inputs: From df0b815c2382f572a127e4a35dba30cf1fa9fe45 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Fri, 16 Sep 2022 09:26:33 +0200 Subject: [PATCH 028/179] more explicit Example constructor example (#11489) * make constructor example for Example more explicit * shorten example and add spaces --- website/docs/api/example.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/website/docs/api/example.md b/website/docs/api/example.md index 0228e8935..63768d58f 100644 --- a/website/docs/api/example.md +++ b/website/docs/api/example.md @@ -23,11 +23,13 @@ both documents. > ```python > from spacy.tokens import Doc > from spacy.training import Example -> -> words = ["hello", "world", "!"] -> spaces = [True, False, False] -> predicted = Doc(nlp.vocab, words=words, spaces=spaces) -> reference = parse_gold_doc(my_data) +> pred_words = ["Apply", "some", "sunscreen"] +> pred_spaces = [True, True, False] +> gold_words = ["Apply", "some", "sun", "screen"] +> gold_spaces = [True, True, False, False] +> gold_tags = ["VERB", "DET", "NOUN", "NOUN"] +> predicted = Doc(nlp.vocab, words=pred_words, spaces=pred_spaces) +> reference = Doc(nlp.vocab, words=gold_words, spaces=gold_spaces, tags=gold_tags) > example = Example(predicted, reference) > ``` From 279358be63a6f32c49c2c89d4657a5239f238d9e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 16 Sep 2022 11:50:19 +0200 Subject: [PATCH 029/179] Auto-format code with black (#11513) Co-authored-by: explosion-bot --- spacy/tests/matcher/test_levenshtein.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/spacy/tests/matcher/test_levenshtein.py b/spacy/tests/matcher/test_levenshtein.py index 6c7793f63..d30e36132 100644 --- a/spacy/tests/matcher/test_levenshtein.py +++ b/spacy/tests/matcher/test_levenshtein.py @@ -28,8 +28,16 @@ from spacy.matcher import levenshtein (4, "いあうう", "ううああ"), (3, "いあいい", "ういああ"), (3, "いいああ", "ううあう"), - (166,"TCTGGGCACGGATTCGTCAGATTCCATGTCCATATTTGAGGCTCTTGCAGGCAAAATTTGGGCATGTGAACTCCTTATAGTCCCCGTGC","ATATGGATTGGGGGCATTCAAAGATACGGTTTCCCTTTCTTCAGTTTCGCGCGGCGCACGTCCGGGTGCGAGCCAGTTCGTCTTACTCACATTGTCGACTTCACGAATCGCGCATGATGTGCTTAGCCTGTACTTACGAACGAACTTTCGGTCCAAATACATTCTATCAACACCGAGGTATCCGTGCCACACGCCGAAGCTCGACCGTGTTCGTTGAGAGGTGGAAATGGTAAAAGATGAACATAGTC"), - (111,"GGTTCGGCCGAATTCATAGAGCGTGGTAGTCGACGGTATCCCGCCTGGTAGGGGCCCCTTCTACCTAGCGGAAGTTTGTCAGTACTCTATAACACGAGGGCCTCTCACACCCTAGATCGTCCAGCCACTCGAAGATCGCAGCACCCTTACAGAAAGGCATTAATGTTTCTCCTAGCACTTGTGCAATGGTGAAGGAGTGATG","CGTAACACTTCGCGCTACTGGGCTGCAACGTCTTGGGCATACATGCAAGATTATCTAATGCAAGCTTGAGCCCCGCTTGCGGAATTTCCCTAATCGGGGTCCCTTCCTGTTACGATAAGGACGCGTGCACT"), + ( + 166, + "TCTGGGCACGGATTCGTCAGATTCCATGTCCATATTTGAGGCTCTTGCAGGCAAAATTTGGGCATGTGAACTCCTTATAGTCCCCGTGC", + "ATATGGATTGGGGGCATTCAAAGATACGGTTTCCCTTTCTTCAGTTTCGCGCGGCGCACGTCCGGGTGCGAGCCAGTTCGTCTTACTCACATTGTCGACTTCACGAATCGCGCATGATGTGCTTAGCCTGTACTTACGAACGAACTTTCGGTCCAAATACATTCTATCAACACCGAGGTATCCGTGCCACACGCCGAAGCTCGACCGTGTTCGTTGAGAGGTGGAAATGGTAAAAGATGAACATAGTC", + ), + ( + 111, + "GGTTCGGCCGAATTCATAGAGCGTGGTAGTCGACGGTATCCCGCCTGGTAGGGGCCCCTTCTACCTAGCGGAAGTTTGTCAGTACTCTATAACACGAGGGCCTCTCACACCCTAGATCGTCCAGCCACTCGAAGATCGCAGCACCCTTACAGAAAGGCATTAATGTTTCTCCTAGCACTTGTGCAATGGTGAAGGAGTGATG", + "CGTAACACTTCGCGCTACTGGGCTGCAACGTCTTGGGCATACATGCAAGATTATCTAATGCAAGCTTGAGCCCCGCTTGCGGAATTTCCCTAATCGGGGTCCCTTCCTGTTACGATAAGGACGCGTGCACT", + ), ], ) def test_levenshtein(dist, a, b): From af9b01ef97d934a8601aff46d8341fdaf78b88df Mon Sep 17 00:00:00 2001 From: Raphael Mitsch Date: Fri, 16 Sep 2022 16:54:31 +0200 Subject: [PATCH 030/179] Add dependency check to project step runs (#11226) * Add dependency check to project step running. * Fix dependency mismatch warning. * Remove newline. * Add types-setuptools to setup.cfg. * Move types-setuptools to test requirements. Move warnings into _validate_requirements(). Handle file reading in project_run(). * Remove newline formatting for output of package conflicts. * Show full version conflict message instead of just package name. * Update spacy/cli/project/run.py Co-authored-by: Adriane Boyd * Fix typo. * Re-add rephrasing of message for conflicting packages. Remove requirements path redundancy. * Update spacy/cli/project/run.py Co-authored-by: Adriane Boyd * Update spacy/cli/project/run.py Co-authored-by: Adriane Boyd * Print unified message for requirement conflicts and missing requirements. * Update spacy/cli/project/run.py Co-authored-by: Adriane Boyd * Fix warning message. * Print conflict/missing messages individually. * Print conflict/missing messages individually. * Add check_requirements setting in project.yml to disable requirements check. * Update website/docs/usage/projects.md Co-authored-by: Adriane Boyd * Update website/docs/usage/projects.md Co-authored-by: Adriane Boyd * Update description of project.yml structure in projects.md. * Update website/docs/usage/projects.md Co-authored-by: Sofie Van Landeghem * Prettify projects docs. Co-authored-by: Adriane Boyd Co-authored-by: Sofie Van Landeghem --- requirements.txt | 1 + spacy/cli/project/run.py | 40 +++++++++++++++++++++++++++++++++- website/docs/usage/projects.md | 35 ++++++++++++++++++----------- 3 files changed, 62 insertions(+), 14 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3e8501b2f..e45fde787 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,7 @@ hypothesis>=3.27.0,<7.0.0 mypy>=0.910,<0.970; platform_machine!='aarch64' types-dataclasses>=0.1.3; python_version < "3.7" types-mock>=0.1.1 +types-setuptools>=57.0.0 types-requests types-setuptools>=57.0.0 black>=22.0,<23.0 diff --git a/spacy/cli/project/run.py b/spacy/cli/project/run.py index d42d95465..ebab7471e 100644 --- a/spacy/cli/project/run.py +++ b/spacy/cli/project/run.py @@ -1,5 +1,8 @@ -from typing import Optional, List, Dict, Sequence, Any, Iterable +from typing import Optional, List, Dict, Sequence, Any, Iterable, Tuple +import os.path from pathlib import Path + +import pkg_resources from wasabi import msg from wasabi.util import locale_escape import sys @@ -71,6 +74,12 @@ def project_run( commands = {cmd["name"]: cmd for cmd in config.get("commands", [])} workflows = config.get("workflows", {}) validate_subcommand(list(commands.keys()), list(workflows.keys()), subcommand) + + req_path = project_dir / "requirements.txt" + if config.get("check_requirements", True) and os.path.exists(req_path): + with req_path.open() as requirements_file: + _check_requirements([req.replace("\n", "") for req in requirements_file]) + if subcommand in workflows: msg.info(f"Running workflow '{subcommand}'") for cmd in workflows[subcommand]: @@ -310,3 +319,32 @@ def get_fileinfo(project_dir: Path, paths: List[str]) -> List[Dict[str, Optional md5 = get_checksum(file_path) if file_path.exists() else None data.append({"path": path, "md5": md5}) return data + + +def _check_requirements(requirements: List[str]) -> Tuple[bool, bool]: + """Checks whether requirements are installed and free of version conflicts. + requirements (List[str]): List of requirements. + RETURNS (Tuple[bool, bool]): Whether (1) any packages couldn't be imported, (2) any packages with version conflicts + exist. + """ + + failed_pkgs_msgs: List[str] = [] + conflicting_pkgs_msgs: List[str] = [] + + for req in requirements: + try: + pkg_resources.require(req) + except pkg_resources.DistributionNotFound as dnf: + failed_pkgs_msgs.append(dnf.report()) + except pkg_resources.VersionConflict as vc: + conflicting_pkgs_msgs.append(vc.report()) + + if len(failed_pkgs_msgs) or len(conflicting_pkgs_msgs): + msg.warn( + title="Missing requirements or requirement conflicts detected. Make sure your Python environment is set up " + "correctly and you installed all requirements specified in your project's requirements.txt: " + ) + for pgk_msg in failed_pkgs_msgs + conflicting_pkgs_msgs: + msg.text(pgk_msg) + + return len(failed_pkgs_msgs) > 0, len(conflicting_pkgs_msgs) > 0 diff --git a/website/docs/usage/projects.md b/website/docs/usage/projects.md index 35150035a..4797bbfe3 100644 --- a/website/docs/usage/projects.md +++ b/website/docs/usage/projects.md @@ -148,6 +148,13 @@ skipped. You can also set `--force` to force re-running a command, or `--dry` to perform a "dry run" and see what would happen (without actually running the script). +Since spaCy v3.4.2, `spacy projects run` checks your installed dependencies to +verify that your environment is properly set up and aligns with the project's +`requirements.txt`, if there is one. If missing or conflicting dependencies are +detected, a corresponding warning is displayed. If you'd like to disable the +dependency check, set `check_requirements: false` in your project's +`project.yml`. + ### 4. Run a workflow {#run-workfow} > #### project.yml @@ -226,26 +233,28 @@ pipelines. ```yaml %%GITHUB_PROJECTS/pipelines/tagger_parser_ud/project.yml ``` + > #### Tip: Overriding variables on the CLI > -> If you want to override one or more variables on the CLI and are not already specifying a -> project directory, you need to add `.` as a placeholder: +> If you want to override one or more variables on the CLI and are not already +> specifying a project directory, you need to add `.` as a placeholder: > > ``` > python -m spacy project run test . --vars.foo bar > ``` -| Section | Description | -| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `title` | An optional project title used in `--help` message and [auto-generated docs](#custom-docs). | -| `description` | An optional project description used in [auto-generated docs](#custom-docs). | -| `vars` | A dictionary of variables that can be referenced in paths, URLs and scripts and overriden on the CLI, just like [`config.cfg` variables](/usage/training#config-interpolation). For example, `${vars.name}` will use the value of the variable `name`. Variables need to be defined in the section `vars`, but can be a nested dict, so you're able to reference `${vars.model.name}`. | -| `env` | A dictionary of variables, mapped to the names of environment variables that will be read in when running the project. For example, `${env.name}` will use the value of the environment variable defined as `name`. | -| `directories` | An optional list of [directories](#project-files) that should be created in the project for assets, training outputs, metrics etc. spaCy will make sure that these directories always exist. | -| `assets` | A list of assets that can be fetched with the [`project assets`](/api/cli#project-assets) command. `url` defines a URL or local path, `dest` is the destination file relative to the project directory, and an optional `checksum` ensures that an error is raised if the file's checksum doesn't match. Instead of `url`, you can also provide a `git` block with the keys `repo`, `branch` and `path`, to download from a Git repo. | -| `workflows` | A dictionary of workflow names, mapped to a list of command names, to execute in order. Workflows can be run with the [`project run`](/api/cli#project-run) command. | -| `commands` | A list of named commands. A command can define an optional help message (shown in the CLI when the user adds `--help`) and the `script`, a list of commands to run. The `deps` and `outputs` let you define the created file the command depends on and produces, respectively. This lets spaCy determine whether a command needs to be re-run because its dependencies or outputs changed. Commands can be run as part of a workflow, or separately with the [`project run`](/api/cli#project-run) command. | -| `spacy_version` | Optional spaCy version range like `>=3.0.0,<3.1.0` that the project is compatible with. If it's loaded with an incompatible version, an error is raised when the project is loaded. | +| Section | Description | +| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `title` | An optional project title used in `--help` message and [auto-generated docs](#custom-docs). | +| `description` | An optional project description used in [auto-generated docs](#custom-docs). | +| `vars` | A dictionary of variables that can be referenced in paths, URLs and scripts and overriden on the CLI, just like [`config.cfg` variables](/usage/training#config-interpolation). For example, `${vars.name}` will use the value of the variable `name`. Variables need to be defined in the section `vars`, but can be a nested dict, so you're able to reference `${vars.model.name}`. | +| `env` | A dictionary of variables, mapped to the names of environment variables that will be read in when running the project. For example, `${env.name}` will use the value of the environment variable defined as `name`. | +| `directories` | An optional list of [directories](#project-files) that should be created in the project for assets, training outputs, metrics etc. spaCy will make sure that these directories always exist. | +| `assets` | A list of assets that can be fetched with the [`project assets`](/api/cli#project-assets) command. `url` defines a URL or local path, `dest` is the destination file relative to the project directory, and an optional `checksum` ensures that an error is raised if the file's checksum doesn't match. Instead of `url`, you can also provide a `git` block with the keys `repo`, `branch` and `path`, to download from a Git repo. | +| `workflows` | A dictionary of workflow names, mapped to a list of command names, to execute in order. Workflows can be run with the [`project run`](/api/cli#project-run) command. | +| `commands` | A list of named commands. A command can define an optional help message (shown in the CLI when the user adds `--help`) and the `script`, a list of commands to run. The `deps` and `outputs` let you define the created file the command depends on and produces, respectively. This lets spaCy determine whether a command needs to be re-run because its dependencies or outputs changed. Commands can be run as part of a workflow, or separately with the [`project run`](/api/cli#project-run) command. | +| `spacy_version` | Optional spaCy version range like `>=3.0.0,<3.1.0` that the project is compatible with. If it's loaded with an incompatible version, an error is raised when the project is loaded. | +| `check_requirements` 3.4.2 | A flag determining whether to verify that the installed dependencies align with the project's `requirements.txt`. Defaults to `true`. | ### Data assets {#data-assets} From f40d2fac29678111ec600eb7def9d58b174f14a2 Mon Sep 17 00:00:00 2001 From: Basile Dura Date: Fri, 23 Sep 2022 13:18:51 +0200 Subject: [PATCH 031/179] fix: remove duplicate v3.2 (#11530) --- website/meta/sidebars.json | 1 - 1 file changed, 1 deletion(-) diff --git a/website/meta/sidebars.json b/website/meta/sidebars.json index 1b743636c..06fce7742 100644 --- a/website/meta/sidebars.json +++ b/website/meta/sidebars.json @@ -12,7 +12,6 @@ { "text": "New in v3.0", "url": "/usage/v3" }, { "text": "New in v3.1", "url": "/usage/v3-1" }, { "text": "New in v3.2", "url": "/usage/v3-2" }, - { "text": "New in v3.2", "url": "/usage/v3-2" }, { "text": "New in v3.3", "url": "/usage/v3-3" }, { "text": "New in v3.4", "url": "/usage/v3-4" } ] From 6f692a06d54d53f702def1a2ca20a649b7a1b644 Mon Sep 17 00:00:00 2001 From: Richard Hudson Date: Mon, 26 Sep 2022 15:58:21 +0200 Subject: [PATCH 032/179] Remove side effects from Doc.__init__() (#11506) * Remove side effects from Doc.__init__() * Changes based on review comment * Readd test * Change interface of Doc.__init__() * Simplify test Co-authored-by: Adriane Boyd * Update doc.md Co-authored-by: Adriane Boyd --- spacy/tests/doc/test_doc_api.py | 15 +++++++++++++++ spacy/tokens/doc.pyi | 2 +- spacy/tokens/doc.pyx | 12 ++++++------ website/docs/api/doc.md | 30 +++++++++++++++--------------- 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/spacy/tests/doc/test_doc_api.py b/spacy/tests/doc/test_doc_api.py index a64ab2ba8..38003dea9 100644 --- a/spacy/tests/doc/test_doc_api.py +++ b/spacy/tests/doc/test_doc_api.py @@ -82,6 +82,21 @@ def test_issue2396(en_vocab): assert (span.get_lca_matrix() == matrix).all() +@pytest.mark.issue(11499) +def test_init_args_unmodified(en_vocab): + words = ["A", "sentence"] + ents = ["B-TYPE1", ""] + sent_starts = [True, False] + Doc( + vocab=en_vocab, + words=words, + ents=ents, + sent_starts=sent_starts, + ) + assert ents == ["B-TYPE1", ""] + assert sent_starts == [True, False] + + @pytest.mark.parametrize("text", ["-0.23", "+123,456", "±1"]) @pytest.mark.parametrize("lang_cls", [English, MultiLanguage]) @pytest.mark.issue(2782) diff --git a/spacy/tokens/doc.pyi b/spacy/tokens/doc.pyi index a40fa74aa..f0cdaee87 100644 --- a/spacy/tokens/doc.pyi +++ b/spacy/tokens/doc.pyi @@ -72,7 +72,7 @@ class Doc: lemmas: Optional[List[str]] = ..., heads: Optional[List[int]] = ..., deps: Optional[List[str]] = ..., - sent_starts: Optional[List[Union[bool, None]]] = ..., + sent_starts: Optional[List[Union[bool, int, None]]] = ..., ents: Optional[List[str]] = ..., ) -> None: ... @property diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index 7ba9a3341..d7d2fd8e6 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -217,9 +217,9 @@ cdef class Doc: head in the doc. Defaults to None. deps (Optional[List[str]]): A list of unicode strings, of the same length as words, to assign as token.dep. Defaults to None. - sent_starts (Optional[List[Union[bool, None]]]): A list of values, of - the same length as words, to assign as token.is_sent_start. Will be - overridden by heads if heads is provided. Defaults to None. + sent_starts (Optional[List[Union[bool, int, None]]]): A list of values, + of the same length as words, to assign as token.is_sent_start. Will + be overridden by heads if heads is provided. Defaults to None. ents (Optional[List[str]]): A list of unicode strings, of the same length as words, as IOB tags to assign as token.ent_iob and token.ent_type. Defaults to None. @@ -285,6 +285,7 @@ cdef class Doc: heads = [0] * len(deps) if heads and not deps: raise ValueError(Errors.E1017) + sent_starts = list(sent_starts) if sent_starts is not None else None if sent_starts is not None: for i in range(len(sent_starts)): if sent_starts[i] is True: @@ -300,12 +301,11 @@ cdef class Doc: ent_iobs = None ent_types = None if ents is not None: + ents = [ent if ent != "" else None for ent in ents] iob_strings = Token.iob_strings() # make valid IOB2 out of IOB1 or IOB2 for i, ent in enumerate(ents): - if ent is "": - ents[i] = None - elif ent is not None and not isinstance(ent, str): + if ent is not None and not isinstance(ent, str): raise ValueError(Errors.E177.format(tag=ent)) if i < len(ents) - 1: # OI -> OB diff --git a/website/docs/api/doc.md b/website/docs/api/doc.md index f97f4ad83..f97ed4547 100644 --- a/website/docs/api/doc.md +++ b/website/docs/api/doc.md @@ -31,21 +31,21 @@ Construct a `Doc` object. The most common way to get a `Doc` object is via the > doc = Doc(nlp.vocab, words=words, spaces=spaces) > ``` -| Name | Description | -| ---------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `vocab` | A storage container for lexical types. ~~Vocab~~ | -| `words` | A list of strings or integer hash values to add to the document as words. ~~Optional[List[Union[str,int]]]~~ | -| `spaces` | A list of boolean values indicating whether each word has a subsequent space. Must have the same length as `words`, if specified. Defaults to a sequence of `True`. ~~Optional[List[bool]]~~ | -| _keyword-only_ | | -| `user\_data` | Optional extra data to attach to the Doc. ~~Dict~~ | -| `tags` 3 | A list of strings, of the same length as `words`, to assign as `token.tag` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | -| `pos` 3 | A list of strings, of the same length as `words`, to assign as `token.pos` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | -| `morphs` 3 | A list of strings, of the same length as `words`, to assign as `token.morph` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | -| `lemmas` 3 | A list of strings, of the same length as `words`, to assign as `token.lemma` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | -| `heads` 3 | A list of values, of the same length as `words`, to assign as the head for each word. Head indices are the absolute position of the head in the `Doc`. Defaults to `None`. ~~Optional[List[int]]~~ | -| `deps` 3 | A list of strings, of the same length as `words`, to assign as `token.dep` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | -| `sent_starts` 3 | A list of values, of the same length as `words`, to assign as `token.is_sent_start`. Will be overridden by heads if `heads` is provided. Defaults to `None`. ~~Optional[List[Optional[bool]]]~~ | -| `ents` 3 | A list of strings, of the same length of `words`, to assign the token-based IOB tag. Defaults to `None`. ~~Optional[List[str]]~~ | +| Name | Description | +| ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | A storage container for lexical types. ~~Vocab~~ | +| `words` | A list of strings or integer hash values to add to the document as words. ~~Optional[List[Union[str,int]]]~~ | +| `spaces` | A list of boolean values indicating whether each word has a subsequent space. Must have the same length as `words`, if specified. Defaults to a sequence of `True`. ~~Optional[List[bool]]~~ | +| _keyword-only_ | | +| `user\_data` | Optional extra data to attach to the Doc. ~~Dict~~ | +| `tags` 3 | A list of strings, of the same length as `words`, to assign as `token.tag` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | +| `pos` 3 | A list of strings, of the same length as `words`, to assign as `token.pos` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | +| `morphs` 3 | A list of strings, of the same length as `words`, to assign as `token.morph` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | +| `lemmas` 3 | A list of strings, of the same length as `words`, to assign as `token.lemma` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | +| `heads` 3 | A list of values, of the same length as `words`, to assign as the head for each word. Head indices are the absolute position of the head in the `Doc`. Defaults to `None`. ~~Optional[List[int]]~~ | +| `deps` 3 | A list of strings, of the same length as `words`, to assign as `token.dep` for each word. Defaults to `None`. ~~Optional[List[str]]~~ | +| `sent_starts` 3 | A list of values, of the same length as `words`, to assign as `token.is_sent_start`. Will be overridden by heads if `heads` is provided. Defaults to `None`. ~~Optional[List[Union[bool, int, None]]]~~ | +| `ents` 3 | A list of strings, of the same length of `words`, to assign the token-based IOB tag. Defaults to `None`. ~~Optional[List[str]]~~ | ## Doc.\_\_getitem\_\_ {#getitem tag="method"} From 936a5f0506d5a117aeae000481560e1fc0031036 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Tue, 27 Sep 2022 15:25:24 +0900 Subject: [PATCH 033/179] Fix English pipeline names in 3.4 release notes (#11542) --- website/docs/usage/v3-4.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/docs/usage/v3-4.md b/website/docs/usage/v3-4.md index 7cc4570d5..597fc3cc8 100644 --- a/website/docs/usage/v3-4.md +++ b/website/docs/usage/v3-4.md @@ -65,10 +65,10 @@ The English CNN pipelines have new word vectors: | Package | Model Version | TAG | Parser LAS | NER F | | ----------------------------------------------- | ------------- | ---: | ---------: | ----: | -| [`en_core_news_md`](/models/en#en_core_news_md) | v3.3.0 | 97.3 | 90.1 | 84.6 | -| [`en_core_news_md`](/models/en#en_core_news_lg) | v3.4.0 | 97.2 | 90.3 | 85.5 | -| [`en_core_news_lg`](/models/en#en_core_news_md) | v3.3.0 | 97.4 | 90.1 | 85.3 | -| [`en_core_news_lg`](/models/en#en_core_news_lg) | v3.4.0 | 97.3 | 90.2 | 85.6 | +| [`en_core_web_md`](/models/en#en_core_web_md) | v3.3.0 | 97.3 | 90.1 | 84.6 | +| [`en_core_web_md`](/models/en#en_core_web_lg) | v3.4.0 | 97.2 | 90.3 | 85.5 | +| [`en_core_web_lg`](/models/en#en_core_web_md) | v3.3.0 | 97.4 | 90.1 | 85.3 | +| [`en_core_web_lg`](/models/en#en_core_web_lg) | v3.4.0 | 97.3 | 90.2 | 85.6 | ## Notes about upgrading from v3.3 {#upgrading} From 877671e09a0a72ca20ccbbcd65d7073f588cd320 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 27 Sep 2022 10:16:51 +0200 Subject: [PATCH 034/179] Preserve missing entity annotation in augmenters (#11540) Preserve both `-` and `O` annotation in augmenters rather than relying on `Example.to_dict`'s default support for one option outside of labeled entity spans. This is intended as a temporary workaround for augmenters for v3.4.x. The behavior of `Example` and related IOB utils could be improved in the general case for v3.5. --- spacy/tests/training/test_augmenters.py | 7 +++++-- spacy/training/augment.py | 14 +++++++++++++- spacy/training/iob_utils.py | 8 ++++++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/spacy/tests/training/test_augmenters.py b/spacy/tests/training/test_augmenters.py index e3639c5da..35860a199 100644 --- a/spacy/tests/training/test_augmenters.py +++ b/spacy/tests/training/test_augmenters.py @@ -31,7 +31,7 @@ def doc(nlp): words = ["Sarah", "'s", "sister", "flew", "to", "Silicon", "Valley", "via", "London", "."] tags = ["NNP", "POS", "NN", "VBD", "IN", "NNP", "NNP", "IN", "NNP", "."] pos = ["PROPN", "PART", "NOUN", "VERB", "ADP", "PROPN", "PROPN", "ADP", "PROPN", "PUNCT"] - ents = ["B-PERSON", "I-PERSON", "O", "O", "O", "B-LOC", "I-LOC", "O", "B-GPE", "O"] + ents = ["B-PERSON", "I-PERSON", "O", "", "O", "B-LOC", "I-LOC", "O", "B-GPE", "O"] cats = {"TRAVEL": 1.0, "BAKING": 0.0} # fmt: on doc = Doc(nlp.vocab, words=words, tags=tags, pos=pos, ents=ents) @@ -106,6 +106,7 @@ def test_lowercase_augmenter(nlp, doc): assert [(e.start, e.end, e.label) for e in eg.reference.ents] == ents for ref_ent, orig_ent in zip(eg.reference.ents, doc.ents): assert ref_ent.text == orig_ent.text.lower() + assert [t.ent_iob for t in doc] == [t.ent_iob for t in eg.reference] assert [t.pos_ for t in eg.reference] == [t.pos_ for t in doc] # check that augmentation works when lowercasing leads to different @@ -166,7 +167,7 @@ def test_make_whitespace_variant(nlp): lemmas = ["they", "fly", "to", "New", "York", "City", ".", "\n", "then", "they", "drive", "to", "Washington", ",", "D.C."] heads = [1, 1, 1, 4, 5, 2, 1, 10, 10, 10, 10, 10, 11, 12, 12] deps = ["nsubj", "ROOT", "prep", "compound", "compound", "pobj", "punct", "dep", "advmod", "nsubj", "ROOT", "prep", "pobj", "punct", "appos"] - ents = ["O", "O", "O", "B-GPE", "I-GPE", "I-GPE", "O", "O", "O", "O", "O", "O", "B-GPE", "O", "B-GPE"] + ents = ["O", "", "O", "B-GPE", "I-GPE", "I-GPE", "O", "O", "O", "O", "O", "O", "B-GPE", "O", "B-GPE"] # fmt: on doc = Doc( nlp.vocab, @@ -215,6 +216,8 @@ def test_make_whitespace_variant(nlp): assert mod_ex2.reference[j].head.i == j - 1 # entities are well-formed assert len(doc.ents) == len(mod_ex.reference.ents) + # there is one token with missing entity information + assert any(t.ent_iob == 0 for t in mod_ex.reference) for ent in mod_ex.reference.ents: assert not ent[0].is_space assert not ent[-1].is_space diff --git a/spacy/training/augment.py b/spacy/training/augment.py index 55d780ba4..2fe8c24fb 100644 --- a/spacy/training/augment.py +++ b/spacy/training/augment.py @@ -6,7 +6,7 @@ from functools import partial from ..util import registry from .example import Example -from .iob_utils import split_bilu_label +from .iob_utils import split_bilu_label, _doc_to_biluo_tags_with_partial if TYPE_CHECKING: from ..language import Language # noqa: F401 @@ -62,6 +62,9 @@ def combined_augmenter( if orth_variants and random.random() < orth_level: raw_text = example.text orig_dict = example.to_dict() + orig_dict["doc_annotation"]["entities"] = _doc_to_biluo_tags_with_partial( + example.reference + ) variant_text, variant_token_annot = make_orth_variants( nlp, raw_text, @@ -128,6 +131,9 @@ def lower_casing_augmenter( def make_lowercase_variant(nlp: "Language", example: Example): example_dict = example.to_dict() + example_dict["doc_annotation"]["entities"] = _doc_to_biluo_tags_with_partial( + example.reference + ) doc = nlp.make_doc(example.text.lower()) example_dict["token_annotation"]["ORTH"] = [t.lower_ for t in example.reference] return example.from_dict(doc, example_dict) @@ -146,6 +152,9 @@ def orth_variants_augmenter( else: raw_text = example.text orig_dict = example.to_dict() + orig_dict["doc_annotation"]["entities"] = _doc_to_biluo_tags_with_partial( + example.reference + ) variant_text, variant_token_annot = make_orth_variants( nlp, raw_text, @@ -248,6 +257,9 @@ def make_whitespace_variant( RETURNS (Example): Example with one additional space token. """ example_dict = example.to_dict() + example_dict["doc_annotation"]["entities"] = _doc_to_biluo_tags_with_partial( + example.reference + ) doc_dict = example_dict.get("doc_annotation", {}) token_dict = example_dict.get("token_annotation", {}) # returned unmodified if: diff --git a/spacy/training/iob_utils.py b/spacy/training/iob_utils.py index 61f83a1c3..0d4d246b0 100644 --- a/spacy/training/iob_utils.py +++ b/spacy/training/iob_utils.py @@ -60,6 +60,14 @@ def doc_to_biluo_tags(doc: Doc, missing: str = "O"): ) +def _doc_to_biluo_tags_with_partial(doc: Doc) -> List[str]: + ents = doc_to_biluo_tags(doc, missing="-") + for i, token in enumerate(doc): + if token.ent_iob == 2: + ents[i] = "O" + return ents + + def offsets_to_biluo_tags( doc: Doc, entities: Iterable[Tuple[int, int, Union[str, int]]], missing: str = "O" ) -> List[str]: From a44b7d4622108a42ddb95b62b642df6f142a3450 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Tue, 27 Sep 2022 18:11:23 +0900 Subject: [PATCH 035/179] Add experimental coref docs (#11291) * Add experimental coref docs * Docs cleanup * Apply suggestions from code review Co-authored-by: Sofie Van Landeghem * Apply changes from code review * Fix prettier formatting It seems a period after a number made this think it was a list? * Update docs on examples for initialize * Add docs for coref scorers * Remove 3.4 notes from coref There won't be a "new" tag until it's in core. * Add docs for span cleaner * Fix docs * Fix docs to match spacy-experimental These weren't properly updated when the code was moved out of spacy core. * More doc fixes * Formatting * Update architectures * Fix links * Fix another link Co-authored-by: Sofie Van Landeghem Co-authored-by: svlandeg --- website/docs/api/architectures.md | 92 ++++++- website/docs/api/coref.md | 353 ++++++++++++++++++++++++ website/docs/api/pipeline-functions.md | 33 +++ website/docs/api/scorer.md | 59 ++++ website/docs/api/span-resolver.md | 356 +++++++++++++++++++++++++ website/meta/sidebars.json | 2 + 6 files changed, 889 insertions(+), 6 deletions(-) create mode 100644 website/docs/api/coref.md create mode 100644 website/docs/api/span-resolver.md diff --git a/website/docs/api/architectures.md b/website/docs/api/architectures.md index 2537faff6..4c5447f75 100644 --- a/website/docs/api/architectures.md +++ b/website/docs/api/architectures.md @@ -11,6 +11,7 @@ menu: - ['Text Classification', 'textcat'] - ['Span Classification', 'spancat'] - ['Entity Linking', 'entitylinker'] + - ['Coreference', 'coref-architectures'] --- A **model architecture** is a function that wires up a @@ -587,8 +588,8 @@ consists of either two or three subnetworks: run once for each batch. - **lower**: Construct a feature-specific vector for each `(token, feature)` pair. This is also run once for each batch. Constructing the state - representation is then a matter of summing the component features and - applying the non-linearity. + representation is then a matter of summing the component features and applying + the non-linearity. - **upper** (optional): A feed-forward network that predicts scores from the state representation. If not present, the output from the lower model is used as action scores directly. @@ -628,8 +629,8 @@ same signature, but the `use_upper` argument was `True` by default. > ``` Build a tagger model, using a provided token-to-vector component. The tagger -model adds a linear layer with softmax activation to predict scores given -the token vectors. +model adds a linear layer with softmax activation to predict scores given the +token vectors. | Name | Description | | ----------- | ------------------------------------------------------------------------------------------ | @@ -920,5 +921,84 @@ A function that reads an existing `KnowledgeBase` from file. A function that takes as input a [`KnowledgeBase`](/api/kb) and a [`Span`](/api/span) object denoting a named entity, and returns a list of plausible [`Candidate`](/api/kb/#candidate) objects. The default -`CandidateGenerator` uses the text of a mention to find its potential -aliases in the `KnowledgeBase`. Note that this function is case-dependent. +`CandidateGenerator` uses the text of a mention to find its potential aliases in +the `KnowledgeBase`. Note that this function is case-dependent. + +## Coreference {#coref-architectures tag="experimental"} + +A [`CoreferenceResolver`](/api/coref) component identifies tokens that refer to +the same entity. A [`SpanResolver`](/api/span-resolver) component infers spans +from single tokens. Together these components can be used to reproduce +traditional coreference models. You can also omit the `SpanResolver` if working +with only token-level clusters is acceptable. + +### spacy-experimental.Coref.v1 {#Coref tag="experimental"} + +> #### Example Config +> +> ```ini +> +> [model] +> @architectures = "spacy-experimental.Coref.v1" +> distance_embedding_size = 20 +> dropout = 0.3 +> hidden_size = 1024 +> depth = 2 +> antecedent_limit = 50 +> antecedent_batch_size = 512 +> +> [model.tok2vec] +> @architectures = "spacy-transformers.TransformerListener.v1" +> grad_factor = 1.0 +> upstream = "transformer" +> pooling = {"@layers":"reduce_mean.v1"} +> ``` + +The `Coref` model architecture is a Thinc `Model`. + +| Name | Description | +| ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `tok2vec` | The [`tok2vec`](#tok2vec) layer of the model. ~~Model~~ | +| `distance_embedding_size` | A representation of the distance between candidates. ~~int~~ | +| `dropout` | The dropout to use internally. Unlike some Thinc models, this has separate dropout for the internal PyTorch layers. ~~float~~ | +| `hidden_size` | Size of the main internal layers. ~~int~~ | +| `depth` | Depth of the internal network. ~~int~~ | +| `antecedent_limit` | How many candidate antecedents to keep after rough scoring. This has a significant effect on memory usage. Typical values would be 50 to 200, or higher for very long documents. ~~int~~ | +| `antecedent_batch_size` | Internal batch size. ~~int~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | + +### spacy-experimental.SpanResolver.v1 {#SpanResolver tag="experimental"} + +> #### Example Config +> +> ```ini +> +> [model] +> @architectures = "spacy-experimental.SpanResolver.v1" +> hidden_size = 1024 +> distance_embedding_size = 64 +> conv_channels = 4 +> window_size = 1 +> max_distance = 128 +> prefix = "coref_head_clusters" +> +> [model.tok2vec] +> @architectures = "spacy-transformers.TransformerListener.v1" +> grad_factor = 1.0 +> upstream = "transformer" +> pooling = {"@layers":"reduce_mean.v1"} +> ``` + +The `SpanResolver` model architecture is a Thinc `Model`. Note that +`MentionClusters` is `List[List[Tuple[int, int]]]`. + +| Name | Description | +| ------------------------- | -------------------------------------------------------------------------------------------------------------------- | +| `tok2vec` | The [`tok2vec`](#tok2vec) layer of the model. ~~Model~~ | +| `hidden_size` | Size of the main internal layers. ~~int~~ | +| `distance_embedding_size` | A representation of the distance between two candidates. ~~int~~ | +| `conv_channels` | The number of channels in the internal CNN. ~~int~~ | +| `window_size` | The number of neighboring tokens to consider in the internal CNN. `1` means consider one token on each side. ~~int~~ | +| `max_distance` | The longest possible length of a predicted span. ~~int~~ | +| `prefix` | The prefix that indicates spans to use for input data. ~~string~~ | +| **CREATES** | The model using the architecture. ~~Model[List[Doc], List[MentionClusters]]~~ | diff --git a/website/docs/api/coref.md b/website/docs/api/coref.md new file mode 100644 index 000000000..8f54422d6 --- /dev/null +++ b/website/docs/api/coref.md @@ -0,0 +1,353 @@ +--- +title: CoreferenceResolver +tag: class,experimental +source: spacy-experimental/coref/coref_component.py +teaser: 'Pipeline component for word-level coreference resolution' +api_base_class: /api/pipe +api_string_name: coref +api_trainable: true +--- + +> #### Installation +> +> ```bash +> $ pip install -U spacy-experimental +> ``` + + + +This component is not yet integrated into spaCy core, and is available via the +extension package +[`spacy-experimental`](https://github.com/explosion/spacy-experimental) starting +in version 0.6.0. It exposes the component via +[entry points](/usage/saving-loading/#entry-points), so if you have the package +installed, using `factory = "experimental_coref"` in your +[training config](/usage/training#config) or +`nlp.add_pipe("experimental_coref")` will work out-of-the-box. + + + +A `CoreferenceResolver` component groups tokens into clusters that refer to the +same thing. Clusters are represented as SpanGroups that start with a prefix +(`coref_clusters` by default). + +A `CoreferenceResolver` component can be paired with a +[`SpanResolver`](/api/span-resolver) to expand single tokens to spans. + +## Assigned Attributes {#assigned-attributes} + +Predictions will be saved to `Doc.spans` as a [`SpanGroup`](/api/spangroup). The +span key will be a prefix plus a serial number referring to the coreference +cluster, starting from zero. + +The span key prefix defaults to `"coref_clusters"`, but can be passed as a +parameter. + +| Location | Value | +| ------------------------------------------ | ------------------------------------------------------------------------------------------------------- | +| `Doc.spans[prefix + "_" + cluster_number]` | One coreference cluster, represented as single-token spans. Cluster numbers start from 1. ~~SpanGroup~~ | + +## Config and implementation {#config} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures#coref-architectures) documentation for +details on the architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy_experimental.coref.coref_component import DEFAULT_COREF_MODEL +> from spacy_experimental.coref.coref_util import DEFAULT_CLUSTER_PREFIX +> config={ +> "model": DEFAULT_COREF_MODEL, +> "span_cluster_prefix": DEFAULT_CLUSTER_PREFIX, +> }, +> nlp.add_pipe("experimental_coref", config=config) +> ``` + +| Setting | Description | +| --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [Coref](/api/architectures#Coref). ~~Model~~ | +| `span_cluster_prefix` | The prefix for the keys for clusters saved to `doc.spans`. Defaults to `coref_clusters`. ~~str~~ | + +## CoreferenceResolver.\_\_init\_\_ {#init tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> coref = nlp.add_pipe("experimental_coref") +> +> # Construction via add_pipe with custom model +> config = {"model": {"@architectures": "my_coref.v1"}} +> coref = nlp.add_pipe("experimental_coref", config=config) +> +> # Construction from class +> from spacy_experimental.coref.coref_component import CoreferenceResolver +> coref = CoreferenceResolver(nlp.vocab, model) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#add_pipe). + +| Name | Description | +| --------------------- | --------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `span_cluster_prefix` | The prefix for the key for saving clusters of spans. ~~bool~~ | + +## CoreferenceResolver.\_\_call\_\_ {#call tag="method"} + +Apply the pipe to one document. The document is modified in place and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](/api/coref#call) and [`pipe`](/api/coref#pipe) delegate to the +[`predict`](/api/coref#predict) and +[`set_annotations`](/api/coref#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> coref = nlp.add_pipe("experimental_coref") +> # This usually happens under the hood +> processed = coref(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## CoreferenceResolver.pipe {#pipe tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/coref#call) and +[`pipe`](/api/coref#pipe) delegate to the [`predict`](/api/coref#predict) and +[`set_annotations`](/api/coref#set_annotations) methods. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> for doc in coref.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## CoreferenceResolver.initialize {#initialize tag="method"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize). + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> coref.initialize(lambda: examples, nlp=nlp) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | + +## CoreferenceResolver.predict {#predict tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects, without +modifying them. Clusters are returned as a list of `MentionClusters`, one for +each input `Doc`. A `MentionClusters` instance is just a list of lists of pairs +of `int`s, where each item corresponds to a cluster, and the `int`s correspond +to token indices. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> clusters = coref.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The predicted coreference clusters for the `docs`. ~~List[MentionClusters]~~ | + +## CoreferenceResolver.set_annotations {#set_annotations tag="method"} + +Modify a batch of documents, saving coreference clusters in `Doc.spans`. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> clusters = coref.predict([doc1, doc2]) +> coref.set_annotations([doc1, doc2], clusters) +> ``` + +| Name | Description | +| ---------- | ---------------------------------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `clusters` | The predicted coreference clusters for the `docs`. ~~List[MentionClusters]~~ | + +## CoreferenceResolver.update {#update tag="method"} + +Learn from a batch of [`Example`](/api/example) objects. Delegates to +[`predict`](/api/coref#predict). + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> optimizer = nlp.initialize() +> losses = coref.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## CoreferenceResolver.create_optimizer {#create_optimizer tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> optimizer = coref.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## CoreferenceResolver.use_params {#use_params tag="method, contextmanager"} + +Modify the pipe's model, to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> with coref.use_params(optimizer.averages): +> coref.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## CoreferenceResolver.to_disk {#to_disk tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> coref.to_disk("/path/to/coref") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## CoreferenceResolver.from_disk {#from_disk tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> coref.from_disk("/path/to/coref") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `CoreferenceResolver` object. ~~CoreferenceResolver~~ | + +## CoreferenceResolver.to_bytes {#to_bytes tag="method"} + +> #### Example +> +> ```python +> coref = nlp.add_pipe("experimental_coref") +> coref_bytes = coref.to_bytes() +> ``` + +Serialize the pipe to a bytestring, including the `KnowledgeBase`. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `CoreferenceResolver` object. ~~bytes~~ | + +## CoreferenceResolver.from_bytes {#from_bytes tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> coref_bytes = coref.to_bytes() +> coref = nlp.add_pipe("experimental_coref") +> coref.from_bytes(coref_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `CoreferenceResolver` object. ~~CoreferenceResolver~~ | + +## Serialization fields {#serialization-fields} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = coref.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | diff --git a/website/docs/api/pipeline-functions.md b/website/docs/api/pipeline-functions.md index 1b7017ca7..070292782 100644 --- a/website/docs/api/pipeline-functions.md +++ b/website/docs/api/pipeline-functions.md @@ -153,3 +153,36 @@ whole pipeline has run. | `attrs` | A dict of the `Doc` attributes and the values to set them to. Defaults to `{"tensor": None, "_.trf_data": None}` to clean up after `tok2vec` and `transformer` components. ~~dict~~ | | `silent` | If `False`, show warnings if attributes aren't found or can't be set. Defaults to `True`. ~~bool~~ | | **RETURNS** | The modified `Doc` with the modified attributes. ~~Doc~~ | + +## span_cleaner {#span_cleaner tag="function,experimental"} + +Remove `SpanGroup`s from `doc.spans` based on a key prefix. This is used to +clean up after the [`CoreferenceResolver`](/api/coref) when it's paired with a +[`SpanResolver`](/api/span-resolver). + + + +This pipeline function is not yet integrated into spaCy core, and is available +via the extension package +[`spacy-experimental`](https://github.com/explosion/spacy-experimental) starting +in version 0.6.0. It exposes the component via +[entry points](/usage/saving-loading/#entry-points), so if you have the package +installed, using `factory = "span_cleaner"` in your +[training config](/usage/training#config) or `nlp.add_pipe("span_cleaner")` will +work out-of-the-box. + + + +> #### Example +> +> ```python +> config = {"prefix": "coref_head_clusters"} +> nlp.add_pipe("span_cleaner", config=config) +> doc = nlp("text") +> assert "coref_head_clusters_1" not in doc.spans +> ``` + +| Setting | Description | +| ----------- | ------------------------------------------------------------------------------------------------------------------------- | +| `prefix` | A prefix to check `SpanGroup` keys for. Any matching groups will be removed. Defaults to `"coref_head_clusters"`. ~~str~~ | +| **RETURNS** | The modified `Doc` with any matching spans removed. ~~Doc~~ | diff --git a/website/docs/api/scorer.md b/website/docs/api/scorer.md index 8dbe3b276..ca3462aa9 100644 --- a/website/docs/api/scorer.md +++ b/website/docs/api/scorer.md @@ -270,3 +270,62 @@ Compute micro-PRF and per-entity PRF scores. | Name | Description | | ---------- | ------------------------------------------------------------------------------------------------------------------- | | `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | + +## score_coref_clusters {#score_coref_clusters tag="experimental"} + +Returns LEA ([Moosavi and Strube, 2016](https://aclanthology.org/P16-1060/)) PRF +scores for coreference clusters. + + + +Note this scoring function is not yet included in spaCy core - for details, see +the [CoreferenceResolver](/api/coref) docs. + + + +> #### Example +> +> ```python +> scores = score_coref_clusters( +> examples, +> span_cluster_prefix="coref_clusters", +> ) +> print(scores["coref_f"]) +> ``` + +| Name | Description | +| --------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `span_cluster_prefix` | The prefix used for spans representing coreference clusters. ~~str~~ | +| **RETURNS** | A dictionary containing the scores. ~~Dict[str, Optional[float]]~~ | + +## score_span_predictions {#score_span_predictions tag="experimental"} + +Return accuracy for reconstructions of spans from single tokens. Only exactly +correct predictions are counted as correct, there is no partial credit for near +answers. Used by the [SpanResolver](/api/span-resolver). + + + +Note this scoring function is not yet included in spaCy core - for details, see +the [SpanResolver](/api/span-resolver) docs. + + + +> #### Example +> +> ```python +> scores = score_span_predictions( +> examples, +> output_prefix="coref_clusters", +> ) +> print(scores["span_coref_clusters_accuracy"]) +> ``` + +| Name | Description | +| --------------- | ------------------------------------------------------------------------------------------------------------------- | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `output_prefix` | The prefix used for spans representing the final predicted spans. ~~str~~ | +| **RETURNS** | A dictionary containing the scores. ~~Dict[str, Optional[float]]~~ | diff --git a/website/docs/api/span-resolver.md b/website/docs/api/span-resolver.md new file mode 100644 index 000000000..3e992cd03 --- /dev/null +++ b/website/docs/api/span-resolver.md @@ -0,0 +1,356 @@ +--- +title: SpanResolver +tag: class,experimental +source: spacy-experimental/coref/span_resolver_component.py +teaser: 'Pipeline component for resolving tokens into spans' +api_base_class: /api/pipe +api_string_name: span_resolver +api_trainable: true +--- + +> #### Installation +> +> ```bash +> $ pip install -U spacy-experimental +> ``` + + + +This component not yet integrated into spaCy core, and is available via the +extension package +[`spacy-experimental`](https://github.com/explosion/spacy-experimental) starting +in version 0.6.0. It exposes the component via +[entry points](/usage/saving-loading/#entry-points), so if you have the package +installed, using `factory = "experimental_span_resolver"` in your +[training config](/usage/training#config) or +`nlp.add_pipe("experimental_span_resolver")` will work out-of-the-box. + + + +A `SpanResolver` component takes in tokens (represented as `Span` objects of +length 1) and resolves them into `Span` objects of arbitrary length. The initial +use case is as a post-processing step on word-level +[coreference resolution](/api/coref). The input and output keys used to store +`Span` objects are configurable. + +## Assigned Attributes {#assigned-attributes} + +Predictions will be saved to `Doc.spans` as [`SpanGroup`s](/api/spangroup). + +Input token spans will be read in using an input prefix, by default +`"coref_head_clusters"`, and output spans will be saved using an output prefix +(default `"coref_clusters"`) plus a serial number starting from one. The +prefixes are configurable. + +| Location | Value | +| ------------------------------------------------- | ------------------------------------------------------------------------- | +| `Doc.spans[output_prefix + "_" + cluster_number]` | One group of predicted spans. Cluster number starts from 1. ~~SpanGroup~~ | + +## Config and implementation {#config} + +The default config is defined by the pipeline component factory and describes +how the component should be configured. You can override its settings via the +`config` argument on [`nlp.add_pipe`](/api/language#add_pipe) or in your +[`config.cfg` for training](/usage/training#config). See the +[model architectures](/api/architectures#coref-architectures) documentation for +details on the architectures and their arguments and hyperparameters. + +> #### Example +> +> ```python +> from spacy_experimental.coref.span_resolver_component import DEFAULT_SPAN_RESOLVER_MODEL +> from spacy_experimental.coref.coref_util import DEFAULT_CLUSTER_PREFIX, DEFAULT_CLUSTER_HEAD_PREFIX +> config={ +> "model": DEFAULT_SPAN_RESOLVER_MODEL, +> "input_prefix": DEFAULT_CLUSTER_HEAD_PREFIX, +> "output_prefix": DEFAULT_CLUSTER_PREFIX, +> }, +> nlp.add_pipe("experimental_span_resolver", config=config) +> ``` + +| Setting | Description | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [SpanResolver](/api/architectures#SpanResolver). ~~Model~~ | +| `input_prefix` | The prefix to use for input `SpanGroup`s. Defaults to `coref_head_clusters`. ~~str~~ | +| `output_prefix` | The prefix for predicted `SpanGroup`s. Defaults to `coref_clusters`. ~~str~~ | + +## SpanResolver.\_\_init\_\_ {#init tag="method"} + +> #### Example +> +> ```python +> # Construction via add_pipe with default model +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> +> # Construction via add_pipe with custom model +> config = {"model": {"@architectures": "my_span_resolver.v1"}} +> span_resolver = nlp.add_pipe("experimental_span_resolver", config=config) +> +> # Construction from class +> from spacy_experimental.coref.span_resolver_component import SpanResolver +> span_resolver = SpanResolver(nlp.vocab, model) +> ``` + +Create a new pipeline instance. In your application, you would normally use a +shortcut for this and instantiate the component using its string name and +[`nlp.add_pipe`](/api/language#add_pipe). + +| Name | Description | +| --------------- | --------------------------------------------------------------------------------------------------- | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model~~ | +| `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | +| _keyword-only_ | | +| `input_prefix` | The prefix to use for input `SpanGroup`s. Defaults to `coref_head_clusters`. ~~str~~ | +| `output_prefix` | The prefix for predicted `SpanGroup`s. Defaults to `coref_clusters`. ~~str~~ | + +## SpanResolver.\_\_call\_\_ {#call tag="method"} + +Apply the pipe to one document. The document is modified in place and returned. +This usually happens under the hood when the `nlp` object is called on a text +and all pipeline components are applied to the `Doc` in order. Both +[`__call__`](#call) and [`pipe`](#pipe) delegate to the [`predict`](#predict) +and [`set_annotations`](#set_annotations) methods. + +> #### Example +> +> ```python +> doc = nlp("This is a sentence.") +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> # This usually happens under the hood +> processed = span_resolver(doc) +> ``` + +| Name | Description | +| ----------- | -------------------------------- | +| `doc` | The document to process. ~~Doc~~ | +| **RETURNS** | The processed document. ~~Doc~~ | + +## SpanResolver.pipe {#pipe tag="method"} + +Apply the pipe to a stream of documents. This usually happens under the hood +when the `nlp` object is called on a text and all pipeline components are +applied to the `Doc` in order. Both [`__call__`](/api/span-resolver#call) and +[`pipe`](/api/span-resolver#pipe) delegate to the +[`predict`](/api/span-resolver#predict) and +[`set_annotations`](/api/span-resolver#set_annotations) methods. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> for doc in span_resolver.pipe(docs, batch_size=50): +> pass +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------- | +| `stream` | A stream of documents. ~~Iterable[Doc]~~ | +| _keyword-only_ | | +| `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | +| **YIELDS** | The processed documents in order. ~~Doc~~ | + +## SpanResolver.initialize {#initialize tag="method"} + +Initialize the component for training. `get_examples` should be a function that +returns an iterable of [`Example`](/api/example) objects. **At least one example +should be supplied.** The data examples are used to **initialize the model** of +the component and can either be the full training data or a representative +sample. Initialization includes validating the network, +[inferring missing shapes](https://thinc.ai/docs/usage-models#validation) and +setting up the label scheme based on the data. This method is typically called +by [`Language.initialize`](/api/language#initialize). + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> span_resolver.initialize(lambda: examples, nlp=nlp) +> ``` + +| Name | Description | +| -------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `get_examples` | Function that returns gold-standard annotations in the form of [`Example`](/api/example) objects. Must contain at least one `Example`. ~~Callable[[], Iterable[Example]]~~ | +| _keyword-only_ | | +| `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | + +## SpanResolver.predict {#predict tag="method"} + +Apply the component's model to a batch of [`Doc`](/api/doc) objects, without +modifying them. Predictions are returned as a list of `MentionClusters`, one for +each input `Doc`. A `MentionClusters` instance is just a list of lists of pairs +of `int`s, where each item corresponds to an input `SpanGroup`, and the `int`s +correspond to token indices. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> spans = span_resolver.predict([doc1, doc2]) +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------- | +| `docs` | The documents to predict. ~~Iterable[Doc]~~ | +| **RETURNS** | The predicted spans for the `Doc`s. ~~List[MentionClusters]~~ | + +## SpanResolver.set_annotations {#set_annotations tag="method"} + +Modify a batch of documents, saving predictions using the output prefix in +`Doc.spans`. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> spans = span_resolver.predict([doc1, doc2]) +> span_resolver.set_annotations([doc1, doc2], spans) +> ``` + +| Name | Description | +| ------- | ------------------------------------------------------------- | +| `docs` | The documents to modify. ~~Iterable[Doc]~~ | +| `spans` | The predicted spans for the `docs`. ~~List[MentionClusters]~~ | + +## SpanResolver.update {#update tag="method"} + +Learn from a batch of [`Example`](/api/example) objects. Delegates to +[`predict`](/api/span-resolver#predict). + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> optimizer = nlp.initialize() +> losses = span_resolver.update(examples, sgd=optimizer) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------ | +| `examples` | A batch of [`Example`](/api/example) objects to learn from. ~~Iterable[Example]~~ | +| _keyword-only_ | | +| `drop` | The dropout rate. ~~float~~ | +| `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | +| `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | +| **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | + +## SpanResolver.create_optimizer {#create_optimizer tag="method"} + +Create an optimizer for the pipeline component. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> optimizer = span_resolver.create_optimizer() +> ``` + +| Name | Description | +| ----------- | ---------------------------- | +| **RETURNS** | The optimizer. ~~Optimizer~~ | + +## SpanResolver.use_params {#use_params tag="method, contextmanager"} + +Modify the pipe's model, to use the given parameter values. At the end of the +context, the original parameters are restored. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> with span_resolver.use_params(optimizer.averages): +> span_resolver.to_disk("/best_model") +> ``` + +| Name | Description | +| -------- | -------------------------------------------------- | +| `params` | The parameter values to use in the model. ~~dict~~ | + +## SpanResolver.to_disk {#to_disk tag="method"} + +Serialize the pipe to disk. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> span_resolver.to_disk("/path/to/span_resolver") +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | + +## SpanResolver.from_disk {#from_disk tag="method"} + +Load the pipe from disk. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> span_resolver.from_disk("/path/to/span_resolver") +> ``` + +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------- | +| `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `SpanResolver` object. ~~SpanResolver~~ | + +## SpanResolver.to_bytes {#to_bytes tag="method"} + +> #### Example +> +> ```python +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> span_resolver_bytes = span_resolver.to_bytes() +> ``` + +Serialize the pipe to a bytestring. + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The serialized form of the `SpanResolver` object. ~~bytes~~ | + +## SpanResolver.from_bytes {#from_bytes tag="method"} + +Load the pipe from a bytestring. Modifies the object in place and returns it. + +> #### Example +> +> ```python +> span_resolver_bytes = span_resolver.to_bytes() +> span_resolver = nlp.add_pipe("experimental_span_resolver") +> span_resolver.from_bytes(span_resolver_bytes) +> ``` + +| Name | Description | +| -------------- | ------------------------------------------------------------------------------------------- | +| `bytes_data` | The data to load from. ~~bytes~~ | +| _keyword-only_ | | +| `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The `SpanResolver` object. ~~SpanResolver~~ | + +## Serialization fields {#serialization-fields} + +During serialization, spaCy will export several data fields used to restore +different aspects of the object. If needed, you can exclude them from +serialization by passing in the string names via the `exclude` argument. + +> #### Example +> +> ```python +> data = span_resolver.to_disk("/path", exclude=["vocab"]) +> ``` + +| Name | Description | +| ------- | -------------------------------------------------------------- | +| `vocab` | The shared [`Vocab`](/api/vocab). | +| `cfg` | The config file. You usually don't want to exclude this. | +| `model` | The binary model data. You usually don't want to exclude this. | diff --git a/website/meta/sidebars.json b/website/meta/sidebars.json index 06fce7742..2d8745d77 100644 --- a/website/meta/sidebars.json +++ b/website/meta/sidebars.json @@ -94,6 +94,7 @@ "label": "Pipeline", "items": [ { "text": "AttributeRuler", "url": "/api/attributeruler" }, + { "text": "CoreferenceResolver", "url": "/api/coref" }, { "text": "DependencyParser", "url": "/api/dependencyparser" }, { "text": "EditTreeLemmatizer", "url": "/api/edittreelemmatizer" }, { "text": "EntityLinker", "url": "/api/entitylinker" }, @@ -104,6 +105,7 @@ { "text": "SentenceRecognizer", "url": "/api/sentencerecognizer" }, { "text": "Sentencizer", "url": "/api/sentencizer" }, { "text": "SpanCategorizer", "url": "/api/spancategorizer" }, + { "text": "SpanResolver", "url": "/api/span-resolver" }, { "text": "SpanRuler", "url": "/api/spanruler" }, { "text": "Tagger", "url": "/api/tagger" }, { "text": "TextCategorizer", "url": "/api/textcategorizer" }, From 3e8bc1272f95c89e0aa9e5a19f51e286a7934ffa Mon Sep 17 00:00:00 2001 From: Jacobo Myerston <43222279+jmyerston@users.noreply.github.com> Date: Tue, 27 Sep 2022 02:38:56 -0700 Subject: [PATCH 036/179] add punctuation to grc (#11426) * add punctuation to grc Add support for special editorial punctuation that is common in ancient Greek texts. Ancient Greek texts, as found in digital and print form, have been largely edited by scholars. Restorations and improvements are normally marked with special characters that need to be handled properly by the tokenizer. * add unit tests * simplify regex * move generic quotes to char classes * rename unit test * fix regex Co-authored-by: Adriane Boyd Co-authored-by: svlandeg Co-authored-by: Sofie Van Landeghem Co-authored-by: Adriane Boyd --- spacy/lang/char_classes.py | 2 +- spacy/lang/grc/__init__.py | 4 +++ spacy/lang/grc/punctuation.py | 46 ++++++++++++++++++++++++++ spacy/tests/lang/grc/test_tokenizer.py | 18 ++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 spacy/lang/grc/punctuation.py create mode 100644 spacy/tests/lang/grc/test_tokenizer.py diff --git a/spacy/lang/char_classes.py b/spacy/lang/char_classes.py index 1d204c46c..37c58c85f 100644 --- a/spacy/lang/char_classes.py +++ b/spacy/lang/char_classes.py @@ -280,7 +280,7 @@ _currency = ( _punct = ( r"… …… , : ; \! \? ¿ ؟ ¡ \( \) \[ \] \{ \} < > _ # \* & 。 ？ ！ ， 、 ； ： ～ · । ، ۔ ؛ ٪" ) -_quotes = r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉' +_quotes = r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉 〈 〉 ⟦ ⟧' _hyphens = "- – — -- --- —— ~" # Various symbols like dingbats, but also emoji diff --git a/spacy/lang/grc/__init__.py b/spacy/lang/grc/__init__.py index e83f0c5a5..019b3802e 100644 --- a/spacy/lang/grc/__init__.py +++ b/spacy/lang/grc/__init__.py @@ -1,11 +1,15 @@ from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES from ...language import Language, BaseDefaults class AncientGreekDefaults(BaseDefaults): tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + suffixes = TOKENIZER_SUFFIXES + infixes = TOKENIZER_INFIXES lex_attr_getters = LEX_ATTRS stop_words = STOP_WORDS diff --git a/spacy/lang/grc/punctuation.py b/spacy/lang/grc/punctuation.py new file mode 100644 index 000000000..8f3589e9a --- /dev/null +++ b/spacy/lang/grc/punctuation.py @@ -0,0 +1,46 @@ +from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_CURRENCY +from ..char_classes import LIST_ICONS, ALPHA_LOWER, ALPHA_UPPER, ALPHA, HYPHENS +from ..char_classes import CONCAT_QUOTES + +_prefixes = ( + [ + "†", + "⸏", + ] + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_CURRENCY + + LIST_ICONS +) + +_suffixes = ( + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + [ + "†", + "⸎", + r"(?<=[\u1F00-\u1FFF\u0370-\u03FF])[\-\.⸏]", + ] +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + r"(?<=[\u1F00-\u1FFF\u0370-\u03FF])—", + ] +) + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/tests/lang/grc/test_tokenizer.py b/spacy/tests/lang/grc/test_tokenizer.py new file mode 100644 index 000000000..3df5b546b --- /dev/null +++ b/spacy/tests/lang/grc/test_tokenizer.py @@ -0,0 +1,18 @@ +import pytest + + +# fmt: off +GRC_TOKEN_EXCEPTION_TESTS = [ + ("τὸ 〈τῆς〉 φιλοσοφίας ἔργον ἔνιοί φασιν ἀπὸ ⟦βαρβάρων⟧ ἄρξαι.", ["τὸ", "〈", "τῆς", "〉", "φιλοσοφίας", "ἔργον", "ἔνιοί", "φασιν", "ἀπὸ", "⟦", "βαρβάρων", "⟧", "ἄρξαι", "."]), + ("τὴν δὲ τῶν Αἰγυπτίων φιλοσοφίαν εἶναι τοιαύτην περί τε †θεῶν† καὶ ὑπὲρ δικαιοσύνης.", ["τὴν", "δὲ", "τῶν", "Αἰγυπτίων", "φιλοσοφίαν", "εἶναι", "τοιαύτην", "περί", "τε", "†", "θεῶν", "†", "καὶ", "ὑπὲρ", "δικαιοσύνης", "."]), + ("⸏πόσις δ' Ἐρεχθεύς ἐστί μοι σεσωσμένος⸏", ["⸏", "πόσις", "δ'", "Ἐρεχθεύς", "ἐστί", "μοι", "σεσωσμένος", "⸏"]), + ("⸏ὔπνον ἴδωμεν⸎", ["⸏", "ὔπνον", "ἴδωμεν", "⸎"]), +] +# fmt: on + + +@pytest.mark.parametrize("text,expected_tokens", GRC_TOKEN_EXCEPTION_TESTS) +def test_grc_tokenizer(grc_tokenizer, text, expected_tokens): + tokens = grc_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list From 9557b0fb01612f5b32823dfc52cae71af37f0bd8 Mon Sep 17 00:00:00 2001 From: Taniguchi Yasufumi Date: Tue, 27 Sep 2022 21:11:50 +0900 Subject: [PATCH 037/179] Add spacy-partial-tagger to spaCy Universe (#11538) --- website/meta/universe.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/website/meta/universe.json b/website/meta/universe.json index 9145855c6..9ec0d6c0e 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -3984,7 +3984,21 @@ }, "category": ["pipeline"], "tags": ["interpretation", "ja"] + }, + { + "id": "spacy-partial-tagger", + "title": "spaCy - Partial Tagger", + "slogan": "Sequence Tagger for Partially Annotated Dataset in spaCy", + "description": "This is a library to build a CRF tagger with a partially annotated dataset in spaCy. You can build your own tagger only from dictionary.", + "github": "doccano/spacy-partial-tagger", + "pip": "spacy-partial-tagger", + "category": ["pipeline", "training"], + "author": "Yasufumi Taniguchi", + "author_links": { + "github": "yasufumy" + } } + ], "categories": [ From aea16719be04d4d6ab889cd20fe0e323b2c7ffee Mon Sep 17 00:00:00 2001 From: Raphael Mitsch Date: Tue, 27 Sep 2022 14:22:36 +0200 Subject: [PATCH 038/179] Simplify and clarify enable/disable behavior of spacy.load() (#11459) * Change enable/disable behavior so that arguments take precedence over config options. Extend error message on conflict. Add warning message in case of overwriting config option with arguments. * Fix tests in test_serialize_pipeline.py to reflect changes to handling of enable/disable. * Fix type issue. * Move comment. * Move comment. * Issue UserWarning instead of printing wasabi message. Adjust test. * Added pytest.warns(UserWarning) for expected warning to fix tests. * Update warning message. * Move type handling out of fetch_pipes_status(). * Add global variable for default value. Use id() to determine whether used values are default value. * Fix default value for disable. * Rename DEFAULT_PIPE_STATUS to _DEFAULT_EMPTY_PIPES. --- spacy/__init__.py | 6 +- spacy/errors.py | 7 ++- spacy/language.py | 59 ++++++++++++------- spacy/tests/pipeline/test_pipe_methods.py | 33 +++++++++-- .../serialize/test_serialize_pipeline.py | 7 ++- spacy/util.py | 23 ++++---- 6 files changed, 92 insertions(+), 43 deletions(-) diff --git a/spacy/__init__.py b/spacy/__init__.py index d60f46b96..c3568bc5c 100644 --- a/spacy/__init__.py +++ b/spacy/__init__.py @@ -31,9 +31,9 @@ def load( name: Union[str, Path], *, vocab: Union[Vocab, bool] = True, - disable: Union[str, Iterable[str]] = util.SimpleFrozenList(), - enable: Union[str, Iterable[str]] = util.SimpleFrozenList(), - exclude: Union[str, Iterable[str]] = util.SimpleFrozenList(), + disable: Union[str, Iterable[str]] = util._DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = util._DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = util._DEFAULT_EMPTY_PIPES, config: Union[Dict[str, Any], Config] = util.SimpleFrozenDict(), ) -> Language: """Load a spaCy model from an installed package or a local path. diff --git a/spacy/errors.py b/spacy/errors.py index f55b378e9..c035f684d 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -212,6 +212,8 @@ class Warnings(metaclass=ErrorsWithCodes): W121 = ("Attempting to trace non-existent method '{method}' in pipe '{pipe}'") W122 = ("Couldn't trace method '{method}' in pipe '{pipe}'. This can happen if the pipe class " "is a Cython extension type.") + W123 = ("Argument {arg} with value {arg_value} is used instead of {config_value} as specified in the config. Be " + "aware that this might affect other components in your pipeline.") class Errors(metaclass=ErrorsWithCodes): @@ -937,8 +939,9 @@ class Errors(metaclass=ErrorsWithCodes): E1040 = ("Doc.from_json requires all tokens to have the same attributes. " "Some tokens do not contain annotation for: {partial_attrs}") E1041 = ("Expected a string, Doc, or bytes as input, but got: {type}") - E1042 = ("Function was called with `{arg1}`={arg1_values} and " - "`{arg2}`={arg2_values} but these arguments are conflicting.") + E1042 = ("`enable={enable}` and `disable={disable}` are inconsistent with each other.\nIf you only passed " + "one of `enable` or `disable`, the other argument is specified in your pipeline's configuration.\nIn that " + "case pass an empty list for the previously not specified argument to avoid this error.") E1043 = ("Expected None or a value in range [{range_start}, {range_end}] for entity linker threshold, but got " "{value}.") diff --git a/spacy/language.py b/spacy/language.py index 34a06e576..d391f15ab 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -1,4 +1,4 @@ -from typing import Iterator, Optional, Any, Dict, Callable, Iterable, Collection +from typing import Iterator, Optional, Any, Dict, Callable, Iterable from typing import Union, Tuple, List, Set, Pattern, Sequence from typing import NoReturn, TYPE_CHECKING, TypeVar, cast, overload @@ -10,6 +10,7 @@ from contextlib import contextmanager from copy import deepcopy from pathlib import Path import warnings + from thinc.api import get_current_ops, Config, CupyOps, Optimizer import srsly import multiprocessing as mp @@ -24,7 +25,7 @@ from .pipe_analysis import validate_attrs, analyze_pipes, print_pipe_analysis from .training import Example, validate_examples from .training.initialize import init_vocab, init_tok2vec from .scorer import Scorer -from .util import registry, SimpleFrozenList, _pipe, raise_error +from .util import registry, SimpleFrozenList, _pipe, raise_error, _DEFAULT_EMPTY_PIPES from .util import SimpleFrozenDict, combine_score_weights, CONFIG_SECTION_ORDER from .util import warn_if_jupyter_cupy from .lang.tokenizer_exceptions import URL_MATCH, BASE_EXCEPTIONS @@ -1698,9 +1699,9 @@ class Language: config: Union[Dict[str, Any], Config] = {}, *, vocab: Union[Vocab, bool] = True, - disable: Union[str, Iterable[str]] = SimpleFrozenList(), - enable: Union[str, Iterable[str]] = SimpleFrozenList(), - exclude: Union[str, Iterable[str]] = SimpleFrozenList(), + disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, meta: Dict[str, Any] = SimpleFrozenDict(), auto_fill: bool = True, validate: bool = True, @@ -1727,12 +1728,6 @@ class Language: DOCS: https://spacy.io/api/language#from_config """ - if isinstance(disable, str): - disable = [disable] - if isinstance(enable, str): - enable = [enable] - if isinstance(exclude, str): - exclude = [exclude] if auto_fill: config = Config( cls.default_config, section_order=CONFIG_SECTION_ORDER @@ -1877,9 +1872,38 @@ class Language: nlp.vocab.from_bytes(vocab_b) # Resolve disabled/enabled settings. + if isinstance(disable, str): + disable = [disable] + if isinstance(enable, str): + enable = [enable] + if isinstance(exclude, str): + exclude = [exclude] + + def fetch_pipes_status(value: Iterable[str], key: str) -> Iterable[str]: + """Fetch value for `enable` or `disable` w.r.t. the specified config and passed arguments passed to + .load(). If both arguments and config specified values for this field, the passed arguments take precedence + and a warning is printed. + value (Iterable[str]): Passed value for `enable` or `disable`. + key (str): Key for field in config (either "enabled" or "disabled"). + RETURN (Iterable[str]): + """ + # We assume that no argument was passed if the value is the specified default value. + if id(value) == id(_DEFAULT_EMPTY_PIPES): + return config["nlp"].get(key, []) + else: + if len(config["nlp"].get(key, [])): + warnings.warn( + Warnings.W123.format( + arg=key[:-1], + arg_value=value, + config_value=config["nlp"][key], + ) + ) + return value + disabled_pipes = cls._resolve_component_status( - [*config["nlp"]["disabled"], *disable], - [*config["nlp"].get("enabled", []), *enable], + fetch_pipes_status(disable, "disabled"), + fetch_pipes_status(enable, "enabled"), config["nlp"]["pipeline"], ) nlp._disabled = set(p for p in disabled_pipes if p not in exclude) @@ -2064,14 +2088,7 @@ class Language: pipe_name for pipe_name in pipe_names if pipe_name not in enable ] if disable and disable != to_disable: - raise ValueError( - Errors.E1042.format( - arg1="enable", - arg2="disable", - arg1_values=enable, - arg2_values=disable, - ) - ) + raise ValueError(Errors.E1042.format(enable=enable, disable=disable)) return tuple(to_disable) diff --git a/spacy/tests/pipeline/test_pipe_methods.py b/spacy/tests/pipeline/test_pipe_methods.py index b946061f6..14a7a36e5 100644 --- a/spacy/tests/pipeline/test_pipe_methods.py +++ b/spacy/tests/pipeline/test_pipe_methods.py @@ -605,10 +605,35 @@ def test_update_with_annotates(): assert results[component] == "" -def test_load_disable_enable() -> None: - """ - Tests spacy.load() with dis-/enabling components. - """ +@pytest.mark.issue(11443) +def test_enable_disable_conflict_with_config(): + """Test conflict between enable/disable w.r.t. `nlp.disabled` set in the config.""" + nlp = English() + nlp.add_pipe("tagger") + nlp.add_pipe("senter") + nlp.add_pipe("sentencizer") + + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + # Expected to fail, as config and arguments conflict. + with pytest.raises(ValueError): + spacy.load( + tmp_dir, enable=["tagger"], config={"nlp": {"disabled": ["senter"]}} + ) + # Expected to succeed without warning due to the lack of a conflicting config option. + spacy.load(tmp_dir, enable=["tagger"]) + # Expected to succeed with a warning, as disable=[] should override the config setting. + with pytest.warns(UserWarning): + spacy.load( + tmp_dir, + enable=["tagger"], + disable=[], + config={"nlp": {"disabled": ["senter"]}}, + ) + + +def test_load_disable_enable(): + """Tests spacy.load() with dis-/enabling components.""" base_nlp = English() for pipe in ("sentencizer", "tagger", "parser"): diff --git a/spacy/tests/serialize/test_serialize_pipeline.py b/spacy/tests/serialize/test_serialize_pipeline.py index 9fcf18e2d..b948bb76c 100644 --- a/spacy/tests/serialize/test_serialize_pipeline.py +++ b/spacy/tests/serialize/test_serialize_pipeline.py @@ -404,10 +404,11 @@ def test_serialize_pipeline_disable_enable(): assert nlp3.component_names == ["ner", "tagger"] with make_tempdir() as d: nlp3.to_disk(d) - nlp4 = spacy.load(d, disable=["ner"]) - assert nlp4.pipe_names == [] + with pytest.warns(UserWarning): + nlp4 = spacy.load(d, disable=["ner"]) + assert nlp4.pipe_names == ["tagger"] assert nlp4.component_names == ["ner", "tagger"] - assert nlp4.disabled == ["ner", "tagger"] + assert nlp4.disabled == ["ner"] with make_tempdir() as d: nlp.to_disk(d) nlp5 = spacy.load(d, exclude=["tagger"]) diff --git a/spacy/util.py b/spacy/util.py index 4e1a62d05..3034808ba 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -67,7 +67,6 @@ LEXEME_NORM_LANGS = ["cs", "da", "de", "el", "en", "id", "lb", "mk", "pt", "ru", CONFIG_SECTION_ORDER = ["paths", "variables", "system", "nlp", "components", "corpora", "training", "pretraining", "initialize"] # fmt: on - logger = logging.getLogger("spacy") logger_stream_handler = logging.StreamHandler() logger_stream_handler.setFormatter( @@ -394,13 +393,17 @@ def get_module_path(module: ModuleType) -> Path: return file_path.parent +# Default value for passed enable/disable values. +_DEFAULT_EMPTY_PIPES = SimpleFrozenList() + + def load_model( name: Union[str, Path], *, vocab: Union["Vocab", bool] = True, - disable: Union[str, Iterable[str]] = SimpleFrozenList(), - enable: Union[str, Iterable[str]] = SimpleFrozenList(), - exclude: Union[str, Iterable[str]] = SimpleFrozenList(), + disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), ) -> "Language": """Load a model from a package or data path. @@ -470,9 +473,9 @@ def load_model_from_path( *, meta: Optional[Dict[str, Any]] = None, vocab: Union["Vocab", bool] = True, - disable: Union[str, Iterable[str]] = SimpleFrozenList(), - enable: Union[str, Iterable[str]] = SimpleFrozenList(), - exclude: Union[str, Iterable[str]] = SimpleFrozenList(), + disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), ) -> "Language": """Load a model from a data directory path. Creates Language class with @@ -516,9 +519,9 @@ def load_model_from_config( *, meta: Dict[str, Any] = SimpleFrozenDict(), vocab: Union["Vocab", bool] = True, - disable: Union[str, Iterable[str]] = SimpleFrozenList(), - enable: Union[str, Iterable[str]] = SimpleFrozenList(), - exclude: Union[str, Iterable[str]] = SimpleFrozenList(), + disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, auto_fill: bool = False, validate: bool = True, ) -> "Language": From e794d4ae39b65aed341fa588ed4d473644aec672 Mon Sep 17 00:00:00 2001 From: Peter Baumgartner <5107405+pmbaumgartner@users.noreply.github.com> Date: Wed, 28 Sep 2022 11:16:05 -0400 Subject: [PATCH 039/179] `debug data` Spancat Table Improvements (#11504) * update * fix format function * pull out _format_number * format with black --- spacy/cli/_util.py | 9 +++++++++ spacy/cli/debug_data.py | 29 ++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py index ae43b991b..897964a88 100644 --- a/spacy/cli/_util.py +++ b/spacy/cli/_util.py @@ -573,3 +573,12 @@ def setup_gpu(use_gpu: int, silent=None) -> None: local_msg.info("Using CPU") if gpu_is_available(): local_msg.info("To switch to GPU 0, use the option: --gpu-id 0") + + +def _format_number(number: Union[int, float], ndigits: int = 2) -> str: + """Formats a number (float or int) rounding to `ndigits`, without truncating trailing 0s, + as happens with `round(number, ndigits)`""" + if isinstance(number, float): + return f"{number:.{ndigits}f}" + else: + return str(number) diff --git a/spacy/cli/debug_data.py b/spacy/cli/debug_data.py index bd05471b1..963d5b926 100644 --- a/spacy/cli/debug_data.py +++ b/spacy/cli/debug_data.py @@ -9,7 +9,7 @@ import typer import math from ._util import app, Arg, Opt, show_validation_error, parse_config_overrides -from ._util import import_code, debug_cli +from ._util import import_code, debug_cli, _format_number from ..training import Example, remove_bilu_prefix from ..training.initialize import get_sourced_components from ..schemas import ConfigSchemaTraining @@ -989,7 +989,8 @@ def _get_kl_divergence(p: Counter, q: Counter) -> float: def _format_span_row(span_data: List[Dict], labels: List[str]) -> List[Any]: """Compile into one list for easier reporting""" d = { - label: [label] + list(round(d[label], 2) for d in span_data) for label in labels + label: [label] + list(_format_number(d[label]) for d in span_data) + for label in labels } return list(d.values()) @@ -1004,6 +1005,10 @@ def _get_span_characteristics( label: _gmean(l) for label, l in compiled_gold["spans_length"][spans_key].items() } + spans_per_type = { + label: len(spans) + for label, spans in compiled_gold["spans_per_type"][spans_key].items() + } min_lengths = [min(l) for l in compiled_gold["spans_length"][spans_key].values()] max_lengths = [max(l) for l in compiled_gold["spans_length"][spans_key].values()] @@ -1031,6 +1036,7 @@ def _get_span_characteristics( return { "sd": span_distinctiveness, "bd": sb_distinctiveness, + "spans_per_type": spans_per_type, "lengths": span_length, "min_length": min(min_lengths), "max_length": max(max_lengths), @@ -1045,12 +1051,15 @@ def _get_span_characteristics( def _print_span_characteristics(span_characteristics: Dict[str, Any]): """Print all span characteristics into a table""" - headers = ("Span Type", "Length", "SD", "BD") + headers = ("Span Type", "Length", "SD", "BD", "N") + # Wasabi has this at 30 by default, but we might have some long labels + max_col = max(30, max(len(label) for label in span_characteristics["labels"])) # Prepare table data with all span characteristics table_data = [ span_characteristics["lengths"], span_characteristics["sd"], span_characteristics["bd"], + span_characteristics["spans_per_type"], ] table = _format_span_row( span_data=table_data, labels=span_characteristics["labels"] @@ -1061,8 +1070,18 @@ def _print_span_characteristics(span_characteristics: Dict[str, Any]): span_characteristics["avg_sd"], span_characteristics["avg_bd"], ] - footer = ["Wgt. Average"] + [str(round(f, 2)) for f in footer_data] - msg.table(table, footer=footer, header=headers, divider=True) + + footer = ( + ["Wgt. Average"] + ["{:.2f}".format(round(f, 2)) for f in footer_data] + ["-"] + ) + msg.table( + table, + footer=footer, + header=headers, + divider=True, + aligns=["l"] + ["r"] * (len(footer_data) + 1), + max_col=max_col, + ) def _get_spans_length_freq_dist( From 6d7630c5d372cda53b88a18b10bb893ce478d294 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Thu, 29 Sep 2022 10:44:06 +0200 Subject: [PATCH 040/179] Allow overriding spacy_version in spacy package meta (#11552) --- spacy/cli/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/cli/package.py b/spacy/cli/package.py index b8c8397b6..324c5d1bb 100644 --- a/spacy/cli/package.py +++ b/spacy/cli/package.py @@ -299,8 +299,8 @@ def get_meta( } nlp = util.load_model_from_path(Path(model_path)) meta.update(nlp.meta) - meta.update(existing_meta) meta["spacy_version"] = util.get_minor_version_range(about.__version__) + meta.update(existing_meta) meta["vectors"] = { "width": nlp.vocab.vectors_length, "vectors": len(nlp.vocab.vectors), From ba63f57f81441d049da52c5d398e5b226019a1a6 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Thu, 29 Sep 2022 18:50:29 +0900 Subject: [PATCH 041/179] Update docs to reflect Doc input to Language (#11555) --- website/docs/api/language.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/website/docs/api/language.md b/website/docs/api/language.md index ed763e36a..767a7450a 100644 --- a/website/docs/api/language.md +++ b/website/docs/api/language.md @@ -164,6 +164,9 @@ examples, see the Apply the pipeline to some text. The text can span multiple sentences, and can contain arbitrary whitespace. Alignment into the original string is preserved. +Instead of text, a `Doc` can be passed as input, in which case tokenization is +skipped, but the rest of the pipeline is run. + > #### Example > > ```python @@ -173,7 +176,7 @@ contain arbitrary whitespace. Alignment into the original string is preserved. | Name | Description | | --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `text` | The text to be processed. ~~str~~ | +| `text` | The text to be processed, or a Doc. ~~Union[str, Doc]~~ | | _keyword-only_ | | | `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). ~~List[str]~~ | | `component_cfg` | Optional dictionary of keyword arguments for components, keyed by component names. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Any]]]~~ | @@ -184,6 +187,9 @@ contain arbitrary whitespace. Alignment into the original string is preserved. Process texts as a stream, and yield `Doc` objects in order. This is usually more efficient than processing texts one-by-one. +Instead of text, a `Doc` object can be passed as input. In this case +tokenization is skipped but the rest of the pipeline is run. + > #### Example > > ```python @@ -194,7 +200,7 @@ more efficient than processing texts one-by-one. | Name | Description | | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `texts` | A sequence of strings. ~~Iterable[str]~~ | +| `texts` | A sequence of strings (or `Doc` objects). ~~Iterable[Union[str, Doc]]~~ | | _keyword-only_ | | | `as_tuples` | If set to `True`, inputs should be a sequence of `(text, context)` tuples. Output will then be a sequence of `(doc, context)` tuples. Defaults to `False`. ~~bool~~ | | `batch_size` | The number of texts to buffer. ~~Optional[int]~~ | From bcda8bc1e720e999243d23ce620181fcad7e8e46 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Thu, 29 Sep 2022 14:24:40 +0200 Subject: [PATCH 042/179] update mypy to latest version (#11546) * update mypy and disable it for python 3.6 * ignoring mypy's type redefinition error --- .github/azure-steps.yml | 2 +- requirements.txt | 2 +- spacy/pipeline/entityruler.py | 5 ++--- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml index c7722391f..9d57219ca 100644 --- a/.github/azure-steps.yml +++ b/.github/azure-steps.yml @@ -27,7 +27,7 @@ steps: - script: python -m mypy spacy displayName: 'Run mypy' - condition: ne(variables['python_version'], '3.10') + condition: ne(variables['python_version'], '3.6') - task: DeleteFiles@1 inputs: diff --git a/requirements.txt b/requirements.txt index e45fde787..446560c06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ pytest-timeout>=1.3.0,<2.0.0 mock>=2.0.0,<3.0.0 flake8>=3.8.0,<3.10.0 hypothesis>=3.27.0,<7.0.0 -mypy>=0.910,<0.970; platform_machine!='aarch64' +mypy>=0.980,<0.990; platform_machine != "aarch64" and python_version >= "3.7" types-dataclasses>=0.1.3; python_version < "3.7" types-mock>=0.1.1 types-setuptools>=57.0.0 diff --git a/spacy/pipeline/entityruler.py b/spacy/pipeline/entityruler.py index 3cb1ca676..8154a077d 100644 --- a/spacy/pipeline/entityruler.py +++ b/spacy/pipeline/entityruler.py @@ -1,6 +1,5 @@ -import warnings from typing import Optional, Union, List, Dict, Tuple, Iterable, Any, Callable, Sequence -from typing import cast +import warnings from collections import defaultdict from pathlib import Path import srsly @@ -317,7 +316,7 @@ class EntityRuler(Pipe): phrase_pattern["id"] = ent_id phrase_patterns.append(phrase_pattern) for entry in token_patterns + phrase_patterns: # type: ignore[operator] - label = entry["label"] + label = entry["label"] # type: ignore if "id" in entry: ent_label = label label = self._create_label(label, entry["id"]) From ff9002b726cfdae083a9a0206e1ef615f19a6088 Mon Sep 17 00:00:00 2001 From: Gabriele Picco Date: Thu, 29 Sep 2022 16:34:44 +0100 Subject: [PATCH 043/179] Add Zshot Spacy plugin (#11557) * Add Zshot Spacy plugin Add Zshot (Zero and Few shot named entity & relationships recognition) Spacy plugin * Update website/meta/universe.json Co-authored-by: Adriane Boyd * Update website/meta/universe.json Co-authored-by: Adriane Boyd Co-authored-by: Adriane Boyd --- website/meta/universe.json | 57 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/website/meta/universe.json b/website/meta/universe.json index 9ec0d6c0e..a6a1a0fc7 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -1,5 +1,62 @@ { "resources": [ + { + "id": "Zshot", + "title": "Zshot", + "slogan": "Zero and Few shot named entity & relationships recognition", + "github": "ibm/zshot", + "pip": "zshot", + "code_example": [ + "import spacy", + "from zshot import PipelineConfig, displacy", + "from zshot.linker import LinkerRegen", + "from zshot.mentions_extractor import MentionsExtractorSpacy", + "from zshot.utils.data_models import Entity", + "", + "nlp = spacy.load('en_core_web_sm')", + "# zero shot definition of entities", + "nlp_config = PipelineConfig(", + " mentions_extractor=MentionsExtractorSpacy(),", + " linker=LinkerRegen(),", + " entities=[", + " Entity(name='Paris',", + " description='Paris is located in northern central France, in a north-bending arc of the river Seine'),", + " Entity(name='IBM',", + " description='International Business Machines Corporation (IBM) is an American multinational technology corporation headquartered in Armonk, New York'),", + " Entity(name='New York', description='New York is a city in U.S. state'),", + " Entity(name='Florida', description='southeasternmost U.S. state'),", + " Entity(name='American',", + " description='American, something of, from, or related to the United States of America, commonly known as the United States or America'),", + " Entity(name='Chemical formula',", + " description='In chemistry, a chemical formula is a way of presenting information about the chemical proportions of atoms that constitute a particular chemical compound or molecul'),", + " Entity(name='Acetamide',", + " description='Acetamide (systematic name: ethanamide) is an organic compound with the formula CH3CONH2. It is the simplest amide derived from acetic acid. It finds some use as a plasticizer and as an industrial solvent.'),", + " Entity(name='Armonk',", + " description='Armonk is a hamlet and census-designated place (CDP) in the town of North Castle, located in Westchester County, New York, United States.'),", + " Entity(name='Acetic Acid',", + " description='Acetic acid, systematically named ethanoic acid, is an acidic, colourless liquid and organic compound with the chemical formula CH3COOH'),", + " Entity(name='Industrial solvent',", + " description='Acetamide (systematic name: ethanamide) is an organic compound with the formula CH3CONH2. It is the simplest amide derived from acetic acid. It finds some use as a plasticizer and as an industrial solvent.'),", + " ]", + ")", + "nlp.add_pipe('zshot', config=nlp_config, last=True)", + "", + "text = 'International Business Machines Corporation (IBM) is an American multinational technology corporation' \\", + " ' headquartered in Armonk, New York, with operations in over 171 countries.'", + "", + "doc = nlp(text)", + "displacy.serve(doc, style='ent')" + ], + "thumb": "https://ibm.github.io/zshot/img/graph.png", + "url": "https://ibm.github.io/zshot/", + "author": "IBM Research", + "author_links": { + "github": "ibm", + "twitter": "IBMResearch", + "website": "https://research.ibm.com/labs/ireland/" + }, + "category": ["scientific", "models", "research"] + }, { "id": "concepcy", "title": "concepCy", From 087cc74c6abdd43e04e4313cdcf292edf6187f4b Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Mon, 3 Oct 2022 18:53:21 +0900 Subject: [PATCH 044/179] Remove mention of 1.7 from issue template (#11570) It's rare to have anyone using v1 anymore, so this message is no longer helpful. --- .github/ISSUE_TEMPLATE/01_bugs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/01_bugs.md b/.github/ISSUE_TEMPLATE/01_bugs.md index 255a5241e..f0d0ba912 100644 --- a/.github/ISSUE_TEMPLATE/01_bugs.md +++ b/.github/ISSUE_TEMPLATE/01_bugs.md @@ -10,7 +10,7 @@ about: Use this template if you came across a bug or unexpected behaviour differ ## Your Environment - + * Operating System: * Python Version Used: * spaCy Version Used: From 70e21dfcad28b044903ba33b2b8831d925151b76 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Mon, 3 Oct 2022 13:04:03 +0200 Subject: [PATCH 045/179] PR to test importlib-metadata (#11569) * empty commit * restrict importlib-metadata to lower than 5.0.0 * restrict importlib-metadata also for validate CI step * set fixed version for CI * try flake8 5.0.4 in CI validation step * from importlib-metadata from requirements again --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f475b7fdd..2f5201614 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -31,7 +31,7 @@ jobs: inputs: versionSpec: "3.7" - script: | - pip install flake8==3.9.2 + pip install flake8==5.0.4 python -m flake8 spacy --count --select=E901,E999,F821,F822,F823,W605 --show-source --statistics displayName: "flake8" From b187076a2dd0f034c1a8918c9b332711688b5dc2 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Mon, 3 Oct 2022 17:01:04 +0200 Subject: [PATCH 046/179] fix docs (#11573) --- website/docs/api/kb_in_memory.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/api/kb_in_memory.md b/website/docs/api/kb_in_memory.md index c9ce624f0..9e3279e6a 100644 --- a/website/docs/api/kb_in_memory.md +++ b/website/docs/api/kb_in_memory.md @@ -21,9 +21,9 @@ Create the knowledge base. > #### Example > > ```python -> from spacy.kb import KnowledgeBase +> from spacy.kb import InMemoryLookupKB > vocab = nlp.vocab -> kb = KnowledgeBase(vocab=vocab, entity_vector_length=64) +> kb = InMemoryLookupKB(vocab=vocab, entity_vector_length=64) > ``` | Name | Description | From 8cd77dd54cfc89c2f67ca2412490ef9b49a98518 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 4 Oct 2022 11:23:04 +0200 Subject: [PATCH 047/179] Sync flake8 version across requirements (#11580) --- .pre-commit-config.yaml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b959262e3..df59697b1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ repos: language_version: python3.7 additional_dependencies: ['click==8.0.4'] - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 + rev: 5.0.4 hooks: - id: flake8 args: diff --git a/requirements.txt b/requirements.txt index 446560c06..14847ff21 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ cython>=0.25,<3.0 pytest>=5.2.0,!=7.1.0 pytest-timeout>=1.3.0,<2.0.0 mock>=2.0.0,<3.0.0 -flake8>=3.8.0,<3.10.0 +flake8>=3.8.0,<6.0.0 hypothesis>=3.27.0,<7.0.0 mypy>=0.980,<0.990; platform_machine != "aarch64" and python_version >= "3.7" types-dataclasses>=0.1.3; python_version < "3.7" From ef74f8f5e447dec10ab69d2a7e94f0e09165db75 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Tue, 11 Oct 2022 14:15:22 +0200 Subject: [PATCH 048/179] Fix mypy error in edittree lemmatizer (#11612) * cleanup imports * try limiting Thinc to previous release * remove Model specification * fix code and revert Thinc constraint --- spacy/pipeline/edit_tree_lemmatizer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spacy/pipeline/edit_tree_lemmatizer.py b/spacy/pipeline/edit_tree_lemmatizer.py index b7d615f6d..7f6367c75 100644 --- a/spacy/pipeline/edit_tree_lemmatizer.py +++ b/spacy/pipeline/edit_tree_lemmatizer.py @@ -1,7 +1,6 @@ from typing import cast, Any, Callable, Dict, Iterable, List, Optional -from typing import Sequence, Tuple, Union +from typing import Tuple from collections import Counter -from copy import deepcopy from itertools import islice import numpy as np @@ -150,7 +149,7 @@ class EditTreeLemmatizer(TrainablePipe): # Handle cases where there are no tokens in any docs. n_labels = len(self.cfg["labels"]) guesses: List[Ints2d] = [ - self.model.ops.alloc((0, n_labels), dtype="i") for doc in docs + self.model.ops.alloc2i(0, n_labels, dtype="i") for _ in docs ] assert len(guesses) == n_docs return guesses From 29649589fc889a58c8b631d569d4ae378a10aa2b Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Tue, 11 Oct 2022 15:25:05 +0200 Subject: [PATCH 049/179] remove dtype (#11615) --- spacy/pipeline/edit_tree_lemmatizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/pipeline/edit_tree_lemmatizer.py b/spacy/pipeline/edit_tree_lemmatizer.py index 7f6367c75..76b0e0bc9 100644 --- a/spacy/pipeline/edit_tree_lemmatizer.py +++ b/spacy/pipeline/edit_tree_lemmatizer.py @@ -149,7 +149,7 @@ class EditTreeLemmatizer(TrainablePipe): # Handle cases where there are no tokens in any docs. n_labels = len(self.cfg["labels"]) guesses: List[Ints2d] = [ - self.model.ops.alloc2i(0, n_labels, dtype="i") for _ in docs + self.model.ops.alloc2i(0, n_labels) for _ in docs ] assert len(guesses) == n_docs return guesses From 2e52479eec987367117d27fb4f049df2efb2518d Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Tue, 11 Oct 2022 23:45:05 +0900 Subject: [PATCH 050/179] Fix example code for spacy-wordnet (#11593) * Fix example code for spacy-wordnet It looks like in the most recent version, 0.1.0, it's no longer possible to pass the lang parameter to the component separately. Doing so will raise an error. * Apply suggestions from code review Co-authored-by: Sofie Van Landeghem * Cleanup * More cleanup Co-authored-by: Sofie Van Landeghem --- website/meta/universe.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/website/meta/universe.json b/website/meta/universe.json index a6a1a0fc7..637e9d6ce 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -2460,20 +2460,20 @@ "import spacy", "from spacy_wordnet.wordnet_annotator import WordnetAnnotator ", "", - "# Load an spacy model (supported models are \"es\" and \"en\") ", - "nlp = spacy.load('en')", - "# Spacy 3.x", - "nlp.add_pipe(\"spacy_wordnet\", after='tagger', config={'lang': nlp.lang})", - "# Spacy 2.x", + "# Load a spaCy model (supported languages are \"es\" and \"en\") ", + "nlp = spacy.load('en_core_web_sm')", + "# spaCy 3.x", + "nlp.add_pipe(\"spacy_wordnet\", after='tagger')", + "# spaCy 2.x", "# nlp.add_pipe(WordnetAnnotator(nlp.lang), after='tagger')", "token = nlp('prices')[0]", "", - "# wordnet object link spacy token with nltk wordnet interface by giving acces to", + "# WordNet object links spaCy token with NLTK WordNet interface by giving access to", "# synsets and lemmas ", "token._.wordnet.synsets()", "token._.wordnet.lemmas()", "", - "# And automatically tags with wordnet domains", + "# And automatically add info about WordNet domains", "token._.wordnet.wordnet_domains()" ], "author": "recognai", From fe06e037bcd733708401bce082863994b1fc48bd Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 12 Oct 2022 12:18:39 +0200 Subject: [PATCH 051/179] Fix init for pymorphy2_lookup lemmatizer mode (#11631) --- spacy/lang/ru/lemmatizer.py | 2 +- spacy/lang/uk/lemmatizer.py | 2 +- spacy/tests/conftest.py | 17 +++++++++++++++++ spacy/tests/lang/ru/test_lemmatizer.py | 14 ++++++++++++++ spacy/tests/lang/uk/test_lemmatizer.py | 8 ++++++++ 5 files changed, 41 insertions(+), 2 deletions(-) diff --git a/spacy/lang/ru/lemmatizer.py b/spacy/lang/ru/lemmatizer.py index 85180b1e4..5bf685d44 100644 --- a/spacy/lang/ru/lemmatizer.py +++ b/spacy/lang/ru/lemmatizer.py @@ -23,7 +23,7 @@ class RussianLemmatizer(Lemmatizer): overwrite: bool = False, scorer: Optional[Callable] = lemmatizer_score, ) -> None: - if mode == "pymorphy2": + if mode in {"pymorphy2", "pymorphy2_lookup"}: try: from pymorphy2 import MorphAnalyzer except ImportError: diff --git a/spacy/lang/uk/lemmatizer.py b/spacy/lang/uk/lemmatizer.py index a8bc56057..d4f8cc9e5 100644 --- a/spacy/lang/uk/lemmatizer.py +++ b/spacy/lang/uk/lemmatizer.py @@ -18,7 +18,7 @@ class UkrainianLemmatizer(RussianLemmatizer): overwrite: bool = False, scorer: Optional[Callable] = lemmatizer_score, ) -> None: - if mode == "pymorphy2": + if mode in {"pymorphy2", "pymorphy2_lookup"}: try: from pymorphy2 import MorphAnalyzer except ImportError: diff --git a/spacy/tests/conftest.py b/spacy/tests/conftest.py index 742bfcc6a..394ef00d3 100644 --- a/spacy/tests/conftest.py +++ b/spacy/tests/conftest.py @@ -343,6 +343,14 @@ def ru_lemmatizer(): return get_lang_class("ru")().add_pipe("lemmatizer") +@pytest.fixture +def ru_lookup_lemmatizer(): + pytest.importorskip("pymorphy2") + return get_lang_class("ru")().add_pipe( + "lemmatizer", config={"mode": "pymorphy2_lookup"} + ) + + @pytest.fixture(scope="session") def sa_tokenizer(): return get_lang_class("sa")().tokenizer @@ -422,6 +430,15 @@ def uk_lemmatizer(): return get_lang_class("uk")().add_pipe("lemmatizer") +@pytest.fixture +def uk_lookup_lemmatizer(): + pytest.importorskip("pymorphy2") + pytest.importorskip("pymorphy2_dicts_uk") + return get_lang_class("uk")().add_pipe( + "lemmatizer", config={"mode": "pymorphy2_lookup"} + ) + + @pytest.fixture(scope="session") def ur_tokenizer(): return get_lang_class("ur")().tokenizer diff --git a/spacy/tests/lang/ru/test_lemmatizer.py b/spacy/tests/lang/ru/test_lemmatizer.py index 9ca7f441b..e82fd4f8c 100644 --- a/spacy/tests/lang/ru/test_lemmatizer.py +++ b/spacy/tests/lang/ru/test_lemmatizer.py @@ -78,3 +78,17 @@ def test_ru_lemmatizer_punct(ru_lemmatizer): assert ru_lemmatizer.pymorphy2_lemmatize(doc[0]) == ['"'] doc = Doc(ru_lemmatizer.vocab, words=["»"], pos=["PUNCT"]) assert ru_lemmatizer.pymorphy2_lemmatize(doc[0]) == ['"'] + + +def test_ru_doc_lookup_lemmatization(ru_lookup_lemmatizer): + words = ["мама", "мыла", "раму"] + pos = ["NOUN", "VERB", "NOUN"] + morphs = [ + "Animacy=Anim|Case=Nom|Gender=Fem|Number=Sing", + "Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act", + "Animacy=Anim|Case=Acc|Gender=Fem|Number=Sing", + ] + doc = Doc(ru_lookup_lemmatizer.vocab, words=words, pos=pos, morphs=morphs) + doc = ru_lookup_lemmatizer(doc) + lemmas = [token.lemma_ for token in doc] + assert lemmas == ["мама", "мыла", "раму"] diff --git a/spacy/tests/lang/uk/test_lemmatizer.py b/spacy/tests/lang/uk/test_lemmatizer.py index 57dd4198a..788744aa1 100644 --- a/spacy/tests/lang/uk/test_lemmatizer.py +++ b/spacy/tests/lang/uk/test_lemmatizer.py @@ -9,3 +9,11 @@ def test_uk_lemmatizer(uk_lemmatizer): """Check that the default uk lemmatizer runs.""" doc = Doc(uk_lemmatizer.vocab, words=["a", "b", "c"]) uk_lemmatizer(doc) + assert [token.lemma for token in doc] + + +def test_uk_lookup_lemmatizer(uk_lookup_lemmatizer): + """Check that the lookup uk lemmatizer runs.""" + doc = Doc(uk_lookup_lemmatizer.vocab, words=["a", "b", "c"]) + uk_lookup_lemmatizer(doc) + assert [token.lemma for token in doc] From 4d869fcc111151bcefa08ee1a2b7b49dc5ecd677 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Wed, 12 Oct 2022 15:17:40 +0200 Subject: [PATCH 052/179] Small fixes to docstrings (#11610) * add missing scorer arg to docstring * fix class names in textcat_multilabel * add missing scorer to docstrings --- spacy/pipeline/spancat.py | 3 +++ spacy/pipeline/textcat_multilabel.py | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/spacy/pipeline/spancat.py b/spacy/pipeline/spancat.py index 1b7a9eecb..ca9f1dab0 100644 --- a/spacy/pipeline/spancat.py +++ b/spacy/pipeline/spancat.py @@ -133,6 +133,9 @@ def make_spancat( spans_key (str): Key of the doc.spans dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. + scorer (Optional[Callable]): The scoring method. Defaults to + Scorer.score_spans for the Doc.spans[spans_key] with overlapping + spans allowed. threshold (float): Minimum probability to consider a prediction positive. Spans with a positive prediction will be saved on the Doc. Defaults to 0.5. diff --git a/spacy/pipeline/textcat_multilabel.py b/spacy/pipeline/textcat_multilabel.py index e33a885f8..119ae3310 100644 --- a/spacy/pipeline/textcat_multilabel.py +++ b/spacy/pipeline/textcat_multilabel.py @@ -96,8 +96,8 @@ def make_multilabel_textcat( model: Model[List[Doc], List[Floats2d]], threshold: float, scorer: Optional[Callable], -) -> "TextCategorizer": - """Create a TextCategorizer component. The text categorizer predicts categories +) -> "MultiLabel_TextCategorizer": + """Create a MultiLabel_TextCategorizer component. The text categorizer predicts categories over a whole document. It can learn one or more labels, and the labels are considered to be non-mutually exclusive, which means that there can be zero or more labels per doc). @@ -105,6 +105,7 @@ def make_multilabel_textcat( model (Model[List[Doc], List[Floats2d]]): A model instance that predicts scores for each category. threshold (float): Cutoff to consider a prediction "positive". + scorer (Optional[Callable]): The scoring method. """ return MultiLabel_TextCategorizer( nlp.vocab, model, name, threshold=threshold, scorer=scorer @@ -147,6 +148,7 @@ class MultiLabel_TextCategorizer(TextCategorizer): name (str): The component instance name, used to add entries to the losses during training. threshold (float): Cutoff to consider a prediction "positive". + scorer (Optional[Callable]): The scoring method. DOCS: https://spacy.io/api/textcategorizer#init """ From 6b5a3e72198aa9735587b0712e3eb2c24234b463 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 14 Oct 2022 08:16:49 +0200 Subject: [PATCH 053/179] Extend to pydantic v1.10 (#11635) * Update types in `spacy.schemas` for updated pydantic+mypy --- requirements.txt | 2 +- setup.cfg | 2 +- spacy/schemas.py | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/requirements.txt b/requirements.txt index 14847ff21..9d6bbb2c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ pathy>=0.3.5 numpy>=1.15.0 requests>=2.13.0,<3.0.0 tqdm>=4.38.0,<5.0.0 -pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0 +pydantic>=1.7.4,!=1.8,!=1.8.1,<1.11.0 jinja2 langcodes>=3.2.0,<4.0.0 # Official Python utilities diff --git a/setup.cfg b/setup.cfg index 2dc5e7042..c2653feba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,7 +56,7 @@ install_requires = tqdm>=4.38.0,<5.0.0 numpy>=1.15.0 requests>=2.13.0,<3.0.0 - pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0 + pydantic>=1.7.4,!=1.8,!=1.8.1,<1.11.0 jinja2 # Official Python utilities setuptools diff --git a/spacy/schemas.py b/spacy/schemas.py index 048082134..ab71b2016 100644 --- a/spacy/schemas.py +++ b/spacy/schemas.py @@ -181,12 +181,12 @@ class TokenPatternNumber(BaseModel): IS_SUBSET: Optional[List[StrictInt]] = Field(None, alias="is_subset") IS_SUPERSET: Optional[List[StrictInt]] = Field(None, alias="is_superset") INTERSECTS: Optional[List[StrictInt]] = Field(None, alias="intersects") - EQ: Union[StrictInt, StrictFloat] = Field(None, alias="==") - NEQ: Union[StrictInt, StrictFloat] = Field(None, alias="!=") - GEQ: Union[StrictInt, StrictFloat] = Field(None, alias=">=") - LEQ: Union[StrictInt, StrictFloat] = Field(None, alias="<=") - GT: Union[StrictInt, StrictFloat] = Field(None, alias=">") - LT: Union[StrictInt, StrictFloat] = Field(None, alias="<") + EQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="==") + NEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="!=") + GEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias=">=") + LEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="<=") + GT: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias=">") + LT: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="<") class Config: extra = "forbid" @@ -430,7 +430,7 @@ class ProjectConfigAssetURL(BaseModel): # fmt: off dest: StrictStr = Field(..., title="Destination of downloaded asset") url: Optional[StrictStr] = Field(None, title="URL of asset") - checksum: str = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})") + checksum: Optional[str] = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})") description: StrictStr = Field("", title="Description of asset") # fmt: on @@ -438,7 +438,7 @@ class ProjectConfigAssetURL(BaseModel): class ProjectConfigAssetGit(BaseModel): # fmt: off git: ProjectConfigAssetGitItem = Field(..., title="Git repo information") - checksum: str = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})") + checksum: Optional[str] = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})") description: Optional[StrictStr] = Field(None, title="Description of asset") # fmt: on @@ -508,7 +508,7 @@ class DocJSONSchema(BaseModel): None, title="Indices of sentences' start and end indices" ) text: StrictStr = Field(..., title="Document text") - spans: Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]] = Field( + spans: Optional[Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]]] = Field( None, title="Span information - end/start indices, label, KB ID" ) tokens: List[Dict[StrictStr, Union[StrictStr, StrictInt]]] = Field( From ceb62352bfcad49b3ad63e3e65ef12dabab645b3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 14 Oct 2022 18:04:55 +0900 Subject: [PATCH 054/179] Auto-format code with black (#11649) Co-authored-by: explosion-bot --- spacy/pipeline/edit_tree_lemmatizer.py | 4 +--- spacy/schemas.py | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/spacy/pipeline/edit_tree_lemmatizer.py b/spacy/pipeline/edit_tree_lemmatizer.py index 76b0e0bc9..12f9b73a3 100644 --- a/spacy/pipeline/edit_tree_lemmatizer.py +++ b/spacy/pipeline/edit_tree_lemmatizer.py @@ -148,9 +148,7 @@ class EditTreeLemmatizer(TrainablePipe): if not any(len(doc) for doc in docs): # Handle cases where there are no tokens in any docs. n_labels = len(self.cfg["labels"]) - guesses: List[Ints2d] = [ - self.model.ops.alloc2i(0, n_labels) for _ in docs - ] + guesses: List[Ints2d] = [self.model.ops.alloc2i(0, n_labels) for _ in docs] assert len(guesses) == n_docs return guesses scores = self.model.predict(docs) diff --git a/spacy/schemas.py b/spacy/schemas.py index ab71b2016..a67d96d9d 100644 --- a/spacy/schemas.py +++ b/spacy/schemas.py @@ -508,9 +508,9 @@ class DocJSONSchema(BaseModel): None, title="Indices of sentences' start and end indices" ) text: StrictStr = Field(..., title="Document text") - spans: Optional[Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]]] = Field( - None, title="Span information - end/start indices, label, KB ID" - ) + spans: Optional[ + Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]] + ] = Field(None, title="Span information - end/start indices, label, KB ID") tokens: List[Dict[StrictStr, Union[StrictStr, StrictInt]]] = Field( ..., title="Token information - ID, start, annotations" ) From 2ce6aadda2d455cf2f2a1aef494b2bafe3e07119 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Mon, 17 Oct 2022 12:10:03 +0200 Subject: [PATCH 055/179] update default configs to recent versions (#11618) --- spacy/pipeline/spancat.py | 6 +++--- spacy/pipeline/textcat_multilabel.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/spacy/pipeline/spancat.py b/spacy/pipeline/spancat.py index 1b7a9eecb..5ede622c2 100644 --- a/spacy/pipeline/spancat.py +++ b/spacy/pipeline/spancat.py @@ -26,17 +26,17 @@ scorer = {"@layers": "spacy.LinearLogistic.v1"} hidden_size = 128 [model.tok2vec] -@architectures = "spacy.Tok2Vec.v1" +@architectures = "spacy.Tok2Vec.v2" [model.tok2vec.embed] -@architectures = "spacy.MultiHashEmbed.v1" +@architectures = "spacy.MultiHashEmbed.v2" width = 96 rows = [5000, 2000, 1000, 1000] attrs = ["ORTH", "PREFIX", "SUFFIX", "SHAPE"] include_static_vectors = false [model.tok2vec.encode] -@architectures = "spacy.MaxoutWindowEncoder.v1" +@architectures = "spacy.MaxoutWindowEncoder.v2" width = ${model.tok2vec.embed.width} window_size = 1 maxout_pieces = 3 diff --git a/spacy/pipeline/textcat_multilabel.py b/spacy/pipeline/textcat_multilabel.py index e33a885f8..10aef46aa 100644 --- a/spacy/pipeline/textcat_multilabel.py +++ b/spacy/pipeline/textcat_multilabel.py @@ -19,7 +19,7 @@ multi_label_default_config = """ @architectures = "spacy.TextCatEnsemble.v2" [model.tok2vec] -@architectures = "spacy.Tok2Vec.v1" +@architectures = "spacy.Tok2Vec.v2" [model.tok2vec.embed] @architectures = "spacy.MultiHashEmbed.v2" @@ -29,7 +29,7 @@ attrs = ["ORTH", "LOWER", "PREFIX", "SUFFIX", "SHAPE", "ID"] include_static_vectors = false [model.tok2vec.encode] -@architectures = "spacy.MaxoutWindowEncoder.v1" +@architectures = "spacy.MaxoutWindowEncoder.v2" width = ${model.tok2vec.embed.width} window_size = 1 maxout_pieces = 3 From 858565a5671de61334443d6a2348164bc39216e1 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Tue, 18 Oct 2022 15:11:39 +0900 Subject: [PATCH 056/179] Fix issues with DVC commands (#11592) * Fix flag handling in dvc Prior to this commit, if a flag (--verbose or --quiet) was passed to DVC, it would be added to the end of the generated dvc command line. This would result in the command being interpreted as part of the actual command to run, rather than an argument to dvc. This would result in command lines like: spacy project run preprocess --verbose That would fail with an error that there's no such directory as `--verbose`. This change puts the flags at the front of the dvc command so that they are interpreted correctly. It removes the `run_dvc_commands` function, which had been reduced to just a for loop and wasn't used elsewhere. A separate problem is that there's no way to specify the quiet behaviour to dvc from the command line, though it's unclear if that's a bug. * Add dvc quiet flag to docs * Handle case in DVC where no commands are appropriate If only have commands with no deps or outputs (admittedly unlikely), you get a weird error about the dvc file not existing. This gives explicit output instead. * Add support for quiet flag * Fix command execution Commands are strings now because they're joined further up. --- spacy/cli/project/dvc.py | 57 +++++++++++++++++++++------------------- website/docs/api/cli.md | 3 ++- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/spacy/cli/project/dvc.py b/spacy/cli/project/dvc.py index 83dc5efbf..a15353855 100644 --- a/spacy/cli/project/dvc.py +++ b/spacy/cli/project/dvc.py @@ -25,6 +25,7 @@ def project_update_dvc_cli( project_dir: Path = Arg(Path.cwd(), help="Location of project directory. Defaults to current working directory.", exists=True, file_okay=False), workflow: Optional[str] = Arg(None, help=f"Name of workflow defined in {PROJECT_FILE}. Defaults to first workflow if not set."), verbose: bool = Opt(False, "--verbose", "-V", help="Print more info"), + quiet: bool = Opt(False, "--quiet", "-q", help="Print less info"), force: bool = Opt(False, "--force", "-F", help="Force update DVC config"), # fmt: on ): @@ -36,7 +37,7 @@ def project_update_dvc_cli( DOCS: https://spacy.io/api/cli#project-dvc """ - project_update_dvc(project_dir, workflow, verbose=verbose, force=force) + project_update_dvc(project_dir, workflow, verbose=verbose, quiet=quiet, force=force) def project_update_dvc( @@ -44,6 +45,7 @@ def project_update_dvc( workflow: Optional[str] = None, *, verbose: bool = False, + quiet: bool = False, force: bool = False, ) -> None: """Update the auto-generated Data Version Control (DVC) config file. A DVC @@ -54,11 +56,12 @@ def project_update_dvc( workflow (Optional[str]): Optional name of workflow defined in project.yml. If not set, the first workflow will be used. verbose (bool): Print more info. + quiet (bool): Print less info. force (bool): Force update DVC config. """ config = load_project_config(project_dir) updated = update_dvc_config( - project_dir, config, workflow, verbose=verbose, force=force + project_dir, config, workflow, verbose=verbose, quiet=quiet, force=force ) help_msg = "To execute the workflow with DVC, run: dvc repro" if updated: @@ -72,7 +75,7 @@ def update_dvc_config( config: Dict[str, Any], workflow: Optional[str] = None, verbose: bool = False, - silent: bool = False, + quiet: bool = False, force: bool = False, ) -> bool: """Re-run the DVC commands in dry mode and update dvc.yaml file in the @@ -83,7 +86,7 @@ def update_dvc_config( path (Path): The path to the project directory. config (Dict[str, Any]): The loaded project.yml. verbose (bool): Whether to print additional info (via DVC). - silent (bool): Don't output anything (via DVC). + quiet (bool): Don't output anything (via DVC). force (bool): Force update, even if hashes match. RETURNS (bool): Whether the DVC config file was updated. """ @@ -105,6 +108,14 @@ def update_dvc_config( dvc_config_path.unlink() dvc_commands = [] config_commands = {cmd["name"]: cmd for cmd in config.get("commands", [])} + + # some flags that apply to every command + flags = [] + if verbose: + flags.append("--verbose") + if quiet: + flags.append("--quiet") + for name in workflows[workflow]: command = config_commands[name] deps = command.get("deps", []) @@ -118,14 +129,26 @@ def update_dvc_config( deps_cmd = [c for cl in [["-d", p] for p in deps] for c in cl] outputs_cmd = [c for cl in [["-o", p] for p in outputs] for c in cl] outputs_nc_cmd = [c for cl in [["-O", p] for p in outputs_no_cache] for c in cl] - dvc_cmd = ["run", "-n", name, "-w", str(path), "--no-exec"] + + dvc_cmd = ["run", *flags, "-n", name, "-w", str(path), "--no-exec"] if command.get("no_skip"): dvc_cmd.append("--always-changed") full_cmd = [*dvc_cmd, *deps_cmd, *outputs_cmd, *outputs_nc_cmd, *project_cmd] dvc_commands.append(join_command(full_cmd)) + + if not dvc_commands: + # If we don't check for this, then there will be an error when reading the + # config, since DVC wouldn't create it. + msg.fail( + "No usable commands for DVC found. This can happen if none of your " + "commands have dependencies or outputs.", + exits=1, + ) + with working_dir(path): - dvc_flags = {"--verbose": verbose, "--quiet": silent} - run_dvc_commands(dvc_commands, flags=dvc_flags) + for c in dvc_commands: + dvc_command = "dvc " + c + run_command(dvc_command) with dvc_config_path.open("r+", encoding="utf8") as f: content = f.read() f.seek(0, 0) @@ -133,26 +156,6 @@ def update_dvc_config( return True -def run_dvc_commands( - commands: Iterable[str] = SimpleFrozenList(), flags: Dict[str, bool] = {} -) -> None: - """Run a sequence of DVC commands in a subprocess, in order. - - commands (List[str]): The string commands without the leading "dvc". - flags (Dict[str, bool]): Conditional flags to be added to command. Makes it - easier to pass flags like --quiet that depend on a variable or - command-line setting while avoiding lots of nested conditionals. - """ - for c in commands: - command = split_command(c) - dvc_command = ["dvc", *command] - # Add the flags if they are set to True - for flag, is_active in flags.items(): - if is_active: - dvc_command.append(flag) - run_command(dvc_command) - - def check_workflows(workflows: List[str], workflow: Optional[str] = None) -> None: """Validate workflows provided in project.yml and check that a given workflow can be used to generate a DVC config. diff --git a/website/docs/api/cli.md b/website/docs/api/cli.md index e5cd3089b..fc2c46022 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.md @@ -1482,7 +1482,7 @@ You'll also need to add the assets you want to track with ```cli -$ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] +$ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] [--quiet] ``` > #### Example @@ -1499,6 +1499,7 @@ $ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] | `workflow` | Name of workflow defined in `project.yml`. Defaults to first workflow if not set. ~~Optional[str] \(option)~~ | | `--force`, `-F` | Force-updating config file. ~~bool (flag)~~ | | `--verbose`, `-V` | Print more output generated by DVC. ~~bool (flag)~~ | +| `--quiet`, `-q` | Print no output generated by DVC. ~~bool (flag)~~ | | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | A `dvc.yaml` file in the project directory, based on the steps defined in the given workflow. | From a1eacaa8db055322d4a066a08b730243a2f5b969 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 18 Oct 2022 14:36:06 +0200 Subject: [PATCH 057/179] Add python 3.11.0rc2 to CI (#11667) --- .github/azure-steps.yml | 1 + azure-pipelines.yml | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml index 9d57219ca..cc0247b3a 100644 --- a/.github/azure-steps.yml +++ b/.github/azure-steps.yml @@ -10,6 +10,7 @@ steps: inputs: versionSpec: ${{ parameters.python_version }} architecture: ${{ parameters.architecture }} + allowUnstable: true - bash: | echo "##vso[task.setvariable variable=python_version]${{ parameters.python_version }}" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2f5201614..357cce835 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -85,6 +85,15 @@ jobs: Python310Mac: imageName: "macos-latest" python.version: "3.10" + Python311Linux: + imageName: 'ubuntu-latest' + python.version: '3.11.0-rc.2' + Python311Windows: + imageName: 'windows-latest' + python.version: '3.11.0-rc.2' + Python311Mac: + imageName: 'macos-latest' + python.version: '3.11.0-rc.2' maxParallel: 4 pool: vmImage: $(imageName) From d66ccb8eb08cd515904045de84351546065fb3ed Mon Sep 17 00:00:00 2001 From: Edward <43848523+thomashacker@users.noreply.github.com> Date: Wed, 19 Oct 2022 15:52:47 +0200 Subject: [PATCH 058/179] Fix multiple entries per custom extension in doc json (#11551) * Fix multiple extensions and character offset * Rename token_start/end to start/end * Refactor Doc.from_json based on review * Iterate over user_data items * Only add non-empty underscore entries Co-authored-by: Adriane Boyd --- spacy/schemas.py | 4 +- spacy/tests/doc/test_json_doc_conversion.py | 25 +++++++---- spacy/tokens/doc.pyx | 48 ++++++++++----------- 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/spacy/schemas.py b/spacy/schemas.py index a67d96d9d..c824d76b9 100644 --- a/spacy/schemas.py +++ b/spacy/schemas.py @@ -519,9 +519,9 @@ class DocJSONSchema(BaseModel): title="Any custom data stored in the document's _ attribute", alias="_", ) - underscore_token: Optional[Dict[StrictStr, Dict[StrictStr, Any]]] = Field( + underscore_token: Optional[Dict[StrictStr, List[Dict[StrictStr, Any]]]] = Field( None, title="Any custom data stored in the token's _ attribute" ) - underscore_span: Optional[Dict[StrictStr, Dict[StrictStr, Any]]] = Field( + underscore_span: Optional[Dict[StrictStr, List[Dict[StrictStr, Any]]]] = Field( None, title="Any custom data stored in the span's _ attribute" ) diff --git a/spacy/tests/doc/test_json_doc_conversion.py b/spacy/tests/doc/test_json_doc_conversion.py index 0d7c061c9..19698cfb2 100644 --- a/spacy/tests/doc/test_json_doc_conversion.py +++ b/spacy/tests/doc/test_json_doc_conversion.py @@ -128,7 +128,9 @@ def test_doc_to_json_with_token_span_attributes(doc): doc._.json_test1 = "hello world" doc._.json_test2 = [1, 2, 3] doc[0:1]._.span_test = "span_attribute" + doc[0:2]._.span_test = "span_attribute_2" doc[0]._.token_test = 117 + doc[1]._.token_test = 118 doc.spans["span_group"] = [doc[0:1]] json_doc = doc.to_json( underscore=["json_test1", "json_test2", "token_test", "span_test"] @@ -139,8 +141,10 @@ def test_doc_to_json_with_token_span_attributes(doc): assert json_doc["_"]["json_test2"] == [1, 2, 3] assert "underscore_token" in json_doc assert "underscore_span" in json_doc - assert json_doc["underscore_token"]["token_test"]["value"] == 117 - assert json_doc["underscore_span"]["span_test"]["value"] == "span_attribute" + assert json_doc["underscore_token"]["token_test"][0]["value"] == 117 + assert json_doc["underscore_token"]["token_test"][1]["value"] == 118 + assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute" + assert json_doc["underscore_span"]["span_test"][1]["value"] == "span_attribute_2" assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc @@ -161,8 +165,8 @@ def test_doc_to_json_with_custom_user_data(doc): assert json_doc["_"]["json_test"] == "hello world" assert "underscore_token" in json_doc assert "underscore_span" in json_doc - assert json_doc["underscore_token"]["token_test"]["value"] == 117 - assert json_doc["underscore_span"]["span_test"]["value"] == "span_attribute" + assert json_doc["underscore_token"]["token_test"][0]["value"] == 117 + assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute" assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc @@ -181,8 +185,8 @@ def test_doc_to_json_with_token_span_same_identifier(doc): assert json_doc["_"]["my_ext"] == "hello world" assert "underscore_token" in json_doc assert "underscore_span" in json_doc - assert json_doc["underscore_token"]["my_ext"]["value"] == 117 - assert json_doc["underscore_span"]["my_ext"]["value"] == "span_attribute" + assert json_doc["underscore_token"]["my_ext"][0]["value"] == 117 + assert json_doc["underscore_span"]["my_ext"][0]["value"] == "span_attribute" assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc @@ -195,10 +199,9 @@ def test_doc_to_json_with_token_attributes_missing(doc): doc[0]._.token_test = 117 json_doc = doc.to_json(underscore=["span_test"]) - assert "underscore_token" in json_doc assert "underscore_span" in json_doc - assert json_doc["underscore_span"]["span_test"]["value"] == "span_attribute" - assert "token_test" not in json_doc["underscore_token"] + assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute" + assert "underscore_token" not in json_doc assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 @@ -283,7 +286,9 @@ def test_json_to_doc_with_token_span_attributes(doc): doc._.json_test1 = "hello world" doc._.json_test2 = [1, 2, 3] doc[0:1]._.span_test = "span_attribute" + doc[0:2]._.span_test = "span_attribute_2" doc[0]._.token_test = 117 + doc[1]._.token_test = 118 json_doc = doc.to_json( underscore=["json_test1", "json_test2", "token_test", "span_test"] @@ -295,7 +300,9 @@ def test_json_to_doc_with_token_span_attributes(doc): assert new_doc._.json_test1 == "hello world" assert new_doc._.json_test2 == [1, 2, 3] assert new_doc[0]._.token_test == 117 + assert new_doc[1]._.token_test == 118 assert new_doc[0:1]._.span_test == "span_attribute" + assert new_doc[0:2]._.span_test == "span_attribute_2" assert new_doc.user_data == doc.user_data assert new_doc.to_bytes(exclude=["user_data"]) == doc.to_bytes( exclude=["user_data"] diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index d7d2fd8e6..295f91c28 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -1608,24 +1608,20 @@ cdef class Doc: Doc.set_extension(attr) self._.set(attr, doc_json["_"][attr]) - if doc_json.get("underscore_token", {}): - for token_attr in doc_json["underscore_token"]: - token_start = doc_json["underscore_token"][token_attr]["token_start"] - value = doc_json["underscore_token"][token_attr]["value"] - - if not Token.has_extension(token_attr): - Token.set_extension(token_attr) - self[token_start]._.set(token_attr, value) + for token_attr in doc_json.get("underscore_token", {}): + if not Token.has_extension(token_attr): + Token.set_extension(token_attr) + for token_data in doc_json["underscore_token"][token_attr]: + start = token_by_char(self.c, self.length, token_data["start"]) + value = token_data["value"] + self[start]._.set(token_attr, value) - if doc_json.get("underscore_span", {}): - for span_attr in doc_json["underscore_span"]: - token_start = doc_json["underscore_span"][span_attr]["token_start"] - token_end = doc_json["underscore_span"][span_attr]["token_end"] - value = doc_json["underscore_span"][span_attr]["value"] - - if not Span.has_extension(span_attr): - Span.set_extension(span_attr) - self[token_start:token_end]._.set(span_attr, value) + for span_attr in doc_json.get("underscore_span", {}): + if not Span.has_extension(span_attr): + Span.set_extension(span_attr) + for span_data in doc_json["underscore_span"][span_attr]: + value = span_data["value"] + self.char_span(span_data["start"], span_data["end"])._.set(span_attr, value) return self def to_json(self, underscore=None): @@ -1673,30 +1669,34 @@ cdef class Doc: if underscore: user_keys = set() if self.user_data: - data["_"] = {} - data["underscore_token"] = {} - data["underscore_span"] = {} - for data_key in self.user_data: + for data_key, value in self.user_data.copy().items(): if type(data_key) == tuple and len(data_key) >= 4 and data_key[0] == "._.": attr = data_key[1] start = data_key[2] end = data_key[3] if attr in underscore: user_keys.add(attr) - value = self.user_data[data_key] if not srsly.is_json_serializable(value): raise ValueError(Errors.E107.format(attr=attr, value=repr(value))) # Check if doc attribute if start is None: + if "_" not in data: + data["_"] = {} data["_"][attr] = value # Check if token attribute elif end is None: + if "underscore_token" not in data: + data["underscore_token"] = {} if attr not in data["underscore_token"]: - data["underscore_token"][attr] = {"token_start": start, "value": value} + data["underscore_token"][attr] = [] + data["underscore_token"][attr].append({"start": start, "value": value}) # Else span attribute else: + if "underscore_span" not in data: + data["underscore_span"] = {} if attr not in data["underscore_span"]: - data["underscore_span"][attr] = {"token_start": start, "token_end": end, "value": value} + data["underscore_span"][attr] = [] + data["underscore_span"][attr].append({"start": start, "end": end, "value": value}) for attr in underscore: if attr not in user_keys: From 3d0e895363921d4acb7f89a5b708472681e6fc1b Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 19 Oct 2022 17:33:55 +0200 Subject: [PATCH 059/179] Set version to v3.4.2 (#11672) --- spacy/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/about.py b/spacy/about.py index 843c15aba..ce86e6294 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -1,6 +1,6 @@ # fmt: off __title__ = "spacy" -__version__ = "3.4.1" +__version__ = "3.4.2" __download_url__ = "https://github.com/explosion/spacy-models/releases/download" __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json" __projects__ = "https://github.com/explosion/projects" From bf83f6872a55e307da289fb901db3c16dd35e8d1 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Thu, 20 Oct 2022 20:35:03 +0900 Subject: [PATCH 060/179] Add detailed example of env dict usage (#11677) * Add detailed example of env dict usage * Mark code blocks as yaml --- website/docs/usage/projects.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/website/docs/usage/projects.md b/website/docs/usage/projects.md index 4797bbfe3..90b612358 100644 --- a/website/docs/usage/projects.md +++ b/website/docs/usage/projects.md @@ -243,6 +243,27 @@ pipelines. > python -m spacy project run test . --vars.foo bar > ``` +> #### Tip: Environment Variables +> +> Commands in a project file are not executed in a shell, so they don't have +> direct access to environment variables. But you can insert environment +> variables using the `env` dictionary to make values available for +> interpolation, just like values in `vars`. Here's an example `env` dict that +> makes `$PATH` available as `ENV_PATH`: +> +> ```yaml +> env: +> ENV_PATH: PATH +> ``` +> +> This can be used in a project command like so: +> +> ```yaml +> - name: "echo-path" +> script: +> - "echo ${env.ENV_PATH}" +> ``` + | Section | Description | | --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `title` | An optional project title used in `--help` message and [auto-generated docs](#custom-docs). | From b69d249a223fa4e633e11babc0830f3b68df57e2 Mon Sep 17 00:00:00 2001 From: Cellan Hall <60790416+Ce11an@users.noreply.github.com> Date: Thu, 20 Oct 2022 12:38:29 +0100 Subject: [PATCH 061/179] Adding `spacy-cleaner` to the spaCy universe (#11674) * added spacy-cleaner to the spaCy universe * Move data to righ section of universe.json * Cleanup - fix typo ("replacers") - spaCy doesn't need to be marked as code - lemma of "Hello" is lower case Co-authored-by: Paul O'Leary McCann --- website/meta/universe.json | 41 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/website/meta/universe.json b/website/meta/universe.json index 637e9d6ce..d7c99956b 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -1,5 +1,46 @@ { "resources": [ + { + "id": "spacy-cleaner", + "title": "spacy-cleaner", + "slogan": "Easily clean text with spaCy!", + "description": "**spacy-cleaner** utilises spaCy `Language` models to replace, remove, and \n mutate spaCy tokens. Cleaning actions available are:\n\n* Remove/replace stopwords.\n* Remove/replace punctuation.\n* Remove/replace numbers.\n* Remove/replace emails.\n* Remove/replace URLs.\n* Perform lemmatisation.\n\nSee our [docs](https://ce11an.github.io/spacy-cleaner/) for more information.", + "github": "Ce11an/spacy-cleaner", + "pip": "spacy-cleaner", + "code_example": [ + "import spacy", + "import spacy_cleaner", + "from spacy_cleaner.processing import removers, replacers, mutators", + "", + "model = spacy.load(\"en_core_web_sm\")", + "pipeline = spacy_cleaner.Pipeline(", + " model,", + " removers.remove_stopword_token,", + " replacers.replace_punctuation_token,", + " mutators.mutate_lemma_token,", + ")", + "", + "texts = [\"Hello, my name is Cellan! I love to swim!\"]", + "", + "pipeline.clean(texts)", + "# ['hello _IS_PUNCT_ Cellan _IS_PUNCT_ love swim _IS_PUNCT_']" + ], + "code_language": "python", + "url": "https://ce11an.github.io/spacy-cleaner/", + "image": "https://raw.githubusercontent.com/Ce11an/spacy-cleaner/main/docs/assets/images/spacemen.png", + "author": "Cellan Hall", + "author_links": { + "twitter": "Ce11an", + "github": "Ce11an", + "website": "https://www.linkedin.com/in/cellan-hall/" + }, + "category": [ + "extension" + ], + "tags": [ + "text-processing" + ] + }, { "id": "Zshot", "title": "Zshot", From 84d9cb6b387572293c8bcf26b0e71b508104b165 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 21 Oct 2022 11:54:17 +0200 Subject: [PATCH 062/179] Auto-format code with black (#11687) Co-authored-by: explosion-bot --- spacy/tests/pipeline/test_tok2vec.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/spacy/tests/pipeline/test_tok2vec.py b/spacy/tests/pipeline/test_tok2vec.py index 659274db9..e423d9a19 100644 --- a/spacy/tests/pipeline/test_tok2vec.py +++ b/spacy/tests/pipeline/test_tok2vec.py @@ -231,7 +231,7 @@ def test_tok2vec_listener_callback(): def test_tok2vec_listener_overfitting(): - """ Test that a pipeline with a listener properly overfits, even if 'tok2vec' is in the annotating components """ + """Test that a pipeline with a listener properly overfits, even if 'tok2vec' is in the annotating components""" orig_config = Config().from_str(cfg_string) nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) train_examples = [] @@ -264,7 +264,7 @@ def test_tok2vec_listener_overfitting(): def test_tok2vec_frozen_not_annotating(): - """ Test that a pipeline with a frozen tok2vec raises an error when the tok2vec is not annotating """ + """Test that a pipeline with a frozen tok2vec raises an error when the tok2vec is not annotating""" orig_config = Config().from_str(cfg_string) nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) train_examples = [] @@ -274,12 +274,16 @@ def test_tok2vec_frozen_not_annotating(): for i in range(2): losses = {} - with pytest.raises(ValueError, match=r"the tok2vec embedding layer is not updated"): - nlp.update(train_examples, sgd=optimizer, losses=losses, exclude=["tok2vec"]) + with pytest.raises( + ValueError, match=r"the tok2vec embedding layer is not updated" + ): + nlp.update( + train_examples, sgd=optimizer, losses=losses, exclude=["tok2vec"] + ) def test_tok2vec_frozen_overfitting(): - """ Test that a pipeline with a frozen & annotating tok2vec can still overfit """ + """Test that a pipeline with a frozen & annotating tok2vec can still overfit""" orig_config = Config().from_str(cfg_string) nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) train_examples = [] @@ -289,7 +293,13 @@ def test_tok2vec_frozen_overfitting(): for i in range(100): losses = {} - nlp.update(train_examples, sgd=optimizer, losses=losses, exclude=["tok2vec"], annotates=["tok2vec"]) + nlp.update( + train_examples, + sgd=optimizer, + losses=losses, + exclude=["tok2vec"], + annotates=["tok2vec"], + ) assert losses["tagger"] < 0.0001 # test the trained model From 88d35450dcedd89fa739640d8a8d3e62f3643b4a Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 25 Oct 2022 14:53:18 +0200 Subject: [PATCH 063/179] Rename test helper method with non-test_ name (#11701) --- spacy/tests/test_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/tests/test_models.py b/spacy/tests/test_models.py index 2306cabb7..d91ed1201 100644 --- a/spacy/tests/test_models.py +++ b/spacy/tests/test_models.py @@ -23,7 +23,7 @@ def get_textcat_bow_kwargs(): def get_textcat_cnn_kwargs(): - return {"tok2vec": test_tok2vec(), "exclusive_classes": False, "nO": 13} + return {"tok2vec": make_test_tok2vec(), "exclusive_classes": False, "nO": 13} def get_all_params(model): @@ -65,7 +65,7 @@ def get_tok2vec_kwargs(): } -def test_tok2vec(): +def make_test_tok2vec(): return build_Tok2Vec_model(**get_tok2vec_kwargs()) From 8740e4341f03fe2720f50c64e2f94a339d6bd4be Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 25 Oct 2022 14:54:54 +0200 Subject: [PATCH 064/179] Update languages and version in README and website (#11694) --- README.md | 6 +++--- website/meta/languages.json | 28 ++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d9ef83e01..abfc3da67 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ be used in real products. spaCy comes with [pretrained pipelines](https://spacy.io/models) and -currently supports tokenization and training for **60+ languages**. It features +currently supports tokenization and training for **70+ languages**. It features state-of-the-art speed and **neural network models** for tagging, parsing, **named entity recognition**, **text classification** and more, multi-task learning with pretrained **transformers** like BERT, as well as a @@ -16,7 +16,7 @@ production-ready [**training system**](https://spacy.io/usage/training) and easy model packaging, deployment and workflow management. spaCy is commercial open-source software, released under the MIT license. -💫 **Version 3.4.0 out now!** +💫 **Version 3.4 out now!** [Check out the release notes here.](https://github.com/explosion/spaCy/releases) [![Azure Pipelines](https://img.shields.io/azure-devops/build/explosion-ai/public/8/master.svg?logo=azure-pipelines&style=flat-square&label=build)](https://dev.azure.com/explosion-ai/public/_build?definitionId=8) @@ -79,7 +79,7 @@ more people can benefit from it. ## Features -- Support for **60+ languages** +- Support for **70+ languages** - **Trained pipelines** for different languages and tasks - Multi-task learning with pretrained **transformers** like BERT - Support for pretrained **word vectors** and embeddings diff --git a/website/meta/languages.json b/website/meta/languages.json index 0028b4a5f..bd1535c90 100644 --- a/website/meta/languages.json +++ b/website/meta/languages.json @@ -4,12 +4,22 @@ "code": "af", "name": "Afrikaans" }, + { + "code": "am", + "name": "Amharic", + "has_examples": true + }, { "code": "ar", "name": "Arabic", "example": "هذه جملة", "has_examples": true }, + { + "code": "az", + "name": "Azerbaijani", + "has_examples": true + }, { "code": "bg", "name": "Bulgarian", @@ -65,7 +75,7 @@ { "code": "dsb", "name": "Lower Sorbian", - "has_examples": true + "has_examples": true }, { "code": "el", @@ -142,6 +152,11 @@ "code": "ga", "name": "Irish" }, + { + "code": "grc", + "name": "Ancient Greek", + "has_examples": true + }, { "code": "gu", "name": "Gujarati", @@ -172,7 +187,7 @@ { "code": "hsb", "name": "Upper Sorbian", - "has_examples": true + "has_examples": true }, { "code": "hu", @@ -260,6 +275,10 @@ "example": "Адамга эң кыйыны — күн сайын адам болуу", "has_examples": true }, + { + "code": "la", + "name": "Latin" + }, { "code": "lb", "name": "Luxembourgish", @@ -448,6 +467,11 @@ "example": "นี่คือประโยค", "has_examples": true }, + { + "code": "ti", + "name": "Tigrinya", + "has_examples": true + }, { "code": "tl", "name": "Tagalog" From 0a9859ba01c8a51842218e1817dff7ff784951df Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 25 Oct 2022 19:38:23 +0200 Subject: [PATCH 065/179] Reduce python 3.10 in CI to one OS (#11703) --- azure-pipelines.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 357cce835..eea07cb7a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -76,15 +76,15 @@ jobs: # Python39Mac: # imageName: "macos-latest" # python.version: "3.9" - Python310Linux: - imageName: "ubuntu-latest" - python.version: "3.10" + # Python310Linux: + # imageName: "ubuntu-latest" + # python.version: "3.10" Python310Windows: imageName: "windows-latest" python.version: "3.10" - Python310Mac: - imageName: "macos-latest" - python.version: "3.10" + # Python310Mac: + # imageName: "macos-latest" + # python.version: "3.10" Python311Linux: imageName: 'ubuntu-latest' python.version: '3.11.0-rc.2' From a9139907a943f0cc91dac0338aa43caa38939778 Mon Sep 17 00:00:00 2001 From: Ryn Daniels <397565+ryndaniels@users.noreply.github.com> Date: Wed, 26 Oct 2022 09:15:13 +0300 Subject: [PATCH 066/179] update github actions to deal with deprecations (#11702) --- .github/workflows/autoblack.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/autoblack.yml b/.github/workflows/autoblack.yml index 8d0282650..3ad4cf408 100644 --- a/.github/workflows/autoblack.yml +++ b/.github/workflows/autoblack.yml @@ -12,10 +12,10 @@ jobs: if: github.repository_owner == 'explosion' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: ref: ${{ github.head_ref }} - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v3 - run: pip install black - name: Auto-format code if needed run: black spacy @@ -23,10 +23,11 @@ jobs: # code and makes GitHub think the action failed - name: Check for modified files id: git-check - run: echo ::set-output name=modified::$(if git diff-index --quiet HEAD --; then echo "false"; else echo "true"; fi) + run: echo modified=$(if git diff-index --quiet HEAD --; then echo "false"; else echo "true"; fi) >> $GITHUB_OUTPUT + - name: Create Pull Request if: steps.git-check.outputs.modified == 'true' - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@v4 with: title: Auto-format code with black labels: meta From 865691d169c3be413007f0d7324e03a7aac3b3cb Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 26 Oct 2022 08:43:00 +0200 Subject: [PATCH 067/179] Adjust default attrs for textcat configs (#11698) --- spacy/pipeline/textcat.py | 4 ++-- spacy/pipeline/textcat_multilabel.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spacy/pipeline/textcat.py b/spacy/pipeline/textcat.py index c45f819fc..59549ad99 100644 --- a/spacy/pipeline/textcat.py +++ b/spacy/pipeline/textcat.py @@ -24,8 +24,8 @@ single_label_default_config = """ [model.tok2vec.embed] @architectures = "spacy.MultiHashEmbed.v2" width = 64 -rows = [2000, 2000, 1000, 1000, 1000, 1000] -attrs = ["ORTH", "LOWER", "PREFIX", "SUFFIX", "SHAPE", "ID"] +rows = [2000, 2000, 500, 1000, 500] +attrs = ["NORM", "LOWER", "PREFIX", "SUFFIX", "SHAPE"] include_static_vectors = false [model.tok2vec.encode] diff --git a/spacy/pipeline/textcat_multilabel.py b/spacy/pipeline/textcat_multilabel.py index 493c440c3..eb83d9cb7 100644 --- a/spacy/pipeline/textcat_multilabel.py +++ b/spacy/pipeline/textcat_multilabel.py @@ -24,8 +24,8 @@ multi_label_default_config = """ [model.tok2vec.embed] @architectures = "spacy.MultiHashEmbed.v2" width = 64 -rows = [2000, 2000, 1000, 1000, 1000, 1000] -attrs = ["ORTH", "LOWER", "PREFIX", "SUFFIX", "SHAPE", "ID"] +rows = [2000, 2000, 500, 1000, 500] +attrs = ["NORM", "LOWER", "PREFIX", "SUFFIX", "SHAPE"] include_static_vectors = false [model.tok2vec.encode] From 6b78135b9e158e5bc02e39c1a73ef28bb360a44f Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Thu, 27 Oct 2022 22:08:24 +0900 Subject: [PATCH 068/179] Add warning to install widget for M1 GPUs (#11666) * Add warning to install widget for M1 GPUs * Use Thinc tracking issue instead * Update website/src/widgets/quickstart-install.js Co-authored-by: Adriane Boyd * Underline URL in warning * Update website/src/widgets/quickstart-install.js Co-authored-by: Adriane Boyd * Don't install cupy on m1 gpus Co-authored-by: Adriane Boyd --- website/src/styles/quickstart.module.sass | 3 +++ website/src/widgets/quickstart-install.js | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/website/src/styles/quickstart.module.sass b/website/src/styles/quickstart.module.sass index 8ad106a78..d0f9db551 100644 --- a/website/src/styles/quickstart.module.sass +++ b/website/src/styles/quickstart.module.sass @@ -149,6 +149,9 @@ & > span display: block + a + text-decoration: underline + .small font-size: var(--font-size-code) line-height: 1.65 diff --git a/website/src/widgets/quickstart-install.js b/website/src/widgets/quickstart-install.js index 0d2186acb..28dd14ecc 100644 --- a/website/src/widgets/quickstart-install.js +++ b/website/src/widgets/quickstart-install.js @@ -159,6 +159,9 @@ const QuickstartInstall = ({ id, title }) => { setters={setters} showDropdown={showDropdown} > + + # Note M1 GPU support is experimental, see Thinc issue #792 + python -m venv .env @@ -198,7 +201,13 @@ const QuickstartInstall = ({ id, title }) => { {nightly ? ' --pre' : ''} conda install -c conda-forge spacy - + + conda install -c conda-forge cupy + + + conda install -c conda-forge cupy + + conda install -c conda-forge cupy From d61e742960ef230b423dfa157449b291a03bd119 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Fri, 28 Oct 2022 17:25:34 +0900 Subject: [PATCH 069/179] Handle Docs with no entities in EntityLinker (#11640) * Handle docs with no entities If a whole batch contains no entities it won't make it to the model, but it's possible for individual Docs to have no entities. Before this commit, those Docs would cause an error when attempting to concatenate arrays because the dimensions didn't match. It turns out the process of preparing the Ragged at the end of the span maker forward was a little different from list2ragged, which just uses the flatten function directly. Letting list2ragged do the conversion avoids the dimension issue. This did not come up before because in NEL demo projects it's typical for data with no entities to be discarded before it reaches the NEL component. This includes a simple direct test that shows the issue and checks it's resolved. It doesn't check if there are any downstream changes, so a more complete test could be added. A full run was tested by adding an example with no entities to the Emerson sample project. * Add a blank instance to default training data in tests Rather than adding a specific test, since not failing on instances with no entities is basic functionality, it makes sense to add it to the default set. * Fix without modifying architecture If the architecture is modified this would have to be a new version, but this change isn't big enough to merit that. --- spacy/ml/models/entity_linker.py | 7 +++---- spacy/tests/pipeline/test_entity_linker.py | 22 +++++++++++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/spacy/ml/models/entity_linker.py b/spacy/ml/models/entity_linker.py index 4d18d216a..299b6bb52 100644 --- a/spacy/ml/models/entity_linker.py +++ b/spacy/ml/models/entity_linker.py @@ -71,11 +71,10 @@ def span_maker_forward(model, docs: List[Doc], is_train) -> Tuple[Ragged, Callab cands.append((start_token, end_token)) candidates.append(ops.asarray2i(cands)) - candlens = ops.asarray1i([len(cands) for cands in candidates]) - candidates = ops.xp.concatenate(candidates) - outputs = Ragged(candidates, candlens) + lengths = model.ops.asarray1i([len(cands) for cands in candidates]) + out = Ragged(model.ops.flatten(candidates), lengths) # because this is just rearranging docs, the backprop does nothing - return outputs, lambda x: [] + return out, lambda x: [] @registry.misc("spacy.KBFromFile.v1") diff --git a/spacy/tests/pipeline/test_entity_linker.py b/spacy/tests/pipeline/test_entity_linker.py index 4d683acc5..99f164f15 100644 --- a/spacy/tests/pipeline/test_entity_linker.py +++ b/spacy/tests/pipeline/test_entity_linker.py @@ -9,6 +9,7 @@ from spacy.compat import pickle from spacy.kb import Candidate, InMemoryLookupKB, get_candidates, KnowledgeBase from spacy.lang.en import English from spacy.ml import load_kb +from spacy.ml.models.entity_linker import build_span_maker from spacy.pipeline import EntityLinker from spacy.pipeline.legacy import EntityLinker_v1 from spacy.pipeline.tok2vec import DEFAULT_TOK2VEC_MODEL @@ -715,7 +716,11 @@ TRAIN_DATA = [ ("Russ Cochran was a member of University of Kentucky's golf team.", {"links": {(0, 12): {"Q7381115": 0.0, "Q2146908": 1.0}}, "entities": [(0, 12, "PERSON"), (43, 51, "LOC")], - "sent_starts": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}) + "sent_starts": [1, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}), + # having a blank instance shouldn't break things + ("The weather is nice today.", + {"links": {}, "entities": [], + "sent_starts": [1, -1, 0, 0, 0, 0]}) ] GOLD_entities = ["Q2146908", "Q7381115", "Q7381115", "Q2146908"] # fmt: on @@ -1196,3 +1201,18 @@ def test_threshold(meet_threshold: bool, config: Dict[str, Any]): assert len(doc.ents) == 1 assert doc.ents[0].kb_id_ == entity_id if meet_threshold else EntityLinker.NIL + + +def test_span_maker_forward_with_empty(): + """The forward pass of the span maker may have a doc with no entities.""" + nlp = English() + doc1 = nlp("a b c") + ent = doc1[0:1] + ent.label_ = "X" + doc1.ents = [ent] + # no entities + doc2 = nlp("x y z") + + # just to get a model + span_maker = build_span_maker() + span_maker([doc1, doc2], False) From d25f09468c4eca20eb464d78d35e439474ed2dbc Mon Sep 17 00:00:00 2001 From: Aaron Zipp <15341396+aaronzipp@users.noreply.github.com> Date: Mon, 31 Oct 2022 05:27:12 +0100 Subject: [PATCH 070/179] Spelling mistake in rule-based-matching.md (#11717) Changed retokenize to retokenizer --- website/docs/usage/rule-based-matching.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/usage/rule-based-matching.md b/website/docs/usage/rule-based-matching.md index f096890cb..64bbf8e7b 100644 --- a/website/docs/usage/rule-based-matching.md +++ b/website/docs/usage/rule-based-matching.md @@ -1792,7 +1792,7 @@ the entity `Span` – for example `._.orgs` or `._.prev_orgs` and > [`Doc.retokenize`](/api/doc#retokenize) context manager: > > ```python -> with doc.retokenize() as retokenize: +> with doc.retokenize() as retokenizer: > for ent in doc.ents: > retokenizer.merge(ent) > ``` From f7edd84b44a37b78d87fe6815399a576f1980b8b Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 2 Nov 2022 13:42:20 +0100 Subject: [PATCH 071/179] Switch CI to Python 3.11.0 (#11737) --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index eea07cb7a..bf3672b8b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -87,13 +87,13 @@ jobs: # python.version: "3.10" Python311Linux: imageName: 'ubuntu-latest' - python.version: '3.11.0-rc.2' + python.version: '3.11.0' Python311Windows: imageName: 'windows-latest' - python.version: '3.11.0-rc.2' + python.version: '3.11.0' Python311Mac: imageName: 'macos-latest' - python.version: '3.11.0-rc.2' + python.version: '3.11.0' maxParallel: 4 pool: vmImage: $(imageName) From 420b1d854be86e899088bb136f1daf23fc61ed1d Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 2 Nov 2022 15:35:04 +0100 Subject: [PATCH 072/179] Update textcat scorer threshold behavior (#11696) * Update textcat scorer threshold behavior For `textcat` (with exclusive classes) the scorer should always use a threshold of 0.0 because there should be one predicted label per doc and the numeric score for that particular label should not matter. * Rename to test_textcat_multilabel_threshold * Remove all uses of threshold for multi_label=False * Update Scorer.score_cats API docs * Add tests for score_cats with thresholds * Update textcat API docs * Fix types * Convert threshold back to float * Fix threshold type in docstring * Improve formatting in Scorer API docs --- spacy/pipeline/textcat.py | 7 +++-- spacy/scorer.py | 12 +++---- spacy/tests/pipeline/test_textcat.py | 6 ++-- spacy/tests/test_scorer.py | 47 ++++++++++++++++++++++++++++ website/docs/api/scorer.md | 21 +++++++------ website/docs/api/textcategorizer.md | 5 ++- 6 files changed, 73 insertions(+), 25 deletions(-) diff --git a/spacy/pipeline/textcat.py b/spacy/pipeline/textcat.py index 59549ad99..238a768ed 100644 --- a/spacy/pipeline/textcat.py +++ b/spacy/pipeline/textcat.py @@ -72,7 +72,7 @@ subword_features = true "textcat", assigns=["doc.cats"], default_config={ - "threshold": 0.5, + "threshold": 0.0, "model": DEFAULT_SINGLE_TEXTCAT_MODEL, "scorer": {"@scorers": "spacy.textcat_scorer.v1"}, }, @@ -144,7 +144,8 @@ class TextCategorizer(TrainablePipe): model (thinc.api.Model): The Thinc Model powering the pipeline component. name (str): The component instance name, used to add entries to the losses during training. - threshold (float): Cutoff to consider a prediction "positive". + threshold (float): Unused, not needed for single-label (exclusive + classes) classification. scorer (Optional[Callable]): The scoring method. Defaults to Scorer.score_cats for the attribute "cats". @@ -154,7 +155,7 @@ class TextCategorizer(TrainablePipe): self.model = model self.name = name self._rehearsal_model = None - cfg = {"labels": [], "threshold": threshold, "positive_label": None} + cfg: Dict[str, Any] = {"labels": [], "threshold": threshold, "positive_label": None} self.cfg = dict(cfg) self.scorer = scorer diff --git a/spacy/scorer.py b/spacy/scorer.py index 8cd755ac4..16fc303a0 100644 --- a/spacy/scorer.py +++ b/spacy/scorer.py @@ -446,7 +446,7 @@ class Scorer: labels (Iterable[str]): The set of possible labels. Defaults to []. multi_label (bool): Whether the attribute allows multiple labels. Defaults to True. When set to False (exclusive labels), missing - gold labels are interpreted as 0.0. + gold labels are interpreted as 0.0 and the threshold is set to 0.0. positive_label (str): The positive label for a binary task with exclusive classes. Defaults to None. threshold (float): Cutoff to consider a prediction "positive". Defaults @@ -471,6 +471,8 @@ class Scorer: """ if threshold is None: threshold = 0.5 if multi_label else 0.0 + if not multi_label: + threshold = 0.0 f_per_type = {label: PRFScore() for label in labels} auc_per_type = {label: ROCAUCScore() for label in labels} labels = set(labels) @@ -505,20 +507,18 @@ class Scorer: # Get the highest-scoring for each. pred_label, pred_score = max(pred_cats.items(), key=lambda it: it[1]) gold_label, gold_score = max(gold_cats.items(), key=lambda it: it[1]) - if pred_label == gold_label and pred_score >= threshold: + if pred_label == gold_label: f_per_type[pred_label].tp += 1 else: f_per_type[gold_label].fn += 1 - if pred_score >= threshold: - f_per_type[pred_label].fp += 1 + f_per_type[pred_label].fp += 1 elif gold_cats: gold_label, gold_score = max(gold_cats, key=lambda it: it[1]) if gold_score > 0: f_per_type[gold_label].fn += 1 elif pred_cats: pred_label, pred_score = max(pred_cats.items(), key=lambda it: it[1]) - if pred_score >= threshold: - f_per_type[pred_label].fp += 1 + f_per_type[pred_label].fp += 1 micro_prf = PRFScore() for label_prf in f_per_type.values(): micro_prf.tp += label_prf.tp diff --git a/spacy/tests/pipeline/test_textcat.py b/spacy/tests/pipeline/test_textcat.py index 0bb036a33..d359b77db 100644 --- a/spacy/tests/pipeline/test_textcat.py +++ b/spacy/tests/pipeline/test_textcat.py @@ -823,10 +823,10 @@ def test_textcat_loss(multi_label: bool, expected_loss: float): assert loss == expected_loss -def test_textcat_threshold(): +def test_textcat_multilabel_threshold(): # Ensure the scorer can be called with a different threshold nlp = English() - nlp.add_pipe("textcat") + nlp.add_pipe("textcat_multilabel") train_examples = [] for text, annotations in TRAIN_DATA_SINGLE_LABEL: @@ -849,7 +849,7 @@ def test_textcat_threshold(): ) pos_f = scores["cats_score"] assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 1.0 - assert pos_f > macro_f + assert pos_f >= macro_f def test_textcat_multi_threshold(): diff --git a/spacy/tests/test_scorer.py b/spacy/tests/test_scorer.py index 6e15fa2de..b903f1669 100644 --- a/spacy/tests/test_scorer.py +++ b/spacy/tests/test_scorer.py @@ -474,3 +474,50 @@ def test_prf_score(): assert (a.precision, a.recall, a.fscore) == approx( (c.precision, c.recall, c.fscore) ) + + +def test_score_cats(en_tokenizer): + text = "some text" + gold_doc = en_tokenizer(text) + gold_doc.cats = {"POSITIVE": 1.0, "NEGATIVE": 0.0} + pred_doc = en_tokenizer(text) + pred_doc.cats = {"POSITIVE": 0.75, "NEGATIVE": 0.25} + example = Example(pred_doc, gold_doc) + # threshold is ignored for multi_label=False + scores1 = Scorer.score_cats( + [example], + "cats", + labels=list(gold_doc.cats.keys()), + multi_label=False, + positive_label="POSITIVE", + threshold=0.1, + ) + scores2 = Scorer.score_cats( + [example], + "cats", + labels=list(gold_doc.cats.keys()), + multi_label=False, + positive_label="POSITIVE", + threshold=0.9, + ) + assert scores1["cats_score"] == 1.0 + assert scores2["cats_score"] == 1.0 + assert scores1 == scores2 + # threshold is relevant for multi_label=True + scores = Scorer.score_cats( + [example], + "cats", + labels=list(gold_doc.cats.keys()), + multi_label=True, + threshold=0.9, + ) + assert scores["cats_macro_f"] == 0.0 + # threshold is relevant for multi_label=True + scores = Scorer.score_cats( + [example], + "cats", + labels=list(gold_doc.cats.keys()), + multi_label=True, + threshold=0.1, + ) + assert scores["cats_macro_f"] == 0.5 diff --git a/website/docs/api/scorer.md b/website/docs/api/scorer.md index ca3462aa9..9ef36e6fc 100644 --- a/website/docs/api/scorer.md +++ b/website/docs/api/scorer.md @@ -229,16 +229,17 @@ The reported `{attr}_score` depends on the classification properties: > print(scores["cats_macro_auc"]) > ``` -| Name | Description | -| ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | -| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | -| `attr` | The attribute to score. ~~str~~ | -| _keyword-only_ | | -| `getter` | Defaults to `getattr`. If provided, `getter(doc, attr)` should return the cats for an individual `Doc`. ~~Callable[[Doc, str], Dict[str, float]]~~ | -| labels | The set of possible labels. Defaults to `[]`. ~~Iterable[str]~~ | -| `multi_label` | Whether the attribute allows multiple labels. Defaults to `True`. ~~bool~~ | -| `positive_label` | The positive label for a binary task with exclusive classes. Defaults to `None`. ~~Optional[str]~~ | -| **RETURNS** | A dictionary containing the scores, with inapplicable scores as `None`. ~~Dict[str, Optional[float]]~~ | +| Name | Description | +| ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | +| `attr` | The attribute to score. ~~str~~ | +| _keyword-only_ | | +| `getter` | Defaults to `getattr`. If provided, `getter(doc, attr)` should return the cats for an individual `Doc`. ~~Callable[[Doc, str], Dict[str, float]]~~ | +| labels | The set of possible labels. Defaults to `[]`. ~~Iterable[str]~~ | +| `multi_label` | Whether the attribute allows multiple labels. Defaults to `True`. When set to `False` (exclusive labels), missing gold labels are interpreted as `0.0` and the threshold is set to `0.0`. ~~bool~~ | +| `positive_label` | The positive label for a binary task with exclusive classes. Defaults to `None`. ~~Optional[str]~~ | +| `threshold` | Cutoff to consider a prediction "positive". Defaults to `0.5` for multi-label, and `0.0` (i.e. whatever's highest scoring) otherwise. ~~float~~ | +| **RETURNS** | A dictionary containing the scores, with inapplicable scores as `None`. ~~Dict[str, Optional[float]]~~ | ## Scorer.score_links {#score_links tag="staticmethod" new="3"} diff --git a/website/docs/api/textcategorizer.md b/website/docs/api/textcategorizer.md index 042b4ab76..f5f8706ec 100644 --- a/website/docs/api/textcategorizer.md +++ b/website/docs/api/textcategorizer.md @@ -63,7 +63,6 @@ architectures and their arguments and hyperparameters. > ```python > from spacy.pipeline.textcat import DEFAULT_SINGLE_TEXTCAT_MODEL > config = { -> "threshold": 0.5, > "model": DEFAULT_SINGLE_TEXTCAT_MODEL, > } > nlp.add_pipe("textcat", config=config) @@ -82,7 +81,7 @@ architectures and their arguments and hyperparameters. | Setting | Description | | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `threshold` | Cutoff to consider a prediction "positive", relevant when printing accuracy results. ~~float~~ | +| `threshold` | Cutoff to consider a prediction "positive", relevant for `textcat_multilabel` when calculating accuracy scores. ~~float~~ | | `model` | A model instance that predicts scores for each category. Defaults to [TextCatEnsemble](/api/architectures#TextCatEnsemble). ~~Model[List[Doc], List[Floats2d]]~~ | | `scorer` | The scoring method. Defaults to [`Scorer.score_cats`](/api/scorer#score_cats) for the attribute `"cats"`. ~~Optional[Callable]~~ | @@ -123,7 +122,7 @@ shortcut for this and instantiate the component using its string name and | `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model[List[Doc], List[Floats2d]]~~ | | `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | | _keyword-only_ | | -| `threshold` | Cutoff to consider a prediction "positive", relevant when printing accuracy results. ~~float~~ | +| `threshold` | Cutoff to consider a prediction "positive", relevant for `textcat_multilabel` when calculating accuracy scores. ~~float~~ | | `scorer` | The scoring method. Defaults to [`Scorer.score_cats`](/api/scorer#score_cats) for the attribute `"cats"`. ~~Optional[Callable]~~ | ## TextCategorizer.\_\_call\_\_ {#call tag="method"} From 2fb7e4dc74bd491ecec43971b2b29b0d28efd492 Mon Sep 17 00:00:00 2001 From: Ryn Daniels <397565+ryndaniels@users.noreply.github.com> Date: Wed, 2 Nov 2022 16:36:30 +0200 Subject: [PATCH 073/179] More version updates for github action deprecation warnings (#11705) * More version updates for github action deprecation warnings * fix the deprecated set-output commands * bump explosion-bot to run on ubuntu-latest --- .github/workflows/autoblack.yml | 2 +- .github/workflows/explosionbot.yml | 6 +++--- .github/workflows/slowtests.yml | 6 +++--- .github/workflows/spacy_universe_alert.yml | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/autoblack.yml b/.github/workflows/autoblack.yml index 3ad4cf408..70882c3cc 100644 --- a/.github/workflows/autoblack.yml +++ b/.github/workflows/autoblack.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v3 with: ref: ${{ github.head_ref }} - - uses: actions/setup-python@v3 + - uses: actions/setup-python@v4 - run: pip install black - name: Auto-format code if needed run: black spacy diff --git a/.github/workflows/explosionbot.yml b/.github/workflows/explosionbot.yml index d585ecd9c..6b472cd12 100644 --- a/.github/workflows/explosionbot.yml +++ b/.github/workflows/explosionbot.yml @@ -8,14 +8,14 @@ on: jobs: explosion-bot: - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - name: Dump GitHub context env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" - - uses: actions/checkout@v1 - - uses: actions/setup-python@v1 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 - name: Install and run explosion-bot run: | pip install git+https://${{ secrets.EXPLOSIONBOT_TOKEN }}@github.com/explosion/explosion-bot diff --git a/.github/workflows/slowtests.yml b/.github/workflows/slowtests.yml index 38ceb18c6..f9fd3e817 100644 --- a/.github/workflows/slowtests.yml +++ b/.github/workflows/slowtests.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v3 with: ref: ${{ matrix.branch }} - name: Get commits from past 24 hours @@ -23,9 +23,9 @@ jobs: today=$(date '+%Y-%m-%d %H:%M:%S') yesterday=$(date -d "yesterday" '+%Y-%m-%d %H:%M:%S') if git log --after="$yesterday" --before="$today" | grep commit ; then - echo "::set-output name=run_tests::true" + echo run_tests=true >> $GITHUB_OUTPUT else - echo "::set-output name=run_tests::false" + echo run_tests=false >> $GITHUB_OUTPUT fi - name: Trigger buildkite build diff --git a/.github/workflows/spacy_universe_alert.yml b/.github/workflows/spacy_universe_alert.yml index cbbf14c6e..f507e0594 100644 --- a/.github/workflows/spacy_universe_alert.yml +++ b/.github/workflows/spacy_universe_alert.yml @@ -17,8 +17,8 @@ jobs: run: | echo "$GITHUB_CONTEXT" - - uses: actions/checkout@v1 - - uses: actions/setup-python@v1 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 - name: Install Bernadette app dependency and send an alert env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} From 1211552f0ec84aef0b55f834d76899ab07e2c5cc Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Thu, 3 Nov 2022 09:29:46 +0100 Subject: [PATCH 074/179] Modernize and simplify CI steps (#11738) * Use `build` instead of `python setup.py sdist` * Remove in-place build with `setup.py` * Remove `gpu` parameter and GPU tests * Keep `architecture` and `num_build_jobs` in azure steps with CI defaults * Fix use of `num_build_jobs` parameters * Remove now-unused `prefix` parameter * Test imports and CLI before installing test requirements * Remove `*.egg-info` directory in addition to source directory for an warning-free `import spacy` * Switch `thinc-apple-ops` test to python 3.11 (as most recent python that is tested across platforms) --- .github/azure-steps.yml | 70 +++++++++++++++++++---------------------- azure-pipelines.yml | 17 ---------- 2 files changed, 33 insertions(+), 54 deletions(-) diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml index cc0247b3a..b2bc80dd6 100644 --- a/.github/azure-steps.yml +++ b/.github/azure-steps.yml @@ -1,9 +1,7 @@ parameters: python_version: '' - architecture: '' - prefix: '' - gpu: false - num_build_jobs: 1 + architecture: 'x64' + num_build_jobs: 2 steps: - task: UsePythonVersion@0 @@ -17,16 +15,16 @@ steps: displayName: 'Set variables' - script: | - ${{ parameters.prefix }} python -m pip install -U pip setuptools - ${{ parameters.prefix }} python -m pip install -U -r requirements.txt + python -m pip install -U build pip setuptools + python -m pip install -U -r requirements.txt displayName: "Install dependencies" - script: | - ${{ parameters.prefix }} python setup.py build_ext --inplace -j ${{ parameters.num_build_jobs }} - ${{ parameters.prefix }} python setup.py sdist --formats=gztar - displayName: "Compile and build sdist" + python -m build --sdist + displayName: "Build sdist" - - script: python -m mypy spacy + - script: | + python -m mypy spacy displayName: 'Run mypy' condition: ne(variables['python_version'], '3.6') @@ -35,35 +33,24 @@ steps: contents: "spacy" displayName: "Delete source directory" + - task: DeleteFiles@1 + inputs: + contents: "*.egg-info" + displayName: "Delete egg-info directory" + - script: | - ${{ parameters.prefix }} python -m pip freeze --exclude torch --exclude cupy-cuda110 > installed.txt - ${{ parameters.prefix }} python -m pip uninstall -y -r installed.txt + python -m pip freeze > installed.txt + python -m pip uninstall -y -r installed.txt displayName: "Uninstall all packages" - bash: | - ${{ parameters.prefix }} SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1) - ${{ parameters.prefix }} SPACY_NUM_BUILD_JOBS=2 python -m pip install dist/$SDIST + SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1) + SPACY_NUM_BUILD_JOBS=${{ parameters.num_build_jobs }} python -m pip install dist/$SDIST displayName: "Install from sdist" - script: | - ${{ parameters.prefix }} python -m pip install -U -r requirements.txt - displayName: "Install test requirements" - - - script: | - ${{ parameters.prefix }} python -m pip install -U cupy-cuda110 -f https://github.com/cupy/cupy/releases/v9.0.0 - ${{ parameters.prefix }} python -m pip install "torch==1.7.1+cu110" -f https://download.pytorch.org/whl/torch_stable.html - displayName: "Install GPU requirements" - condition: eq(${{ parameters.gpu }}, true) - - - script: | - ${{ parameters.prefix }} python -m pytest --pyargs spacy -W error - displayName: "Run CPU tests" - condition: eq(${{ parameters.gpu }}, false) - - - script: | - ${{ parameters.prefix }} python -m pytest --pyargs spacy -W error -p spacy.tests.enable_gpu - displayName: "Run GPU tests" - condition: eq(${{ parameters.gpu }}, true) + python -W error -c "import spacy" + displayName: "Test import" - script: | python -m spacy download ca_core_news_sm @@ -106,13 +93,22 @@ steps: displayName: 'Test assemble CLI vectors warning' condition: eq(variables['python_version'], '3.8') + - script: | + python -m pip install -U -r requirements.txt + displayName: "Install test requirements" + + - script: | + python -m pytest --pyargs spacy -W error + displayName: "Run CPU tests" + + - script: | + python -m pip install --pre thinc-apple-ops + python -m pytest --pyargs spacy + displayName: "Run CPU tests with thinc-apple-ops" + condition: and(startsWith(variables['imageName'], 'macos'), eq(variables['python.version'], '3.11')) + - script: | python .github/validate_universe_json.py website/meta/universe.json displayName: 'Test website/meta/universe.json' condition: eq(variables['python_version'], '3.8') - - script: | - ${{ parameters.prefix }} python -m pip install --pre thinc-apple-ops - ${{ parameters.prefix }} python -m pytest --pyargs spacy - displayName: "Run CPU tests with thinc-apple-ops" - condition: and(startsWith(variables['imageName'], 'macos'), eq(variables['python.version'], '3.10')) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bf3672b8b..3499042cb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -101,20 +101,3 @@ jobs: - template: .github/azure-steps.yml parameters: python_version: '$(python.version)' - architecture: 'x64' - -# - job: "TestGPU" -# dependsOn: "Validate" -# strategy: -# matrix: -# Python38LinuxX64_GPU: -# python.version: '3.8' -# pool: -# name: "LinuxX64_GPU" -# steps: -# - template: .github/azure-steps.yml -# parameters: -# python_version: '$(python.version)' -# architecture: 'x64' -# gpu: true -# num_build_jobs: 24 From db56600536e2d615a766fc2fc973a6cc9e0f1a52 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Thu, 3 Nov 2022 18:52:59 +0900 Subject: [PATCH 075/179] Fix default parameters for load functions (fix #11706) (#11713) * Fix default parameters for load functions Some load functions used SimpleFrozenList() directly instead of the _DEFAULT_EMPTY_PIPES parameter. That mostly worked as intended, but the changes in #11459 check for equality using identity, not value, so a warning is incorrectly raised sometimes, as in #11706. This change just has all the load functions use the singleton value instead. * Add test that there are no warnings on module-based load This will succeed due to changes in this branch, but local tests with the latest release failed as intended. * Try reverting commit and see if CI changes There is an error in CI that is probably unrelated. Revert "Fix default parameters for load functions" This reverts commit dc46b35687e92e4793e64edb11997d44b88c6a8b. * Revert "Try reverting commit and see if CI changes" This reverts commit 2514ed07ef29851b5ac60015442a7ce44c69decc. Co-authored-by: Adriane Boyd --- .github/azure-steps.yml | 5 +++++ spacy/util.py | 12 ++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml index b2bc80dd6..e8bd0d212 100644 --- a/.github/azure-steps.yml +++ b/.github/azure-steps.yml @@ -59,6 +59,11 @@ steps: displayName: 'Test download CLI' condition: eq(variables['python_version'], '3.8') + - script: | + python -W error -c "import ca_core_news_sm; nlp = ca_core_news_sm.load(); doc=nlp('test')" + displayName: 'Test no warnings on load (#11713)' + condition: eq(variables['python_version'], '3.8') + - script: | python -m spacy convert extra/example_data/ner_example_data/ner-token-per-line-conll2003.json . displayName: 'Test convert CLI' diff --git a/spacy/util.py b/spacy/util.py index 3034808ba..76a1e0bfa 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -443,9 +443,9 @@ def load_model_from_package( name: str, *, vocab: Union["Vocab", bool] = True, - disable: Union[str, Iterable[str]] = SimpleFrozenList(), - enable: Union[str, Iterable[str]] = SimpleFrozenList(), - exclude: Union[str, Iterable[str]] = SimpleFrozenList(), + disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), ) -> "Language": """Load a model from an installed package. @@ -619,9 +619,9 @@ def load_model_from_init_py( init_file: Union[Path, str], *, vocab: Union["Vocab", bool] = True, - disable: Union[str, Iterable[str]] = SimpleFrozenList(), - enable: Union[str, Iterable[str]] = SimpleFrozenList(), - exclude: Union[str, Iterable[str]] = SimpleFrozenList(), + disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, + exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, config: Union[Dict[str, Any], Config] = SimpleFrozenDict(), ) -> "Language": """Helper function to use in the `load()` method of a model package's From 40e1000db08858e8c928efacab8f710e027dde61 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Thu, 3 Nov 2022 11:49:08 +0100 Subject: [PATCH 076/179] Restore Doc attr getter values in Doc.to_json (#11700) --- spacy/tests/doc/test_json_doc_conversion.py | 9 +++++++ spacy/tokens/doc.pyx | 27 ++++++++++++++------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/spacy/tests/doc/test_json_doc_conversion.py b/spacy/tests/doc/test_json_doc_conversion.py index 19698cfb2..11a1817e6 100644 --- a/spacy/tests/doc/test_json_doc_conversion.py +++ b/spacy/tests/doc/test_json_doc_conversion.py @@ -370,3 +370,12 @@ def test_json_to_doc_validation_error(doc): doc_json.pop("tokens") with pytest.raises(ValueError): Doc(doc.vocab).from_json(doc_json, validate=True) + + +def test_to_json_underscore_doc_getters(doc): + def get_text_length(doc): + return len(doc.text) + + Doc.set_extension("text_length", getter=get_text_length) + doc_json = doc.to_json(underscore=["text_length"]) + assert doc_json["_"]["text_length"] == get_text_length(doc) diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index 295f91c28..f2621292c 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -1668,6 +1668,20 @@ cdef class Doc: if underscore: user_keys = set() + # Handle doc attributes with .get to include values from getters + # and not only values stored in user_data, for backwards + # compatibility + for attr in underscore: + if self.has_extension(attr): + if "_" not in data: + data["_"] = {} + value = self._.get(attr) + if not srsly.is_json_serializable(value): + raise ValueError(Errors.E107.format(attr=attr, value=repr(value))) + data["_"][attr] = value + user_keys.add(attr) + # Token and span attributes only include values stored in user_data + # and not values generated by getters if self.user_data: for data_key, value in self.user_data.copy().items(): if type(data_key) == tuple and len(data_key) >= 4 and data_key[0] == "._.": @@ -1678,20 +1692,15 @@ cdef class Doc: user_keys.add(attr) if not srsly.is_json_serializable(value): raise ValueError(Errors.E107.format(attr=attr, value=repr(value))) - # Check if doc attribute - if start is None: - if "_" not in data: - data["_"] = {} - data["_"][attr] = value - # Check if token attribute - elif end is None: + # Token attribute + if start is not None and end is None: if "underscore_token" not in data: data["underscore_token"] = {} if attr not in data["underscore_token"]: data["underscore_token"][attr] = [] data["underscore_token"][attr].append({"start": start, "value": value}) - # Else span attribute - else: + # Span attribute + elif start is not None and end is not None: if "underscore_span" not in data: data["underscore_span"] = {} if attr not in data["underscore_span"]: From bbf64cfc4391cccba447346badaacca4d42e583d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 4 Nov 2022 11:17:43 +0100 Subject: [PATCH 077/179] Auto-format code with black (#11749) Co-authored-by: explosion-bot --- spacy/pipeline/textcat.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spacy/pipeline/textcat.py b/spacy/pipeline/textcat.py index 238a768ed..4023c4456 100644 --- a/spacy/pipeline/textcat.py +++ b/spacy/pipeline/textcat.py @@ -155,7 +155,11 @@ class TextCategorizer(TrainablePipe): self.model = model self.name = name self._rehearsal_model = None - cfg: Dict[str, Any] = {"labels": [], "threshold": threshold, "positive_label": None} + cfg: Dict[str, Any] = { + "labels": [], + "threshold": threshold, + "positive_label": None, + } self.cfg = dict(cfg) self.scorer = scorer From ea326cf47d5324cff14bef983b0da122b9f0d1ed Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Mon, 7 Nov 2022 08:11:13 +0100 Subject: [PATCH 078/179] Fix types for Span.id and Span.id_ (#11744) --- spacy/tokens/span.pyi | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/spacy/tokens/span.pyi b/spacy/tokens/span.pyi index 617e3d19d..0a6f306a6 100644 --- a/spacy/tokens/span.pyi +++ b/spacy/tokens/span.pyi @@ -117,15 +117,13 @@ class Span: end_char: int label: int kb_id: int + id: int ent_id: int ent_id_: str @property - def id(self) -> int: ... - @property - def id_(self) -> str: ... - @property def orth_(self) -> str: ... @property def lemma_(self) -> str: ... label_: str kb_id_: str + id_: str From b76222e56adb49e33d7d0471674dfe2f207b2020 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Mon, 7 Nov 2022 16:11:55 +0900 Subject: [PATCH 079/179] Raise Typer limit (#11720) * Raise typer limit to <0.7.0 * Raise limit to <0.8.0 --- requirements.txt | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9d6bbb2c4..d91a3b3d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ murmurhash>=0.28.0,<1.1.0 wasabi>=0.9.1,<1.1.0 srsly>=2.4.3,<3.0.0 catalogue>=2.0.6,<2.1.0 -typer>=0.3.0,<0.5.0 +typer>=0.3.0,<0.8.0 pathy>=0.3.5 # Third party dependencies numpy>=1.15.0 diff --git a/setup.cfg b/setup.cfg index c2653feba..82d4d2758 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,7 @@ install_requires = srsly>=2.4.3,<3.0.0 catalogue>=2.0.6,<2.1.0 # Third-party dependencies - typer>=0.3.0,<0.5.0 + typer>=0.3.0,<0.8.0 pathy>=0.3.5 tqdm>=4.38.0,<5.0.0 numpy>=1.15.0 From e91b47a22655c0384202f797e9d50d3660596d32 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Mon, 7 Nov 2022 10:43:34 +0100 Subject: [PATCH 080/179] Check for unsafe paths in tarfile.extractall (CVE-2007-4559) (#11746) * Adding tarfile member sanitization to extractall() * Format * Simplify and add error message * Fix import * Add comment about CVE Co-authored-by: TrellixVulnTeam --- spacy/cli/project/remote_storage.py | 19 ++++++++++++++++++- spacy/errors.py | 2 ++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/spacy/cli/project/remote_storage.py b/spacy/cli/project/remote_storage.py index 336a4bcb3..12e252b3c 100644 --- a/spacy/cli/project/remote_storage.py +++ b/spacy/cli/project/remote_storage.py @@ -10,6 +10,7 @@ from .._util import get_hash, get_checksum, download_file, ensure_pathy from ...util import make_tempdir, get_minor_version, ENV_VARS, check_bool_env_var from ...git_info import GIT_VERSION from ... import about +from ...errors import Errors if TYPE_CHECKING: from pathy import Pathy # noqa: F401 @@ -84,7 +85,23 @@ class RemoteStorage: with tarfile.open(tar_loc, mode=mode_string) as tar_file: # This requires that the path is added correctly, relative # to root. This is how we set things up in push() - tar_file.extractall(self.root) + + # Disallow paths outside the current directory for the tar + # file (CVE-2007-4559, directory traversal vulnerability) + def is_within_directory(directory, target): + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + prefix = os.path.commonprefix([abs_directory, abs_target]) + return prefix == abs_directory + + def safe_extract(tar, path): + for member in tar.getmembers(): + member_path = os.path.join(path, member.name) + if not is_within_directory(path, member_path): + raise ValueError(Errors.E852) + tar.extractall(path) + + safe_extract(tar_file, self.root) return url def find( diff --git a/spacy/errors.py b/spacy/errors.py index e0628819d..2f8a3996f 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -544,6 +544,8 @@ class Errors(metaclass=ErrorsWithCodes): "during training, make sure to include it in 'annotating components'") # New errors added in v3.x + E852 = ("The tar file pulled from the remote attempted an unsafe path " + "traversal.") E853 = ("Unsupported component factory name '{name}'. The character '.' is " "not permitted in factory names.") E854 = ("Unable to set doc.ents. Check that the 'ents_filter' does not " From 6105f20d8a10a18a0e5985d310664812198840a8 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Mon, 7 Nov 2022 13:25:40 +0100 Subject: [PATCH 081/179] Switch CI to python 3.11 (#11765) --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3499042cb..9c3b92f06 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -87,13 +87,13 @@ jobs: # python.version: "3.10" Python311Linux: imageName: 'ubuntu-latest' - python.version: '3.11.0' + python.version: '3.11' Python311Windows: imageName: 'windows-latest' - python.version: '3.11.0' + python.version: '3.11' Python311Mac: imageName: 'macos-latest' - python.version: '3.11.0' + python.version: '3.11' maxParallel: 4 pool: vmImage: $(imageName) From e116395f890a70447c75109026e7b37f20c142c2 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Mon, 7 Nov 2022 14:46:08 +0100 Subject: [PATCH 082/179] Add fallback in requirements check, only check once (#11735) * Add fallback in requirements check, only check once * Rename to skip_requirements_check * Update spacy/cli/project/run.py Co-authored-by: Paul O'Leary McCann Co-authored-by: Paul O'Leary McCann --- spacy/cli/project/run.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/spacy/cli/project/run.py b/spacy/cli/project/run.py index ebab7471e..638e7fab1 100644 --- a/spacy/cli/project/run.py +++ b/spacy/cli/project/run.py @@ -53,6 +53,7 @@ def project_run( force: bool = False, dry: bool = False, capture: bool = False, + skip_requirements_check: bool = False, ) -> None: """Run a named script defined in the project.yml. If the script is part of the default pipeline (defined in the "run" section), DVC is used to @@ -69,6 +70,7 @@ def project_run( sys.exit will be called with the return code. You should use capture=False when you want to turn over execution to the command, and capture=True when you want to run the command more like a function. + skip_requirements_check (bool): Whether to skip the requirements check. """ config = load_project_config(project_dir, overrides=overrides) commands = {cmd["name"]: cmd for cmd in config.get("commands", [])} @@ -76,9 +78,10 @@ def project_run( validate_subcommand(list(commands.keys()), list(workflows.keys()), subcommand) req_path = project_dir / "requirements.txt" - if config.get("check_requirements", True) and os.path.exists(req_path): - with req_path.open() as requirements_file: - _check_requirements([req.replace("\n", "") for req in requirements_file]) + if not skip_requirements_check: + if config.get("check_requirements", True) and os.path.exists(req_path): + with req_path.open() as requirements_file: + _check_requirements([req.strip() for req in requirements_file]) if subcommand in workflows: msg.info(f"Running workflow '{subcommand}'") @@ -90,6 +93,7 @@ def project_run( force=force, dry=dry, capture=capture, + skip_requirements_check=True, ) else: cmd = commands[subcommand] @@ -338,6 +342,11 @@ def _check_requirements(requirements: List[str]) -> Tuple[bool, bool]: failed_pkgs_msgs.append(dnf.report()) except pkg_resources.VersionConflict as vc: conflicting_pkgs_msgs.append(vc.report()) + except Exception: + msg.warn(f"Unable to check requirement: {req} " + "Check that the requirement is formatted according to PEP " + "440, in particular that URLs are formatted as " + "'package_name @ URL'") if len(failed_pkgs_msgs) or len(conflicting_pkgs_msgs): msg.warn( From 2e3cfd758ea414497802843970666a18ed4d123e Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 8 Nov 2022 04:46:19 +0100 Subject: [PATCH 083/179] Use python 3.10 for GHA universe alert (#11768) --- .github/workflows/spacy_universe_alert.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/spacy_universe_alert.yml b/.github/workflows/spacy_universe_alert.yml index f507e0594..837aaeb33 100644 --- a/.github/workflows/spacy_universe_alert.yml +++ b/.github/workflows/spacy_universe_alert.yml @@ -19,6 +19,8 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 + with: + python-version: '3.10' - name: Install Bernadette app dependency and send an alert env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} From 20bbbe3e44f14d42a4861d1399ad98d6e1707d84 Mon Sep 17 00:00:00 2001 From: Raphael Mitsch Date: Tue, 8 Nov 2022 14:58:10 +0100 Subject: [PATCH 084/179] Revert disable/disabled merging behavior (#11745) * Merge disable with disabled. Adjust warnings, errors and tests. * Replace any() with set operation. * Update spacy/tests/pipeline/test_pipe_methods.py Co-authored-by: Adriane Boyd * Update docs. * Remve reference to config entry nlp.enabled from docs. Co-authored-by: Adriane Boyd --- spacy/errors.py | 4 +- spacy/language.py | 45 ++++++++----------- spacy/tests/pipeline/test_pipe_methods.py | 18 ++++---- .../serialize/test_serialize_pipeline.py | 7 ++- website/docs/api/language.md | 24 +++++----- website/docs/api/top-level.md | 20 ++++----- website/docs/usage/processing-pipelines.md | 3 +- 7 files changed, 56 insertions(+), 65 deletions(-) diff --git a/spacy/errors.py b/spacy/errors.py index 2f8a3996f..278e5496a 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -212,8 +212,8 @@ class Warnings(metaclass=ErrorsWithCodes): W121 = ("Attempting to trace non-existent method '{method}' in pipe '{pipe}'") W122 = ("Couldn't trace method '{method}' in pipe '{pipe}'. This can happen if the pipe class " "is a Cython extension type.") - W123 = ("Argument {arg} with value {arg_value} is used instead of {config_value} as specified in the config. Be " - "aware that this might affect other components in your pipeline.") + W123 = ("Argument `enable` with value {enable} does not contain all values specified in the config option " + "`enabled` ({enabled}). Be aware that this might affect other components in your pipeline.") class Errors(metaclass=ErrorsWithCodes): diff --git a/spacy/language.py b/spacy/language.py index d391f15ab..967af1e62 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -1879,31 +1879,22 @@ class Language: if isinstance(exclude, str): exclude = [exclude] - def fetch_pipes_status(value: Iterable[str], key: str) -> Iterable[str]: - """Fetch value for `enable` or `disable` w.r.t. the specified config and passed arguments passed to - .load(). If both arguments and config specified values for this field, the passed arguments take precedence - and a warning is printed. - value (Iterable[str]): Passed value for `enable` or `disable`. - key (str): Key for field in config (either "enabled" or "disabled"). - RETURN (Iterable[str]): - """ - # We assume that no argument was passed if the value is the specified default value. - if id(value) == id(_DEFAULT_EMPTY_PIPES): - return config["nlp"].get(key, []) - else: - if len(config["nlp"].get(key, [])): - warnings.warn( - Warnings.W123.format( - arg=key[:-1], - arg_value=value, - config_value=config["nlp"][key], - ) + # `enable` should not be merged with `enabled` (the opposite is true for `disable`/`disabled`). If the config + # specifies values for `enabled` not included in `enable`, emit warning. + if id(enable) != id(_DEFAULT_EMPTY_PIPES): + enabled = config["nlp"].get("enabled", []) + if len(enabled) and not set(enabled).issubset(enable): + warnings.warn( + Warnings.W123.format( + enable=enable, + enabled=enabled, ) - return value + ) + # Ensure sets of disabled/enabled pipe names are not contradictory. disabled_pipes = cls._resolve_component_status( - fetch_pipes_status(disable, "disabled"), - fetch_pipes_status(enable, "enabled"), + list({*disable, *config["nlp"].get("disabled", [])}), + enable, config["nlp"]["pipeline"], ) nlp._disabled = set(p for p in disabled_pipes if p not in exclude) @@ -2084,10 +2075,12 @@ class Language: if enable: if isinstance(enable, str): enable = [enable] - to_disable = [ - pipe_name for pipe_name in pipe_names if pipe_name not in enable - ] - if disable and disable != to_disable: + to_disable = { + *[pipe_name for pipe_name in pipe_names if pipe_name not in enable], + *disable, + } + # If any pipe to be enabled is in to_disable, the specification is inconsistent. + if len(set(enable) & to_disable): raise ValueError(Errors.E1042.format(enable=enable, disable=disable)) return tuple(to_disable) diff --git a/spacy/tests/pipeline/test_pipe_methods.py b/spacy/tests/pipeline/test_pipe_methods.py index 14a7a36e5..4dd7bae16 100644 --- a/spacy/tests/pipeline/test_pipe_methods.py +++ b/spacy/tests/pipeline/test_pipe_methods.py @@ -615,20 +615,18 @@ def test_enable_disable_conflict_with_config(): with make_tempdir() as tmp_dir: nlp.to_disk(tmp_dir) - # Expected to fail, as config and arguments conflict. - with pytest.raises(ValueError): - spacy.load( - tmp_dir, enable=["tagger"], config={"nlp": {"disabled": ["senter"]}} - ) + # Expected to succeed, as config and arguments do not conflict. + assert spacy.load( + tmp_dir, enable=["tagger"], config={"nlp": {"disabled": ["senter"]}} + ).disabled == ["senter", "sentencizer"] # Expected to succeed without warning due to the lack of a conflicting config option. spacy.load(tmp_dir, enable=["tagger"]) - # Expected to succeed with a warning, as disable=[] should override the config setting. - with pytest.warns(UserWarning): + # Expected to fail due to conflict between enable and disabled. + with pytest.raises(ValueError): spacy.load( tmp_dir, - enable=["tagger"], - disable=[], - config={"nlp": {"disabled": ["senter"]}}, + enable=["senter"], + config={"nlp": {"disabled": ["senter", "tagger"]}}, ) diff --git a/spacy/tests/serialize/test_serialize_pipeline.py b/spacy/tests/serialize/test_serialize_pipeline.py index b948bb76c..9fcf18e2d 100644 --- a/spacy/tests/serialize/test_serialize_pipeline.py +++ b/spacy/tests/serialize/test_serialize_pipeline.py @@ -404,11 +404,10 @@ def test_serialize_pipeline_disable_enable(): assert nlp3.component_names == ["ner", "tagger"] with make_tempdir() as d: nlp3.to_disk(d) - with pytest.warns(UserWarning): - nlp4 = spacy.load(d, disable=["ner"]) - assert nlp4.pipe_names == ["tagger"] + nlp4 = spacy.load(d, disable=["ner"]) + assert nlp4.pipe_names == [] assert nlp4.component_names == ["ner", "tagger"] - assert nlp4.disabled == ["ner"] + assert nlp4.disabled == ["ner", "tagger"] with make_tempdir() as d: nlp.to_disk(d) nlp5 = spacy.load(d, exclude=["tagger"]) diff --git a/website/docs/api/language.md b/website/docs/api/language.md index 767a7450a..504640d57 100644 --- a/website/docs/api/language.md +++ b/website/docs/api/language.md @@ -63,18 +63,18 @@ spaCy loads a model under the hood based on its > nlp = Language.from_config(config) > ``` -| Name | Description | -| ------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `config` | The loaded config. ~~Union[Dict[str, Any], Config]~~ | -| _keyword-only_ | | -| `vocab` | A `Vocab` object. If `True`, a vocab is created using the default language data settings. ~~Vocab~~ | -| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | -| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled, but can be enabled again using [`nlp.enable_pipe`](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | -| `exclude` | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | -| `meta` | [Meta data](/api/data-formats#meta) overrides. ~~Dict[str, Any]~~ | -| `auto_fill` | Whether to automatically fill in missing values in the config, based on defaults and function argument annotations. Defaults to `True`. ~~bool~~ | -| `validate` | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | -| **RETURNS** | The initialized object. ~~Language~~ | +| Name | Description | +| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `config` | The loaded config. ~~Union[Dict[str, Any], Config]~~ | +| _keyword-only_ | | +| `vocab` | A `Vocab` object. If `True`, a vocab is created using the default language data settings. ~~Vocab~~ | +| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). Is merged with the config entry `nlp.disabled`. ~~Union[str, Iterable[str]]~~ | +| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled, but can be enabled again using [nlp.enable_pipe](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `exclude` | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | +| `meta` | [Meta data](/api/data-formats#meta) overrides. ~~Dict[str, Any]~~ | +| `auto_fill` | Whether to automatically fill in missing values in the config, based on defaults and function argument annotations. Defaults to `True`. ~~bool~~ | +| `validate` | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | +| **RETURNS** | The initialized object. ~~Language~~ | ## Language.component {#component tag="classmethod" new="3"} diff --git a/website/docs/api/top-level.md b/website/docs/api/top-level.md index bc53fc868..c798f2a8d 100644 --- a/website/docs/api/top-level.md +++ b/website/docs/api/top-level.md @@ -45,16 +45,16 @@ specified separately using the new `exclude` keyword argument. > nlp = spacy.load("en_core_web_sm", exclude=["parser", "tagger"]) > ``` -| Name | Description | -| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `name` | Pipeline to load, i.e. package name or path. ~~Union[str, Path]~~ | -| _keyword-only_ | | -| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | -| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | -| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled. ~~Union[str, Iterable[str]]~~ | -| `exclude` 3 | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | -| `config` 3 | Optional config overrides, either as nested dict or dict keyed by section value in dot notation, e.g. `"components.name.value"`. ~~Union[Dict[str, Any], Config]~~ | -| **RETURNS** | A `Language` object with the loaded pipeline. ~~Language~~ | +| Name | Description | +| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Pipeline to load, i.e. package name or path. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | +| `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). Is merged with the config entry `nlp.disabled`. ~~Union[str, Iterable[str]]~~ | +| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled. ~~Union[str, Iterable[str]]~~ | +| `exclude` 3 | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | +| `config` 3 | Optional config overrides, either as nested dict or dict keyed by section value in dot notation, e.g. `"components.name.value"`. ~~Union[Dict[str, Any], Config]~~ | +| **RETURNS** | A `Language` object with the loaded pipeline. ~~Language~~ | Essentially, `spacy.load()` is a convenience wrapper that reads the pipeline's [`config.cfg`](/api/data-formats#config), uses the language and pipeline diff --git a/website/docs/usage/processing-pipelines.md b/website/docs/usage/processing-pipelines.md index bd28810ae..0b63cdcb8 100644 --- a/website/docs/usage/processing-pipelines.md +++ b/website/docs/usage/processing-pipelines.md @@ -363,7 +363,8 @@ nlp.enable_pipe("tagger") ``` In addition to `disable`, `spacy.load()` also accepts `enable`. If `enable` is -set, all components except for those in `enable` are disabled. +set, all components except for those in `enable` are disabled. If `enable` and +`disable` conflict (i.e. the same component is included in both), an error is raised. ```python # Load the complete pipeline, but disable all components except for tok2vec and tagger From 03eebe9d1c79d39a632876205e93f023fc096d85 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 9 Nov 2022 10:59:28 +0100 Subject: [PATCH 085/179] Update warning, add tests for project requirements check (#11777) * Update warning, add tests for project requirements check * Make warning more general for differences between PEP 508 and pip * Add tests for _check_requirements * Parameterize test --- spacy/cli/project/run.py | 5 ++--- spacy/tests/test_cli.py | 41 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/spacy/cli/project/run.py b/spacy/cli/project/run.py index 638e7fab1..5db9e14f4 100644 --- a/spacy/cli/project/run.py +++ b/spacy/cli/project/run.py @@ -344,9 +344,8 @@ def _check_requirements(requirements: List[str]) -> Tuple[bool, bool]: conflicting_pkgs_msgs.append(vc.report()) except Exception: msg.warn(f"Unable to check requirement: {req} " - "Check that the requirement is formatted according to PEP " - "440, in particular that URLs are formatted as " - "'package_name @ URL'") + "Checks are currently limited to requirement specifiers " + "(PEP 508)") if len(failed_pkgs_msgs) or len(conflicting_pkgs_msgs): msg.warn( diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py index 838e00369..8225e14f1 100644 --- a/spacy/tests/test_cli.py +++ b/spacy/tests/test_cli.py @@ -1,5 +1,6 @@ import os import math +import pkg_resources from random import sample from typing import Counter @@ -25,6 +26,7 @@ from spacy.cli.download import get_compatibility, get_version from spacy.cli.init_config import RECOMMENDATIONS, init_config, fill_config from spacy.cli.package import get_third_party_dependencies from spacy.cli.package import _is_permitted_package_name +from spacy.cli.project.run import _check_requirements from spacy.cli.validate import get_model_pkgs from spacy.lang.en import English from spacy.lang.nl import Dutch @@ -855,3 +857,42 @@ def test_span_length_freq_dist_output_must_be_correct(): span_freqs = _get_spans_length_freq_dist(sample_span_lengths, threshold) assert sum(span_freqs.values()) >= threshold assert list(span_freqs.keys()) == [3, 1, 4, 5, 2] + + +@pytest.mark.parametrize( + "reqs,output", + [ + [ + """ + spacy + + # comment + + thinc""", + (False, False), + ], + [ + """# comment + --some-flag + spacy""", + (False, False), + ], + [ + """# comment + --some-flag + spacy; python_version >= '3.6'""", + (False, False), + ], + [ + """# comment + spacyunknowndoesnotexist12345""", + (True, False), + ], + ], +) +def test_project_check_requirements(reqs, output): + # excessive guard against unlikely package name + try: + pkg_resources.require("spacyunknowndoesnotexist12345") + except pkg_resources.DistributionNotFound: + assert output == _check_requirements([req.strip() for req in reqs.split("\n")]) From 322b5dc1df7031139780963cebaa081a75384834 Mon Sep 17 00:00:00 2001 From: Jacobo Myerston <43222279+jmyerston@users.noreply.github.com> Date: Wed, 9 Nov 2022 20:21:20 -0800 Subject: [PATCH 086/179] Add greCy to Universe (#11774) * Update universe.json * Update universe.json fixes Github value --- website/meta/universe.json | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/website/meta/universe.json b/website/meta/universe.json index d7c99956b..fa765f640 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -1,5 +1,31 @@ { "resources": [ + { + "id": "grecy", + "title": "greCy", + "slogan": "Ancient Greek pipelines for spaCy", + "description": "greCy offers state-of-the-art pipelines for ancient Greek NLP. The repository makes language models available in various sizes, some of them containing floret word vectors and a BERT transformer layer.", + "github": "jmyerston/greCy", + "code_example": [ + "import spacy", + "#After installing the grc_ud_proiel_trf wheel package from the greCy repository", + "", + "nlp = spacy.load('grc_ud_proiel_trf')", + "doc = nlp('δοκῶ μοι περὶ ὧν πυνθάνεσθε οὐκ ἀμελέτητος εἶναι.')", + "", + "for token in doc:", + " print(token.text, token.norm_, token.lemma_, token.pos_, token.tag_)" + ], + "code_language": "python", + "author": "Jacobo Myerston", + "author_links": { + "twitter": "@jcbmyrstn", + "github": "jmyerston", + "website": "https://huggingface.co/spaces/Jacobo/syntax" + }, + "category": ["pipeline", "research"], + "tags": ["ancient Greek"] + }, { "id": "spacy-cleaner", "title": "spacy-cleaner", From 188a7d00eb552faaa70ba6ee3032757eecefbb5a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 11 Nov 2022 09:58:31 +0100 Subject: [PATCH 087/179] Auto-format code with black (#11792) Co-authored-by: explosion-bot --- spacy/cli/project/run.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/spacy/cli/project/run.py b/spacy/cli/project/run.py index 5db9e14f4..a109c4a5a 100644 --- a/spacy/cli/project/run.py +++ b/spacy/cli/project/run.py @@ -343,9 +343,11 @@ def _check_requirements(requirements: List[str]) -> Tuple[bool, bool]: except pkg_resources.VersionConflict as vc: conflicting_pkgs_msgs.append(vc.report()) except Exception: - msg.warn(f"Unable to check requirement: {req} " - "Checks are currently limited to requirement specifiers " - "(PEP 508)") + msg.warn( + f"Unable to check requirement: {req} " + "Checks are currently limited to requirement specifiers " + "(PEP 508)" + ) if len(failed_pkgs_msgs) or len(conflicting_pkgs_msgs): msg.warn( From 3478ff1eb0fd57c48a332e7787efe6ea47492e13 Mon Sep 17 00:00:00 2001 From: Edward <43848523+thomashacker@users.noreply.github.com> Date: Mon, 14 Nov 2022 09:41:01 +0100 Subject: [PATCH 088/179] remove new v2 tags (#11780) --- website/README.md | 4 +- website/docs/api/cli.md | 81 ++++++------ website/docs/api/doc.md | 50 ++++---- website/docs/api/language.md | 72 +++++------ website/docs/api/lexeme.md | 82 ++++++------ website/docs/api/matcher.md | 14 +-- website/docs/api/phrasematcher.md | 10 +- website/docs/api/span.md | 62 +++++----- website/docs/api/token.md | 144 +++++++++++----------- website/docs/api/top-level.md | 52 ++++---- website/docs/api/vocab.md | 22 ++-- website/docs/usage/rule-based-matching.md | 6 +- website/docs/usage/saving-loading.md | 12 +- 13 files changed, 305 insertions(+), 306 deletions(-) diff --git a/website/README.md b/website/README.md index db050cf03..66bc20ad9 100644 --- a/website/README.md +++ b/website/README.md @@ -155,7 +155,7 @@ import Tag from 'components/tag' > ```jsx > method -> 2.1 +> 4 > tagger, parser > ``` @@ -170,7 +170,7 @@ installed. -method 2 tagger, +method 4 tagger, parser diff --git a/website/docs/api/cli.md b/website/docs/api/cli.md index fc2c46022..024450920 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.md @@ -53,7 +53,7 @@ $ python -m spacy download [model] [--direct] [--sdist] [pip_args] | `--direct`, `-D` | Force direct download of exact package version. ~~bool (flag)~~ | | `--sdist`, `-S` 3 | Download the source package (`.tar.gz` archive) instead of the default pre-built binary wheel. ~~bool (flag)~~ | | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | -| pip args 2.1 | Additional installation options to be passed to `pip install` when installing the pipeline package. For example, `--user` to install to the user home directory or `--no-deps` to not install package dependencies. ~~Any (option/flag)~~ | +| pip args | Additional installation options to be passed to `pip install` when installing the pipeline package. For example, `--user` to install to the user home directory or `--no-deps` to not install package dependencies. ~~Any (option/flag)~~ | | **CREATES** | The installed pipeline package in your `site-packages` directory. | ## info {#info tag="command"} @@ -77,15 +77,15 @@ $ python -m spacy info [--markdown] [--silent] [--exclude] $ python -m spacy info [model] [--markdown] [--silent] [--exclude] ``` -| Name | Description | -| ------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------- | -| `model` | A trained pipeline, i.e. package name or path (optional). ~~Optional[str] \(option)~~ | -| `--markdown`, `-md` | Print information as Markdown. ~~bool (flag)~~ | -| `--silent`, `-s` 2.0.12 | Don't print anything, just return the values. ~~bool (flag)~~ | -| `--exclude`, `-e` | Comma-separated keys to exclude from the print-out. Defaults to `"labels"`. ~~Optional[str]~~ | -| `--url`, `-u` 3.5.0 | Print the URL to download the most recent compatible version of the pipeline. Requires a pipeline name. ~~bool (flag)~~ | -| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | -| **PRINTS** | Information about your spaCy installation. | +| Name | Description | +| -------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | +| `model` | A trained pipeline, i.e. package name or path (optional). ~~Optional[str] \(option)~~ | +| `--markdown`, `-md` | Print information as Markdown. ~~bool (flag)~~ | +| `--silent`, `-s` | Don't print anything, just return the values. ~~bool (flag)~~ | +| `--exclude`, `-e` | Comma-separated keys to exclude from the print-out. Defaults to `"labels"`. ~~Optional[str]~~ | +| `--url`, `-u` 3.5.0 | Print the URL to download the most recent compatible version of the pipeline. Requires a pipeline name. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **PRINTS** | Information about your spaCy installation. | ## validate {#validate new="2" tag="command"} @@ -260,22 +260,22 @@ chosen based on the file extension of the input file. $ python -m spacy convert [input_file] [output_dir] [--converter] [--file-type] [--n-sents] [--seg-sents] [--base] [--morphology] [--merge-subtokens] [--ner-map] [--lang] ``` -| Name | Description | -| ------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------- | -| `input_path` | Input file or directory. ~~Path (positional)~~ | -| `output_dir` | Output directory for converted file. Defaults to `"-"`, meaning data will be written to `stdout`. ~~Optional[Path] \(option)~~ | -| `--converter`, `-c` 2 | Name of converter to use (see below). ~~str (option)~~ | -| `--file-type`, `-t` 2.1 | Type of file to create. Either `spacy` (default) for binary [`DocBin`](/api/docbin) data or `json` for v2.x JSON format. ~~str (option)~~ | -| `--n-sents`, `-n` | Number of sentences per document. Supported for: `conll`, `conllu`, `iob`, `ner` ~~int (option)~~ | -| `--seg-sents`, `-s` 2.2 | Segment sentences. Supported for: `conll`, `ner` ~~bool (flag)~~ | -| `--base`, `-b`, `--model` | Trained spaCy pipeline for sentence segmentation to use as base (for `--seg-sents`). ~~Optional[str](option)~~ | -| `--morphology`, `-m` | Enable appending morphology to tags. Supported for: `conllu` ~~bool (flag)~~ | -| `--merge-subtokens`, `-T` | Merge CoNLL-U subtokens ~~bool (flag)~~ | -| `--ner-map`, `-nm` | NER tag mapping (as JSON-encoded dict of entity types). Supported for: `conllu` ~~Optional[Path](option)~~ | -| `--lang`, `-l` 2.1 | Language code (if tokenizer required). ~~Optional[str] \(option)~~ | -| `--concatenate`, `-C` | Concatenate output to a single file ~~bool (flag)~~ | -| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | -| **CREATES** | Binary [`DocBin`](/api/docbin) training data that can be used with [`spacy train`](/api/cli#train). | +| Name | Description | +| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `input_path` | Input file or directory. ~~Path (positional)~~ | +| `output_dir` | Output directory for converted file. Defaults to `"-"`, meaning data will be written to `stdout`. ~~Optional[Path] \(option)~~ | +| `--converter`, `-c` | Name of converter to use (see below). ~~str (option)~~ | +| `--file-type`, `-t` | Type of file to create. Either `spacy` (default) for binary [`DocBin`](/api/docbin) data or `json` for v2.x JSON format. ~~str (option)~~ | +| `--n-sents`, `-n` | Number of sentences per document. Supported for: `conll`, `conllu`, `iob`, `ner` ~~int (option)~~ | +| `--seg-sents`, `-s` | Segment sentences. Supported for: `conll`, `ner` ~~bool (flag)~~ | +| `--base`, `-b`, `--model` | Trained spaCy pipeline for sentence segmentation to use as base (for `--seg-sents`). ~~Optional[str](option)~~ | +| `--morphology`, `-m` | Enable appending morphology to tags. Supported for: `conllu` ~~bool (flag)~~ | +| `--merge-subtokens`, `-T` | Merge CoNLL-U subtokens ~~bool (flag)~~ | +| `--ner-map`, `-nm` | NER tag mapping (as JSON-encoded dict of entity types). Supported for: `conllu` ~~Optional[Path](option)~~ | +| `--lang`, `-l` | Language code (if tokenizer required). ~~Optional[str] \(option)~~ | +| `--concatenate`, `-C` | Concatenate output to a single file ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | Binary [`DocBin`](/api/docbin) training data that can be used with [`spacy train`](/api/cli#train). | ### Converters {#converters} @@ -474,8 +474,7 @@ report span characteristics such as the average span length and the span (or span boundary) distinctiveness. The distinctiveness measure shows how different the tokens are with respect to the rest of the corpus using the KL-divergence of the token distributions. To learn more, you can check out Papay et al.'s work on -[*Dissecting Span Identification Tasks with Performance Prediction* (EMNLP -2020)](https://aclanthology.org/2020.emnlp-main.396/). +[*Dissecting Span Identification Tasks with Performance Prediction* (EMNLP 2020)](https://aclanthology.org/2020.emnlp-main.396/). @@ -1229,19 +1228,19 @@ $ python -m spacy package [input_dir] [output_dir] [--code] [--meta-path] [--cre > $ pip install dist/en_pipeline-0.0.0.tar.gz > ``` -| Name | Description | -| ------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `input_dir` | Path to directory containing pipeline data. ~~Path (positional)~~ | -| `output_dir` | Directory to create package folder in. ~~Path (positional)~~ | -| `--code`, `-c` 3 | Comma-separated paths to Python files to be included in the package and imported in its `__init__.py`. This allows including [registering functions](/usage/training#custom-functions) and [custom components](/usage/processing-pipelines#custom-components). ~~str (option)~~ | -| `--meta-path`, `-m` 2 | Path to [`meta.json`](/api/data-formats#meta) file (optional). ~~Optional[Path] \(option)~~ | -| `--create-meta`, `-C` 2 | Create a `meta.json` file on the command line, even if one already exists in the directory. If an existing file is found, its entries will be shown as the defaults in the command line prompt. ~~bool (flag)~~ | -| `--build`, `-b` 3 | Comma-separated artifact formats to build. Can be `sdist` (for a `.tar.gz` archive) and/or `wheel` (for a binary `.whl` file), or `none` if you want to run this step manually. The generated artifacts can be installed by `pip install`. Defaults to `sdist`. ~~str (option)~~ | -| `--name`, `-n` 3 | Package name to override in meta. ~~Optional[str] \(option)~~ | -| `--version`, `-v` 3 | Package version to override in meta. Useful when training new versions, as it doesn't require editing the meta template. ~~Optional[str] \(option)~~ | -| `--force`, `-f` | Force overwriting of existing folder in output directory. ~~bool (flag)~~ | -| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | -| **CREATES** | A Python package containing the spaCy pipeline. | +| Name | Description | +| -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `input_dir` | Path to directory containing pipeline data. ~~Path (positional)~~ | +| `output_dir` | Directory to create package folder in. ~~Path (positional)~~ | +| `--code`, `-c` 3 | Comma-separated paths to Python files to be included in the package and imported in its `__init__.py`. This allows including [registering functions](/usage/training#custom-functions) and [custom components](/usage/processing-pipelines#custom-components). ~~str (option)~~ | +| `--meta-path`, `-m` | Path to [`meta.json`](/api/data-formats#meta) file (optional). ~~Optional[Path] \(option)~~ | +| `--create-meta`, `-C` | Create a `meta.json` file on the command line, even if one already exists in the directory. If an existing file is found, its entries will be shown as the defaults in the command line prompt. ~~bool (flag)~~ | +| `--build`, `-b` 3 | Comma-separated artifact formats to build. Can be `sdist` (for a `.tar.gz` archive) and/or `wheel` (for a binary `.whl` file), or `none` if you want to run this step manually. The generated artifacts can be installed by `pip install`. Defaults to `sdist`. ~~str (option)~~ | +| `--name`, `-n` 3 | Package name to override in meta. ~~Optional[str] \(option)~~ | +| `--version`, `-v` 3 | Package version to override in meta. Useful when training new versions, as it doesn't require editing the meta template. ~~Optional[str] \(option)~~ | +| `--force`, `-f` | Force overwriting of existing folder in output directory. ~~bool (flag)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | A Python package containing the spaCy pipeline. | ## project {#project new="3"} diff --git a/website/docs/api/doc.md b/website/docs/api/doc.md index f97ed4547..090489d83 100644 --- a/website/docs/api/doc.md +++ b/website/docs/api/doc.md @@ -209,15 +209,15 @@ alignment mode `"strict". > assert span.text == "New York" > ``` -| Name | Description | -| ------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `start` | The index of the first character of the span. ~~int~~ | -| `end` | The index of the last character after the span. ~~int~~ | -| `label` | A label to attach to the span, e.g. for named entities. ~~Union[int, str]~~ | -| `kb_id` 2.2 | An ID from a knowledge base to capture the meaning of a named entity. ~~Union[int, str]~~ | -| `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -| `alignment_mode` | How character indices snap to token boundaries. Options: `"strict"` (no snapping), `"contract"` (span of all tokens completely within the character span), `"expand"` (span of all tokens at least partially covered by the character span). Defaults to `"strict"`. ~~str~~ | -| **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ | +| Name | Description | +| ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `start` | The index of the first character of the span. ~~int~~ | +| `end` | The index of the last character after the span. ~~int~~ | +| `label` | A label to attach to the span, e.g. for named entities. ~~Union[int, str]~~ | +| `kb_id` | An ID from a knowledge base to capture the meaning of a named entity. ~~Union[int, str]~~ | +| `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | +| `alignment_mode` | How character indices snap to token boundaries. Options: `"strict"` (no snapping), `"contract"` (span of all tokens completely within the character span), `"expand"` (span of all tokens at least partially covered by the character span). Defaults to `"strict"`. ~~str~~ | +| **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ | ## Doc.set_ents {#set_ents tag="method" new="3"} @@ -751,22 +751,22 @@ The L2 norm of the document's vector representation. ## Attributes {#attributes} -| Name | Description | -| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------- | -| `text` | A string representation of the document text. ~~str~~ | -| `text_with_ws` | An alias of `Doc.text`, provided for duck-type compatibility with `Span` and `Token`. ~~str~~ | -| `mem` | The document's local memory heap, for all C data it owns. ~~cymem.Pool~~ | -| `vocab` | The store of lexical types. ~~Vocab~~ | -| `tensor` 2 | Container for dense vector representations. ~~numpy.ndarray~~ | -| `user_data` | A generic storage area, for user custom data. ~~Dict[str, Any]~~ | -| `lang` 2.1 | Language of the document's vocabulary. ~~int~~ | -| `lang_` 2.1 | Language of the document's vocabulary. ~~str~~ | -| `sentiment` | The document's positivity/negativity score, if available. ~~float~~ | -| `user_hooks` | A dictionary that allows customization of the `Doc`'s properties. ~~Dict[str, Callable]~~ | -| `user_token_hooks` | A dictionary that allows customization of properties of `Token` children. ~~Dict[str, Callable]~~ | -| `user_span_hooks` | A dictionary that allows customization of properties of `Span` children. ~~Dict[str, Callable]~~ | -| `has_unknown_spaces` | Whether the document was constructed without known spacing between tokens (typically when created from gold tokenization). ~~bool~~ | -| `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ | +| Name | Description | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| `text` | A string representation of the document text. ~~str~~ | +| `text_with_ws` | An alias of `Doc.text`, provided for duck-type compatibility with `Span` and `Token`. ~~str~~ | +| `mem` | The document's local memory heap, for all C data it owns. ~~cymem.Pool~~ | +| `vocab` | The store of lexical types. ~~Vocab~~ | +| `tensor` | Container for dense vector representations. ~~numpy.ndarray~~ | +| `user_data` | A generic storage area, for user custom data. ~~Dict[str, Any]~~ | +| `lang` | Language of the document's vocabulary. ~~int~~ | +| `lang_` | Language of the document's vocabulary. ~~str~~ | +| `sentiment` | The document's positivity/negativity score, if available. ~~float~~ | +| `user_hooks` | A dictionary that allows customization of the `Doc`'s properties. ~~Dict[str, Callable]~~ | +| `user_token_hooks` | A dictionary that allows customization of properties of `Token` children. ~~Dict[str, Callable]~~ | +| `user_span_hooks` | A dictionary that allows customization of properties of `Span` children. ~~Dict[str, Callable]~~ | +| `has_unknown_spaces` | Whether the document was constructed without known spacing between tokens (typically when created from gold tokenization). ~~bool~~ | +| `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ | ## Serialization fields {#serialization-fields} diff --git a/website/docs/api/language.md b/website/docs/api/language.md index 504640d57..ad0ac2a46 100644 --- a/website/docs/api/language.md +++ b/website/docs/api/language.md @@ -63,18 +63,18 @@ spaCy loads a model under the hood based on its > nlp = Language.from_config(config) > ``` -| Name | Description | -| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `config` | The loaded config. ~~Union[Dict[str, Any], Config]~~ | -| _keyword-only_ | | -| `vocab` | A `Vocab` object. If `True`, a vocab is created using the default language data settings. ~~Vocab~~ | +| Name | Description | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `config` | The loaded config. ~~Union[Dict[str, Any], Config]~~ | +| _keyword-only_ | | +| `vocab` | A `Vocab` object. If `True`, a vocab is created using the default language data settings. ~~Vocab~~ | | `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). Is merged with the config entry `nlp.disabled`. ~~Union[str, Iterable[str]]~~ | -| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled, but can be enabled again using [nlp.enable_pipe](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | -| `exclude` | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | -| `meta` | [Meta data](/api/data-formats#meta) overrides. ~~Dict[str, Any]~~ | -| `auto_fill` | Whether to automatically fill in missing values in the config, based on defaults and function argument annotations. Defaults to `True`. ~~bool~~ | -| `validate` | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | -| **RETURNS** | The initialized object. ~~Language~~ | +| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled, but can be enabled again using [nlp.enable_pipe](/api/language#enable_pipe). ~~Union[str, Iterable[str]]~~ | +| `exclude` | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | +| `meta` | [Meta data](/api/data-formats#meta) overrides. ~~Dict[str, Any]~~ | +| `auto_fill` | Whether to automatically fill in missing values in the config, based on defaults and function argument annotations. Defaults to `True`. ~~bool~~ | +| `validate` | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | +| **RETURNS** | The initialized object. ~~Language~~ | ## Language.component {#component tag="classmethod" new="3"} @@ -198,16 +198,16 @@ tokenization is skipped but the rest of the pipeline is run. > assert doc.has_annotation("DEP") > ``` -| Name | Description | -| ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `texts` | A sequence of strings (or `Doc` objects). ~~Iterable[Union[str, Doc]]~~ | -| _keyword-only_ | | -| `as_tuples` | If set to `True`, inputs should be a sequence of `(text, context)` tuples. Output will then be a sequence of `(doc, context)` tuples. Defaults to `False`. ~~bool~~ | -| `batch_size` | The number of texts to buffer. ~~Optional[int]~~ | -| `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). ~~List[str]~~ | -| `component_cfg` | Optional dictionary of keyword arguments for components, keyed by component names. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Any]]]~~ | -| `n_process` 2.2.2 | Number of processors to use. Defaults to `1`. ~~int~~ | -| **YIELDS** | Documents in the order of the original text. ~~Doc~~ | +| Name | Description | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `texts` | A sequence of strings (or `Doc` objects). ~~Iterable[Union[str, Doc]]~~ | +| _keyword-only_ | | +| `as_tuples` | If set to `True`, inputs should be a sequence of `(text, context)` tuples. Output will then be a sequence of `(doc, context)` tuples. Defaults to `False`. ~~bool~~ | +| `batch_size` | The number of texts to buffer. ~~Optional[int]~~ | +| `disable` | Names of pipeline components to [disable](/usage/processing-pipelines#disabling). ~~List[str]~~ | +| `component_cfg` | Optional dictionary of keyword arguments for components, keyed by component names. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Any]]]~~ | +| `n_process` | Number of processors to use. Defaults to `1`. ~~int~~ | +| **YIELDS** | Documents in the order of the original text. ~~Doc~~ | ## Language.set_error_handler {#set_error_handler tag="method" new="3"} @@ -1030,21 +1030,21 @@ details. ## Attributes {#attributes} -| Name | Description | -| --------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| `vocab` | A container for the lexical types. ~~Vocab~~ | -| `tokenizer` | The tokenizer. ~~Tokenizer~~ | -| `make_doc` | Callable that takes a string and returns a `Doc`. ~~Callable[[str], Doc]~~ | -| `pipeline` | List of `(name, component)` tuples describing the current processing pipeline, in order. ~~List[Tuple[str, Callable[[Doc], Doc]]]~~ | -| `pipe_names` 2 | List of pipeline component names, in order. ~~List[str]~~ | -| `pipe_labels` 2.2 | List of labels set by the pipeline components, if available, keyed by component name. ~~Dict[str, List[str]]~~ | -| `pipe_factories` 2.2 | Dictionary of pipeline component names, mapped to their factory names. ~~Dict[str, str]~~ | -| `factories` | All available factory functions, keyed by name. ~~Dict[str, Callable[[...], Callable[[Doc], Doc]]]~~ | -| `factory_names` 3 | List of all available factory names. ~~List[str]~~ | -| `components` 3 | List of all available `(name, component)` tuples, including components that are currently disabled. ~~List[Tuple[str, Callable[[Doc], Doc]]]~~ | -| `component_names` 3 | List of all available component names, including components that are currently disabled. ~~List[str]~~ | -| `disabled` 3 | Names of components that are currently disabled and don't run as part of the pipeline. ~~List[str]~~ | -| `path` 2 | Path to the pipeline data directory, if a pipeline is loaded from a path or package. Otherwise `None`. ~~Optional[Path]~~ | +| Name | Description | +| -------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | A container for the lexical types. ~~Vocab~~ | +| `tokenizer` | The tokenizer. ~~Tokenizer~~ | +| `make_doc` | Callable that takes a string and returns a `Doc`. ~~Callable[[str], Doc]~~ | +| `pipeline` | List of `(name, component)` tuples describing the current processing pipeline, in order. ~~List[Tuple[str, Callable[[Doc], Doc]]]~~ | +| `pipe_names` | List of pipeline component names, in order. ~~List[str]~~ | +| `pipe_labels` | List of labels set by the pipeline components, if available, keyed by component name. ~~Dict[str, List[str]]~~ | +| `pipe_factories` | Dictionary of pipeline component names, mapped to their factory names. ~~Dict[str, str]~~ | +| `factories` | All available factory functions, keyed by name. ~~Dict[str, Callable[[...], Callable[[Doc], Doc]]]~~ | +| `factory_names` 3 | List of all available factory names. ~~List[str]~~ | +| `components` 3 | List of all available `(name, component)` tuples, including components that are currently disabled. ~~List[Tuple[str, Callable[[Doc], Doc]]]~~ | +| `component_names` 3 | List of all available component names, including components that are currently disabled. ~~List[str]~~ | +| `disabled` 3 | Names of components that are currently disabled and don't run as part of the pipeline. ~~List[str]~~ | +| `path` | Path to the pipeline data directory, if a pipeline is loaded from a path or package. Otherwise `None`. ~~Optional[Path]~~ | ## Class attributes {#class-attributes} diff --git a/website/docs/api/lexeme.md b/website/docs/api/lexeme.md index c5d4b7544..eb76afa90 100644 --- a/website/docs/api/lexeme.md +++ b/website/docs/api/lexeme.md @@ -121,44 +121,44 @@ The L2 norm of the lexeme's vector representation. ## Attributes {#attributes} -| Name | Description | -| -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `vocab` | The lexeme's vocabulary. ~~Vocab~~ | -| `text` | Verbatim text content. ~~str~~ | -| `orth` | ID of the verbatim text content. ~~int~~ | -| `orth_` | Verbatim text content (identical to `Lexeme.text`). Exists mostly for consistency with the other attributes. ~~str~~ | -| `rank` | Sequential ID of the lexeme's lexical type, used to index into tables, e.g. for word vectors. ~~int~~ | -| `flags` | Container of the lexeme's binary flags. ~~int~~ | -| `norm` | The lexeme's norm, i.e. a normalized form of the lexeme text. ~~int~~ | -| `norm_` | The lexeme's norm, i.e. a normalized form of the lexeme text. ~~str~~ | -| `lower` | Lowercase form of the word. ~~int~~ | -| `lower_` | Lowercase form of the word. ~~str~~ | -| `shape` | Transform of the word's string, to show orthographic features. Alphabetic characters are replaced by `x` or `X`, and numeric characters are replaced by `d`, and sequences of the same character are truncated after length 4. For example,`"Xxxx"`or`"dd"`. ~~int~~ | -| `shape_` | Transform of the word's string, to show orthographic features. Alphabetic characters are replaced by `x` or `X`, and numeric characters are replaced by `d`, and sequences of the same character are truncated after length 4. For example,`"Xxxx"`or`"dd"`. ~~str~~ | -| `prefix` | Length-N substring from the start of the word. Defaults to `N=1`. ~~int~~ | -| `prefix_` | Length-N substring from the start of the word. Defaults to `N=1`. ~~str~~ | -| `suffix` | Length-N substring from the end of the word. Defaults to `N=3`. ~~int~~ | -| `suffix_` | Length-N substring from the start of the word. Defaults to `N=3`. ~~str~~ | -| `is_alpha` | Does the lexeme consist of alphabetic characters? Equivalent to `lexeme.text.isalpha()`. ~~bool~~ | -| `is_ascii` | Does the lexeme consist of ASCII characters? Equivalent to `[any(ord(c) >= 128 for c in lexeme.text)]`. ~~bool~~ | -| `is_digit` | Does the lexeme consist of digits? Equivalent to `lexeme.text.isdigit()`. ~~bool~~ | -| `is_lower` | Is the lexeme in lowercase? Equivalent to `lexeme.text.islower()`. ~~bool~~ | -| `is_upper` | Is the lexeme in uppercase? Equivalent to `lexeme.text.isupper()`. ~~bool~~ | -| `is_title` | Is the lexeme in titlecase? Equivalent to `lexeme.text.istitle()`. ~~bool~~ | -| `is_punct` | Is the lexeme punctuation? ~~bool~~ | -| `is_left_punct` | Is the lexeme a left punctuation mark, e.g. `(`? ~~bool~~ | -| `is_right_punct` | Is the lexeme a right punctuation mark, e.g. `)`? ~~bool~~ | -| `is_space` | Does the lexeme consist of whitespace characters? Equivalent to `lexeme.text.isspace()`. ~~bool~~ | -| `is_bracket` | Is the lexeme a bracket? ~~bool~~ | -| `is_quote` | Is the lexeme a quotation mark? ~~bool~~ | -| `is_currency` 2.0.8 | Is the lexeme a currency symbol? ~~bool~~ | -| `like_url` | Does the lexeme resemble a URL? ~~bool~~ | -| `like_num` | Does the lexeme represent a number? e.g. "10.9", "10", "ten", etc. ~~bool~~ | -| `like_email` | Does the lexeme resemble an email address? ~~bool~~ | -| `is_oov` | Is the lexeme out-of-vocabulary (i.e. does it not have a word vector)? ~~bool~~ | -| `is_stop` | Is the lexeme part of a "stop list"? ~~bool~~ | -| `lang` | Language of the parent vocabulary. ~~int~~ | -| `lang_` | Language of the parent vocabulary. ~~str~~ | -| `prob` | Smoothed log probability estimate of the lexeme's word type (context-independent entry in the vocabulary). ~~float~~ | -| `cluster` | Brown cluster ID. ~~int~~ | -| `sentiment` | A scalar value indicating the positivity or negativity of the lexeme. ~~float~~ | +| Name | Description | +| ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `vocab` | The lexeme's vocabulary. ~~Vocab~~ | +| `text` | Verbatim text content. ~~str~~ | +| `orth` | ID of the verbatim text content. ~~int~~ | +| `orth_` | Verbatim text content (identical to `Lexeme.text`). Exists mostly for consistency with the other attributes. ~~str~~ | +| `rank` | Sequential ID of the lexeme's lexical type, used to index into tables, e.g. for word vectors. ~~int~~ | +| `flags` | Container of the lexeme's binary flags. ~~int~~ | +| `norm` | The lexeme's norm, i.e. a normalized form of the lexeme text. ~~int~~ | +| `norm_` | The lexeme's norm, i.e. a normalized form of the lexeme text. ~~str~~ | +| `lower` | Lowercase form of the word. ~~int~~ | +| `lower_` | Lowercase form of the word. ~~str~~ | +| `shape` | Transform of the word's string, to show orthographic features. Alphabetic characters are replaced by `x` or `X`, and numeric characters are replaced by `d`, and sequences of the same character are truncated after length 4. For example,`"Xxxx"`or`"dd"`. ~~int~~ | +| `shape_` | Transform of the word's string, to show orthographic features. Alphabetic characters are replaced by `x` or `X`, and numeric characters are replaced by `d`, and sequences of the same character are truncated after length 4. For example,`"Xxxx"`or`"dd"`. ~~str~~ | +| `prefix` | Length-N substring from the start of the word. Defaults to `N=1`. ~~int~~ | +| `prefix_` | Length-N substring from the start of the word. Defaults to `N=1`. ~~str~~ | +| `suffix` | Length-N substring from the end of the word. Defaults to `N=3`. ~~int~~ | +| `suffix_` | Length-N substring from the start of the word. Defaults to `N=3`. ~~str~~ | +| `is_alpha` | Does the lexeme consist of alphabetic characters? Equivalent to `lexeme.text.isalpha()`. ~~bool~~ | +| `is_ascii` | Does the lexeme consist of ASCII characters? Equivalent to `[any(ord(c) >= 128 for c in lexeme.text)]`. ~~bool~~ | +| `is_digit` | Does the lexeme consist of digits? Equivalent to `lexeme.text.isdigit()`. ~~bool~~ | +| `is_lower` | Is the lexeme in lowercase? Equivalent to `lexeme.text.islower()`. ~~bool~~ | +| `is_upper` | Is the lexeme in uppercase? Equivalent to `lexeme.text.isupper()`. ~~bool~~ | +| `is_title` | Is the lexeme in titlecase? Equivalent to `lexeme.text.istitle()`. ~~bool~~ | +| `is_punct` | Is the lexeme punctuation? ~~bool~~ | +| `is_left_punct` | Is the lexeme a left punctuation mark, e.g. `(`? ~~bool~~ | +| `is_right_punct` | Is the lexeme a right punctuation mark, e.g. `)`? ~~bool~~ | +| `is_space` | Does the lexeme consist of whitespace characters? Equivalent to `lexeme.text.isspace()`. ~~bool~~ | +| `is_bracket` | Is the lexeme a bracket? ~~bool~~ | +| `is_quote` | Is the lexeme a quotation mark? ~~bool~~ | +| `is_currency` | Is the lexeme a currency symbol? ~~bool~~ | +| `like_url` | Does the lexeme resemble a URL? ~~bool~~ | +| `like_num` | Does the lexeme represent a number? e.g. "10.9", "10", "ten", etc. ~~bool~~ | +| `like_email` | Does the lexeme resemble an email address? ~~bool~~ | +| `is_oov` | Is the lexeme out-of-vocabulary (i.e. does it not have a word vector)? ~~bool~~ | +| `is_stop` | Is the lexeme part of a "stop list"? ~~bool~~ | +| `lang` | Language of the parent vocabulary. ~~int~~ | +| `lang_` | Language of the parent vocabulary. ~~str~~ | +| `prob` | Smoothed log probability estimate of the lexeme's word type (context-independent entry in the vocabulary). ~~float~~ | +| `cluster` | Brown cluster ID. ~~int~~ | +| `sentiment` | A scalar value indicating the positivity or negativity of the lexeme. ~~float~~ | diff --git a/website/docs/api/matcher.md b/website/docs/api/matcher.md index 8cc446c6a..cd7bfa070 100644 --- a/website/docs/api/matcher.md +++ b/website/docs/api/matcher.md @@ -33,7 +33,7 @@ rule-based matching are: | Attribute | Description | | ---------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | | `ORTH` | The exact verbatim text of a token. ~~str~~ | -| `TEXT` 2.1 | The exact verbatim text of a token. ~~str~~ | +| `TEXT` | The exact verbatim text of a token. ~~str~~ | | `NORM` | The normalized form of the token text. ~~str~~ | | `LOWER` | The lowercase form of the token text. ~~str~~ | | `LENGTH` | The length of the token text. ~~int~~ | @@ -48,7 +48,7 @@ rule-based matching are: | `ENT_IOB` | The IOB part of the token's entity tag. ~~str~~ | | `ENT_ID` | The token's entity ID (`ent_id`). ~~str~~ | | `ENT_KB_ID` | The token's entity knowledge base ID (`ent_kb_id`). ~~str~~ | -| `_` 2.1 | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ | +| `_` | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ | | `OP` | Operator or quantifier to determine how often to match a token pattern. ~~str~~ | Operators and quantifiers define **how often** a token pattern should be @@ -64,7 +64,7 @@ matched: > ``` | OP | Description | -|---------|------------------------------------------------------------------------| +| ------- | ---------------------------------------------------------------------- | | `!` | Negate the pattern, by requiring it to match exactly 0 times. | | `?` | Make the pattern optional, by allowing it to match 0 or 1 times. | | `+` | Require the pattern to match 1 or more times. | @@ -109,10 +109,10 @@ string where an integer is expected) or unexpected property names. > matcher = Matcher(nlp.vocab) > ``` -| Name | Description | -| --------------------------------------- | ----------------------------------------------------------------------------------------------------- | -| `vocab` | The vocabulary object, which must be shared with the documents the matcher will operate on. ~~Vocab~~ | -| `validate` 2.1 | Validate all patterns added to this matcher. ~~bool~~ | +| Name | Description | +| ---------- | ----------------------------------------------------------------------------------------------------- | +| `vocab` | The vocabulary object, which must be shared with the documents the matcher will operate on. ~~Vocab~~ | +| `validate` | Validate all patterns added to this matcher. ~~bool~~ | ## Matcher.\_\_call\_\_ {#call tag="method"} diff --git a/website/docs/api/phrasematcher.md b/website/docs/api/phrasematcher.md index 2cef9ac2a..cd419ae5c 100644 --- a/website/docs/api/phrasematcher.md +++ b/website/docs/api/phrasematcher.md @@ -36,11 +36,11 @@ be shown. > matcher = PhraseMatcher(nlp.vocab) > ``` -| Name | Description | -| --------------------------------------- | ------------------------------------------------------------------------------------------------------ | -| `vocab` | The vocabulary object, which must be shared with the documents the matcher will operate on. ~~Vocab~~ | -| `attr` 2.1 | The token attribute to match on. Defaults to `ORTH`, i.e. the verbatim token text. ~~Union[int, str]~~ | -| `validate` 2.1 | Validate patterns added to the matcher. ~~bool~~ | +| Name | Description | +| ---------- | ------------------------------------------------------------------------------------------------------ | +| `vocab` | The vocabulary object, which must be shared with the documents the matcher will operate on. ~~Vocab~~ | +| `attr` | The token attribute to match on. Defaults to `ORTH`, i.e. the verbatim token text. ~~Union[int, str]~~ | +| `validate` | Validate patterns added to the matcher. ~~bool~~ | ## PhraseMatcher.\_\_call\_\_ {#call tag="method"} diff --git a/website/docs/api/span.md b/website/docs/api/span.md index 89f608994..69bbe8db1 100644 --- a/website/docs/api/span.md +++ b/website/docs/api/span.md @@ -186,14 +186,14 @@ the character indices don't map to a valid span. > assert span.text == "New York" > ``` -| Name | Description | -| ------------------------------------ | ----------------------------------------------------------------------------------------- | -| `start` | The index of the first character of the span. ~~int~~ | -| `end` | The index of the last character after the span. ~~int~~ | -| `label` | A label to attach to the span, e.g. for named entities. ~~Union[int, str]~~ | -| `kb_id` 2.2 | An ID from a knowledge base to capture the meaning of a named entity. ~~Union[int, str]~~ | -| `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -| **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ | +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------- | +| `start` | The index of the first character of the span. ~~int~~ | +| `end` | The index of the last character after the span. ~~int~~ | +| `label` | A label to attach to the span, e.g. for named entities. ~~Union[int, str]~~ | +| `kb_id` | An ID from a knowledge base to capture the meaning of a named entity. ~~Union[int, str]~~ | +| `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | +| **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ | ## Span.similarity {#similarity tag="method" model="vectors"} @@ -544,26 +544,26 @@ overlaps with will be returned. ## Attributes {#attributes} -| Name | Description | -| --------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | -| `doc` | The parent document. ~~Doc~~ | -| `tensor` 2.1.7 | The span's slice of the parent `Doc`'s tensor. ~~numpy.ndarray~~ | -| `start` | The token offset for the start of the span. ~~int~~ | -| `end` | The token offset for the end of the span. ~~int~~ | -| `start_char` | The character offset for the start of the span. ~~int~~ | -| `end_char` | The character offset for the end of the span. ~~int~~ | -| `text` | A string representation of the span text. ~~str~~ | -| `text_with_ws` | The text content of the span with a trailing whitespace character if the last token has one. ~~str~~ | -| `orth` | ID of the verbatim text content. ~~int~~ | -| `orth_` | Verbatim text content (identical to `Span.text`). Exists mostly for consistency with the other attributes. ~~str~~ | -| `label` | The hash value of the span's label. ~~int~~ | -| `label_` | The span's label. ~~str~~ | -| `lemma_` | The span's lemma. Equivalent to `"".join(token.text_with_ws for token in span)`. ~~str~~ | -| `kb_id` | The hash value of the knowledge base ID referred to by the span. ~~int~~ | -| `kb_id_` | The knowledge base ID referred to by the span. ~~str~~ | -| `ent_id` | The hash value of the named entity the root token is an instance of. ~~int~~ | -| `ent_id_` | The string ID of the named entity the root token is an instance of. ~~str~~ | -| `id` | The hash value of the span's ID. ~~int~~ | -| `id_` | The span's ID. ~~str~~ | -| `sentiment` | A scalar value indicating the positivity or negativity of the span. ~~float~~ | -| `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ | +| Name | Description | +| -------------- | ----------------------------------------------------------------------------------------------------------------------------- | +| `doc` | The parent document. ~~Doc~~ | +| `tensor` | The span's slice of the parent `Doc`'s tensor. ~~numpy.ndarray~~ | +| `start` | The token offset for the start of the span. ~~int~~ | +| `end` | The token offset for the end of the span. ~~int~~ | +| `start_char` | The character offset for the start of the span. ~~int~~ | +| `end_char` | The character offset for the end of the span. ~~int~~ | +| `text` | A string representation of the span text. ~~str~~ | +| `text_with_ws` | The text content of the span with a trailing whitespace character if the last token has one. ~~str~~ | +| `orth` | ID of the verbatim text content. ~~int~~ | +| `orth_` | Verbatim text content (identical to `Span.text`). Exists mostly for consistency with the other attributes. ~~str~~ | +| `label` | The hash value of the span's label. ~~int~~ | +| `label_` | The span's label. ~~str~~ | +| `lemma_` | The span's lemma. Equivalent to `"".join(token.text_with_ws for token in span)`. ~~str~~ | +| `kb_id` | The hash value of the knowledge base ID referred to by the span. ~~int~~ | +| `kb_id_` | The knowledge base ID referred to by the span. ~~str~~ | +| `ent_id` | The hash value of the named entity the root token is an instance of. ~~int~~ | +| `ent_id_` | The string ID of the named entity the root token is an instance of. ~~str~~ | +| `id` | The hash value of the span's ID. ~~int~~ | +| `id_` | The span's ID. ~~str~~ | +| `sentiment` | A scalar value indicating the positivity or negativity of the span. ~~float~~ | +| `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ | diff --git a/website/docs/api/token.md b/website/docs/api/token.md index d43cd3ff1..89bd77447 100644 --- a/website/docs/api/token.md +++ b/website/docs/api/token.md @@ -403,75 +403,75 @@ The L2 norm of the token's vector representation. ## Attributes {#attributes} -| Name | Description | -| -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `doc` | The parent document. ~~Doc~~ | -| `lex` 3 | The underlying lexeme. ~~Lexeme~~ | -| `sent` 2.0.12 | The sentence span that this token is a part of. ~~Span~~ | -| `text` | Verbatim text content. ~~str~~ | -| `text_with_ws` | Text content, with trailing space character if present. ~~str~~ | -| `whitespace_` | Trailing space character if present. ~~str~~ | -| `orth` | ID of the verbatim text content. ~~int~~ | -| `orth_` | Verbatim text content (identical to `Token.text`). Exists mostly for consistency with the other attributes. ~~str~~ | -| `vocab` | The vocab object of the parent `Doc`. ~~vocab~~ | -| `tensor` 2.1.7 | The token's slice of the parent `Doc`'s tensor. ~~numpy.ndarray~~ | -| `head` | The syntactic parent, or "governor", of this token. ~~Token~~ | -| `left_edge` | The leftmost token of this token's syntactic descendants. ~~Token~~ | -| `right_edge` | The rightmost token of this token's syntactic descendants. ~~Token~~ | -| `i` | The index of the token within the parent document. ~~int~~ | -| `ent_type` | Named entity type. ~~int~~ | -| `ent_type_` | Named entity type. ~~str~~ | -| `ent_iob` | IOB code of named entity tag. `3` means the token begins an entity, `2` means it is outside an entity, `1` means it is inside an entity, and `0` means no entity tag is set. ~~int~~ | -| `ent_iob_` | IOB code of named entity tag. "B" means the token begins an entity, "I" means it is inside an entity, "O" means it is outside an entity, and "" means no entity tag is set. ~~str~~ | -| `ent_kb_id` 2.2 | Knowledge base ID that refers to the named entity this token is a part of, if any. ~~int~~ | -| `ent_kb_id_` 2.2 | Knowledge base ID that refers to the named entity this token is a part of, if any. ~~str~~ | -| `ent_id` | ID of the entity the token is an instance of, if any. Currently not used, but potentially for coreference resolution. ~~int~~ | -| `ent_id_` | ID of the entity the token is an instance of, if any. Currently not used, but potentially for coreference resolution. ~~str~~ | -| `lemma` | Base form of the token, with no inflectional suffixes. ~~int~~ | -| `lemma_` | Base form of the token, with no inflectional suffixes. ~~str~~ | -| `norm` | The token's norm, i.e. a normalized form of the token text. Can be set in the language's [tokenizer exceptions](/usage/linguistic-features#language-data). ~~int~~ | -| `norm_` | The token's norm, i.e. a normalized form of the token text. Can be set in the language's [tokenizer exceptions](/usage/linguistic-features#language-data). ~~str~~ | -| `lower` | Lowercase form of the token. ~~int~~ | -| `lower_` | Lowercase form of the token text. Equivalent to `Token.text.lower()`. ~~str~~ | -| `shape` | Transform of the token's string to show orthographic features. Alphabetic characters are replaced by `x` or `X`, and numeric characters are replaced by `d`, and sequences of the same character are truncated after length 4. For example,`"Xxxx"`or`"dd"`. ~~int~~ | -| `shape_` | Transform of the token's string to show orthographic features. Alphabetic characters are replaced by `x` or `X`, and numeric characters are replaced by `d`, and sequences of the same character are truncated after length 4. For example,`"Xxxx"`or`"dd"`. ~~str~~ | -| `prefix` | Hash value of a length-N substring from the start of the token. Defaults to `N=1`. ~~int~~ | -| `prefix_` | A length-N substring from the start of the token. Defaults to `N=1`. ~~str~~ | -| `suffix` | Hash value of a length-N substring from the end of the token. Defaults to `N=3`. ~~int~~ | -| `suffix_` | Length-N substring from the end of the token. Defaults to `N=3`. ~~str~~ | -| `is_alpha` | Does the token consist of alphabetic characters? Equivalent to `token.text.isalpha()`. ~~bool~~ | -| `is_ascii` | Does the token consist of ASCII characters? Equivalent to `all(ord(c) < 128 for c in token.text)`. ~~bool~~ | -| `is_digit` | Does the token consist of digits? Equivalent to `token.text.isdigit()`. ~~bool~~ | -| `is_lower` | Is the token in lowercase? Equivalent to `token.text.islower()`. ~~bool~~ | -| `is_upper` | Is the token in uppercase? Equivalent to `token.text.isupper()`. ~~bool~~ | -| `is_title` | Is the token in titlecase? Equivalent to `token.text.istitle()`. ~~bool~~ | -| `is_punct` | Is the token punctuation? ~~bool~~ | -| `is_left_punct` | Is the token a left punctuation mark, e.g. `"("` ? ~~bool~~ | -| `is_right_punct` | Is the token a right punctuation mark, e.g. `")"` ? ~~bool~~ | -| `is_sent_start` | Does the token start a sentence? ~~bool~~ or `None` if unknown. Defaults to `True` for the first token in the `Doc`. | -| `is_sent_end` | Does the token end a sentence? ~~bool~~ or `None` if unknown. | -| `is_space` | Does the token consist of whitespace characters? Equivalent to `token.text.isspace()`. ~~bool~~ | -| `is_bracket` | Is the token a bracket? ~~bool~~ | -| `is_quote` | Is the token a quotation mark? ~~bool~~ | -| `is_currency` 2.0.8 | Is the token a currency symbol? ~~bool~~ | -| `like_url` | Does the token resemble a URL? ~~bool~~ | -| `like_num` | Does the token represent a number? e.g. "10.9", "10", "ten", etc. ~~bool~~ | -| `like_email` | Does the token resemble an email address? ~~bool~~ | -| `is_oov` | Is the token out-of-vocabulary (i.e. does it not have a word vector)? ~~bool~~ | -| `is_stop` | Is the token part of a "stop list"? ~~bool~~ | -| `pos` | Coarse-grained part-of-speech from the [Universal POS tag set](https://universaldependencies.org/u/pos/). ~~int~~ | -| `pos_` | Coarse-grained part-of-speech from the [Universal POS tag set](https://universaldependencies.org/u/pos/). ~~str~~ | -| `tag` | Fine-grained part-of-speech. ~~int~~ | -| `tag_` | Fine-grained part-of-speech. ~~str~~ | -| `morph` 3 | Morphological analysis. ~~MorphAnalysis~~ | -| `dep` | Syntactic dependency relation. ~~int~~ | -| `dep_` | Syntactic dependency relation. ~~str~~ | -| `lang` | Language of the parent document's vocabulary. ~~int~~ | -| `lang_` | Language of the parent document's vocabulary. ~~str~~ | -| `prob` | Smoothed log probability estimate of token's word type (context-independent entry in the vocabulary). ~~float~~ | -| `idx` | The character offset of the token within the parent document. ~~int~~ | -| `sentiment` | A scalar value indicating the positivity or negativity of the token. ~~float~~ | -| `lex_id` | Sequential ID of the token's lexical type, used to index into tables, e.g. for word vectors. ~~int~~ | -| `rank` | Sequential ID of the token's lexical type, used to index into tables, e.g. for word vectors. ~~int~~ | -| `cluster` | Brown cluster ID. ~~int~~ | -| `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ | +| Name | Description | +| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `doc` | The parent document. ~~Doc~~ | +| `lex` 3 | The underlying lexeme. ~~Lexeme~~ | +| `sent` | The sentence span that this token is a part of. ~~Span~~ | +| `text` | Verbatim text content. ~~str~~ | +| `text_with_ws` | Text content, with trailing space character if present. ~~str~~ | +| `whitespace_` | Trailing space character if present. ~~str~~ | +| `orth` | ID of the verbatim text content. ~~int~~ | +| `orth_` | Verbatim text content (identical to `Token.text`). Exists mostly for consistency with the other attributes. ~~str~~ | +| `vocab` | The vocab object of the parent `Doc`. ~~vocab~~ | +| `tensor` | The token's slice of the parent `Doc`'s tensor. ~~numpy.ndarray~~ | +| `head` | The syntactic parent, or "governor", of this token. ~~Token~~ | +| `left_edge` | The leftmost token of this token's syntactic descendants. ~~Token~~ | +| `right_edge` | The rightmost token of this token's syntactic descendants. ~~Token~~ | +| `i` | The index of the token within the parent document. ~~int~~ | +| `ent_type` | Named entity type. ~~int~~ | +| `ent_type_` | Named entity type. ~~str~~ | +| `ent_iob` | IOB code of named entity tag. `3` means the token begins an entity, `2` means it is outside an entity, `1` means it is inside an entity, and `0` means no entity tag is set. ~~int~~ | +| `ent_iob_` | IOB code of named entity tag. "B" means the token begins an entity, "I" means it is inside an entity, "O" means it is outside an entity, and "" means no entity tag is set. ~~str~~ | +| `ent_kb_id` | Knowledge base ID that refers to the named entity this token is a part of, if any. ~~int~~ | +| `ent_kb_id_` | Knowledge base ID that refers to the named entity this token is a part of, if any. ~~str~~ | +| `ent_id` | ID of the entity the token is an instance of, if any. Currently not used, but potentially for coreference resolution. ~~int~~ | +| `ent_id_` | ID of the entity the token is an instance of, if any. Currently not used, but potentially for coreference resolution. ~~str~~ | +| `lemma` | Base form of the token, with no inflectional suffixes. ~~int~~ | +| `lemma_` | Base form of the token, with no inflectional suffixes. ~~str~~ | +| `norm` | The token's norm, i.e. a normalized form of the token text. Can be set in the language's [tokenizer exceptions](/usage/linguistic-features#language-data). ~~int~~ | +| `norm_` | The token's norm, i.e. a normalized form of the token text. Can be set in the language's [tokenizer exceptions](/usage/linguistic-features#language-data). ~~str~~ | +| `lower` | Lowercase form of the token. ~~int~~ | +| `lower_` | Lowercase form of the token text. Equivalent to `Token.text.lower()`. ~~str~~ | +| `shape` | Transform of the token's string to show orthographic features. Alphabetic characters are replaced by `x` or `X`, and numeric characters are replaced by `d`, and sequences of the same character are truncated after length 4. For example,`"Xxxx"`or`"dd"`. ~~int~~ | +| `shape_` | Transform of the token's string to show orthographic features. Alphabetic characters are replaced by `x` or `X`, and numeric characters are replaced by `d`, and sequences of the same character are truncated after length 4. For example,`"Xxxx"`or`"dd"`. ~~str~~ | +| `prefix` | Hash value of a length-N substring from the start of the token. Defaults to `N=1`. ~~int~~ | +| `prefix_` | A length-N substring from the start of the token. Defaults to `N=1`. ~~str~~ | +| `suffix` | Hash value of a length-N substring from the end of the token. Defaults to `N=3`. ~~int~~ | +| `suffix_` | Length-N substring from the end of the token. Defaults to `N=3`. ~~str~~ | +| `is_alpha` | Does the token consist of alphabetic characters? Equivalent to `token.text.isalpha()`. ~~bool~~ | +| `is_ascii` | Does the token consist of ASCII characters? Equivalent to `all(ord(c) < 128 for c in token.text)`. ~~bool~~ | +| `is_digit` | Does the token consist of digits? Equivalent to `token.text.isdigit()`. ~~bool~~ | +| `is_lower` | Is the token in lowercase? Equivalent to `token.text.islower()`. ~~bool~~ | +| `is_upper` | Is the token in uppercase? Equivalent to `token.text.isupper()`. ~~bool~~ | +| `is_title` | Is the token in titlecase? Equivalent to `token.text.istitle()`. ~~bool~~ | +| `is_punct` | Is the token punctuation? ~~bool~~ | +| `is_left_punct` | Is the token a left punctuation mark, e.g. `"("` ? ~~bool~~ | +| `is_right_punct` | Is the token a right punctuation mark, e.g. `")"` ? ~~bool~~ | +| `is_sent_start` | Does the token start a sentence? ~~bool~~ or `None` if unknown. Defaults to `True` for the first token in the `Doc`. | +| `is_sent_end` | Does the token end a sentence? ~~bool~~ or `None` if unknown. | +| `is_space` | Does the token consist of whitespace characters? Equivalent to `token.text.isspace()`. ~~bool~~ | +| `is_bracket` | Is the token a bracket? ~~bool~~ | +| `is_quote` | Is the token a quotation mark? ~~bool~~ | +| `is_currency` | Is the token a currency symbol? ~~bool~~ | +| `like_url` | Does the token resemble a URL? ~~bool~~ | +| `like_num` | Does the token represent a number? e.g. "10.9", "10", "ten", etc. ~~bool~~ | +| `like_email` | Does the token resemble an email address? ~~bool~~ | +| `is_oov` | Is the token out-of-vocabulary (i.e. does it not have a word vector)? ~~bool~~ | +| `is_stop` | Is the token part of a "stop list"? ~~bool~~ | +| `pos` | Coarse-grained part-of-speech from the [Universal POS tag set](https://universaldependencies.org/u/pos/). ~~int~~ | +| `pos_` | Coarse-grained part-of-speech from the [Universal POS tag set](https://universaldependencies.org/u/pos/). ~~str~~ | +| `tag` | Fine-grained part-of-speech. ~~int~~ | +| `tag_` | Fine-grained part-of-speech. ~~str~~ | +| `morph` 3 | Morphological analysis. ~~MorphAnalysis~~ | +| `dep` | Syntactic dependency relation. ~~int~~ | +| `dep_` | Syntactic dependency relation. ~~str~~ | +| `lang` | Language of the parent document's vocabulary. ~~int~~ | +| `lang_` | Language of the parent document's vocabulary. ~~str~~ | +| `prob` | Smoothed log probability estimate of token's word type (context-independent entry in the vocabulary). ~~float~~ | +| `idx` | The character offset of the token within the parent document. ~~int~~ | +| `sentiment` | A scalar value indicating the positivity or negativity of the token. ~~float~~ | +| `lex_id` | Sequential ID of the token's lexical type, used to index into tables, e.g. for word vectors. ~~int~~ | +| `rank` | Sequential ID of the token's lexical type, used to index into tables, e.g. for word vectors. ~~int~~ | +| `cluster` | Brown cluster ID. ~~int~~ | +| `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ | diff --git a/website/docs/api/top-level.md b/website/docs/api/top-level.md index c798f2a8d..211affa4a 100644 --- a/website/docs/api/top-level.md +++ b/website/docs/api/top-level.md @@ -45,16 +45,16 @@ specified separately using the new `exclude` keyword argument. > nlp = spacy.load("en_core_web_sm", exclude=["parser", "tagger"]) > ``` -| Name | Description | -| ------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `name` | Pipeline to load, i.e. package name or path. ~~Union[str, Path]~~ | -| _keyword-only_ | | -| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | +| Name | Description | +| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `name` | Pipeline to load, i.e. package name or path. ~~Union[str, Path]~~ | +| _keyword-only_ | | +| `vocab` | Optional shared vocab to pass in on initialization. If `True` (default), a new `Vocab` object will be created. ~~Union[Vocab, bool]~~ | | `disable` | Name(s) of pipeline component(s) to [disable](/usage/processing-pipelines#disabling). Disabled pipes will be loaded but they won't be run unless you explicitly enable them by calling [nlp.enable_pipe](/api/language#enable_pipe). Is merged with the config entry `nlp.disabled`. ~~Union[str, Iterable[str]]~~ | -| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled. ~~Union[str, Iterable[str]]~~ | -| `exclude` 3 | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | -| `config` 3 | Optional config overrides, either as nested dict or dict keyed by section value in dot notation, e.g. `"components.name.value"`. ~~Union[Dict[str, Any], Config]~~ | -| **RETURNS** | A `Language` object with the loaded pipeline. ~~Language~~ | +| `enable` 3.4 | Name(s) of pipeline component(s) to [enable](/usage/processing-pipelines#disabling). All other pipes will be disabled. ~~Union[str, Iterable[str]]~~ | +| `exclude` 3 | Name(s) of pipeline component(s) to [exclude](/usage/processing-pipelines#disabling). Excluded components won't be loaded. ~~Union[str, Iterable[str]]~~ | +| `config` 3 | Optional config overrides, either as nested dict or dict keyed by section value in dot notation, e.g. `"components.name.value"`. ~~Union[Dict[str, Any], Config]~~ | +| **RETURNS** | A `Language` object with the loaded pipeline. ~~Language~~ | Essentially, `spacy.load()` is a convenience wrapper that reads the pipeline's [`config.cfg`](/api/data-formats#config), uses the language and pipeline @@ -354,22 +354,22 @@ If a setting is not present in the options, the default value will be used. > displacy.serve(doc, style="dep", options=options) > ``` -| Name | Description | -| ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------- | -| `fine_grained` | Use fine-grained part-of-speech tags (`Token.tag_`) instead of coarse-grained tags (`Token.pos_`). Defaults to `False`. ~~bool~~ | -| `add_lemma` 2.2.4 | Print the lemmas in a separate row below the token texts. Defaults to `False`. ~~bool~~ | -| `collapse_punct` | Attach punctuation to tokens. Can make the parse more readable, as it prevents long arcs to attach punctuation. Defaults to `True`. ~~bool~~ | -| `collapse_phrases` | Merge noun phrases into one token. Defaults to `False`. ~~bool~~ | -| `compact` | "Compact mode" with square arrows that takes up less space. Defaults to `False`. ~~bool~~ | -| `color` | Text color (HEX, RGB or color names). Defaults to `"#000000"`. ~~str~~ | -| `bg` | Background color (HEX, RGB or color names). Defaults to `"#ffffff"`. ~~str~~ | -| `font` | Font name or font family for all text. Defaults to `"Arial"`. ~~str~~ | -| `offset_x` | Spacing on left side of the SVG in px. Defaults to `50`. ~~int~~ | -| `arrow_stroke` | Width of arrow path in px. Defaults to `2`. ~~int~~ | -| `arrow_width` | Width of arrow head in px. Defaults to `10` in regular mode and `8` in compact mode. ~~int~~ | -| `arrow_spacing` | Spacing between arrows in px to avoid overlaps. Defaults to `20` in regular mode and `12` in compact mode. ~~int~~ | -| `word_spacing` | Vertical spacing between words and arcs in px. Defaults to `45`. ~~int~~ | -| `distance` | Distance between words in px. Defaults to `175` in regular mode and `150` in compact mode. ~~int~~ | +| Name | Description | +| ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------- | +| `fine_grained` | Use fine-grained part-of-speech tags (`Token.tag_`) instead of coarse-grained tags (`Token.pos_`). Defaults to `False`. ~~bool~~ | +| `add_lemma` | Print the lemmas in a separate row below the token texts. Defaults to `False`. ~~bool~~ | +| `collapse_punct` | Attach punctuation to tokens. Can make the parse more readable, as it prevents long arcs to attach punctuation. Defaults to `True`. ~~bool~~ | +| `collapse_phrases` | Merge noun phrases into one token. Defaults to `False`. ~~bool~~ | +| `compact` | "Compact mode" with square arrows that takes up less space. Defaults to `False`. ~~bool~~ | +| `color` | Text color (HEX, RGB or color names). Defaults to `"#000000"`. ~~str~~ | +| `bg` | Background color (HEX, RGB or color names). Defaults to `"#ffffff"`. ~~str~~ | +| `font` | Font name or font family for all text. Defaults to `"Arial"`. ~~str~~ | +| `offset_x` | Spacing on left side of the SVG in px. Defaults to `50`. ~~int~~ | +| `arrow_stroke` | Width of arrow path in px. Defaults to `2`. ~~int~~ | +| `arrow_width` | Width of arrow head in px. Defaults to `10` in regular mode and `8` in compact mode. ~~int~~ | +| `arrow_spacing` | Spacing between arrows in px to avoid overlaps. Defaults to `20` in regular mode and `12` in compact mode. ~~int~~ | +| `word_spacing` | Vertical spacing between words and arcs in px. Defaults to `45`. ~~int~~ | +| `distance` | Distance between words in px. Defaults to `175` in regular mode and `150` in compact mode. ~~int~~ | #### Named Entity Visualizer options {#displacy_options-ent} @@ -385,7 +385,7 @@ If a setting is not present in the options, the default value will be used. | ------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `ents` | Entity types to highlight or `None` for all types (default). ~~Optional[List[str]]~~ | | `colors` | Color overrides. Entity types should be mapped to color names or values. ~~Dict[str, str]~~ | -| `template` 2.2 | Optional template to overwrite the HTML used to render entity spans. Should be a format string and can use `{bg}`, `{text}` and `{label}`. See [`templates.py`](%%GITHUB_SPACY/spacy/displacy/templates.py) for examples. ~~Optional[str]~~ | +| `template` | Optional template to overwrite the HTML used to render entity spans. Should be a format string and can use `{bg}`, `{text}` and `{label}`. See [`templates.py`](%%GITHUB_SPACY/spacy/displacy/templates.py) for examples. ~~Optional[str]~~ | | `kb_url_template` 3.2.1 | Optional template to construct the KB url for the entity to link to. Expects a python f-string format with single field to fill in. ~~Optional[str]~~ | #### Span Visualizer options {#displacy_options-span} diff --git a/website/docs/api/vocab.md b/website/docs/api/vocab.md index 2e4a206ec..afbd1301d 100644 --- a/website/docs/api/vocab.md +++ b/website/docs/api/vocab.md @@ -21,15 +21,15 @@ Create the vocabulary. > vocab = Vocab(strings=["hello", "world"]) > ``` -| Name | Description | -| ------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `lex_attr_getters` | A dictionary mapping attribute IDs to functions to compute them. Defaults to `None`. ~~Optional[Dict[str, Callable[[str], Any]]]~~ | -| `strings` | A [`StringStore`](/api/stringstore) that maps strings to hash values, and vice versa, or a list of strings. ~~Union[List[str], StringStore]~~ | -| `lookups` | A [`Lookups`](/api/lookups) that stores the `lexeme_norm` and other large lookup tables. Defaults to `None`. ~~Optional[Lookups]~~ | -| `oov_prob` | The default OOV probability. Defaults to `-20.0`. ~~float~~ | -| `vectors_name` 2.2 | A name to identify the vectors table. ~~str~~ | -| `writing_system` | A dictionary describing the language's writing system. Typically provided by [`Language.Defaults`](/api/language#defaults). ~~Dict[str, Any]~~ | -| `get_noun_chunks` | A function that yields base noun phrases used for [`Doc.noun_chunks`](/api/doc#noun_chunks). ~~Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]~~ | +| Name | Description | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `lex_attr_getters` | A dictionary mapping attribute IDs to functions to compute them. Defaults to `None`. ~~Optional[Dict[str, Callable[[str], Any]]]~~ | +| `strings` | A [`StringStore`](/api/stringstore) that maps strings to hash values, and vice versa, or a list of strings. ~~Union[List[str], StringStore]~~ | +| `lookups` | A [`Lookups`](/api/lookups) that stores the `lexeme_norm` and other large lookup tables. Defaults to `None`. ~~Optional[Lookups]~~ | +| `oov_prob` | The default OOV probability. Defaults to `-20.0`. ~~float~~ | +| `vectors_name` | A name to identify the vectors table. ~~str~~ | +| `writing_system` | A dictionary describing the language's writing system. Typically provided by [`Language.Defaults`](/api/language#defaults). ~~Dict[str, Any]~~ | +| `get_noun_chunks` | A function that yields base noun phrases used for [`Doc.noun_chunks`](/api/doc#noun_chunks). ~~Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]~~ | ## Vocab.\_\_len\_\_ {#len tag="method"} @@ -311,10 +311,10 @@ Load state from a binary string. | Name | Description | | ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `strings` | A table managing the string-to-int mapping. ~~StringStore~~ | -| `vectors` 2 | A table associating word IDs to word vectors. ~~Vectors~~ | +| `vectors` | A table associating word IDs to word vectors. ~~Vectors~~ | | `vectors_length` | Number of dimensions for each word vector. ~~int~~ | | `lookups` | The available lookup tables in this vocab. ~~Lookups~~ | -| `writing_system` 2.1 | A dict with information about the language's writing system. ~~Dict[str, Any]~~ | +| `writing_system` | A dict with information about the language's writing system. ~~Dict[str, Any]~~ | | `get_noun_chunks` 3.0 | A function that yields base noun phrases used for [`Doc.noun_chunks`](/ap/doc#noun_chunks). ~~Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]~~ | ## Serialization fields {#serialization-fields} diff --git a/website/docs/usage/rule-based-matching.md b/website/docs/usage/rule-based-matching.md index 64bbf8e7b..ad8ea27f3 100644 --- a/website/docs/usage/rule-based-matching.md +++ b/website/docs/usage/rule-based-matching.md @@ -162,7 +162,7 @@ rule-based matching are: | Attribute | Description | | ---------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `ORTH` | The exact verbatim text of a token. ~~str~~ | -| `TEXT` 2.1 | The exact verbatim text of a token. ~~str~~ | +| `TEXT` | The exact verbatim text of a token. ~~str~~ | | `NORM` | The normalized form of the token text. ~~str~~ | | `LOWER` | The lowercase form of the token text. ~~str~~ | | `LENGTH` | The length of the token text. ~~int~~ | @@ -174,7 +174,7 @@ rule-based matching are: | `SPACY` | Token has a trailing space. ~~bool~~ | | `POS`, `TAG`, `MORPH`, `DEP`, `LEMMA`, `SHAPE` | The token's simple and extended part-of-speech tag, morphological analysis, dependency label, lemma, shape. Note that the values of these attributes are case-sensitive. For a list of available part-of-speech tags and dependency labels, see the [Annotation Specifications](/api/annotation). ~~str~~ | | `ENT_TYPE` | The token's entity label. ~~str~~ | -| `_` 2.1 | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ | +| `_` | Properties in [custom extension attributes](/usage/processing-pipelines#custom-components-attributes). ~~Dict[str, Any]~~ | | `OP` | [Operator or quantifier](#quantifiers) to determine how often to match a token pattern. ~~str~~ | @@ -375,7 +375,7 @@ scoped quantifiers – instead, you can build those behaviors with `on_match` callbacks. | OP | Description | -|---------|------------------------------------------------------------------------| +| ------- | ---------------------------------------------------------------------- | | `!` | Negate the pattern, by requiring it to match exactly 0 times. | | `?` | Make the pattern optional, by allowing it to match 0 or 1 times. | | `+` | Require the pattern to match 1 or more times. | diff --git a/website/docs/usage/saving-loading.md b/website/docs/usage/saving-loading.md index 0fd713a49..29870a2e3 100644 --- a/website/docs/usage/saving-loading.md +++ b/website/docs/usage/saving-loading.md @@ -306,12 +306,12 @@ pipeline component factories, language classes and other settings. To make spaCy use your entry points, your package needs to expose them and it needs to be installed in the same environment – that's it. -| Entry point | Description | -| ------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`spacy_factories`](#entry-points-components) | Group of entry points for pipeline component factories, keyed by component name. Can be used to expose custom components defined by another package. | -| [`spacy_languages`](#entry-points-languages) | Group of entry points for custom [`Language` subclasses](/usage/linguistic-features#language-data), keyed by language shortcut. | -| `spacy_lookups` 2.2 | Group of entry points for custom [`Lookups`](/api/lookups), including lemmatizer data. Used by spaCy's [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) package. | -| [`spacy_displacy_colors`](#entry-points-displacy) 2.2 | Group of entry points of custom label colors for the [displaCy visualizer](/usage/visualizers#ent). The key name doesn't matter, but it should point to a dict of labels and color values. Useful for custom models that predict different entity types. | +| Entry point | Description | +| ------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`spacy_factories`](#entry-points-components) | Group of entry points for pipeline component factories, keyed by component name. Can be used to expose custom components defined by another package. | +| [`spacy_languages`](#entry-points-languages) | Group of entry points for custom [`Language` subclasses](/usage/linguistic-features#language-data), keyed by language shortcut. | +| `spacy_lookups` | Group of entry points for custom [`Lookups`](/api/lookups), including lemmatizer data. Used by spaCy's [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) package. | +| [`spacy_displacy_colors`](#entry-points-displacy) | Group of entry points of custom label colors for the [displaCy visualizer](/usage/visualizers#ent). The key name doesn't matter, but it should point to a dict of labels and color values. Useful for custom models that predict different entity types. | ### Custom components via entry points {#entry-points-components} From bb523d4d9105d417e240e6f8f83b63ed3dcc565e Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Mon, 14 Nov 2022 19:58:38 +0900 Subject: [PATCH 089/179] Remove spacy-ray from docs (#11781) * Remove spacy ray from cli docs * Remove more ray docs * Remove ray from universe --- website/docs/api/cli.md | 45 --------------------- website/docs/usage/index.md | 1 - website/docs/usage/projects.md | 48 ----------------------- website/docs/usage/training.md | 71 ---------------------------------- website/meta/universe.json | 11 ------ 5 files changed, 176 deletions(-) diff --git a/website/docs/api/cli.md b/website/docs/api/cli.md index 024450920..6e581b903 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.md @@ -15,7 +15,6 @@ menu: - ['assemble', 'assemble'] - ['package', 'package'] - ['project', 'project'] - - ['ray', 'ray'] - ['huggingface-hub', 'huggingface-hub'] --- @@ -1502,50 +1501,6 @@ $ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] [-- | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | A `dvc.yaml` file in the project directory, based on the steps defined in the given workflow. | -## ray {#ray new="3"} - -The `spacy ray` CLI includes commands for parallel and distributed computing via -[Ray](https://ray.io). - - - -To use this command, you need the -[`spacy-ray`](https://github.com/explosion/spacy-ray) package installed. -Installing the package will automatically add the `ray` command to the spaCy -CLI. - - - -### ray train {#ray-train tag="command"} - -Train a spaCy pipeline using [Ray](https://ray.io) for parallel training. The -command works just like [`spacy train`](/api/cli#train). For more details and -examples, see the usage guide on -[parallel training](/usage/training#parallel-training) and the spaCy project -[integration](/usage/projects#ray). - -```cli -$ python -m spacy ray train [config_path] [--code] [--output] [--n-workers] [--address] [--gpu-id] [--verbose] [overrides] -``` - -> #### Example -> -> ```cli -> $ python -m spacy ray train config.cfg --n-workers 2 -> ``` - -| Name | Description | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `config_path` | Path to [training config](/api/data-formats#config) file containing all settings and hyperparameters. ~~Path (positional)~~ | -| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | -| `--output`, `-o` | Directory or remote storage URL for saving trained pipeline. The directory will be created if it doesn't exist. ~~Optional[Path] \(option)~~ | -| `--n-workers`, `-n` | The number of workers. Defaults to `1`. ~~int (option)~~ | -| `--address`, `-a` | Optional address of the Ray cluster. If not set (default), Ray will run locally. ~~Optional[str] \(option)~~ | -| `--gpu-id`, `-g` | GPU ID or `-1` for CPU. Defaults to `-1`. ~~int (option)~~ | -| `--verbose`, `-V` | Display more information for debugging purposes. ~~bool (flag)~~ | -| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | -| overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | - ## huggingface-hub {#huggingface-hub new="3.1"} The `spacy huggingface-cli` CLI includes commands for uploading your trained diff --git a/website/docs/usage/index.md b/website/docs/usage/index.md index 1f4869606..dff5a16ba 100644 --- a/website/docs/usage/index.md +++ b/website/docs/usage/index.md @@ -75,7 +75,6 @@ spaCy's [`setup.cfg`](%%GITHUB_SPACY/setup.cfg) for details on what's included. | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `lookups` | Install [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data) for data tables for lemmatization and lexeme normalization. The data is serialized with trained pipelines, so you only need this package if you want to train your own models. | | `transformers` | Install [`spacy-transformers`](https://github.com/explosion/spacy-transformers). The package will be installed automatically when you install a transformer-based pipeline. | -| `ray` | Install [`spacy-ray`](https://github.com/explosion/spacy-ray) to add CLI commands for [parallel training](/usage/training#parallel-training). | | `cuda`, ... | Install spaCy with GPU support provided by [CuPy](https://cupy.chainer.org) for your given CUDA version. See the GPU [installation instructions](#gpu) for details and options. | | `apple` | Install [`thinc-apple-ops`](https://github.com/explosion/thinc-apple-ops) to improve performance on an Apple M1. | | `ja`, `ko`, `th` | Install additional dependencies required for tokenization for the [languages](/usage/models#languages). | diff --git a/website/docs/usage/projects.md b/website/docs/usage/projects.md index 90b612358..34315e4e7 100644 --- a/website/docs/usage/projects.md +++ b/website/docs/usage/projects.md @@ -1014,54 +1014,6 @@ https://github.com/explosion/projects/blob/v3/integrations/fastapi/scripts/main. --- -### Ray {#ray} - -> #### Installation -> -> ```cli -> $ pip install -U %%SPACY_PKG_NAME[ray]%%SPACY_PKG_FLAGS -> # Check that the CLI is registered -> $ python -m spacy ray --help -> ``` - -[Ray](https://ray.io/) is a fast and simple framework for building and running -**distributed applications**. You can use Ray for parallel and distributed -training with spaCy via our lightweight -[`spacy-ray`](https://github.com/explosion/spacy-ray) extension package. If the -package is installed in the same environment as spaCy, it will automatically add -[`spacy ray`](/api/cli#ray) commands to your spaCy CLI. See the usage guide on -[parallel training](/usage/training#parallel-training) for more details on how -it works under the hood. - - - -Get started with parallel training using our project template. It trains a -simple model on a Universal Dependencies Treebank and lets you parallelize the -training with Ray. - - - -You can integrate [`spacy ray train`](/api/cli#ray-train) into your -`project.yml` just like the regular training command and pass it the config, and -optional output directory or remote storage URL and config overrides if needed. - - -```yaml -### project.yml -commands: - - name: "ray" - help: "Train a model via parallel training with Ray" - script: - - "python -m spacy ray train configs/config.cfg -o training/ --paths.train corpus/train.spacy --paths.dev corpus/dev.spacy" - deps: - - "corpus/train.spacy" - - "corpus/dev.spacy" - outputs: - - "training/model-best" -``` - ---- - ### Weights & Biases {#wandb} [Weights & Biases](https://www.wandb.com/) is a popular platform for experiment diff --git a/website/docs/usage/training.md b/website/docs/usage/training.md index 27a8bbca7..e40a395c4 100644 --- a/website/docs/usage/training.md +++ b/website/docs/usage/training.md @@ -1572,77 +1572,6 @@ token-based annotations like the dependency parse or entity labels, you'll need to take care to adjust the `Example` object so its annotations match and remain valid. -## Parallel & distributed training with Ray {#parallel-training} - -> #### Installation -> -> ```cli -> $ pip install -U %%SPACY_PKG_NAME[ray]%%SPACY_PKG_FLAGS -> # Check that the CLI is registered -> $ python -m spacy ray --help -> ``` - -[Ray](https://ray.io/) is a fast and simple framework for building and running -**distributed applications**. You can use Ray to train spaCy on one or more -remote machines, potentially speeding up your training process. Parallel -training won't always be faster though – it depends on your batch size, models, -and hardware. - - - -To use Ray with spaCy, you need the -[`spacy-ray`](https://github.com/explosion/spacy-ray) package installed. -Installing the package will automatically add the `ray` command to the spaCy -CLI. - - - -The [`spacy ray train`](/api/cli#ray-train) command follows the same API as -[`spacy train`](/api/cli#train), with a few extra options to configure the Ray -setup. You can optionally set the `--address` option to point to your Ray -cluster. If it's not set, Ray will run locally. - -```cli -python -m spacy ray train config.cfg --n-workers 2 -``` - - - -Get started with parallel training using our project template. It trains a -simple model on a Universal Dependencies Treebank and lets you parallelize the -training with Ray. - - - -### How parallel training works {#parallel-training-details} - -Each worker receives a shard of the **data** and builds a copy of the **model -and optimizer** from the [`config.cfg`](#config). It also has a communication -channel to **pass gradients and parameters** to the other workers. Additionally, -each worker is given ownership of a subset of the parameter arrays. Every -parameter array is owned by exactly one worker, and the workers are given a -mapping so they know which worker owns which parameter. - -![Illustration of setup](../images/spacy-ray.svg) - -As training proceeds, every worker will be computing gradients for **all** of -the model parameters. When they compute gradients for parameters they don't own, -they'll **send them to the worker** that does own that parameter, along with a -version identifier so that the owner can decide whether to discard the gradient. -Workers use the gradients they receive and the ones they compute locally to -update the parameters they own, and then broadcast the updated array and a new -version ID to the other workers. - -This training procedure is **asynchronous** and **non-blocking**. Workers always -push their gradient increments and parameter updates, they do not have to pull -them and block on the result, so the transfers can happen in the background, -overlapped with the actual training work. The workers also do not have to stop -and wait for each other ("synchronize") at the start of each batch. This is very -useful for spaCy, because spaCy is often trained on long documents, which means -**batches can vary in size** significantly. Uneven workloads make synchronous -gradient descent inefficient, because if one batch is slow, all of the other -workers are stuck waiting for it to complete before they can continue. - ## Internal training API {#api} diff --git a/website/meta/universe.json b/website/meta/universe.json index fa765f640..661f5da12 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -557,17 +557,6 @@ "tags": ["sentiment", "textblob"], "spacy_version": 3 }, - { - "id": "spacy-ray", - "title": "spacy-ray", - "slogan": "Parallel and distributed training with spaCy and Ray", - "description": "[Ray](https://ray.io/) is a fast and simple framework for building and running **distributed applications**. This very lightweight extension package lets you use Ray for parallel and distributed training with spaCy. If `spacy-ray` is installed in the same environment as spaCy, it will automatically add `spacy ray` commands to your spaCy CLI.", - "github": "explosion/spacy-ray", - "pip": "spacy-ray", - "category": ["training"], - "author": "Explosion / Anyscale", - "thumb": "https://i.imgur.com/7so6ZpS.png" - }, { "id": "spacy-sentence-bert", "title": "spaCy - sentence-transformers", From 9baa686f827eeaecf28bf6d75836eeaec090cd69 Mon Sep 17 00:00:00 2001 From: Peter Baumgartner <5107405+pmbaumgartner@users.noreply.github.com> Date: Mon, 14 Nov 2022 10:53:14 -0500 Subject: [PATCH 090/179] remove migration support form (#11802) --- website/docs/usage/v3.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/website/docs/usage/v3.md b/website/docs/usage/v3.md index 971779ed3..64f93b7c0 100644 --- a/website/docs/usage/v3.md +++ b/website/docs/usage/v3.md @@ -15,18 +15,6 @@ menu: > To help you make the transition from v2.x to v3.0, we've uploaded the old > website to [**v2.spacy.io**](https://v2.spacy.io/docs). - - -Want to make the transition from spaCy v2 to spaCy v3 as smooth as possible for -you and your organization? We're now offering commercial **migration support** -for your spaCy pipelines! We've put a lot of work into making it easy to upgrade -your existing code and training workflows – but custom projects may always need -some custom work, especially when it comes to taking advantage of the new -capabilities. -[**Details & application →**](https://form.typeform.com/to/vMs2zSjM) - - -
From 7e684ad691992e759e71026a11c1ddd77c401f39 Mon Sep 17 00:00:00 2001 From: Denis Bezykornov Date: Tue, 15 Nov 2022 13:37:25 +0300 Subject: [PATCH 091/179] Update russian tokenizer exceptions (#11753) * Fix typos, add couple of new abbreviations, remove nonbreaking spaces * Remove space from abbreviation Co-authored-by: Adriane Boyd --- spacy/lang/ru/tokenizer_exceptions.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/spacy/lang/ru/tokenizer_exceptions.py b/spacy/lang/ru/tokenizer_exceptions.py index f3756e26c..e1889f785 100644 --- a/spacy/lang/ru/tokenizer_exceptions.py +++ b/spacy/lang/ru/tokenizer_exceptions.py @@ -61,6 +61,11 @@ for abbr in [ {ORTH: "2к23", NORM: "2023"}, {ORTH: "2к24", NORM: "2024"}, {ORTH: "2к25", NORM: "2025"}, + {ORTH: "2к26", NORM: "2026"}, + {ORTH: "2к27", NORM: "2027"}, + {ORTH: "2к28", NORM: "2028"}, + {ORTH: "2к29", NORM: "2029"}, + {ORTH: "2к30", NORM: "2030"}, ]: _exc[abbr[ORTH]] = [abbr] @@ -268,8 +273,8 @@ for abbr in [ {ORTH: "з-ка", NORM: "заимка"}, {ORTH: "п-к", NORM: "починок"}, {ORTH: "киш.", NORM: "кишлак"}, - {ORTH: "п. ст. ", NORM: "поселок станция"}, - {ORTH: "п. ж/д ст. ", NORM: "поселок при железнодорожной станции"}, + {ORTH: "п. ст.", NORM: "поселок станция"}, + {ORTH: "п. ж/д ст.", NORM: "поселок при железнодорожной станции"}, {ORTH: "ж/д бл-ст", NORM: "железнодорожный блокпост"}, {ORTH: "ж/д б-ка", NORM: "железнодорожная будка"}, {ORTH: "ж/д в-ка", NORM: "железнодорожная ветка"}, @@ -280,12 +285,12 @@ for abbr in [ {ORTH: "ж/д п.п.", NORM: "железнодорожный путевой пост"}, {ORTH: "ж/д о.п.", NORM: "железнодорожный остановочный пункт"}, {ORTH: "ж/д рзд.", NORM: "железнодорожный разъезд"}, - {ORTH: "ж/д ст. ", NORM: "железнодорожная станция"}, + {ORTH: "ж/д ст.", NORM: "железнодорожная станция"}, {ORTH: "м-ко", NORM: "местечко"}, {ORTH: "д.", NORM: "деревня"}, {ORTH: "с.", NORM: "село"}, {ORTH: "сл.", NORM: "слобода"}, - {ORTH: "ст. ", NORM: "станция"}, + {ORTH: "ст.", NORM: "станция"}, {ORTH: "ст-ца", NORM: "станица"}, {ORTH: "у.", NORM: "улус"}, {ORTH: "х.", NORM: "хутор"}, @@ -388,8 +393,9 @@ for abbr in [ {ORTH: "прим.", NORM: "примечание"}, {ORTH: "прим.ред.", NORM: "примечание редакции"}, {ORTH: "см. также", NORM: "смотри также"}, - {ORTH: "кв.м.", NORM: "квадрантный метр"}, - {ORTH: "м2", NORM: "квадрантный метр"}, + {ORTH: "см.", NORM: "смотри"}, + {ORTH: "кв.м.", NORM: "квадратный метр"}, + {ORTH: "м2", NORM: "квадратный метр"}, {ORTH: "б/у", NORM: "бывший в употреблении"}, {ORTH: "сокр.", NORM: "сокращение"}, {ORTH: "чел.", NORM: "человек"}, From caa9efad5991d574cf2bdc69fabfc6d952d5cba9 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Tue, 15 Nov 2022 14:15:00 +0100 Subject: [PATCH 092/179] prevent rewriting an already raw URL (#11810) --- spacy/cli/project/assets.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/spacy/cli/project/assets.py b/spacy/cli/project/assets.py index 61438d1a8..8f35b2d23 100644 --- a/spacy/cli/project/assets.py +++ b/spacy/cli/project/assets.py @@ -189,7 +189,11 @@ def convert_asset_url(url: str) -> str: RETURNS (str): The converted URL. """ # If the asset URL is a regular GitHub URL it's likely a mistake - if re.match(r"(http(s?)):\/\/github.com", url) and "releases/download" not in url: + if ( + re.match(r"(http(s?)):\/\/github.com", url) + and "releases/download" not in url + and "/raw/" not in url + ): converted = url.replace("github.com", "raw.githubusercontent.com") converted = re.sub(r"/(tree|blob)/", "/", converted) msg.warn( From c0c54e44bc70ca737b421def1f6ce3c30809a54b Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Wed, 16 Nov 2022 17:44:42 +0900 Subject: [PATCH 093/179] Add equality definition for vectors (#11806) * Add equality definition for vectors This re-uses the check from sourcing components. * Use the equality check * Format Co-authored-by: Adriane Boyd --- spacy/language.py | 8 +------- spacy/tests/vocab_vectors/test_vectors.py | 20 ++++++++++++++++++++ spacy/vectors.pyx | 9 +++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/spacy/language.py b/spacy/language.py index 967af1e62..836f3abf9 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -706,13 +706,7 @@ class Language: # Check source type if not isinstance(source, Language): raise ValueError(Errors.E945.format(name=source_name, source=type(source))) - # Check vectors, with faster checks first - if ( - self.vocab.vectors.shape != source.vocab.vectors.shape - or self.vocab.vectors.key2row != source.vocab.vectors.key2row - or self.vocab.vectors.to_bytes(exclude=["strings"]) - != source.vocab.vectors.to_bytes(exclude=["strings"]) - ): + if self.vocab.vectors != source.vocab.vectors: warnings.warn(Warnings.W113.format(name=source_name)) if source_name not in source.component_names: raise KeyError( diff --git a/spacy/tests/vocab_vectors/test_vectors.py b/spacy/tests/vocab_vectors/test_vectors.py index dd2cfc596..70835816d 100644 --- a/spacy/tests/vocab_vectors/test_vectors.py +++ b/spacy/tests/vocab_vectors/test_vectors.py @@ -626,3 +626,23 @@ def test_floret_vectors(floret_vectors_vec_str, floret_vectors_hashvec_str): OPS.to_numpy(vocab_r[word].vector), decimal=6, ) + + +def test_equality(): + vectors1 = Vectors(shape=(10, 10)) + vectors2 = Vectors(shape=(10, 8)) + + assert vectors1 != vectors2 + + vectors2 = Vectors(shape=(10, 10)) + assert vectors1 == vectors2 + + vectors1.add("hello", row=2) + assert vectors1 != vectors2 + + vectors2.add("hello", row=2) + assert vectors1 == vectors2 + + vectors1.resize((5, 9)) + vectors2.resize((5, 9)) + assert vectors1 == vectors2 diff --git a/spacy/vectors.pyx b/spacy/vectors.pyx index 8300220c1..be0f6db09 100644 --- a/spacy/vectors.pyx +++ b/spacy/vectors.pyx @@ -243,6 +243,15 @@ cdef class Vectors: else: return key in self.key2row + def __eq__(self, other): + # Check for equality, with faster checks first + return ( + self.shape == other.shape + and self.key2row == other.key2row + and self.to_bytes(exclude=["strings"]) + == other.to_bytes(exclude=["strings"]) + ) + def resize(self, shape, inplace=False): """Resize the underlying vectors array. If inplace=True, the memory is reallocated. This may cause other references to the data to become From 317b6ef99c0e3512466d31a8274f9fe6a2894355 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 16 Nov 2022 14:09:10 +0100 Subject: [PATCH 094/179] Update to mypy 0.990 (#11801) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d91a3b3d4..23bfa6f14 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ pytest-timeout>=1.3.0,<2.0.0 mock>=2.0.0,<3.0.0 flake8>=3.8.0,<6.0.0 hypothesis>=3.27.0,<7.0.0 -mypy>=0.980,<0.990; platform_machine != "aarch64" and python_version >= "3.7" +mypy>=0.990,<0.1000; platform_machine != "aarch64" and python_version >= "3.7" types-dataclasses>=0.1.3; python_version < "3.7" types-mock>=0.1.1 types-setuptools>=57.0.0 From 75bb7ad541a94c74127b57ffd6d674841767478c Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Thu, 17 Nov 2022 18:25:01 +0900 Subject: [PATCH 095/179] Check textcat values for validity (#11763) * Check textcat values for validity * Fix error numbers * Clean up vals reference * Check category value validity through training The _validate_categories is called in update, which for multilabel is inherited from the single label component. * Formatting --- spacy/errors.py | 2 ++ spacy/pipeline/textcat.py | 10 +++++++--- spacy/pipeline/textcat_multilabel.py | 8 +++++++- spacy/tests/pipeline/test_textcat.py | 24 ++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/spacy/errors.py b/spacy/errors.py index 278e5496a..1d29f0e17 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -544,6 +544,8 @@ class Errors(metaclass=ErrorsWithCodes): "during training, make sure to include it in 'annotating components'") # New errors added in v3.x + E851 = ("The 'textcat' component labels should only have values of 0 or 1, " + "but found value of '{val}'.") E852 = ("The tar file pulled from the remote attempted an unsafe path " "traversal.") E853 = ("Unsupported component factory name '{name}'. The character '.' is " diff --git a/spacy/pipeline/textcat.py b/spacy/pipeline/textcat.py index 4023c4456..a86eb99d2 100644 --- a/spacy/pipeline/textcat.py +++ b/spacy/pipeline/textcat.py @@ -293,7 +293,7 @@ class TextCategorizer(TrainablePipe): bp_scores(gradient) if sgd is not None: self.finish_update(sgd) - losses[self.name] += (gradient**2).sum() + losses[self.name] += (gradient ** 2).sum() return losses def _examples_to_truth( @@ -327,7 +327,7 @@ class TextCategorizer(TrainablePipe): not_missing = self.model.ops.asarray(not_missing) # type: ignore d_scores = scores - truths d_scores *= not_missing - mean_square_error = (d_scores**2).mean() + mean_square_error = (d_scores ** 2).mean() return float(mean_square_error), d_scores def add_label(self, label: str) -> int: @@ -401,5 +401,9 @@ class TextCategorizer(TrainablePipe): def _validate_categories(self, examples: Iterable[Example]): """Check whether the provided examples all have single-label cats annotations.""" for ex in examples: - if list(ex.reference.cats.values()).count(1.0) > 1: + vals = list(ex.reference.cats.values()) + if vals.count(1.0) > 1: raise ValueError(Errors.E895.format(value=ex.reference.cats)) + for val in vals: + if not (val == 1.0 or val == 0.0): + raise ValueError(Errors.E851.format(val=val)) diff --git a/spacy/pipeline/textcat_multilabel.py b/spacy/pipeline/textcat_multilabel.py index eb83d9cb7..ef9bd6557 100644 --- a/spacy/pipeline/textcat_multilabel.py +++ b/spacy/pipeline/textcat_multilabel.py @@ -192,6 +192,8 @@ class MultiLabel_TextCategorizer(TextCategorizer): for label in labels: self.add_label(label) subbatch = list(islice(get_examples(), 10)) + self._validate_categories(subbatch) + doc_sample = [eg.reference for eg in subbatch] label_sample, _ = self._examples_to_truth(subbatch) self._require_labels() @@ -202,4 +204,8 @@ class MultiLabel_TextCategorizer(TextCategorizer): def _validate_categories(self, examples: Iterable[Example]): """This component allows any type of single- or multi-label annotations. This method overwrites the more strict one from 'textcat'.""" - pass + # check that annotation values are valid + for ex in examples: + for val in ex.reference.cats.values(): + if not (val == 1.0 or val == 0.0): + raise ValueError(Errors.E851.format(val=val)) diff --git a/spacy/tests/pipeline/test_textcat.py b/spacy/tests/pipeline/test_textcat.py index d359b77db..2eda9deaf 100644 --- a/spacy/tests/pipeline/test_textcat.py +++ b/spacy/tests/pipeline/test_textcat.py @@ -360,6 +360,30 @@ def test_label_types(name): nlp.initialize() +@pytest.mark.parametrize( + "name,get_examples", + [ + ("textcat", make_get_examples_single_label), + ("textcat_multilabel", make_get_examples_multi_label), + ], +) +def test_invalid_label_value(name, get_examples): + nlp = Language() + textcat = nlp.add_pipe(name) + example_getter = get_examples(nlp) + + def invalid_examples(): + # make one example with an invalid score + examples = example_getter() + ref = examples[0].reference + key = list(ref.cats.keys())[0] + ref.cats[key] = 2.0 + return examples + + with pytest.raises(ValueError): + nlp.initialize(get_examples=invalid_examples) + + @pytest.mark.parametrize("name", ["textcat", "textcat_multilabel"]) def test_no_label(name): nlp = Language() From a83463c5e07035ae5832e6790a0c0170e3746bd1 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 18 Nov 2022 08:15:27 +0100 Subject: [PATCH 096/179] Add transformer recommendation for ca (#11819) Model recommendation from @cayorodriguez. --- .../templates/quickstart_training_recommendations.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/spacy/cli/templates/quickstart_training_recommendations.yml b/spacy/cli/templates/quickstart_training_recommendations.yml index 27945e27a..4f214d22d 100644 --- a/spacy/cli/templates/quickstart_training_recommendations.yml +++ b/spacy/cli/templates/quickstart_training_recommendations.yml @@ -37,6 +37,15 @@ bn: accuracy: name: sagorsarker/bangla-bert-base size_factor: 3 +ca: + word_vectors: null + transformer: + efficiency: + name: projecte-aina/roberta-base-ca-v2 + size_factor: 3 + accuracy: + name: projecte-aina/roberta-base-ca-v2 + size_factor: 3 da: word_vectors: da_core_news_lg transformer: From e3173bd86d65a534f92578b85b0e5058a5c845f4 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Fri, 18 Nov 2022 16:24:22 +0900 Subject: [PATCH 097/179] Remove spikex from Universe (#11825) --- website/meta/universe.json | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/website/meta/universe.json b/website/meta/universe.json index 661f5da12..57bf2d3e3 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -461,37 +461,6 @@ }, "category": ["standalone"] }, - { - "id": "spikex", - "title": "SpikeX - SpaCy Pipes for Knowledge Extraction", - "slogan": "Use SpikeX to build knowledge extraction tools with almost-zero effort", - "description": "SpikeX is a collection of pipes ready to be plugged in a spaCy pipeline. It aims to help in building knowledge extraction tools with almost-zero effort.", - "github": "erre-quadro/spikex", - "pip": "spikex", - "code_example": [ - "from spacy import load as spacy_load", - "from spikex.wikigraph import load as wg_load", - "from spikex.pipes import WikiPageX", - "", - "# load a spacy model and get a doc", - "nlp = spacy_load('en_core_web_sm')", - "doc = nlp('An apple a day keeps the doctor away')", - "# load a WikiGraph", - "wg = wg_load('simplewiki_core')", - "# get a WikiPageX and extract all pages", - "wikipagex = WikiPageX(wg)", - "doc = wikipagex(doc)", - "# see all pages extracted from the doc", - "for span in doc._.wiki_spans:", - " print(span._.wiki_pages)" - ], - "category": ["pipeline", "standalone"], - "author": "Erre Quadro", - "author_links": { - "github": "erre-quadro", - "website": "https://www.errequadrosrl.com" - } - }, { "id": "spacy-dbpedia-spotlight", "title": "DBpedia Spotlight for SpaCy", From 89bfd06fbd89cc00ca2007bf795326538126f937 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 18 Nov 2022 18:24:13 +0900 Subject: [PATCH 098/179] Auto-format code with black (#11826) Co-authored-by: explosion-bot --- spacy/pipeline/textcat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/pipeline/textcat.py b/spacy/pipeline/textcat.py index a86eb99d2..9490e3cb1 100644 --- a/spacy/pipeline/textcat.py +++ b/spacy/pipeline/textcat.py @@ -293,7 +293,7 @@ class TextCategorizer(TrainablePipe): bp_scores(gradient) if sgd is not None: self.finish_update(sgd) - losses[self.name] += (gradient ** 2).sum() + losses[self.name] += (gradient**2).sum() return losses def _examples_to_truth( @@ -327,7 +327,7 @@ class TextCategorizer(TrainablePipe): not_missing = self.model.ops.asarray(not_missing) # type: ignore d_scores = scores - truths d_scores *= not_missing - mean_square_error = (d_scores ** 2).mean() + mean_square_error = (d_scores**2).mean() return float(mean_square_error), d_scores def add_label(self, label: str) -> int: From f0d8309a289015ae44f994e8c0207cdfe41583ec Mon Sep 17 00:00:00 2001 From: Marco Edward Gorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Mon, 21 Nov 2022 07:12:03 +0000 Subject: [PATCH 099/179] fix comparison of constants (#11834) Co-authored-by: MarcoGorelli <> --- .pre-commit-config.yaml | 2 +- spacy/tests/vocab_vectors/test_vocab_api.py | 21 +++++++++++++++++++++ spacy/vocab.pyx | 4 ++-- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df59697b1..e2c5e98fd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ repos: - id: black language_version: python3.7 additional_dependencies: ['click==8.0.4'] -- repo: https://gitlab.com/pycqa/flake8 +- repo: https://github.com/pycqa/flake8 rev: 5.0.4 hooks: - id: flake8 diff --git a/spacy/tests/vocab_vectors/test_vocab_api.py b/spacy/tests/vocab_vectors/test_vocab_api.py index 16cf80a08..b9c386eb8 100644 --- a/spacy/tests/vocab_vectors/test_vocab_api.py +++ b/spacy/tests/vocab_vectors/test_vocab_api.py @@ -1,8 +1,13 @@ +import os + import pytest from spacy.attrs import IS_ALPHA, LEMMA, ORTH +from spacy.lang.en import English from spacy.parts_of_speech import NOUN, VERB from spacy.vocab import Vocab +from ..util import make_tempdir + @pytest.mark.issue(1868) def test_issue1868(): @@ -59,3 +64,19 @@ def test_vocab_api_contains(en_vocab, text): def test_vocab_writing_system(en_vocab): assert en_vocab.writing_system["direction"] == "ltr" assert en_vocab.writing_system["has_case"] is True + + +def test_to_disk(): + nlp = English() + with make_tempdir() as d: + nlp.vocab.to_disk(d) + assert "vectors" in os.listdir(d) + assert "lookups.bin" in os.listdir(d) + + +def test_to_disk_exclude(): + nlp = English() + with make_tempdir() as d: + nlp.vocab.to_disk(d, exclude=("vectors", "lookups")) + assert "vectors" not in os.listdir(d) + assert "lookups.bin" not in os.listdir(d) diff --git a/spacy/vocab.pyx b/spacy/vocab.pyx index 428cadd82..27f8e5f98 100644 --- a/spacy/vocab.pyx +++ b/spacy/vocab.pyx @@ -468,9 +468,9 @@ cdef class Vocab: setters = ["strings", "vectors"] if "strings" not in exclude: self.strings.to_disk(path / "strings.json") - if "vectors" not in "exclude": + if "vectors" not in exclude: self.vectors.to_disk(path, exclude=["strings"]) - if "lookups" not in "exclude": + if "lookups" not in exclude: self.lookups.to_disk(path) def from_disk(self, path, *, exclude=tuple()): From 9d96e44a87fc2646bddc7e4e8d0357d48caf42e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Bl=C3=A4ttermann?= Date: Sun, 11 Sep 2022 22:04:00 +0200 Subject: [PATCH 100/179] Apply Prettier to `README.md` --- website/README.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/website/README.md b/website/README.md index 66bc20ad9..42419fbb7 100644 --- a/website/README.md +++ b/website/README.md @@ -555,13 +555,13 @@ extensions for your code editor. The file in the root defines the settings used in this codebase. ## Building & developing the site with Docker {#docker} -Sometimes it's hard to get a local environment working due to rapid updates to node dependencies, -so it may be easier to use docker for building the docs. -If you'd like to do this, -**be sure you do *not* include your local `node_modules` folder**, -since there are some dependencies that need to be built for the image system. -Rename it before using. +Sometimes it's hard to get a local environment working due to rapid updates to +node dependencies, so it may be easier to use docker for building the docs. + +If you'd like to do this, **be sure you do _not_ include your local +`node_modules` folder**, since there are some dependencies that need to be built +for the image system. Rename it before using. ```bash docker run -it \ @@ -571,13 +571,13 @@ docker run -it \ gatsby develop -H 0.0.0.0 ``` -This will allow you to access the built website at http://0.0.0.0:8000/ -in your browser, and still edit code in your editor while having the site -reflect those changes. +This will allow you to access the built website at http://0.0.0.0:8000/ in your +browser, and still edit code in your editor while having the site reflect those +changes. -**Note**: If you're working on a Mac with an M1 processor, -you might see segfault errors from `qemu` if you use the default image. -To fix this use the `arm64` tagged image in the `docker run` command +**Note**: If you're working on a Mac with an M1 processor, you might see +segfault errors from `qemu` if you use the default image. To fix this use the +`arm64` tagged image in the `docker run` command (ghcr.io/explosion/spacy-io:arm64). ### Building the Docker image {#docker-build} @@ -588,7 +588,8 @@ If you'd like to build the image locally, you can do so like this: docker build -t spacy-io . ``` -This will take some time, so if you want to use the prebuilt image you'll save a bit of time. +This will take some time, so if you want to use the prebuilt image you'll save a +bit of time. ## Markdown reference {#markdown} From 96218a1e8f3af6a2ebcfaf6a28c7cadb4a60c203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Bl=C3=A4ttermann?= Date: Tue, 22 Nov 2022 15:55:39 +0100 Subject: [PATCH 101/179] Delete `styleguide.md` This is in intermediate commit, so the content of `/README.md`can be moved to the styleguid, but the history is kept --- website/docs/styleguide.md | 30 ------------------------------ 1 file changed, 30 deletions(-) delete mode 100644 website/docs/styleguide.md diff --git a/website/docs/styleguide.md b/website/docs/styleguide.md deleted file mode 100644 index ed6f9d99b..000000000 --- a/website/docs/styleguide.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -title: Styleguide -section: styleguide -search_exclude: true -menu: - - ['Logo', 'logo'] - - ['Colors', 'colors'] - - ['Typography', 'typography'] - - ['Elements', 'elements'] - - ['Components', 'components'] - - ['Setup & Installation', 'setup'] - - ['Markdown Reference', 'markdown'] - - ['Project Structure', 'structure'] - - ['Editorial', 'editorial'] -sidebar: - - label: Styleguide - items: - - text: '' - url: '/styleguide' - - label: Resources - items: - - text: Website Source - url: https://github.com/explosion/spacy/tree/master/website - - text: Contributing Guide - url: https://github.com/explosion/spaCy/blob/master/CONTRIBUTING.md ---- - -import Readme from 'README.md' - - From 0794e5c6cce6282434576cd311126b0c3bfebc35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Bl=C3=A4ttermann?= Date: Wed, 23 Nov 2022 01:22:27 +0100 Subject: [PATCH 102/179] Add missing files to project structure in `README.md` --- website/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/README.md b/website/README.md index 42419fbb7..b2f640543 100644 --- a/website/README.md +++ b/website/README.md @@ -648,8 +648,10 @@ In addition to the native markdown elements, you can use the components | ├── languages.json # supported languages and statistical models | ├── sidebars.json # sidebar navigations for different sections | ├── site.json # general site metadata +| ├── type-annotations.json # Type annotations | └── universe.json # data for the spaCy universe section ├── public # compiled site +├── setup # Jinja setup ├── src # source | ├── components # React components | ├── fonts # webfonts @@ -662,6 +664,8 @@ In addition to the native markdown elements, you can use the components | | ├── models.js # layout template for model pages | | └── universe.js # layout templates for universe | └── widgets # non-reusable components with content, e.g. changelog +├── .eslintrc.json # ESLint config file +├── .prettierrc # Prettier config file ├── gatsby-browser.js # browser-specific hooks for Gatsby ├── gatsby-config.js # Gatsby configuration ├── gatsby-node.js # Node-specific hooks for Gatsby From 8c0ceca637d486624cb32fbf9c875e69f81dcf83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Bl=C3=A4ttermann?= Date: Tue, 22 Nov 2022 15:56:21 +0100 Subject: [PATCH 103/179] Move `README.md` content to styleguide --- website/{README.md => docs/styleguide.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename website/{README.md => docs/styleguide.md} (100%) diff --git a/website/README.md b/website/docs/styleguide.md similarity index 100% rename from website/README.md rename to website/docs/styleguide.md From 5659eeaadd750e404152c731728ebf922c824226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Bl=C3=A4ttermann?= Date: Tue, 22 Nov 2022 15:58:39 +0100 Subject: [PATCH 104/179] Remove styleguide content from `README.md` --- website/README.md | 608 ---------------------------------------------- 1 file changed, 608 deletions(-) diff --git a/website/README.md b/website/README.md index b2f640543..743e61acd 100644 --- a/website/README.md +++ b/website/README.md @@ -11,520 +11,6 @@ rendered version is available at https://spacy.io/styleguide._ -The [spacy.io](https://spacy.io) website is implemented using -[Gatsby](https://www.gatsbyjs.org) with -[Remark](https://github.com/remarkjs/remark) and [MDX](https://mdxjs.com/). This -allows authoring content in **straightforward Markdown** without the usual -limitations. Standard elements can be overwritten with powerful -[React](http://reactjs.org/) components and wherever Markdown syntax isn't -enough, JSX components can be used. - -> #### Contributing to the site -> -> The docs can always use another example or more detail, and they should always -> be up to date and not misleading. We always appreciate a -> [pull request](https://github.com/explosion/spaCy/pulls). To quickly find the -> correct file to edit, simply click on the "Suggest edits" button at the bottom -> of a page. -> -> For more details on editing the site locally, see the installation -> instructions and markdown reference below. - -## Logo {#logo source="website/src/images/logo.svg"} - -import { Logos } from 'widgets/styleguide' - -If you would like to use the spaCy logo on your site, please get in touch and -ask us first. However, if you want to show support and tell others that your -project is using spaCy, you can grab one of our -[spaCy badges](/usage/spacy-101#faq-project-with-spacy). - - - -## Colors {#colors} - -import { Colors, Patterns } from 'widgets/styleguide' - - - -### Patterns - - - -## Typography {#typography} - -import { H1, H2, H3, H4, H5, Label, InlineList, Comment } from -'components/typography' - -> #### Markdown -> -> ```markdown_ -> ## Headline 2 -> ## Headline 2 {#some_id} -> ## Headline 2 {#some_id tag="method"} -> ``` -> -> #### JSX -> -> ```jsx ->

Headline 2

->

Headline 2

->

Headline 2

-> ``` - -Headlines are set in -[HK Grotesk](http://cargocollective.com/hanken/HK-Grotesk-Open-Source-Font) by -Hanken Design. All other body text and code uses the best-matching default -system font to provide a "native" reading experience. All code uses the -[JetBrains Mono](https://www.jetbrains.com/lp/mono/) typeface by JetBrains. - - - -Level 2 headings are automatically wrapped in `
` elements at compile -time, using a custom -[Markdown transformer](https://github.com/explosion/spaCy/tree/master/website/plugins/remark-wrap-section.js). -This makes it easier to highlight the section that's currently in the viewpoint -in the sidebar menu. - - - -
-

Headline 1

-

Headline 2

-

Headline 3

-

Headline 4

-
Headline 5
- -
- ---- - -The following optional attributes can be set on the headline to modify it. For -example, to add a tag for the documented type or mark features that have been -introduced in a specific version or require statistical models to be loaded. -Tags are also available as standalone `` components. - -| Argument | Example | Result | -| -------- | -------------------------- | ----------------------------------------- | -| `tag` | `{tag="method"}` | method | -| `new` | `{new="3"}` | 3 | -| `model` | `{model="tagger, parser"}` | tagger, parser | -| `hidden` | `{hidden="true"}` | | - -## Elements {#elements} - -### Links {#links} - -> #### Markdown -> -> ```markdown -> [I am a link](https://spacy.io) -> ``` -> -> #### JSX -> -> ```jsx -> I am a link -> ``` - -Special link styles are used depending on the link URL. - -- [I am a regular external link](https://explosion.ai) -- [I am a link to the documentation](/api/doc) -- [I am a link to an architecture](/api/architectures#HashEmbedCNN) -- [I am a link to a model](/models/en#en_core_web_sm) -- [I am a link to GitHub](https://github.com/explosion/spaCy) - -### Abbreviations {#abbr} - -import { Abbr } from 'components/typography' - -> #### JSX -> -> ```jsx -> Abbreviation -> ``` - -Some text with an abbreviation. On small -screens, I collapse and the explanation text is displayed next to the -abbreviation. - -### Tags {#tags} - -import Tag from 'components/tag' - -> ```jsx -> method -> 4 -> tagger, parser -> ``` - -Tags can be used together with headlines, or next to properties across the -documentation, and combined with tooltips to provide additional information. An -optional `variant` argument can be used for special tags. `variant="new"` makes -the tag take a version number to mark new features. Using the component, -visibility of this tag can later be toggled once the feature isn't considered -new anymore. Setting `variant="model"` takes a description of model capabilities -and can be used to mark features that require a respective model to be -installed. - - - -method 4 tagger, -parser - - - -### Buttons {#buttons} - -import Button from 'components/button' - -> ```jsx -> -> -> ``` - -Link buttons come in two variants, `primary` and `secondary` and two sizes, with -an optional `large` size modifier. Since they're mostly used as enhanced links, -the buttons are implemented as styled links instead of native button elements. - - - - -
- - - - -## Components - -### Table {#table} - -> #### Markdown -> -> ```markdown_ -> | Header 1 | Header 2 | -> | -------- | -------- | -> | Column 1 | Column 2 | -> ``` -> -> #### JSX -> -> ```markup -> -> -> ->
Header 1Header 2
Column 1Column 2
-> ``` - -Tables are used to present data and API documentation. Certain keywords can be -used to mark a footer row with a distinct style, for example to visualize the -return values of a documented function. - -| Header 1 | Header 2 | Header 3 | Header 4 | -| ----------- | -------- | :------: | -------: | -| Column 1 | Column 2 | Column 3 | Column 4 | -| Column 1 | Column 2 | Column 3 | Column 4 | -| Column 1 | Column 2 | Column 3 | Column 4 | -| Column 1 | Column 2 | Column 3 | Column 4 | -| **RETURNS** | Column 2 | Column 3 | Column 4 | - -Tables also support optional "divider" rows that are typically used to denote -keyword-only arguments in API documentation. To turn a row into a dividing -headline, it should only include content in its first cell, and its value should -be italicized: - -> #### Markdown -> -> ```markdown_ -> | Header 1 | Header 2 | Header 3 | -> | -------- | -------- | -------- | -> | Column 1 | Column 2 | Column 3 | -> | _Hello_ | | | -> | Column 1 | Column 2 | Column 3 | -> ``` - -| Header 1 | Header 2 | Header 3 | -| -------- | -------- | -------- | -| Column 1 | Column 2 | Column 3 | -| _Hello_ | | | -| Column 1 | Column 2 | Column 3 | - -### Type Annotations {#type-annotations} - -> #### Markdown -> -> ```markdown_ -> ~~Model[List[Doc], Floats2d]~~ -> ``` -> -> #### JSX -> -> ```markup -> Model[List[Doc], Floats2d] -> ``` - -Type annotations are special inline code blocks are used to describe Python -types in the [type hints](https://docs.python.org/3/library/typing.html) format. -The special component will split the type, apply syntax highlighting and link -all types that specify links in `meta/type-annotations.json`. Types can link to -internal or external documentation pages. To make it easy to represent the type -annotations in Markdown, the rendering "hijacks" the `~~` tags that would -typically be converted to a `` element – but in this case, text surrounded -by `~~` becomes a type annotation. - -- ~~Dict[str, List[Union[Doc, Span]]]~~ -- ~~Model[List[Doc], List[numpy.ndarray]]~~ - -Type annotations support a special visual style in tables and will render as a -separate row, under the cell text. This allows the API docs to display complex -types without taking up too much space in the cell. The type annotation should -always be the **last element** in the row. - -> #### Markdown -> -> ```markdown_ -> | Header 1 | Header 2 | -> | -------- | ----------------------- | -> | Column 1 | Column 2 ~~List[Doc]~~ | -> ``` - -| Name | Description | -| ----------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `vocab` | The shared vocabulary. ~~Vocab~~ | -| `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) wrapping the transformer. ~~Model[List[Doc], FullTransformerBatch]~~ | -| `set_extra_annotations` | Function that takes a batch of `Doc` objects and transformer outputs and can set additional annotations on the `Doc`. ~~Callable[[List[Doc], FullTransformerBatch], None]~~ | - -### List {#list} - -> #### Markdown -> -> ```markdown_ -> 1. One -> 2. Two -> ``` -> -> #### JSX -> -> ```markup ->
    ->
  1. One
    2. ->
  2. Two
    3. ->
-> ``` - -Lists are available as bulleted and numbered. Markdown lists are transformed -automatically. - -- I am a bulleted list -- I have nice bullets -- Lorem ipsum dolor -- consectetur adipiscing elit - -1. I am an ordered list -2. I have nice numbers -3. Lorem ipsum dolor -4. consectetur adipiscing elit - -### Aside {#aside} - -> #### Markdown -> -> ```markdown_ -> > #### Aside title -> > This is aside text. -> ``` -> -> #### JSX -> -> ```jsx -> -> ``` - -Asides can be used to display additional notes and content in the right-hand -column. Asides can contain text, code and other elements if needed. Visually, -asides are moved to the side on the X-axis, and displayed at the same level they -were inserted. On small screens, they collapse and are rendered in their -original position, in between the text. - -To make them easier to use in Markdown, paragraphs formatted as blockquotes will -turn into asides by default. Level 4 headlines (with a leading `####`) will -become aside titles. - -### Code Block {#code-block} - -> #### Markdown -> -> ````markdown_ -> ```python -> ### This is a title -> import spacy -> ``` -> ```` -> -> #### JSX -> -> ```jsx -> -> import spacy -> -> ``` - -Code blocks use the [Prism](http://prismjs.com/) syntax highlighter with a -custom theme. The language can be set individually on each block, and defaults -to raw text with no highlighting. An optional label can be added as the first -line with the prefix `####` (Python-like) and `///` (JavaScript-like). the -indented block as plain text and preserve whitespace. - -```python -### Using spaCy -import spacy -nlp = spacy.load("en_core_web_sm") -doc = nlp("This is a sentence.") -for token in doc: - print(token.text, token.pos_) -``` - -Code blocks and also specify an optional range of line numbers to highlight by -adding `{highlight="..."}` to the headline. Acceptable ranges are spans like -`5-7`, but also `5-7,10` or `5-7,10,13-14`. - -> #### Markdown -> -> ````markdown_ -> ```python -> ### This is a title {highlight="1-2"} -> import spacy -> nlp = spacy.load("en_core_web_sm") -> ``` -> ```` - -```python -### Using the matcher {highlight="5-7"} -import spacy -from spacy.matcher import Matcher - -nlp = spacy.load('en_core_web_sm') -matcher = Matcher(nlp.vocab) -pattern = [{"LOWER": "hello"}, {"IS_PUNCT": True}, {"LOWER": "world"}] -matcher.add("HelloWorld", None, pattern) -doc = nlp("Hello, world! Hello world!") -matches = matcher(doc) -``` - -Adding `{executable="true"}` to the title turns the code into an executable -block, powered by [Binder](https://mybinder.org) and -[Juniper](https://github.com/ines/juniper). If JavaScript is disabled, the -interactive widget defaults to a regular code block. - -> #### Markdown -> -> ````markdown_ -> ```python -> ### {executable="true"} -> import spacy -> nlp = spacy.load("en_core_web_sm") -> ``` -> ```` - -```python -### {executable="true"} -import spacy -nlp = spacy.load("en_core_web_sm") -doc = nlp("This is a sentence.") -for token in doc: - print(token.text, token.pos_) -``` - -If a code block only contains a URL to a GitHub file, the raw file contents are -embedded automatically and syntax highlighting is applied. The link to the -original file is shown at the top of the widget. - -> #### Markdown -> -> ````markdown_ -> ```python -> https://github.com/... -> ``` -> ```` -> -> #### JSX -> -> ```jsx -> -> ``` - -```python -https://github.com/explosion/spaCy/tree/master/spacy/language.py -``` - -### Infobox {#infobox} - -import Infobox from 'components/infobox' - -> #### JSX -> -> ```jsx -> Regular infobox -> This is a warning. -> This is dangerous. -> ``` - -Infoboxes can be used to add notes, updates, warnings or additional information -to a page or section. Semantically, they're implemented and interpreted as an -`aside` element. Infoboxes can take an optional `title` argument, as well as an -optional `variant` (either `"warning"` or `"danger"`). - - - -If needed, an infobox can contain regular text, `inline code`, lists and other -blocks. - - - - - -If needed, an infobox can contain regular text, `inline code`, lists and other -blocks. - - - - - -If needed, an infobox can contain regular text, `inline code`, lists and other -blocks. - - - -### Accordion {#accordion} - -import Accordion from 'components/accordion' - -> #### JSX -> -> ```jsx -> -> Accordion content goes here. -> -> ``` - -Accordions are collapsible sections that are mostly used for lengthy tables, -like the tag and label annotation schemes for different languages. They all need -to be presented – but chances are the user doesn't actually care about _all_ of -them, especially not at the same time. So it's fairly reasonable to hide them -begin a click. This particular implementation was inspired by the amazing -[Inclusive Components blog](https://inclusive-components.design/collapsible-sections/). - - - -Lorem ipsum dolor sit amet, consectetur adipiscing elit. Quisque enim ante, -pretium a orci eget, varius dignissim augue. Nam eu dictum mauris, id tincidunt -nisi. Integer commodo pellentesque tincidunt. Nam at turpis finibus tortor -gravida sodales tincidunt sit amet est. Nullam euismod arcu in tortor auctor, -sit amet dignissim justo congue. - - - ## Setup and installation {#setup} Before running the setup, make sure your versions of @@ -591,54 +77,6 @@ docker build -t spacy-io . This will take some time, so if you want to use the prebuilt image you'll save a bit of time. -## Markdown reference {#markdown} - -All page content and page meta lives in the `.md` files in the `/docs` -directory. The frontmatter block at the top of each file defines the page title -and other settings like the sidebar menu. - -````markdown ---- -title: Page title ---- - -## Headline starting a section {#some_id} - -This is a regular paragraph with a [link](https://spacy.io) and **bold text**. - -> #### This is an aside title -> -> This is aside text. - -### Subheadline - -| Header 1 | Header 2 | -| -------- | -------- | -| Column 1 | Column 2 | - -```python -### Code block title {highlight="2-3"} -import spacy -nlp = spacy.load("en_core_web_sm") -doc = nlp("Hello world") -``` - - - -This is content in the infobox. - - -```` - -In addition to the native markdown elements, you can use the components -[``][infobox], [``][accordion], [``][abbr] and -[``][tag] via their JSX syntax. - -[infobox]: https://spacy.io/styleguide#infobox -[accordion]: https://spacy.io/styleguide#accordion -[abbr]: https://spacy.io/styleguide#abbr -[tag]: https://spacy.io/styleguide#tag - ## Project structure {#structure} ```yaml @@ -671,49 +109,3 @@ In addition to the native markdown elements, you can use the components ├── gatsby-node.js # Node-specific hooks for Gatsby └── package.json # package settings and dependencies ``` - -## Editorial {#editorial} - -- "spaCy" should always be spelled with a lowercase "s" and a capital "C", - unless it specifically refers to the Python package or Python import `spacy` - (in which case it should be formatted as code). - - ✅ spaCy is a library for advanced NLP in Python. - - ❌ Spacy is a library for advanced NLP in Python. - - ✅ First, you need to install the `spacy` package from pip. -- Mentions of code, like function names, classes, variable names etc. in inline - text should be formatted as `code`. - - ✅ "Calling the `nlp` object on a text returns a `Doc`." -- Objects that have pages in the [API docs](/api) should be linked – for - example, [`Doc`](/api/doc) or [`Language.to_disk`](/api/language#to_disk). The - mentions should still be formatted as code within the link. Links pointing to - the API docs will automatically receive a little icon. However, if a paragraph - includes many references to the API, the links can easily get messy. In that - case, we typically only link the first mention of an object and not any - subsequent ones. - - ✅ The [`Span`](/api/span) and [`Token`](/api/token) objects are views of a - [`Doc`](/api/doc). [`Span.as_doc`](/api/span#as_doc) creates a `Doc` object - from a `Span`. - - ❌ The [`Span`](/api/span) and [`Token`](/api/token) objects are views of a - [`Doc`](/api/doc). [`Span.as_doc`](/api/span#as_doc) creates a - [`Doc`](/api/doc) object from a [`Span`](/api/span). - -* Other things we format as code are: references to trained pipeline packages - like `en_core_web_sm` or file names like `code.py` or `meta.json`. - - - ✅ After training, the `config.cfg` is saved to disk. - -* [Type annotations](#type-annotations) are a special type of code formatting, - expressed by wrapping the text in `~~` instead of backticks. The result looks - like this: ~~List[Doc]~~. All references to known types will be linked - automatically. - - - ✅ The model has the input type ~~List[Doc]~~ and it outputs a - ~~List[Array2d]~~. - -* We try to keep links meaningful but short. - - ✅ For details, see the usage guide on - [training with custom code](/usage/training#custom-code). - - ❌ For details, see - [the usage guide on training with custom code](/usage/training#custom-code). - - ❌ For details, see the usage guide on training with custom code - [here](/usage/training#custom-code). From ecbf052abde2ab9373be1d7652e20d50b096e49d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Bl=C3=A4ttermann?= Date: Tue, 22 Nov 2022 16:00:08 +0100 Subject: [PATCH 105/179] Remove `README.md` content from styleguide --- website/docs/styleguide.md | 112 ------------------------------------- 1 file changed, 112 deletions(-) diff --git a/website/docs/styleguide.md b/website/docs/styleguide.md index b2f640543..dbc5258db 100644 --- a/website/docs/styleguide.md +++ b/website/docs/styleguide.md @@ -1,16 +1,3 @@ - - -# spacy.io website and docs - -![Netlify Status](https://api.netlify.com/api/v1/badges/d65fe97d-99ab-47f8-a339-1d8987251da0/deploy-status) - -_This page contains the documentation and styleguide for the spaCy website. Its -rendered version is available at https://spacy.io/styleguide._ - ---- - - - The [spacy.io](https://spacy.io) website is implemented using [Gatsby](https://www.gatsbyjs.org) with [Remark](https://github.com/remarkjs/remark) and [MDX](https://mdxjs.com/). This @@ -525,72 +512,6 @@ sit amet dignissim justo congue. -## Setup and installation {#setup} - -Before running the setup, make sure your versions of -[Node](https://nodejs.org/en/) and [npm](https://www.npmjs.com/) are up to date. -Node v10.15 or later is required. - -```bash -# Clone the repository -git clone https://github.com/explosion/spaCy -cd spaCy/website - -# Install Gatsby's command-line tool -npm install --global gatsby-cli - -# Install the dependencies -npm install - -# Start the development server -npm run dev -``` - -If you are planning on making edits to the site, you should also set up the -[Prettier](https://prettier.io/) code formatter. It takes care of formatting -Markdown and other files automatically. -[See here](https://prettier.io/docs/en/editors.html) for the available -extensions for your code editor. The -[`.prettierrc`](https://github.com/explosion/spaCy/tree/master/website/.prettierrc) -file in the root defines the settings used in this codebase. - -## Building & developing the site with Docker {#docker} - -Sometimes it's hard to get a local environment working due to rapid updates to -node dependencies, so it may be easier to use docker for building the docs. - -If you'd like to do this, **be sure you do _not_ include your local -`node_modules` folder**, since there are some dependencies that need to be built -for the image system. Rename it before using. - -```bash -docker run -it \ - -v $(pwd):/spacy-io/website \ - -p 8000:8000 \ - ghcr.io/explosion/spacy-io \ - gatsby develop -H 0.0.0.0 -``` - -This will allow you to access the built website at http://0.0.0.0:8000/ in your -browser, and still edit code in your editor while having the site reflect those -changes. - -**Note**: If you're working on a Mac with an M1 processor, you might see -segfault errors from `qemu` if you use the default image. To fix this use the -`arm64` tagged image in the `docker run` command -(ghcr.io/explosion/spacy-io:arm64). - -### Building the Docker image {#docker-build} - -If you'd like to build the image locally, you can do so like this: - -```bash -docker build -t spacy-io . -``` - -This will take some time, so if you want to use the prebuilt image you'll save a -bit of time. - ## Markdown reference {#markdown} All page content and page meta lives in the `.md` files in the `/docs` @@ -639,39 +560,6 @@ In addition to the native markdown elements, you can use the components [abbr]: https://spacy.io/styleguide#abbr [tag]: https://spacy.io/styleguide#tag -## Project structure {#structure} - -```yaml -### Directory structure -├── docs # the actual markdown content -├── meta # JSON-formatted site metadata -| ├── languages.json # supported languages and statistical models -| ├── sidebars.json # sidebar navigations for different sections -| ├── site.json # general site metadata -| ├── type-annotations.json # Type annotations -| └── universe.json # data for the spaCy universe section -├── public # compiled site -├── setup # Jinja setup -├── src # source -| ├── components # React components -| ├── fonts # webfonts -| ├── images # images used in the layout -| ├── plugins # custom plugins to transform Markdown -| ├── styles # CSS modules and global styles -| ├── templates # page layouts -| | ├── docs.js # layout template for documentation pages -| | ├── index.js # global layout template -| | ├── models.js # layout template for model pages -| | └── universe.js # layout templates for universe -| └── widgets # non-reusable components with content, e.g. changelog -├── .eslintrc.json # ESLint config file -├── .prettierrc # Prettier config file -├── gatsby-browser.js # browser-specific hooks for Gatsby -├── gatsby-config.js # Gatsby configuration -├── gatsby-node.js # Node-specific hooks for Gatsby -└── package.json # package settings and dependencies -``` - ## Editorial {#editorial} - "spaCy" should always be spelled with a lowercase "s" and a capital "C", From f1ddac187de7e67923e8ee63192787179f70fa4c Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Wed, 23 Nov 2022 18:51:31 +0900 Subject: [PATCH 106/179] Remove unused error object (#11837) --- spacy/language.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/spacy/language.py b/spacy/language.py index 836f3abf9..2789b6690 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -784,14 +784,6 @@ class Language: factory_name, source, name=name ) else: - if not self.has_factory(factory_name): - err = Errors.E002.format( - name=factory_name, - opts=", ".join(self.factory_names), - method="add_pipe", - lang=util.get_object_name(self), - lang_code=self.lang, - ) pipe_component = self.create_pipe( factory_name, name=name, From 8271cfb4cd8a907ff11f12841ee1ceb171b3f528 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Wed, 23 Nov 2022 19:03:18 +0900 Subject: [PATCH 107/179] Remove Learning Path spaCy (#11846) --- website/meta/universe.json | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/website/meta/universe.json b/website/meta/universe.json index 57bf2d3e3..97b53e9c5 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -1993,17 +1993,6 @@ }, "category": ["books"] }, - { - "type": "education", - "id": "learning-path-spacy", - "title": "Learning Path: Mastering spaCy for Natural Language Processing", - "slogan": "O'Reilly, 2017", - "description": "spaCy, a fast, user-friendly library for teaching computers to understand text, simplifies NLP techniques, such as speech tagging and syntactic dependencies, so you can easily extract information, attributes, and objects from massive amounts of text to then document, measure, and analyze. This Learning Path is a hands-on introduction to using spaCy to discover insights through natural language processing. While end-to-end natural language processing solutions can be complex, you’ll learn the linguistics, algorithms, and machine learning skills to get the job done.", - "url": "https://www.safaribooksonline.com/library/view/learning-path-mastering/9781491986653/", - "thumb": "https://i.imgur.com/9MIgMAc.jpg", - "author": "Aaron Kramer", - "category": ["courses"] - }, { "type": "education", "id": "introduction-into-spacy-3", From 5ea14af32b4203bc3087dec63091e63fe4ac95b7 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Wed, 23 Nov 2022 17:54:58 +0100 Subject: [PATCH 108/179] Add `training.before_update` callback (#11739) * Add `training.before_update` callback This callback can be used to implement training paradigms like gradual (un)freezing of components (e.g: the Transformer) after a certain number of training steps to mitigate catastrophic forgetting during fine-tuning. * Fix type annotation, default config value * Generalize arguments passed to the callback * Update schema * Pass `epoch` to callback, rename `current_step` to `step` * Add test * Simplify test * Replace config string with `spacy.blank` * Apply suggestions from code review Co-authored-by: Adriane Boyd * Cleanup imports Co-authored-by: Adriane Boyd --- spacy/default_config.cfg | 2 ++ spacy/schemas.py | 1 + spacy/tests/training/test_training.py | 40 ++++++++++++++++++++++++++- spacy/training/loop.py | 6 ++++ website/docs/api/data-formats.md | 1 + 5 files changed, 49 insertions(+), 1 deletion(-) diff --git a/spacy/default_config.cfg b/spacy/default_config.cfg index 86a72926e..694fb732f 100644 --- a/spacy/default_config.cfg +++ b/spacy/default_config.cfg @@ -90,6 +90,8 @@ dev_corpus = "corpora.dev" train_corpus = "corpora.train" # Optional callback before nlp object is saved to disk after training before_to_disk = null +# Optional callback that is invoked at the start of each training step +before_update = null [training.logger] @loggers = "spacy.ConsoleLogger.v1" diff --git a/spacy/schemas.py b/spacy/schemas.py index c824d76b9..e48fe1702 100644 --- a/spacy/schemas.py +++ b/spacy/schemas.py @@ -329,6 +329,7 @@ class ConfigSchemaTraining(BaseModel): frozen_components: List[str] = Field(..., title="Pipeline components that shouldn't be updated during training") annotating_components: List[str] = Field(..., title="Pipeline components that should set annotations during training") before_to_disk: Optional[Callable[["Language"], "Language"]] = Field(..., title="Optional callback to modify nlp object after training, before it's saved to disk") + before_update: Optional[Callable[["Language", Dict[str, Any]], None]] = Field(..., title="Optional callback that is invoked at the start of each training step") # fmt: on class Config: diff --git a/spacy/tests/training/test_training.py b/spacy/tests/training/test_training.py index 4384a796d..7933ea31f 100644 --- a/spacy/tests/training/test_training.py +++ b/spacy/tests/training/test_training.py @@ -2,6 +2,7 @@ import random import numpy import pytest +import spacy import srsly from spacy.lang.en import English from spacy.tokens import Doc, DocBin @@ -11,9 +12,10 @@ from spacy.training import offsets_to_biluo_tags from spacy.training.alignment_array import AlignmentArray from spacy.training.align import get_alignments from spacy.training.converters import json_to_docs +from spacy.training.loop import train_while_improving from spacy.util import get_words_and_spaces, load_model_from_path, minibatch from spacy.util import load_config_from_str -from thinc.api import compounding +from thinc.api import compounding, Adam from ..util import make_tempdir @@ -1112,3 +1114,39 @@ def test_retokenized_docs(doc): retokenizer.merge(doc1[0:2]) retokenizer.merge(doc1[5:7]) assert example.get_aligned("ORTH", as_string=True) == expected2 + + +def test_training_before_update(doc): + def before_update(nlp, args): + assert args["step"] == 0 + assert args["epoch"] == 1 + + # Raise an error here as the rest of the loop + # will not run to completion due to uninitialized + # models. + raise ValueError("ran_before_update") + + def generate_batch(): + yield 1, [Example(doc, doc)] + + nlp = spacy.blank("en") + nlp.add_pipe("tagger") + optimizer = Adam() + generator = train_while_improving( + nlp, + optimizer, + generate_batch(), + lambda: None, + dropout=0.1, + eval_frequency=100, + accumulate_gradient=10, + patience=10, + max_steps=100, + exclude=[], + annotating_components=[], + before_update=before_update, + ) + + with pytest.raises(ValueError, match="ran_before_update"): + for _ in generator: + pass diff --git a/spacy/training/loop.py b/spacy/training/loop.py index 06372cbb0..885257772 100644 --- a/spacy/training/loop.py +++ b/spacy/training/loop.py @@ -59,6 +59,7 @@ def train( batcher = T["batcher"] train_logger = T["logger"] before_to_disk = create_before_to_disk_callback(T["before_to_disk"]) + before_update = T["before_update"] # Helper function to save checkpoints. This is a closure for convenience, # to avoid passing in all the args all the time. @@ -89,6 +90,7 @@ def train( eval_frequency=T["eval_frequency"], exclude=frozen_components, annotating_components=annotating_components, + before_update=before_update, ) clean_output_dir(output_path) stdout.write(msg.info(f"Pipeline: {nlp.pipe_names}") + "\n") @@ -150,6 +152,7 @@ def train_while_improving( max_steps: int, exclude: List[str], annotating_components: List[str], + before_update: Optional[Callable[["Language", Dict[str, Any]], None]], ): """Train until an evaluation stops improving. Works as a generator, with each iteration yielding a tuple `(batch, info, is_best_checkpoint)`, @@ -198,6 +201,9 @@ def train_while_improving( words_seen = 0 start_time = timer() for step, (epoch, batch) in enumerate(train_data): + if before_update: + before_update_args = {"step": step, "epoch": epoch} + before_update(nlp, before_update_args) dropout = next(dropouts) # type: ignore for subbatch in subdivide_batch(batch, accumulate_gradient): nlp.update( diff --git a/website/docs/api/data-formats.md b/website/docs/api/data-formats.md index ce06c4ea8..768844cf3 100644 --- a/website/docs/api/data-formats.md +++ b/website/docs/api/data-formats.md @@ -186,6 +186,7 @@ process that are used when you run [`spacy train`](/api/cli#train). | `accumulate_gradient` | Whether to divide the batch up into substeps. Defaults to `1`. ~~int~~ | | `batcher` | Callable that takes an iterator of [`Doc`](/api/doc) objects and yields batches of `Doc`s. Defaults to [`batch_by_words`](/api/top-level#batch_by_words). ~~Callable[[Iterator[Doc], Iterator[List[Doc]]]]~~ | | `before_to_disk` | Optional callback to modify `nlp` object right before it is saved to disk during and after training. Can be used to remove or reset config values or disable components. Defaults to `null`. ~~Optional[Callable[[Language], Language]]~~ | +| `before_update` | Optional callback that is invoked at the start of each training step with the `nlp` object and a `Dict` containing the following entries: `step`, `epoch`. Can be used to make deferred changes to components. Defaults to `null`. ~~Optional[Callable[[Language, Dict[str, Any]], None]]~~ | | `dev_corpus` | Dot notation of the config location defining the dev corpus. Defaults to `corpora.dev`. ~~str~~ | | `dropout` | The dropout rate. Defaults to `0.1`. ~~float~~ | | `eval_frequency` | How often to evaluate during training (steps). Defaults to `200`. ~~int~~ | From 8f062b849c846ecdf59263c82632b9fbd4eca9d0 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Thu, 24 Nov 2022 16:03:42 +0100 Subject: [PATCH 109/179] Fix Matcher cython profile=True header (#11867) --- spacy/matcher/matcher.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/matcher/matcher.pyx b/spacy/matcher/matcher.pyx index e1dba01a2..c4a057ca0 100644 --- a/spacy/matcher/matcher.pyx +++ b/spacy/matcher/matcher.pyx @@ -1,4 +1,4 @@ -# cython: infer_types=True, cython: profile=True +# cython: infer_types=True, profile=True from typing import List, Iterable from libcpp.vector cimport vector From 30d31fd335306921aa7e8be081ecb396880aa14b Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 25 Nov 2022 11:12:46 +0100 Subject: [PATCH 110/179] Update Russian and Ukrainian lemmatizers (#11811) * pymorph2 issues #11620, #11626, #11625: - #11620: pymorphy2_lookup - #11626: handle multiple forms pointing to the same normal form + handling empty POS tag - #11625: matching DET that are labelled as PRON by pymorhp2 * Move lemmatizer algorithm changes back into RussianLemmatizer * Fix uk pymorphy3_lookup mode init * Move and update tests for ru/uk lookup lemmatizer modes * Fix typo * Remove traces of previous behavior for uninflected POS * Refactor to private generic-looking pymorphy methods * Remove xfailed uk lemmatizer cases * Update spacy/lang/ru/lemmatizer.py Co-authored-by: Richard Hudson Co-authored-by: Dmytro S Lituiev Co-authored-by: Richard Hudson --- spacy/lang/ru/lemmatizer.py | 51 ++++++++++++++++++-------- spacy/lang/uk/lemmatizer.py | 2 +- spacy/tests/conftest.py | 18 ++++----- spacy/tests/lang/ru/test_lemmatizer.py | 15 ++++++++ spacy/tests/lang/uk/test_lemmatizer.py | 18 ++++++--- 5 files changed, 73 insertions(+), 31 deletions(-) diff --git a/spacy/lang/ru/lemmatizer.py b/spacy/lang/ru/lemmatizer.py index c37a3a91a..f4a35de38 100644 --- a/spacy/lang/ru/lemmatizer.py +++ b/spacy/lang/ru/lemmatizer.py @@ -28,34 +28,39 @@ class RussianLemmatizer(Lemmatizer): from pymorphy2 import MorphAnalyzer except ImportError: raise ImportError( - "The Russian lemmatizer mode 'pymorphy2' requires the " - "pymorphy2 library. Install it with: pip install pymorphy2" + "The lemmatizer mode 'pymorphy2' requires the " + "pymorphy2 library and dictionaries. Install them with: " + "pip install pymorphy2" + "# for Ukrainian dictionaries:" + "pip install pymorphy2-dicts-uk" ) from None if getattr(self, "_morph", None) is None: - self._morph = MorphAnalyzer() - elif mode == "pymorphy3": + self._morph = MorphAnalyzer(lang="ru") + elif mode in {"pymorphy3", "pymorphy3_lookup"}: try: from pymorphy3 import MorphAnalyzer except ImportError: raise ImportError( - "The Russian lemmatizer mode 'pymorphy3' requires the " - "pymorphy3 library. Install it with: pip install pymorphy3" + "The lemmatizer mode 'pymorphy3' requires the " + "pymorphy3 library and dictionaries. Install them with: " + "pip install pymorphy3" + "# for Ukrainian dictionaries:" + "pip install pymorphy3-dicts-uk" ) from None if getattr(self, "_morph", None) is None: - self._morph = MorphAnalyzer() + self._morph = MorphAnalyzer(lang="ru") super().__init__( vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer ) - def pymorphy2_lemmatize(self, token: Token) -> List[str]: + def _pymorphy_lemmatize(self, token: Token) -> List[str]: string = token.text univ_pos = token.pos_ morphology = token.morph.to_dict() if univ_pos == "PUNCT": return [PUNCT_RULES.get(string, string)] if univ_pos not in ("ADJ", "DET", "NOUN", "NUM", "PRON", "PROPN", "VERB"): - # Skip unchangeable pos - return [string.lower()] + return self._pymorphy_lookup_lemmatize(token) analyses = self._morph.parse(string) filtered_analyses = [] for analysis in analyses: @@ -63,8 +68,10 @@ class RussianLemmatizer(Lemmatizer): # Skip suggested parse variant for unknown word for pymorphy continue analysis_pos, _ = oc2ud(str(analysis.tag)) - if analysis_pos == univ_pos or ( - analysis_pos in ("NOUN", "PROPN") and univ_pos in ("NOUN", "PROPN") + if ( + analysis_pos == univ_pos + or (analysis_pos in ("NOUN", "PROPN") and univ_pos in ("NOUN", "PROPN")) + or ((analysis_pos == "PRON") and (univ_pos == "DET")) ): filtered_analyses.append(analysis) if not len(filtered_analyses): @@ -107,15 +114,27 @@ class RussianLemmatizer(Lemmatizer): dict.fromkeys([analysis.normal_form for analysis in filtered_analyses]) ) - def pymorphy2_lookup_lemmatize(self, token: Token) -> List[str]: + def _pymorphy_lookup_lemmatize(self, token: Token) -> List[str]: string = token.text analyses = self._morph.parse(string) - if len(analyses) == 1: - return [analyses[0].normal_form] + # often multiple forms would derive from the same normal form + # thus check _unique_ normal forms + normal_forms = set([an.normal_form for an in analyses]) + if len(normal_forms) == 1: + return [next(iter(normal_forms))] return [string] + def pymorphy2_lemmatize(self, token: Token) -> List[str]: + return self._pymorphy_lemmatize(token) + + def pymorphy2_lookup_lemmatize(self, token: Token) -> List[str]: + return self._pymorphy_lookup_lemmatize(token) + def pymorphy3_lemmatize(self, token: Token) -> List[str]: - return self.pymorphy2_lemmatize(token) + return self._pymorphy_lemmatize(token) + + def pymorphy3_lookup_lemmatize(self, token: Token) -> List[str]: + return self._pymorphy_lookup_lemmatize(token) def oc2ud(oc_tag: str) -> Tuple[str, Dict[str, str]]: diff --git a/spacy/lang/uk/lemmatizer.py b/spacy/lang/uk/lemmatizer.py index 8337e7328..37015cc2a 100644 --- a/spacy/lang/uk/lemmatizer.py +++ b/spacy/lang/uk/lemmatizer.py @@ -29,7 +29,7 @@ class UkrainianLemmatizer(RussianLemmatizer): ) from None if getattr(self, "_morph", None) is None: self._morph = MorphAnalyzer(lang="uk") - elif mode == "pymorphy3": + elif mode in {"pymorphy3", "pymorphy3_lookup"}: try: from pymorphy3 import MorphAnalyzer except ImportError: diff --git a/spacy/tests/conftest.py b/spacy/tests/conftest.py index 0fc74243d..3a5c8e451 100644 --- a/spacy/tests/conftest.py +++ b/spacy/tests/conftest.py @@ -337,17 +337,17 @@ def ru_tokenizer(): return get_lang_class("ru")().tokenizer -@pytest.fixture +@pytest.fixture(scope="session") def ru_lemmatizer(): pytest.importorskip("pymorphy3") return get_lang_class("ru")().add_pipe("lemmatizer") -@pytest.fixture +@pytest.fixture(scope="session") def ru_lookup_lemmatizer(): - pytest.importorskip("pymorphy2") + pytest.importorskip("pymorphy3") return get_lang_class("ru")().add_pipe( - "lemmatizer", config={"mode": "pymorphy2_lookup"} + "lemmatizer", config={"mode": "pymorphy3_lookup"} ) @@ -423,19 +423,19 @@ def uk_tokenizer(): return get_lang_class("uk")().tokenizer -@pytest.fixture +@pytest.fixture(scope="session") def uk_lemmatizer(): pytest.importorskip("pymorphy3") pytest.importorskip("pymorphy3_dicts_uk") return get_lang_class("uk")().add_pipe("lemmatizer") -@pytest.fixture +@pytest.fixture(scope="session") def uk_lookup_lemmatizer(): - pytest.importorskip("pymorphy2") - pytest.importorskip("pymorphy2_dicts_uk") + pytest.importorskip("pymorphy3") + pytest.importorskip("pymorphy3_dicts_uk") return get_lang_class("uk")().add_pipe( - "lemmatizer", config={"mode": "pymorphy2_lookup"} + "lemmatizer", config={"mode": "pymorphy3_lookup"} ) diff --git a/spacy/tests/lang/ru/test_lemmatizer.py b/spacy/tests/lang/ru/test_lemmatizer.py index e82fd4f8c..9a5a9ad68 100644 --- a/spacy/tests/lang/ru/test_lemmatizer.py +++ b/spacy/tests/lang/ru/test_lemmatizer.py @@ -81,6 +81,7 @@ def test_ru_lemmatizer_punct(ru_lemmatizer): def test_ru_doc_lookup_lemmatization(ru_lookup_lemmatizer): + assert ru_lookup_lemmatizer.mode == "pymorphy3_lookup" words = ["мама", "мыла", "раму"] pos = ["NOUN", "VERB", "NOUN"] morphs = [ @@ -92,3 +93,17 @@ def test_ru_doc_lookup_lemmatization(ru_lookup_lemmatizer): doc = ru_lookup_lemmatizer(doc) lemmas = [token.lemma_ for token in doc] assert lemmas == ["мама", "мыла", "раму"] + + +@pytest.mark.parametrize( + "word,lemma", + ( + ("бременем", "бремя"), + ("будешь", "быть"), + ("какая-то", "какой-то"), + ), +) +def test_ru_lookup_lemmatizer(ru_lookup_lemmatizer, word, lemma): + assert ru_lookup_lemmatizer.mode == "pymorphy3_lookup" + doc = Doc(ru_lookup_lemmatizer.vocab, words=[word]) + assert ru_lookup_lemmatizer(doc)[0].lemma_ == lemma diff --git a/spacy/tests/lang/uk/test_lemmatizer.py b/spacy/tests/lang/uk/test_lemmatizer.py index 788744aa1..a65bb25e5 100644 --- a/spacy/tests/lang/uk/test_lemmatizer.py +++ b/spacy/tests/lang/uk/test_lemmatizer.py @@ -8,12 +8,20 @@ pytestmark = pytest.mark.filterwarnings("ignore::DeprecationWarning") def test_uk_lemmatizer(uk_lemmatizer): """Check that the default uk lemmatizer runs.""" doc = Doc(uk_lemmatizer.vocab, words=["a", "b", "c"]) + assert uk_lemmatizer.mode == "pymorphy3" uk_lemmatizer(doc) assert [token.lemma for token in doc] -def test_uk_lookup_lemmatizer(uk_lookup_lemmatizer): - """Check that the lookup uk lemmatizer runs.""" - doc = Doc(uk_lookup_lemmatizer.vocab, words=["a", "b", "c"]) - uk_lookup_lemmatizer(doc) - assert [token.lemma for token in doc] +@pytest.mark.parametrize( + "word,lemma", + ( + ("якийсь", "якийсь"), + ("розповідають", "розповідати"), + ("розповіси", "розповісти"), + ), +) +def test_uk_lookup_lemmatizer(uk_lookup_lemmatizer, word, lemma): + assert uk_lookup_lemmatizer.mode == "pymorphy3_lookup" + doc = Doc(uk_lookup_lemmatizer.vocab, words=[word]) + assert uk_lookup_lemmatizer(doc)[0].lemma_ == lemma From dece775279955e4aa84f718675a72ff34174a7ee Mon Sep 17 00:00:00 2001 From: kadarakos Date: Fri, 25 Nov 2022 11:31:28 +0100 Subject: [PATCH 111/179] correct ndim in docs (#11869) --- website/docs/api/vectors.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/api/vectors.md b/website/docs/api/vectors.md index 9636ea04c..d4702b592 100644 --- a/website/docs/api/vectors.md +++ b/website/docs/api/vectors.md @@ -50,7 +50,7 @@ modified later. | _keyword-only_ | | | `strings` | The string store. A new string store is created if one is not provided. Defaults to `None`. ~~Optional[StringStore]~~ | | `shape` | Size of the table as `(n_entries, n_columns)`, the number of entries and number of columns. Not required if you're initializing the object with `data` and `keys`. ~~Tuple[int, int]~~ | -| `data` | The vector data. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | +| `data` | The vector data. ~~numpy.ndarray[ndim=2, dtype=float32]~~ | | `keys` | A sequence of keys aligned with the data. ~~Iterable[Union[str, int]]~~ | | `name` | A name to identify the vectors table. ~~str~~ | | `mode` 3.2 | Vectors mode: `"default"` or [`"floret"`](https://github.com/explosion/floret) (default: `"default"`). ~~str~~ | From c0fd8a2e71ce5eaad07e0b555fab8a152373fdc6 Mon Sep 17 00:00:00 2001 From: Raphael Mitsch Date: Fri, 25 Nov 2022 11:44:55 +0100 Subject: [PATCH 112/179] find-threshold: CLI command for multi-label classifier threshold tuning (#11280) * Add foundation for find-threshold CLI functionality. * Finish first draft for find-threshold. * Add tests. * Revert adjusted import statements. * Fix mypy errors. * Fix imports. * Harmonize arguments with spacy evaluate command. * Generalize component and threshold handling. Harmonize arguments with 'spacy evaluate' CLI. * Fix Spancat test. * Add beta parameter to Scorer and PRFScore. * Make beta a component scorer setting. * Remove beta. * Update nlp.config (workaround). * Reload pipeline on threshold change. Adjust tests. Remove confection reference. * Remove assumption of component being a Pipe object or having a .cfg attribute. * Adjust test output and reference values. * Remove beta references. Delete universe.json. * Reverting unnecessary changes. Removing unused default values. Renaming variables in find-cli tests. * Update spacy/cli/find_threshold.py Co-authored-by: Adriane Boyd * Remove adding labels in tests. * Remove unused error * Undo changes to PRFScorer * Change default value for n_trials. Log table iteratively. * Add warnings for pointless applications of find_threshold(). * Fix imports. * Adjust type check of TextCategorizer to exclude subclasses. * Change check of if there's only one unique value in scores. * Update spacy/cli/find_threshold.py Co-authored-by: Sofie Van Landeghem * Incorporate feedback. * Fix test issue. Update docstring. * Update docs & docstring. * Update spacy/tests/test_cli.py Co-authored-by: Adriane Boyd * Add examples to docs. Rename _nlp to nlp in tests. * Update spacy/cli/find_threshold.py Co-authored-by: Sofie Van Landeghem * Update spacy/cli/find_threshold.py Co-authored-by: Sofie Van Landeghem Co-authored-by: Adriane Boyd Co-authored-by: Sofie Van Landeghem --- spacy/cli/__init__.py | 1 + spacy/cli/find_threshold.py | 233 ++++++++++++++++++++++++++++++++++++ spacy/errors.py | 1 + spacy/pipeline/spancat.py | 4 +- spacy/tests/test_cli.py | 124 ++++++++++++++++++- website/docs/api/cli.md | 41 +++++++ 6 files changed, 399 insertions(+), 5 deletions(-) create mode 100644 spacy/cli/find_threshold.py diff --git a/spacy/cli/__init__.py b/spacy/cli/__init__.py index ce76ef9a9..aab2c8d12 100644 --- a/spacy/cli/__init__.py +++ b/spacy/cli/__init__.py @@ -27,6 +27,7 @@ from .project.dvc import project_update_dvc # noqa: F401 from .project.push import project_push # noqa: F401 from .project.pull import project_pull # noqa: F401 from .project.document import project_document # noqa: F401 +from .find_threshold import find_threshold # noqa: F401 @app.command("link", no_args_is_help=True, deprecated=True, hidden=True) diff --git a/spacy/cli/find_threshold.py b/spacy/cli/find_threshold.py new file mode 100644 index 000000000..efa664832 --- /dev/null +++ b/spacy/cli/find_threshold.py @@ -0,0 +1,233 @@ +import functools +import operator +from pathlib import Path +import logging +from typing import Optional, Tuple, Any, Dict, List + +import numpy +import wasabi.tables + +from ..pipeline import TextCategorizer, MultiLabel_TextCategorizer +from ..errors import Errors +from ..training import Corpus +from ._util import app, Arg, Opt, import_code, setup_gpu +from .. import util + +_DEFAULTS = { + "n_trials": 11, + "use_gpu": -1, + "gold_preproc": False, +} + + +@app.command( + "find-threshold", + context_settings={"allow_extra_args": False, "ignore_unknown_options": True}, +) +def find_threshold_cli( + # fmt: off + model: str = Arg(..., help="Model name or path"), + data_path: Path = Arg(..., help="Location of binary evaluation data in .spacy format", exists=True), + pipe_name: str = Arg(..., help="Name of pipe to examine thresholds for"), + threshold_key: str = Arg(..., help="Key of threshold attribute in component's configuration"), + scores_key: str = Arg(..., help="Metric to optimize"), + n_trials: int = Opt(_DEFAULTS["n_trials"], "--n_trials", "-n", help="Number of trials to determine optimal thresholds"), + code_path: Optional[Path] = Opt(None, "--code", "-c", help="Path to Python file with additional code (registered functions) to be imported"), + use_gpu: int = Opt(_DEFAULTS["use_gpu"], "--gpu-id", "-g", help="GPU ID or -1 for CPU"), + gold_preproc: bool = Opt(_DEFAULTS["gold_preproc"], "--gold-preproc", "-G", help="Use gold preprocessing"), + verbose: bool = Opt(False, "--silent", "-V", "-VV", help="Display more information for debugging purposes"), + # fmt: on +): + """ + Runs prediction trials for a trained model with varying tresholds to maximize + the specified metric. The search space for the threshold is traversed linearly + from 0 to 1 in `n_trials` steps. Results are displayed in a table on `stdout` + (the corresponding API call to `spacy.cli.find_threshold.find_threshold()` + returns all results). + + This is applicable only for components whose predictions are influenced by + thresholds - e.g. `textcat_multilabel` and `spancat`, but not `textcat`. Note + that the full path to the corresponding threshold attribute in the config has to + be provided. + + DOCS: https://spacy.io/api/cli#find-threshold + """ + + util.logger.setLevel(logging.DEBUG if verbose else logging.INFO) + import_code(code_path) + find_threshold( + model=model, + data_path=data_path, + pipe_name=pipe_name, + threshold_key=threshold_key, + scores_key=scores_key, + n_trials=n_trials, + use_gpu=use_gpu, + gold_preproc=gold_preproc, + silent=False, + ) + + +def find_threshold( + model: str, + data_path: Path, + pipe_name: str, + threshold_key: str, + scores_key: str, + *, + n_trials: int = _DEFAULTS["n_trials"], # type: ignore + use_gpu: int = _DEFAULTS["use_gpu"], # type: ignore + gold_preproc: bool = _DEFAULTS["gold_preproc"], # type: ignore + silent: bool = True, +) -> Tuple[float, float, Dict[float, float]]: + """ + Runs prediction trials for models with varying tresholds to maximize the specified metric. + model (Union[str, Path]): Pipeline to evaluate. Can be a package or a path to a data directory. + data_path (Path): Path to file with DocBin with docs to use for threshold search. + pipe_name (str): Name of pipe to examine thresholds for. + threshold_key (str): Key of threshold attribute in component's configuration. + scores_key (str): Name of score to metric to optimize. + n_trials (int): Number of trials to determine optimal thresholds. + use_gpu (int): GPU ID or -1 for CPU. + gold_preproc (bool): Whether to use gold preprocessing. Gold preprocessing helps the annotations align to the + tokenization, and may result in sequences of more consistent length. However, it may reduce runtime accuracy due + to train/test skew. + silent (bool): Whether to print non-error-related output to stdout. + RETURNS (Tuple[float, float, Dict[float, float]]): Best found threshold, the corresponding score, scores for all + evaluated thresholds. + """ + + setup_gpu(use_gpu, silent=silent) + data_path = util.ensure_path(data_path) + if not data_path.exists(): + wasabi.msg.fail("Evaluation data not found", data_path, exits=1) + nlp = util.load_model(model) + + if pipe_name not in nlp.component_names: + raise AttributeError( + Errors.E001.format(name=pipe_name, opts=nlp.component_names) + ) + pipe = nlp.get_pipe(pipe_name) + if not hasattr(pipe, "scorer"): + raise AttributeError(Errors.E1045) + + if type(pipe) == TextCategorizer: + wasabi.msg.warn( + "The `textcat` component doesn't use a threshold as it's not applicable to the concept of " + "exclusive classes. All thresholds will yield the same results." + ) + + if not silent: + wasabi.msg.info( + title=f"Optimizing for {scores_key} for component '{pipe_name}' with {n_trials} " + f"trials." + ) + + # Load evaluation corpus. + corpus = Corpus(data_path, gold_preproc=gold_preproc) + dev_dataset = list(corpus(nlp)) + config_keys = threshold_key.split(".") + + def set_nested_item( + config: Dict[str, Any], keys: List[str], value: float + ) -> Dict[str, Any]: + """Set item in nested dictionary. Adapted from https://stackoverflow.com/a/54138200. + config (Dict[str, Any]): Configuration dictionary. + keys (List[Any]): Path to value to set. + value (float): Value to set. + RETURNS (Dict[str, Any]): Updated dictionary. + """ + functools.reduce(operator.getitem, keys[:-1], config)[keys[-1]] = value + return config + + def filter_config( + config: Dict[str, Any], keys: List[str], full_key: str + ) -> Dict[str, Any]: + """Filters provided config dictionary so that only the specified keys path remains. + config (Dict[str, Any]): Configuration dictionary. + keys (List[Any]): Path to value to set. + full_key (str): Full user-specified key. + RETURNS (Dict[str, Any]): Filtered dictionary. + """ + if keys[0] not in config: + wasabi.msg.fail( + title=f"Failed to look up `{full_key}` in config: sub-key {[keys[0]]} not found.", + text=f"Make sure you specified {[keys[0]]} correctly. The following sub-keys are available instead: " + f"{list(config.keys())}", + exits=1, + ) + return { + keys[0]: filter_config(config[keys[0]], keys[1:], full_key) + if len(keys) > 1 + else config[keys[0]] + } + + # Evaluate with varying threshold values. + scores: Dict[float, float] = {} + config_keys_full = ["components", pipe_name, *config_keys] + table_col_widths = (10, 10) + thresholds = numpy.linspace(0, 1, n_trials) + print(wasabi.tables.row(["Threshold", f"{scores_key}"], widths=table_col_widths)) + for threshold in thresholds: + # Reload pipeline with overrides specifying the new threshold. + nlp = util.load_model( + model, + config=set_nested_item( + filter_config( + nlp.config, config_keys_full, ".".join(config_keys_full) + ).copy(), + config_keys_full, + threshold, + ), + ) + if hasattr(pipe, "cfg"): + setattr( + nlp.get_pipe(pipe_name), + "cfg", + set_nested_item(getattr(pipe, "cfg"), config_keys, threshold), + ) + + eval_scores = nlp.evaluate(dev_dataset) + if scores_key not in eval_scores: + wasabi.msg.fail( + title=f"Failed to look up score `{scores_key}` in evaluation results.", + text=f"Make sure you specified the correct value for `scores_key`. The following scores are " + f"available: {list(eval_scores.keys())}", + exits=1, + ) + scores[threshold] = eval_scores[scores_key] + + if not isinstance(scores[threshold], (float, int)): + wasabi.msg.fail( + f"Returned score for key '{scores_key}' is not numeric. Threshold optimization only works for numeric " + f"scores.", + exits=1, + ) + print( + wasabi.row( + [round(threshold, 3), round(scores[threshold], 3)], + widths=table_col_widths, + ) + ) + + best_threshold = max(scores.keys(), key=(lambda key: scores[key])) + + # If all scores are identical, emit warning. + if len(set(scores.values())) == 1: + wasabi.msg.warn( + title="All scores are identical. Verify that all settings are correct.", + text="" + if ( + not isinstance(pipe, MultiLabel_TextCategorizer) + or scores_key in ("cats_macro_f", "cats_micro_f") + ) + else "Use `cats_macro_f` or `cats_micro_f` when optimizing the threshold for `textcat_multilabel`.", + ) + + else: + if not silent: + print( + f"\nBest threshold: {round(best_threshold, ndigits=4)} with {scores_key} value of {scores[best_threshold]}." + ) + + return best_threshold, scores[best_threshold], scores diff --git a/spacy/errors.py b/spacy/errors.py index 1d29f0e17..a8de5fb90 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -956,6 +956,7 @@ class Errors(metaclass=ErrorsWithCodes): "sure it's overwritten on the subclass.") E1046 = ("{cls_name} is an abstract class and cannot be instantiated. If you are looking for spaCy's default " "knowledge base, use `InMemoryLookupKB`.") + E1047 = ("`find_threshold()` only supports components with a `scorer` attribute.") # Deprecated model shortcuts, only used in errors and warnings diff --git a/spacy/pipeline/spancat.py b/spacy/pipeline/spancat.py index 956bbb72c..0a84c72fd 100644 --- a/spacy/pipeline/spancat.py +++ b/spacy/pipeline/spancat.py @@ -1,7 +1,7 @@ -from typing import List, Dict, Callable, Tuple, Optional, Iterable, Any, cast +from typing import List, Dict, Callable, Tuple, Optional, Iterable, Any from thinc.api import Config, Model, get_current_ops, set_dropout_rate, Ops from thinc.api import Optimizer -from thinc.types import Ragged, Ints2d, Floats2d, Ints1d +from thinc.types import Ragged, Ints2d, Floats2d import numpy diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py index 8225e14f1..1c4d0c98f 100644 --- a/spacy/tests/test_cli.py +++ b/spacy/tests/test_cli.py @@ -1,9 +1,10 @@ import os import math +from collections import Counter +from typing import Tuple, List, Dict, Any import pkg_resources -from random import sample -from typing import Counter +import numpy import pytest import srsly from click import NoSuchOption @@ -28,11 +29,12 @@ from spacy.cli.package import get_third_party_dependencies from spacy.cli.package import _is_permitted_package_name from spacy.cli.project.run import _check_requirements from spacy.cli.validate import get_model_pkgs +from spacy.cli.find_threshold import find_threshold from spacy.lang.en import English from spacy.lang.nl import Dutch from spacy.language import Language from spacy.schemas import ProjectConfigSchema, RecommendationSchema, validate -from spacy.tokens import Doc +from spacy.tokens import Doc, DocBin from spacy.tokens.span import Span from spacy.training import Example, docs_to_json, offsets_to_biluo_tags from spacy.training.converters import conll_ner_to_docs, conllu_to_docs @@ -859,6 +861,122 @@ def test_span_length_freq_dist_output_must_be_correct(): assert list(span_freqs.keys()) == [3, 1, 4, 5, 2] +def test_cli_find_threshold(capsys): + thresholds = numpy.linspace(0, 1, 10) + + def make_examples(nlp: Language) -> List[Example]: + docs: List[Example] = [] + + for t in [ + ( + "I am angry and confused in the Bank of America.", + { + "cats": {"ANGRY": 1.0, "CONFUSED": 1.0, "HAPPY": 0.0}, + "spans": {"sc": [(31, 46, "ORG")]}, + }, + ), + ( + "I am confused but happy in New York.", + { + "cats": {"ANGRY": 0.0, "CONFUSED": 1.0, "HAPPY": 1.0}, + "spans": {"sc": [(27, 35, "GPE")]}, + }, + ), + ]: + doc = nlp.make_doc(t[0]) + docs.append(Example.from_dict(doc, t[1])) + + return docs + + def init_nlp( + components: Tuple[Tuple[str, Dict[str, Any]], ...] = () + ) -> Tuple[Language, List[Example]]: + new_nlp = English() + new_nlp.add_pipe( # type: ignore + factory_name="textcat_multilabel", + name="tc_multi", + config={"threshold": 0.9}, + ) + + # Append additional components to pipeline. + for cfn, comp_config in components: + new_nlp.add_pipe(cfn, config=comp_config) + + new_examples = make_examples(new_nlp) + new_nlp.initialize(get_examples=lambda: new_examples) + for i in range(5): + new_nlp.update(new_examples) + + return new_nlp, new_examples + + with make_tempdir() as docs_dir: + # Check whether find_threshold() identifies lowest threshold above 0 as (first) ideal threshold, as this matches + # the current model behavior with the examples above. This can break once the model behavior changes and serves + # mostly as a smoke test. + nlp, examples = init_nlp() + DocBin(docs=[example.reference for example in examples]).to_disk( + docs_dir / "docs.spacy" + ) + with make_tempdir() as nlp_dir: + nlp.to_disk(nlp_dir) + res = find_threshold( + model=nlp_dir, + data_path=docs_dir / "docs.spacy", + pipe_name="tc_multi", + threshold_key="threshold", + scores_key="cats_macro_f", + silent=True, + ) + assert res[0] != thresholds[0] + assert thresholds[0] < res[0] < thresholds[9] + assert res[1] == 1.0 + assert res[2][1.0] == 0.0 + + # Test with spancat. + nlp, _ = init_nlp((("spancat", {}),)) + with make_tempdir() as nlp_dir: + nlp.to_disk(nlp_dir) + res = find_threshold( + model=nlp_dir, + data_path=docs_dir / "docs.spacy", + pipe_name="spancat", + threshold_key="threshold", + scores_key="spans_sc_f", + silent=True, + ) + assert res[0] != thresholds[0] + assert thresholds[0] < res[0] < thresholds[8] + assert res[1] >= 0.6 + assert res[2][1.0] == 0.0 + + # Having multiple textcat_multilabel components should work, since the name has to be specified. + nlp, _ = init_nlp((("textcat_multilabel", {}),)) + with make_tempdir() as nlp_dir: + nlp.to_disk(nlp_dir) + assert find_threshold( + model=nlp_dir, + data_path=docs_dir / "docs.spacy", + pipe_name="tc_multi", + threshold_key="threshold", + scores_key="cats_macro_f", + silent=True, + ) + + # Specifying the name of an non-existing pipe should fail. + nlp, _ = init_nlp() + with make_tempdir() as nlp_dir: + nlp.to_disk(nlp_dir) + with pytest.raises(AttributeError): + find_threshold( + model=nlp_dir, + data_path=docs_dir / "docs.spacy", + pipe_name="_", + threshold_key="threshold", + scores_key="cats_macro_f", + silent=True, + ) + + @pytest.mark.parametrize( "reqs,output", [ diff --git a/website/docs/api/cli.md b/website/docs/api/cli.md index 6e581b903..b42ba8a4f 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.md @@ -12,6 +12,7 @@ menu: - ['train', 'train'] - ['pretrain', 'pretrain'] - ['evaluate', 'evaluate'] + - ['find-threshold', 'find-threshold'] - ['assemble', 'assemble'] - ['package', 'package'] - ['project', 'project'] @@ -1161,6 +1162,46 @@ $ python -m spacy evaluate [model] [data_path] [--output] [--code] [--gold-prepr | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | Training results and optional metrics and visualizations. | +## find-threshold {#find-threshold new="3.5" tag="command"} + +Runs prediction trials for a trained model with varying tresholds to maximize +the specified metric. The search space for the threshold is traversed linearly +from 0 to 1 in `n_trials` steps. Results are displayed in a table on `stdout` +(the corresponding API call to `spacy.cli.find_threshold.find_threshold()` +returns all results). + +This is applicable only for components whose predictions are influenced by +thresholds - e.g. `textcat_multilabel` and `spancat`, but not `textcat`. Note +that the full path to the corresponding threshold attribute in the config has to +be provided. + +> #### Examples +> +> ```cli +> # For textcat_multilabel: +> $ python -m spacy find-threshold my_nlp data.spacy textcat_multilabel threshold cats_macro_f +> ``` +> +> ```cli +> # For spancat: +> $ python -m spacy find-threshold my_nlp data.spacy spancat threshold spans_sc_f +> ``` + + +| Name | Description | +| ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | Pipeline to evaluate. Can be a package or a path to a data directory. ~~str (positional)~~ | +| `data_path` | Path to file with DocBin with docs to use for threshold search. ~~Path (positional)~~ | +| `pipe_name` | Name of pipe to examine thresholds for. ~~str (positional)~~ | +| `threshold_key` | Key of threshold attribute in component's configuration. ~~str (positional)~~ | +| `scores_key` | Name of score to metric to optimize. ~~str (positional)~~ | +| `--n_trials`, `-n` | Number of trials to determine optimal thresholds. ~~int (option)~~ | +| `--code`, `-c` | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--gpu-id`, `-g` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ | +| `--gold-preproc`, `-G` | Use gold preprocessing. ~~bool (flag)~~ | +| `--silent`, `-V`, `-VV` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | + ## assemble {#assemble tag="command"} Assemble a pipeline from a config file without additional training. Expects a From 378db0eb1e9231c565faf72078bcfb012f439e9b Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 1 Apr 2022 10:42:25 +0200 Subject: [PATCH 113/179] Temporarily skip tests that require models/compat --- .github/azure-steps.yml | 44 ++++++++++++++++++++--------------------- spacy/tests/test_cli.py | 2 ++ 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml index e8bd0d212..2f77706b8 100644 --- a/.github/azure-steps.yml +++ b/.github/azure-steps.yml @@ -52,17 +52,17 @@ steps: python -W error -c "import spacy" displayName: "Test import" - - script: | - python -m spacy download ca_core_news_sm - python -m spacy download ca_core_news_md - python -c "import spacy; nlp=spacy.load('ca_core_news_sm'); doc=nlp('test')" - displayName: 'Test download CLI' - condition: eq(variables['python_version'], '3.8') - - - script: | - python -W error -c "import ca_core_news_sm; nlp = ca_core_news_sm.load(); doc=nlp('test')" - displayName: 'Test no warnings on load (#11713)' - condition: eq(variables['python_version'], '3.8') +# - script: | +# python -m spacy download ca_core_news_sm +# python -m spacy download ca_core_news_md +# python -c "import spacy; nlp=spacy.load('ca_core_news_sm'); doc=nlp('test')" +# displayName: 'Test download CLI' +# condition: eq(variables['python_version'], '3.8') +# +# - script: | +# python -W error -c "import ca_core_news_sm; nlp = ca_core_news_sm.load(); doc=nlp('test')" +# displayName: 'Test no warnings on load (#11713)' +# condition: eq(variables['python_version'], '3.8') - script: | python -m spacy convert extra/example_data/ner_example_data/ner-token-per-line-conll2003.json . @@ -86,17 +86,17 @@ steps: displayName: 'Test train CLI' condition: eq(variables['python_version'], '3.8') - - script: | - python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')" - PYTHONWARNINGS="error,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir - displayName: 'Test assemble CLI' - condition: eq(variables['python_version'], '3.8') - - - script: | - python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_md'}; config.to_disk('ner_source_md.cfg')" - python -m spacy assemble ner_source_md.cfg output_dir 2>&1 | grep -q W113 - displayName: 'Test assemble CLI vectors warning' - condition: eq(variables['python_version'], '3.8') +# - script: | +# python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')" +# PYTHONWARNINGS="error,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir +# displayName: 'Test assemble CLI' +# condition: eq(variables['python_version'], '3.8') +# +# - script: | +# python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_md'}; config.to_disk('ner_source_md.cfg')" +# python -m spacy assemble ner_source_md.cfg output_dir 2>&1 | grep -q W113 +# displayName: 'Test assemble CLI vectors warning' +# condition: eq(variables['python_version'], '3.8') - script: | python -m pip install -U -r requirements.txt diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py index 8225e14f1..563559cb4 100644 --- a/spacy/tests/test_cli.py +++ b/spacy/tests/test_cli.py @@ -591,6 +591,7 @@ def test_string_to_list_intify(value): assert string_to_list(value, intify=True) == [1, 2, 3] +@pytest.mark.skip(reason="Temporarily skip for dev version") def test_download_compatibility(): spec = SpecifierSet("==" + about.__version__) spec.prereleases = False @@ -601,6 +602,7 @@ def test_download_compatibility(): assert get_minor_version(about.__version__) == get_minor_version(version) +@pytest.mark.skip(reason="Temporarily skip for dev version") def test_validate_compatibility_table(): spec = SpecifierSet("==" + about.__version__) spec.prereleases = False From 32396e0bda3aceda74f2d7d050180032cd381d32 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 25 Nov 2022 11:18:56 +0100 Subject: [PATCH 114/179] Set version to v3.5.0 --- spacy/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/about.py b/spacy/about.py index ce86e6294..640e9e93b 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -1,6 +1,6 @@ # fmt: off __title__ = "spacy" -__version__ = "3.4.2" +__version__ = "3.5.0" __download_url__ = "https://github.com/explosion/spacy-models/releases/download" __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json" __projects__ = "https://github.com/explosion/projects" From 681ec209147ba476a4062e5fec2248c7e0c50d68 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 25 Nov 2022 13:00:57 +0100 Subject: [PATCH 115/179] Add smart_open requirement, update deprecated options (#11864) * Switch from deprecated `ignore_ext` to `compression` * Add upload/download test for local files --- requirements.txt | 1 + setup.cfg | 1 + spacy/cli/_util.py | 2 +- spacy/tests/test_cli.py | 16 ++++++++++++++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 23bfa6f14..dd2eff0c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ srsly>=2.4.3,<3.0.0 catalogue>=2.0.6,<2.1.0 typer>=0.3.0,<0.8.0 pathy>=0.3.5 +smart-open>=5.2.1,<7.0.0 # Third party dependencies numpy>=1.15.0 requests>=2.13.0,<3.0.0 diff --git a/setup.cfg b/setup.cfg index 82d4d2758..330dc8205 100644 --- a/setup.cfg +++ b/setup.cfg @@ -53,6 +53,7 @@ install_requires = # Third-party dependencies typer>=0.3.0,<0.8.0 pathy>=0.3.5 + smart-open>=5.2.1,<7.0.0 tqdm>=4.38.0,<5.0.0 numpy>=1.15.0 requests>=2.13.0,<3.0.0 diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py index 897964a88..872f69c88 100644 --- a/spacy/cli/_util.py +++ b/spacy/cli/_util.py @@ -358,7 +358,7 @@ def download_file(src: Union[str, "Pathy"], dest: Path, *, force: bool = False) if dest.exists() and not force: return None src = str(src) - with smart_open.open(src, mode="rb", ignore_ext=True) as input_file: + with smart_open.open(src, mode="rb", compression="disable") as input_file: with dest.open(mode="wb") as output_file: shutil.copyfileobj(input_file, output_file) diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py index 1c4d0c98f..525c6d255 100644 --- a/spacy/tests/test_cli.py +++ b/spacy/tests/test_cli.py @@ -17,6 +17,7 @@ from spacy.cli._util import is_subpath_of, load_project_config from spacy.cli._util import parse_config_overrides, string_to_list from spacy.cli._util import substitute_project_variables from spacy.cli._util import validate_project_commands +from spacy.cli._util import upload_file, download_file from spacy.cli.debug_data import _compile_gold, _get_labels_from_model from spacy.cli.debug_data import _get_labels_from_spancat from spacy.cli.debug_data import _get_distribution, _get_kl_divergence @@ -1014,3 +1015,18 @@ def test_project_check_requirements(reqs, output): pkg_resources.require("spacyunknowndoesnotexist12345") except pkg_resources.DistributionNotFound: assert output == _check_requirements([req.strip() for req in reqs.split("\n")]) + + +def test_upload_download_local_file(): + with make_tempdir() as d1, make_tempdir() as d2: + filename = "f.txt" + content = "content" + local_file = d1 / filename + remote_file = d2 / filename + with local_file.open(mode="w") as file_: + file_.write(content) + upload_file(local_file, remote_file) + local_file.unlink() + download_file(remote_file, local_file) + with local_file.open(mode="r") as file_: + assert file_.read() == content From c23d54fd261b34ff947a18170a303a305179e7bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Bl=C3=A4ttermann?= Date: Wed, 23 Nov 2022 01:33:20 +0100 Subject: [PATCH 116/179] Remove MDX tags from `README.md` --- website/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/website/README.md b/website/README.md index 743e61acd..e0c0ac450 100644 --- a/website/README.md +++ b/website/README.md @@ -11,7 +11,7 @@ rendered version is available at https://spacy.io/styleguide._ -## Setup and installation {#setup} +## Setup and installation Before running the setup, make sure your versions of [Node](https://nodejs.org/en/) and [npm](https://www.npmjs.com/) are up to date. @@ -40,7 +40,7 @@ extensions for your code editor. The [`.prettierrc`](https://github.com/explosion/spaCy/tree/master/website/.prettierrc) file in the root defines the settings used in this codebase. -## Building & developing the site with Docker {#docker} +## Building & developing the site with Docker Sometimes it's hard to get a local environment working due to rapid updates to node dependencies, so it may be easier to use docker for building the docs. @@ -66,7 +66,7 @@ segfault errors from `qemu` if you use the default image. To fix this use the `arm64` tagged image in the `docker run` command (ghcr.io/explosion/spacy-io:arm64). -### Building the Docker image {#docker-build} +### Building the Docker image If you'd like to build the image locally, you can do so like this: @@ -77,10 +77,9 @@ docker build -t spacy-io . This will take some time, so if you want to use the prebuilt image you'll save a bit of time. -## Project structure {#structure} +## Project structure ```yaml -### Directory structure ├── docs # the actual markdown content ├── meta # JSON-formatted site metadata | ├── languages.json # supported languages and statistical models From 7f2ea20fee67c24eb3c7a10b76f6d554c0df5c88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Bl=C3=A4ttermann?= Date: Tue, 22 Nov 2022 16:16:11 +0100 Subject: [PATCH 117/179] Update `README.md` --- website/README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/website/README.md b/website/README.md index e0c0ac450..890a48ef9 100644 --- a/website/README.md +++ b/website/README.md @@ -1,15 +1,9 @@ - - # spacy.io website and docs ![Netlify Status](https://api.netlify.com/api/v1/badges/d65fe97d-99ab-47f8-a339-1d8987251da0/deploy-status) -_This page contains the documentation and styleguide for the spaCy website. Its -rendered version is available at https://spacy.io/styleguide._ - ---- - - +The styleguide for the spaCy website is available at +[spacy.io/styleguide](https://spacy.io/styleguide). ## Setup and installation From 5c9faf6eea34eade36e465c5493bfbbb039bcbc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcus=20Bl=C3=A4ttermann?= Date: Thu, 24 Nov 2022 21:02:14 +0100 Subject: [PATCH 118/179] Update menu for styleguide This reflects the removed parts from ecbf052abde2ab9373be1d7652e20d50b096e49d --- website/docs/styleguide.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/website/docs/styleguide.md b/website/docs/styleguide.md index 6ae23ff3f..47bca1ed4 100644 --- a/website/docs/styleguide.md +++ b/website/docs/styleguide.md @@ -8,9 +8,7 @@ menu: - ['Typography', 'typography'] - ['Elements', 'elements'] - ['Components', 'components'] - - ['Setup & Installation', 'setup'] - ['Markdown Reference', 'markdown'] - - ['Project Structure', 'structure'] - ['Editorial', 'editorial'] sidebar: - label: Styleguide From 9f986af120717680a1b97b15670dbc427b56ad89 Mon Sep 17 00:00:00 2001 From: Zhangrp Date: Mon, 28 Nov 2022 13:50:30 +0800 Subject: [PATCH 119/179] Add example sentence for Chinese in website meta (#11879) --- website/meta/languages.json | 1 + 1 file changed, 1 insertion(+) diff --git a/website/meta/languages.json b/website/meta/languages.json index bd1535c90..15158df79 100644 --- a/website/meta/languages.json +++ b/website/meta/languages.json @@ -562,6 +562,7 @@ "url": "https://github.com/explosion/spacy-pkuseg" } ], + "example": "这是一个用于示例的句子。", "has_examples": true } ], From f54bfb56c923b94ffeff5f9b774a24479c6dea61 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Mon, 28 Nov 2022 18:01:09 +0900 Subject: [PATCH 120/179] Don't throw an error if using displacy on an unset span key (#11845) * Don't throw an error if using displacy on an unset span key * List available keys in W117 --- spacy/displacy/__init__.py | 5 +++-- spacy/errors.py | 2 +- spacy/tests/test_displacy.py | 10 ++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/spacy/displacy/__init__.py b/spacy/displacy/__init__.py index 7bb300afa..bc32001d7 100644 --- a/spacy/displacy/__init__.py +++ b/spacy/displacy/__init__.py @@ -228,12 +228,13 @@ def parse_spans(doc: Doc, options: Dict[str, Any] = {}) -> Dict[str, Any]: "kb_id": span.kb_id_ if span.kb_id_ else "", "kb_url": kb_url_template.format(span.kb_id_) if kb_url_template else "#", } - for span in doc.spans[spans_key] + for span in doc.spans.get(spans_key, []) ] tokens = [token.text for token in doc] if not spans: - warnings.warn(Warnings.W117.format(spans_key=spans_key)) + keys = list(doc.spans.keys()) + warnings.warn(Warnings.W117.format(spans_key=spans_key, keys=keys)) title = doc.user_data.get("title", None) if hasattr(doc, "user_data") else None settings = get_doc_settings(doc) return { diff --git a/spacy/errors.py b/spacy/errors.py index a8de5fb90..e34614b0f 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -199,7 +199,7 @@ class Warnings(metaclass=ErrorsWithCodes): W117 = ("No spans to visualize found in Doc object with spans_key: '{spans_key}'. If this is " "surprising to you, make sure the Doc was processed using a model " "that supports span categorization, and check the `doc.spans[spans_key]` " - "property manually if necessary.") + "property manually if necessary.\n\nAvailable keys: {keys}") W118 = ("Term '{term}' not found in glossary. It may however be explained in documentation " "for the corpora used to train the language. Please check " "`nlp.meta[\"sources\"]` for any relevant links.") diff --git a/spacy/tests/test_displacy.py b/spacy/tests/test_displacy.py index ccc145b44..f298b38e0 100644 --- a/spacy/tests/test_displacy.py +++ b/spacy/tests/test_displacy.py @@ -203,6 +203,16 @@ def test_displacy_parse_spans_different_spans_key(en_vocab): ] +def test_displacy_parse_empty_spans_key(en_vocab): + """Test that having an unset spans key doesn't raise an error""" + doc = Doc(en_vocab, words=["Welcome", "to", "the", "Bank", "of", "China"]) + doc.spans["custom"] = [Span(doc, 3, 6, "BANK")] + with pytest.warns(UserWarning, match="W117"): + spans = displacy.parse_spans(doc) + + assert isinstance(spans, dict) + + def test_displacy_parse_ents(en_vocab): """Test that named entities on a Doc are converted into displaCy's format.""" doc = Doc(en_vocab, words=["But", "Google", "is", "starting", "from", "behind"]) From 1ebe7db07c8dbb1a55dafb09131b1d08242b79c5 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 29 Nov 2022 11:40:58 +0100 Subject: [PATCH 121/179] Support local filesystem remotes for projects (#11762) * Support local filesystem remotes for projects * Fix support for local filesystem remotes for projects * Use `FluidPath` instead of `Pathy` to support both filesystem and remote paths * Create missing parent directories if required for local filesystem * Add a more general `_file_exists` method to support both `Pathy`, `Path`, and `smart_open`-compatible URLs * Add explicit `smart_open` dependency starting with support for `compression` flag * Update `pathy` dependency to exclude older versions that aren't compatible with required `smart_open` version * Update docs to refer to `Pathy` instead of `smart_open` for project remotes (technically you can still push to any `smart_open`-compatible path but you can't pull from them) * Add tests for local filesystem remotes * Update pathy for general BlobStat sorting * Add import * Remove _file_exists since only Pathy remotes are supported * Format CLI docs * Clean up merge --- requirements.txt | 2 +- setup.cfg | 2 +- spacy/cli/_util.py | 15 +++++--- spacy/cli/project/remote_storage.py | 42 ++++++++++++++-------- spacy/tests/test_cli.py | 56 +++++++++++++++++++++++++++++ website/docs/api/cli.md | 26 +++++++------- website/docs/usage/projects.md | 20 +++++------ 7 files changed, 120 insertions(+), 43 deletions(-) diff --git a/requirements.txt b/requirements.txt index dd2eff0c2..778c05e21 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ wasabi>=0.9.1,<1.1.0 srsly>=2.4.3,<3.0.0 catalogue>=2.0.6,<2.1.0 typer>=0.3.0,<0.8.0 -pathy>=0.3.5 +pathy>=0.10.0 smart-open>=5.2.1,<7.0.0 # Third party dependencies numpy>=1.15.0 diff --git a/setup.cfg b/setup.cfg index 330dc8205..5768c9d3e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,7 +52,7 @@ install_requires = catalogue>=2.0.6,<2.1.0 # Third-party dependencies typer>=0.3.0,<0.8.0 - pathy>=0.3.5 + pathy>=0.10.0 smart-open>=5.2.1,<7.0.0 tqdm>=4.38.0,<5.0.0 numpy>=1.15.0 diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py index 872f69c88..7ce006108 100644 --- a/spacy/cli/_util.py +++ b/spacy/cli/_util.py @@ -23,7 +23,7 @@ from ..util import is_compatible_version, SimpleFrozenDict, ENV_VARS from .. import about if TYPE_CHECKING: - from pathy import Pathy # noqa: F401 + from pathy import FluidPath # noqa: F401 SDIST_SUFFIX = ".tar.gz" @@ -331,7 +331,7 @@ def import_code(code_path: Optional[Union[Path, str]]) -> None: msg.fail(f"Couldn't load Python code: {code_path}", e, exits=1) -def upload_file(src: Path, dest: Union[str, "Pathy"]) -> None: +def upload_file(src: Path, dest: Union[str, "FluidPath"]) -> None: """Upload a file. src (Path): The source path. @@ -339,13 +339,20 @@ def upload_file(src: Path, dest: Union[str, "Pathy"]) -> None: """ import smart_open + # Create parent directories for local paths + if isinstance(dest, Path): + if not dest.parent.exists(): + dest.parent.mkdir(parents=True) + dest = str(dest) with smart_open.open(dest, mode="wb") as output_file: with src.open(mode="rb") as input_file: output_file.write(input_file.read()) -def download_file(src: Union[str, "Pathy"], dest: Path, *, force: bool = False) -> None: +def download_file( + src: Union[str, "FluidPath"], dest: Path, *, force: bool = False +) -> None: """Download a file using smart_open. url (str): The URL of the file. @@ -368,7 +375,7 @@ def ensure_pathy(path): slow and annoying Google Cloud warning).""" from pathy import Pathy # noqa: F811 - return Pathy(path) + return Pathy.fluid(path) def git_checkout( diff --git a/spacy/cli/project/remote_storage.py b/spacy/cli/project/remote_storage.py index 12e252b3c..076541580 100644 --- a/spacy/cli/project/remote_storage.py +++ b/spacy/cli/project/remote_storage.py @@ -5,15 +5,17 @@ import hashlib import urllib.parse import tarfile from pathlib import Path +from wasabi import msg -from .._util import get_hash, get_checksum, download_file, ensure_pathy -from ...util import make_tempdir, get_minor_version, ENV_VARS, check_bool_env_var +from .._util import get_hash, get_checksum, upload_file, download_file +from .._util import ensure_pathy, make_tempdir +from ...util import get_minor_version, ENV_VARS, check_bool_env_var from ...git_info import GIT_VERSION from ... import about from ...errors import Errors if TYPE_CHECKING: - from pathy import Pathy # noqa: F401 + from pathy import FluidPath # noqa: F401 class RemoteStorage: @@ -28,7 +30,7 @@ class RemoteStorage: self.url = ensure_pathy(url) self.compression = compression - def push(self, path: Path, command_hash: str, content_hash: str) -> "Pathy": + def push(self, path: Path, command_hash: str, content_hash: str) -> "FluidPath": """Compress a file or directory within a project and upload it to a remote storage. If an object exists at the full URL, nothing is done. @@ -49,9 +51,7 @@ class RemoteStorage: mode_string = f"w:{self.compression}" if self.compression else "w" with tarfile.open(tar_loc, mode=mode_string) as tar_file: tar_file.add(str(loc), arcname=str(path)) - with tar_loc.open(mode="rb") as input_file: - with url.open(mode="wb") as output_file: - output_file.write(input_file.read()) + upload_file(tar_loc, url) return url def pull( @@ -60,7 +60,7 @@ class RemoteStorage: *, command_hash: Optional[str] = None, content_hash: Optional[str] = None, - ) -> Optional["Pathy"]: + ) -> Optional["FluidPath"]: """Retrieve a file from the remote cache. If the file already exists, nothing is done. @@ -110,25 +110,37 @@ class RemoteStorage: *, command_hash: Optional[str] = None, content_hash: Optional[str] = None, - ) -> Optional["Pathy"]: + ) -> Optional["FluidPath"]: """Find the best matching version of a file within the storage, or `None` if no match can be found. If both the creation and content hash are specified, only exact matches will be returned. Otherwise, the most recent matching file is preferred. """ name = self.encode_name(str(path)) + urls = [] if command_hash is not None and content_hash is not None: - url = self.make_url(path, command_hash, content_hash) + url = self.url / name / command_hash / content_hash urls = [url] if url.exists() else [] elif command_hash is not None: - urls = list((self.url / name / command_hash).iterdir()) + if (self.url / name / command_hash).exists(): + urls = list((self.url / name / command_hash).iterdir()) else: - urls = list((self.url / name).iterdir()) - if content_hash is not None: - urls = [url for url in urls if url.parts[-1] == content_hash] + if (self.url / name).exists(): + for sub_dir in (self.url / name).iterdir(): + urls.extend(sub_dir.iterdir()) + if content_hash is not None: + urls = [url for url in urls if url.parts[-1] == content_hash] + if len(urls) >= 2: + try: + urls.sort(key=lambda x: x.stat().last_modified) # type: ignore + except Exception: + msg.warn( + "Unable to sort remote files by last modified. The file(s) " + "pulled from the cache may not be the most recent." + ) return urls[-1] if urls else None - def make_url(self, path: Path, command_hash: str, content_hash: str) -> "Pathy": + def make_url(self, path: Path, command_hash: str, content_hash: str) -> "FluidPath": """Construct a URL from a subpath, a creation hash and a content hash.""" return self.url / self.encode_name(str(path)) / command_hash / content_hash diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py index 525c6d255..ee3081283 100644 --- a/spacy/tests/test_cli.py +++ b/spacy/tests/test_cli.py @@ -3,6 +3,7 @@ import math from collections import Counter from typing import Tuple, List, Dict, Any import pkg_resources +import time import numpy import pytest @@ -28,6 +29,7 @@ from spacy.cli.download import get_compatibility, get_version from spacy.cli.init_config import RECOMMENDATIONS, init_config, fill_config from spacy.cli.package import get_third_party_dependencies from spacy.cli.package import _is_permitted_package_name +from spacy.cli.project.remote_storage import RemoteStorage from spacy.cli.project.run import _check_requirements from spacy.cli.validate import get_model_pkgs from spacy.cli.find_threshold import find_threshold @@ -862,6 +864,60 @@ def test_span_length_freq_dist_output_must_be_correct(): assert list(span_freqs.keys()) == [3, 1, 4, 5, 2] +def test_local_remote_storage(): + with make_tempdir() as d: + filename = "a.txt" + + content_hashes = ("aaaa", "cccc", "bbbb") + for i, content_hash in enumerate(content_hashes): + # make sure that each subsequent file has a later timestamp + if i > 0: + time.sleep(1) + content = f"{content_hash} content" + loc_file = d / "root" / filename + if not loc_file.parent.exists(): + loc_file.parent.mkdir(parents=True) + with loc_file.open(mode="w") as file_: + file_.write(content) + + # push first version to remote storage + remote = RemoteStorage(d / "root", str(d / "remote")) + remote.push(filename, "aaaa", content_hash) + + # retrieve with full hashes + loc_file.unlink() + remote.pull(filename, command_hash="aaaa", content_hash=content_hash) + with loc_file.open(mode="r") as file_: + assert file_.read() == content + + # retrieve with command hash + loc_file.unlink() + remote.pull(filename, command_hash="aaaa") + with loc_file.open(mode="r") as file_: + assert file_.read() == content + + # retrieve with content hash + loc_file.unlink() + remote.pull(filename, content_hash=content_hash) + with loc_file.open(mode="r") as file_: + assert file_.read() == content + + # retrieve with no hashes + loc_file.unlink() + remote.pull(filename) + with loc_file.open(mode="r") as file_: + assert file_.read() == content + + +def test_local_remote_storage_pull_missing(): + # pulling from a non-existent remote pulls nothing gracefully + with make_tempdir() as d: + filename = "a.txt" + remote = RemoteStorage(d / "root", str(d / "remote")) + assert remote.pull(filename, command_hash="aaaa") is None + assert remote.pull(filename) is None + + def test_cli_find_threshold(capsys): thresholds = numpy.linspace(0, 1, 10) diff --git a/website/docs/api/cli.md b/website/docs/api/cli.md index b42ba8a4f..8823a3bd8 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.md @@ -1391,12 +1391,13 @@ If the contents are different, the new version of the file is uploaded. Deleting obsolete files is left up to you. Remotes can be defined in the `remotes` section of the -[`project.yml`](/usage/projects#project-yml). Under the hood, spaCy uses the -[`smart-open`](https://github.com/RaRe-Technologies/smart_open) library to -communicate with the remote storages, so you can use any protocol that -`smart-open` supports, including [S3](https://aws.amazon.com/s3/), -[Google Cloud Storage](https://cloud.google.com/storage), SSH and more, although -you may need to install extra dependencies to use certain protocols. +[`project.yml`](/usage/projects#project-yml). Under the hood, spaCy uses +[`Pathy`](https://github.com/justindujardin/pathy) to communicate with the +remote storages, so you can use any protocol that `Pathy` supports, including +[S3](https://aws.amazon.com/s3/), +[Google Cloud Storage](https://cloud.google.com/storage), and the local +filesystem, although you may need to install extra dependencies to use certain +protocols. ```cli $ python -m spacy project push [remote] [project_dir] @@ -1435,12 +1436,13 @@ outputs, so if you change the config back, you'll be able to fetch back the result. Remotes can be defined in the `remotes` section of the -[`project.yml`](/usage/projects#project-yml). Under the hood, spaCy uses the -[`smart-open`](https://github.com/RaRe-Technologies/smart_open) library to -communicate with the remote storages, so you can use any protocol that -`smart-open` supports, including [S3](https://aws.amazon.com/s3/), -[Google Cloud Storage](https://cloud.google.com/storage), SSH and more, although -you may need to install extra dependencies to use certain protocols. +[`project.yml`](/usage/projects#project-yml). Under the hood, spaCy uses +[`Pathy`](https://github.com/justindujardin/pathy) to communicate with the +remote storages, so you can use any protocol that `Pathy` supports, including +[S3](https://aws.amazon.com/s3/), +[Google Cloud Storage](https://cloud.google.com/storage), and the local +filesystem, although you may need to install extra dependencies to use certain +protocols. ```cli $ python -m spacy project pull [remote] [project_dir] diff --git a/website/docs/usage/projects.md b/website/docs/usage/projects.md index 34315e4e7..f57578049 100644 --- a/website/docs/usage/projects.md +++ b/website/docs/usage/projects.md @@ -259,9 +259,9 @@ pipelines. > This can be used in a project command like so: > > ```yaml -> - name: "echo-path" -> script: -> - "echo ${env.ENV_PATH}" +> - name: 'echo-path' +> script: +> - 'echo ${env.ENV_PATH}' > ``` | Section | Description | @@ -643,12 +643,13 @@ locally. You can list one or more remotes in the `remotes` section of your [`project.yml`](#project-yml) by mapping a string name to the URL of the -storage. Under the hood, spaCy uses the -[`smart-open`](https://github.com/RaRe-Technologies/smart_open) library to -communicate with the remote storages, so you can use any protocol that -`smart-open` supports, including [S3](https://aws.amazon.com/s3/), -[Google Cloud Storage](https://cloud.google.com/storage), SSH and more, although -you may need to install extra dependencies to use certain protocols. +storage. Under the hood, spaCy uses +[`Pathy`](https://github.com/justindujardin/pathy) to communicate with the +remote storages, so you can use any protocol that `Pathy` supports, including +[S3](https://aws.amazon.com/s3/), +[Google Cloud Storage](https://cloud.google.com/storage), and the local +filesystem, although you may need to install extra dependencies to use certain +protocols. > #### Example > @@ -661,7 +662,6 @@ you may need to install extra dependencies to use certain protocols. remotes: default: 's3://my-spacy-bucket' local: '/mnt/scratch/cache' - stuff: 'ssh://myserver.example.com/whatever' ``` From f1e024345043cdc986e70308a09a7ca383b60dd0 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Tue, 29 Nov 2022 19:50:23 +0900 Subject: [PATCH 122/179] Remove macro auc per type from textcat defaults (#11887) This appears to have been added by mistake and never used. Removing it does not break validation. --- spacy/pipeline/textcat.py | 1 - spacy/pipeline/textcat_multilabel.py | 1 - 2 files changed, 2 deletions(-) diff --git a/spacy/pipeline/textcat.py b/spacy/pipeline/textcat.py index 9490e3cb1..65121114d 100644 --- a/spacy/pipeline/textcat.py +++ b/spacy/pipeline/textcat.py @@ -87,7 +87,6 @@ subword_features = true "cats_macro_f": None, "cats_macro_auc": None, "cats_f_per_type": None, - "cats_macro_auc_per_type": None, }, ) def make_textcat( diff --git a/spacy/pipeline/textcat_multilabel.py b/spacy/pipeline/textcat_multilabel.py index ef9bd6557..328cee723 100644 --- a/spacy/pipeline/textcat_multilabel.py +++ b/spacy/pipeline/textcat_multilabel.py @@ -87,7 +87,6 @@ subword_features = true "cats_macro_f": None, "cats_macro_auc": None, "cats_f_per_type": None, - "cats_macro_auc_per_type": None, }, ) def make_multilabel_textcat( From 6f9d630f7e9c11d8d5f7ba37e3764ef3630c172d Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 29 Nov 2022 13:20:08 +0100 Subject: [PATCH 123/179] Replace Pipe type with Callable in Language (#11803) * Replace Pipe type with Callable in Language * Use Callable[[Doc], Doc] in the docstrings --- spacy/cli/debug_data.py | 2 + spacy/language.py | 55 ++++++++++++++-------------- spacy/tests/pipeline/test_textcat.py | 2 +- spacy/util.py | 7 ++-- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/spacy/cli/debug_data.py b/spacy/cli/debug_data.py index 963d5b926..a85324e87 100644 --- a/spacy/cli/debug_data.py +++ b/spacy/cli/debug_data.py @@ -13,6 +13,7 @@ from ._util import import_code, debug_cli, _format_number from ..training import Example, remove_bilu_prefix from ..training.initialize import get_sourced_components from ..schemas import ConfigSchemaTraining +from ..pipeline import TrainablePipe from ..pipeline._parser_internals import nonproj from ..pipeline._parser_internals.nonproj import DELIMITER from ..pipeline import Morphologizer, SpanCategorizer @@ -934,6 +935,7 @@ def _get_labels_from_model(nlp: Language, factory_name: str) -> Set[str]: labels: Set[str] = set() for pipe_name in pipe_names: pipe = nlp.get_pipe(pipe_name) + assert isinstance(pipe, TrainablePipe) labels.update(pipe.labels) return labels diff --git a/spacy/language.py b/spacy/language.py index 2789b6690..e0abfd5e7 100644 --- a/spacy/language.py +++ b/spacy/language.py @@ -43,8 +43,7 @@ from .lookups import load_lookups from .compat import Literal -if TYPE_CHECKING: - from .pipeline import Pipe # noqa: F401 +PipeCallable = Callable[[Doc], Doc] # This is the base config will all settings (training etc.) @@ -181,7 +180,7 @@ class Language: self.vocab: Vocab = vocab if self.lang is None: self.lang = self.vocab.lang - self._components: List[Tuple[str, "Pipe"]] = [] + self._components: List[Tuple[str, PipeCallable]] = [] self._disabled: Set[str] = set() self.max_length = max_length # Create the default tokenizer from the default config @@ -303,7 +302,7 @@ class Language: return SimpleFrozenList(names) @property - def components(self) -> List[Tuple[str, "Pipe"]]: + def components(self) -> List[Tuple[str, PipeCallable]]: """Get all (name, component) tuples in the pipeline, including the currently disabled components. """ @@ -322,12 +321,12 @@ class Language: return SimpleFrozenList(names, error=Errors.E926.format(attr="component_names")) @property - def pipeline(self) -> List[Tuple[str, "Pipe"]]: + def pipeline(self) -> List[Tuple[str, PipeCallable]]: """The processing pipeline consisting of (name, component) tuples. The components are called on the Doc in order as it passes through the pipeline. - RETURNS (List[Tuple[str, Pipe]]): The pipeline. + RETURNS (List[Tuple[str, Callable[[Doc], Doc]]]): The pipeline. """ pipes = [(n, p) for n, p in self._components if n not in self._disabled] return SimpleFrozenList(pipes, error=Errors.E926.format(attr="pipeline")) @@ -527,7 +526,7 @@ class Language: assigns: Iterable[str] = SimpleFrozenList(), requires: Iterable[str] = SimpleFrozenList(), retokenizes: bool = False, - func: Optional["Pipe"] = None, + func: Optional[PipeCallable] = None, ) -> Callable[..., Any]: """Register a new pipeline component. Can be used for stateless function components that don't require a separate factory. Can be used as a @@ -542,7 +541,7 @@ class Language: e.g. "token.ent_id". Used for pipeline analysis. retokenizes (bool): Whether the component changes the tokenization. Used for pipeline analysis. - func (Optional[Callable]): Factory function if not used as a decorator. + func (Optional[Callable[[Doc], Doc]): Factory function if not used as a decorator. DOCS: https://spacy.io/api/language#component """ @@ -553,11 +552,11 @@ class Language: raise ValueError(Errors.E853.format(name=name)) component_name = name if name is not None else util.get_object_name(func) - def add_component(component_func: "Pipe") -> Callable: + def add_component(component_func: PipeCallable) -> Callable: if isinstance(func, type): # function is a class raise ValueError(Errors.E965.format(name=component_name)) - def factory_func(nlp, name: str) -> "Pipe": + def factory_func(nlp, name: str) -> PipeCallable: return component_func internal_name = cls.get_factory_name(name) @@ -607,7 +606,7 @@ class Language: print_pipe_analysis(analysis, keys=keys) return analysis - def get_pipe(self, name: str) -> "Pipe": + def get_pipe(self, name: str) -> PipeCallable: """Get a pipeline component for a given component name. name (str): Name of pipeline component to get. @@ -628,7 +627,7 @@ class Language: config: Dict[str, Any] = SimpleFrozenDict(), raw_config: Optional[Config] = None, validate: bool = True, - ) -> "Pipe": + ) -> PipeCallable: """Create a pipeline component. Mostly used internally. To create and add a component to the pipeline, you can use nlp.add_pipe. @@ -640,7 +639,7 @@ class Language: raw_config (Optional[Config]): Internals: the non-interpolated config. validate (bool): Whether to validate the component config against the arguments and types expected by the factory. - RETURNS (Pipe): The pipeline component. + RETURNS (Callable[[Doc], Doc]): The pipeline component. DOCS: https://spacy.io/api/language#create_pipe """ @@ -695,13 +694,13 @@ class Language: def create_pipe_from_source( self, source_name: str, source: "Language", *, name: str - ) -> Tuple["Pipe", str]: + ) -> Tuple[PipeCallable, str]: """Create a pipeline component by copying it from an existing model. source_name (str): Name of the component in the source pipeline. source (Language): The source nlp object to copy from. name (str): Optional alternative name to use in current pipeline. - RETURNS (Tuple[Callable, str]): The component and its factory name. + RETURNS (Tuple[Callable[[Doc], Doc], str]): The component and its factory name. """ # Check source type if not isinstance(source, Language): @@ -740,7 +739,7 @@ class Language: config: Dict[str, Any] = SimpleFrozenDict(), raw_config: Optional[Config] = None, validate: bool = True, - ) -> "Pipe": + ) -> PipeCallable: """Add a component to the processing pipeline. Valid components are callables that take a `Doc` object, modify it and return it. Only one of before/after/first/last can be set. Default behaviour is "last". @@ -763,7 +762,7 @@ class Language: raw_config (Optional[Config]): Internals: the non-interpolated config. validate (bool): Whether to validate the component config against the arguments and types expected by the factory. - RETURNS (Pipe): The pipeline component. + RETURNS (Callable[[Doc], Doc]): The pipeline component. DOCS: https://spacy.io/api/language#add_pipe """ @@ -869,7 +868,7 @@ class Language: *, config: Dict[str, Any] = SimpleFrozenDict(), validate: bool = True, - ) -> "Pipe": + ) -> PipeCallable: """Replace a component in the pipeline. name (str): Name of the component to replace. @@ -878,7 +877,7 @@ class Language: component. Will be merged with default config, if available. validate (bool): Whether to validate the component config against the arguments and types expected by the factory. - RETURNS (Pipe): The new pipeline component. + RETURNS (Callable[[Doc], Doc]): The new pipeline component. DOCS: https://spacy.io/api/language#replace_pipe """ @@ -930,11 +929,11 @@ class Language: init_cfg = self._config["initialize"]["components"].pop(old_name) self._config["initialize"]["components"][new_name] = init_cfg - def remove_pipe(self, name: str) -> Tuple[str, "Pipe"]: + def remove_pipe(self, name: str) -> Tuple[str, PipeCallable]: """Remove a component from the pipeline. name (str): Name of the component to remove. - RETURNS (tuple): A `(name, component)` tuple of the removed component. + RETURNS (Tuple[str, Callable[[Doc], Doc]]): A `(name, component)` tuple of the removed component. DOCS: https://spacy.io/api/language#remove_pipe """ @@ -1349,15 +1348,15 @@ class Language: def set_error_handler( self, - error_handler: Callable[[str, "Pipe", List[Doc], Exception], NoReturn], + error_handler: Callable[[str, PipeCallable, List[Doc], Exception], NoReturn], ): - """Set an error handler object for all the components in the pipeline that implement - a set_error_handler function. + """Set an error handler object for all the components in the pipeline + that implement a set_error_handler function. - error_handler (Callable[[str, Pipe, List[Doc], Exception], NoReturn]): - Function that deals with a failing batch of documents. This callable function should take in - the component's name, the component itself, the offending batch of documents, and the exception - that was thrown. + error_handler (Callable[[str, Callable[[Doc], Doc], List[Doc], Exception], NoReturn]): + Function that deals with a failing batch of documents. This callable + function should take in the component's name, the component itself, + the offending batch of documents, and the exception that was thrown. DOCS: https://spacy.io/api/language#set_error_handler """ self.default_error_handler = error_handler diff --git a/spacy/tests/pipeline/test_textcat.py b/spacy/tests/pipeline/test_textcat.py index 2eda9deaf..155ce99a2 100644 --- a/spacy/tests/pipeline/test_textcat.py +++ b/spacy/tests/pipeline/test_textcat.py @@ -838,8 +838,8 @@ def test_textcat_loss(multi_label: bool, expected_loss: float): textcat = nlp.add_pipe("textcat_multilabel") else: textcat = nlp.add_pipe("textcat") - textcat.initialize(lambda: train_examples) assert isinstance(textcat, TextCategorizer) + textcat.initialize(lambda: train_examples) scores = textcat.model.ops.asarray( [[0.0, 0.0, 0.0, 1.0], [0.0, 0.0, 1.0, 1.0]], dtype="f" # type: ignore ) diff --git a/spacy/util.py b/spacy/util.py index 76a1e0bfa..cba403361 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -51,8 +51,7 @@ from . import about if TYPE_CHECKING: # This lets us add type hints for mypy etc. without causing circular imports - from .language import Language # noqa: F401 - from .pipeline import Pipe # noqa: F401 + from .language import Language, PipeCallable # noqa: F401 from .tokens import Doc, Span # noqa: F401 from .vocab import Vocab # noqa: F401 @@ -1642,9 +1641,9 @@ def check_bool_env_var(env_var: str) -> bool: def _pipe( docs: Iterable["Doc"], - proc: "Pipe", + proc: "PipeCallable", name: str, - default_error_handler: Callable[[str, "Pipe", List["Doc"], Exception], NoReturn], + default_error_handler: Callable[[str, "PipeCallable", List["Doc"], Exception], NoReturn], kwargs: Mapping[str, Any], ) -> Iterator["Doc"]: if hasattr(proc, "pipe"): From afd7a2476d2491af864d0723bff96191ea61b429 Mon Sep 17 00:00:00 2001 From: Damian Romero <12145757+damian-romero@users.noreply.github.com> Date: Thu, 1 Dec 2022 07:06:28 -0500 Subject: [PATCH 124/179] Fix typo in vocab.md table (#11908) * Fix typo in vocab.md table Fixes explosion/spaCy/#11907 * Reformat vocab.md with Prettier --- website/docs/api/vocab.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/website/docs/api/vocab.md b/website/docs/api/vocab.md index afbd1301d..5e4de219a 100644 --- a/website/docs/api/vocab.md +++ b/website/docs/api/vocab.md @@ -308,14 +308,14 @@ Load state from a binary string. > assert type(PERSON) == int > ``` -| Name | Description | -| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `strings` | A table managing the string-to-int mapping. ~~StringStore~~ | -| `vectors` | A table associating word IDs to word vectors. ~~Vectors~~ | -| `vectors_length` | Number of dimensions for each word vector. ~~int~~ | -| `lookups` | The available lookup tables in this vocab. ~~Lookups~~ | -| `writing_system` | A dict with information about the language's writing system. ~~Dict[str, Any]~~ | -| `get_noun_chunks` 3.0 | A function that yields base noun phrases used for [`Doc.noun_chunks`](/ap/doc#noun_chunks). ~~Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]~~ | +| Name | Description | +| ---------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `strings` | A table managing the string-to-int mapping. ~~StringStore~~ | +| `vectors` | A table associating word IDs to word vectors. ~~Vectors~~ | +| `vectors_length` | Number of dimensions for each word vector. ~~int~~ | +| `lookups` | The available lookup tables in this vocab. ~~Lookups~~ | +| `writing_system` | A dict with information about the language's writing system. ~~Dict[str, Any]~~ | +| `get_noun_chunks` 3.0 | A function that yields base noun phrases used for [`Doc.noun_chunks`](/api/doc#noun_chunks). ~~Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]~~ | ## Serialization fields {#serialization-fields} From 9cf3fa9711dfff94e88d6e137a52ebabdcceaad8 Mon Sep 17 00:00:00 2001 From: Zhangrp Date: Thu, 1 Dec 2022 20:30:27 +0800 Subject: [PATCH 125/179] Add docs for biluo_to_iob and iob_to_biluo. (#11901) * Add docs for biluo_to_iob and iob_to_biluo. * Fix typos. * Remove redundant links. --- website/docs/api/top-level.md | 48 +++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/website/docs/api/top-level.md b/website/docs/api/top-level.md index 211affa4a..26a5d42f4 100644 --- a/website/docs/api/top-level.md +++ b/website/docs/api/top-level.md @@ -1004,6 +1004,54 @@ This method was previously available as `spacy.gold.spans_from_biluo_tags`. | `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags with each tag describing one token. Each tag string will be of the form of either `""`, `"O"` or `"{action}-{label}"`, where action is one of `"B"`, `"I"`, `"L"`, `"U"`. ~~List[str]~~ | | **RETURNS** | A sequence of `Span` objects with added entity labels. ~~List[Span]~~ | +### training.biluo_to_iob {#biluo_to_iob tag="function"} + +Convert a sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags to +[IOB](/usage/linguistic-features#accessing-ner) tags. This is useful if you want +use the BILUO tags with a model that only supports IOB tags. + +> #### Example +> +> ```python +> from spacy.training import biluo_to_iob +> +> tags = ["O", "O", "B-LOC", "I-LOC", "L-LOC", "O"] +> iob_tags = biluo_to_iob(tags) +> assert iob_tags == ["O", "O", "B-LOC", "I-LOC", "I-LOC", "O"] +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------------------- | +| `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | +| **RETURNS** | A list of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | + +### training.iob_to_biluo {#iob_to_biluo tag="function"} + +Convert a sequence of [IOB](/usage/linguistic-features#accessing-ner) tags to +[BILUO](/usage/linguistic-features#accessing-ner) tags. This is useful if you +want use the IOB tags with a model that only supports BILUO tags. + + + +This method was previously available as `spacy.gold.iob_to_biluo`. + + + +> #### Example +> +> ```python +> from spacy.training import iob_to_biluo +> +> tags = ["O", "O", "B-LOC", "I-LOC", "O"] +> biluo_tags = iob_to_biluo(tags) +> assert biluo_tags == ["O", "O", "B-LOC", "L-LOC", "O"] +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------- | +| `tags` | A sequence of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | +| **RETURNS** | A list of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | + ## Utility functions {#util source="spacy/util.py"} spaCy comes with a small collection of utility functions located in From 445c670a2d537598b3d562fb7f444050164a260b Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 2 Dec 2022 09:33:52 +0100 Subject: [PATCH 126/179] Fix spancat for zero suggestions (#11860) * Add test for spancat predict with zero suggestions * Fix spancat for zero suggestions * Undo changes to extract_spans * Use .sum() as in update --- spacy/pipeline/spancat.py | 5 +++- spacy/tests/pipeline/test_spancat.py | 43 ++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/spacy/pipeline/spancat.py b/spacy/pipeline/spancat.py index 0a84c72fd..a3388e81a 100644 --- a/spacy/pipeline/spancat.py +++ b/spacy/pipeline/spancat.py @@ -272,7 +272,10 @@ class SpanCategorizer(TrainablePipe): DOCS: https://spacy.io/api/spancategorizer#predict """ indices = self.suggester(docs, ops=self.model.ops) - scores = self.model.predict((docs, indices)) # type: ignore + if indices.lengths.sum() == 0: + scores = self.model.ops.alloc2f(0, 0) + else: + scores = self.model.predict((docs, indices)) # type: ignore return indices, scores def set_candidates( diff --git a/spacy/tests/pipeline/test_spancat.py b/spacy/tests/pipeline/test_spancat.py index 15256a763..e9db983d3 100644 --- a/spacy/tests/pipeline/test_spancat.py +++ b/spacy/tests/pipeline/test_spancat.py @@ -372,24 +372,39 @@ def test_overfitting_IO_overlapping(): def test_zero_suggestions(): - # Test with a suggester that returns 0 suggestions + # Test with a suggester that can return 0 suggestions - @registry.misc("test_zero_suggester") - def make_zero_suggester(): - def zero_suggester(docs, *, ops=None): + @registry.misc("test_mixed_zero_suggester") + def make_mixed_zero_suggester(): + def mixed_zero_suggester(docs, *, ops=None): if ops is None: ops = get_current_ops() - return Ragged( - ops.xp.zeros((0, 0), dtype="i"), ops.xp.zeros((len(docs),), dtype="i") - ) + spans = [] + lengths = [] + for doc in docs: + if len(doc) > 0 and len(doc) % 2 == 0: + spans.append((0, 1)) + lengths.append(1) + else: + lengths.append(0) + spans = ops.asarray2i(spans) + lengths_array = ops.asarray1i(lengths) + if len(spans) > 0: + output = Ragged(ops.xp.vstack(spans), lengths_array) + else: + output = Ragged(ops.xp.zeros((0, 0), dtype="i"), lengths_array) + return output - return zero_suggester + return mixed_zero_suggester fix_random_seed(0) nlp = English() spancat = nlp.add_pipe( "spancat", - config={"suggester": {"@misc": "test_zero_suggester"}, "spans_key": SPAN_KEY}, + config={ + "suggester": {"@misc": "test_mixed_zero_suggester"}, + "spans_key": SPAN_KEY, + }, ) train_examples = make_examples(nlp) optimizer = nlp.initialize(get_examples=lambda: train_examples) @@ -397,6 +412,16 @@ def test_zero_suggestions(): assert set(spancat.labels) == {"LOC", "PERSON"} nlp.update(train_examples, sgd=optimizer) + # empty doc + nlp("") + # single doc with zero suggestions + nlp("one") + # single doc with one suggestion + nlp("two two") + # batch with mixed zero/one suggestions + list(nlp.pipe(["one", "two two", "three three three", "", "four four four four"])) + # batch with no suggestions + list(nlp.pipe(["", "one", "three three three"])) def test_set_candidates(): From f9d17a644b3d037924f715c03672ada6d12e4d86 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Fri, 2 Dec 2022 18:17:11 +0900 Subject: [PATCH 127/179] Config generation fails for GPU without transformers (#11899) If you don't have spacy-transformers installed, but try to use `init config` with the GPU flag, you'll get an error. The issue is that the `use_transformers` flag in the config is conflated with the GPU flag, and then there's an attempt to access transformers config info that may not exist. There may be a better way to do this, but this stops the error. --- spacy/cli/templates/quickstart_training.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/cli/templates/quickstart_training.jinja b/spacy/cli/templates/quickstart_training.jinja index 58864883a..b961ac892 100644 --- a/spacy/cli/templates/quickstart_training.jinja +++ b/spacy/cli/templates/quickstart_training.jinja @@ -1,7 +1,7 @@ {# This is a template for training configs used for the quickstart widget in the docs and the init config command. It encodes various best practices and can help generate the best possible configuration, given a user's requirements. #} -{%- set use_transformer = hardware != "cpu" -%} +{%- set use_transformer = hardware != "cpu" and transformer_data -%} {%- set transformer = transformer_data[optimize] if use_transformer else {} -%} {%- set listener_components = ["tagger", "morphologizer", "parser", "ner", "textcat", "textcat_multilabel", "entity_linker", "spancat", "trainable_lemmatizer"] -%} [paths] From df0cb4b77be6e20a62143f5f65c3e165a4a45bcc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 2 Dec 2022 14:49:12 +0100 Subject: [PATCH 128/179] Auto-format code with black (#11913) Co-authored-by: explosion-bot --- spacy/util.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spacy/util.py b/spacy/util.py index cba403361..8d211a9a5 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -1643,7 +1643,9 @@ def _pipe( docs: Iterable["Doc"], proc: "PipeCallable", name: str, - default_error_handler: Callable[[str, "PipeCallable", List["Doc"], Exception], NoReturn], + default_error_handler: Callable[ + [str, "PipeCallable", List["Doc"], Exception], NoReturn + ], kwargs: Mapping[str, Any], ) -> Iterator["Doc"]: if hasattr(proc, "pipe"): From 4b2097a2713b548cba1c841fa5cb8f6f42e3e30f Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Mon, 5 Dec 2022 08:29:13 +0100 Subject: [PATCH 129/179] fix links (#11927) --- website/docs/usage/v3-4.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/usage/v3-4.md b/website/docs/usage/v3-4.md index 597fc3cc8..e10110b71 100644 --- a/website/docs/usage/v3-4.md +++ b/website/docs/usage/v3-4.md @@ -66,8 +66,8 @@ The English CNN pipelines have new word vectors: | Package | Model Version | TAG | Parser LAS | NER F | | ----------------------------------------------- | ------------- | ---: | ---------: | ----: | | [`en_core_web_md`](/models/en#en_core_web_md) | v3.3.0 | 97.3 | 90.1 | 84.6 | -| [`en_core_web_md`](/models/en#en_core_web_lg) | v3.4.0 | 97.2 | 90.3 | 85.5 | -| [`en_core_web_lg`](/models/en#en_core_web_md) | v3.3.0 | 97.4 | 90.1 | 85.3 | +| [`en_core_web_md`](/models/en#en_core_web_md) | v3.4.0 | 97.2 | 90.3 | 85.5 | +| [`en_core_web_lg`](/models/en#en_core_web_lg) | v3.3.0 | 97.4 | 90.1 | 85.3 | | [`en_core_web_lg`](/models/en#en_core_web_lg) | v3.4.0 | 97.3 | 90.2 | 85.6 | ## Notes about upgrading from v3.3 {#upgrading} From 5848656b5e3287d77674ce678e321eadea52f68e Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Mon, 5 Dec 2022 17:43:23 +0900 Subject: [PATCH 130/179] Switch ubuntu-latest to ubuntu-20.04 in main tests (#11928) * Switch ubuntu-latest to ubuntu-20.04 in main tests * Only use 20.04 for 3.6 --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9c3b92f06..0f7ea91f9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -41,7 +41,7 @@ jobs: matrix: # We're only running one platform per Python version to speed up builds Python36Linux: - imageName: "ubuntu-latest" + imageName: "ubuntu-20.04" python.version: "3.6" # Python36Windows: # imageName: "windows-latest" @@ -50,7 +50,7 @@ jobs: # imageName: "macos-latest" # python.version: "3.6" # Python37Linux: - # imageName: "ubuntu-latest" + # imageName: "ubuntu-20.04" # python.version: "3.7" Python37Windows: imageName: "windows-latest" From 6f342bdd72f300cdc431d0e0f2a168c62fd2a861 Mon Sep 17 00:00:00 2001 From: Darigov Research <30328618+darigovresearch@users.noreply.github.com> Date: Mon, 5 Dec 2022 08:49:04 +0000 Subject: [PATCH 131/179] docs: Adds link to license in readme (#11924) Would resolve https://github.com/explosion/spaCy/issues/11923 if merged --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index abfc3da67..7595460fb 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ parsing, **named entity recognition**, **text classification** and more, multi-task learning with pretrained **transformers** like BERT, as well as a production-ready [**training system**](https://spacy.io/usage/training) and easy model packaging, deployment and workflow management. spaCy is commercial -open-source software, released under the MIT license. +open-source software, released under the [MIT license](https://github.com/explosion/spaCy/blob/master/LICENSE). 💫 **Version 3.4 out now!** [Check out the release notes here.](https://github.com/explosion/spaCy/releases) From 8afa8b5a7b8ee51eb42b83dabd0f3c1276369e73 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Mon, 5 Dec 2022 10:00:00 +0100 Subject: [PATCH 132/179] Refactor kwargs in CLI msg for future wasabi compatibility (#11918) Necessary for mypy with wasabi v1+. --- spacy/cli/project/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spacy/cli/project/run.py b/spacy/cli/project/run.py index a109c4a5a..6dd174902 100644 --- a/spacy/cli/project/run.py +++ b/spacy/cli/project/run.py @@ -101,8 +101,8 @@ def project_run( if not (project_dir / dep).exists(): err = f"Missing dependency specified by command '{subcommand}': {dep}" err_help = "Maybe you forgot to run the 'project assets' command or a previous step?" - err_kwargs = {"exits": 1} if not dry else {} - msg.fail(err, err_help, **err_kwargs) + err_exits = 1 if not dry else None + msg.fail(err, err_help, exits=err_exits) check_spacy_commit = check_bool_env_var(ENV_VARS.PROJECT_USE_GIT_VERSION) with working_dir(project_dir) as current_dir: msg.divider(subcommand) From 1aadcfcb37ba166558688782fabbcbe3e32ea020 Mon Sep 17 00:00:00 2001 From: Ryn Daniels <397565+ryndaniels@users.noreply.github.com> Date: Mon, 5 Dec 2022 11:17:10 +0200 Subject: [PATCH 133/179] update lock-threads to v4 (#11930) --- .github/workflows/lock.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index c9833cdba..794adee85 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -15,11 +15,11 @@ jobs: action: runs-on: ubuntu-latest steps: - - uses: dessant/lock-threads@v3 + - uses: dessant/lock-threads@v4 with: process-only: 'issues' issue-inactive-days: '30' - issue-comment: > - This thread has been automatically locked since there - has not been any recent activity after it was closed. + issue-comment: > + This thread has been automatically locked since there + has not been any recent activity after it was closed. Please open a new issue for related bugs. From 23085ffef4bba62aff0de5993ff405cb3ff3528c Mon Sep 17 00:00:00 2001 From: Zhangrp Date: Tue, 6 Dec 2022 16:42:12 +0800 Subject: [PATCH 134/179] Fix interpolation in directory names, see #11235. (#11914) --- spacy/cli/_util.py | 8 ++++---- spacy/tests/test_cli.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py index 7ce006108..9b97a9f19 100644 --- a/spacy/cli/_util.py +++ b/spacy/cli/_util.py @@ -158,15 +158,15 @@ def load_project_config( sys.exit(1) validate_project_version(config) validate_project_commands(config) + if interpolate: + err = f"{PROJECT_FILE} validation error" + with show_validation_error(title=err, hint_fill=False): + config = substitute_project_variables(config, overrides) # Make sure directories defined in config exist for subdir in config.get("directories", []): dir_path = path / subdir if not dir_path.exists(): dir_path.mkdir(parents=True) - if interpolate: - err = f"{PROJECT_FILE} validation error" - with show_validation_error(title=err, hint_fill=False): - config = substitute_project_variables(config, overrides) return config diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py index 2e706458f..3104b49ff 100644 --- a/spacy/tests/test_cli.py +++ b/spacy/tests/test_cli.py @@ -123,6 +123,25 @@ def test_issue7055(): assert "model" in filled_cfg["components"]["ner"] +@pytest.mark.issue(11235) +def test_issue11235(): + """ + Test that the cli handles interpolation in the directory names correctly when loading project config. + """ + lang_var = "en" + variables = {"lang": lang_var} + commands = [{"name": "x", "script": ["hello ${vars.lang}"]}] + directories = ["cfg", "${vars.lang}_model"] + project = {"commands": commands, "vars": variables, "directories": directories} + with make_tempdir() as d: + srsly.write_yaml(d / "project.yml", project) + cfg = load_project_config(d) + # Check that the directories are interpolated and created correctly + assert os.path.exists(d / "cfg") + assert os.path.exists(d / f"{lang_var}_model") + assert cfg["commands"][0]["script"][0] == f"hello {lang_var}" + + def test_cli_info(): nlp = Dutch() nlp.add_pipe("textcat") From 27fac7df2e67a0dbfefd68834c14fb1f9505da49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Wed, 7 Dec 2022 05:53:41 +0100 Subject: [PATCH 135/179] EditTreeLemmatizer: correctly add strings when initializing from labels (#11934) Strings in replacement nodes where not added to the `StringStore` when `EditTreeLemmatizer` was initialized from a set of labels. The corresponding test did not capture this because it added the strings through the examples that were passed to the initialization. This change fixes both this bug in the initialization as the 'shadowing' of the bug in the test. --- spacy/pipeline/edit_tree_lemmatizer.py | 4 +- .../pipeline/test_edit_tree_lemmatizer.py | 37 ++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/spacy/pipeline/edit_tree_lemmatizer.py b/spacy/pipeline/edit_tree_lemmatizer.py index 12f9b73a3..a56c9975e 100644 --- a/spacy/pipeline/edit_tree_lemmatizer.py +++ b/spacy/pipeline/edit_tree_lemmatizer.py @@ -328,9 +328,9 @@ class EditTreeLemmatizer(TrainablePipe): tree = dict(tree) if "orig" in tree: - tree["orig"] = self.vocab.strings[tree["orig"]] + tree["orig"] = self.vocab.strings.add(tree["orig"]) if "orig" in tree: - tree["subst"] = self.vocab.strings[tree["subst"]] + tree["subst"] = self.vocab.strings.add(tree["subst"]) trees.append(tree) diff --git a/spacy/tests/pipeline/test_edit_tree_lemmatizer.py b/spacy/tests/pipeline/test_edit_tree_lemmatizer.py index cf541e301..b12ca5dd4 100644 --- a/spacy/tests/pipeline/test_edit_tree_lemmatizer.py +++ b/spacy/tests/pipeline/test_edit_tree_lemmatizer.py @@ -60,10 +60,45 @@ def test_initialize_from_labels(): nlp2 = Language() lemmatizer2 = nlp2.add_pipe("trainable_lemmatizer") lemmatizer2.initialize( - get_examples=lambda: train_examples, + # We want to check that the strings in replacement nodes are + # added to the string store. Avoid that they get added through + # the examples. + get_examples=lambda: train_examples[:1], labels=lemmatizer.label_data, ) assert lemmatizer2.tree2label == {1: 0, 3: 1, 4: 2, 6: 3} + assert lemmatizer2.label_data == { + "trees": [ + {"orig": "S", "subst": "s"}, + { + "prefix_len": 1, + "suffix_len": 0, + "prefix_tree": 0, + "suffix_tree": 4294967295, + }, + {"orig": "s", "subst": ""}, + { + "prefix_len": 0, + "suffix_len": 1, + "prefix_tree": 4294967295, + "suffix_tree": 2, + }, + { + "prefix_len": 0, + "suffix_len": 0, + "prefix_tree": 4294967295, + "suffix_tree": 4294967295, + }, + {"orig": "E", "subst": "e"}, + { + "prefix_len": 1, + "suffix_len": 0, + "prefix_tree": 5, + "suffix_tree": 4294967295, + }, + ], + "labels": (1, 3, 4, 6), + } def test_no_data(): From 916191848ab7bf90e88f23401451695f61903112 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Wed, 7 Dec 2022 18:09:04 +0900 Subject: [PATCH 136/179] Update scattertext example code (#11937) * Update scattertext example code * Remove PMI Filter Threshold --- website/meta/universe.json | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/website/meta/universe.json b/website/meta/universe.json index 97b53e9c5..8ca657561 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -1468,13 +1468,26 @@ "image": "https://jasonkessler.github.io/2012conventions0.0.2.2.png", "code_example": [ "import spacy", - "import scattertext as st", "", - "nlp = spacy.load('en')", - "corpus = st.CorpusFromPandas(convention_df,", - " category_col='party',", - " text_col='text',", - " nlp=nlp).build()" + "from scattertext import SampleCorpora, produce_scattertext_explorer", + "from scattertext import produce_scattertext_html", + "from scattertext.CorpusFromPandas import CorpusFromPandas", + "", + "nlp = spacy.load('en_core_web_sm')", + "convention_df = SampleCorpora.ConventionData2012.get_data()", + "corpus = CorpusFromPandas(convention_df,", + " category_col='party',", + " text_col='text',", + " nlp=nlp).build()", + "", + "html = produce_scattertext_html(corpus,", + " category='democrat',", + " category_name='Democratic',", + " not_category_name='Republican',", + " minimum_term_frequency=5,", + " width_in_pixels=1000)", + "open('./simple.html', 'wb').write(html.encode('utf-8'))", + "print('Open ./simple.html in Chrome or Firefox.')" ], "author": "Jason Kessler", "author_links": { From 5c3a60e8f4273aff7bd47bce01d62c8224967045 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Wed, 7 Dec 2022 23:52:35 +0900 Subject: [PATCH 137/179] Add in errors used in the beam code that were removed at some point (#11935) I don't think there's any way to use the beam code at the moment, but as long as it's around the errors it refers to should also be present. --- spacy/errors.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spacy/errors.py b/spacy/errors.py index e34614b0f..0e5ef91ed 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -345,6 +345,11 @@ class Errors(metaclass=ErrorsWithCodes): "clear the existing vectors and resize the table.") E074 = ("Error interpreting compiled match pattern: patterns are expected " "to end with the attribute {attr}. Got: {bad_attr}.") + E079 = ("Error computing states in beam: number of predicted beams " + "({pbeams}) does not equal number of gold beams ({gbeams}).") + E080 = ("Duplicate state found in beam: {key}.") + E081 = ("Error getting gradient in beam: number of histories ({n_hist}) " + "does not equal number of losses ({losses}).") E082 = ("Error deprojectivizing parse: number of heads ({n_heads}), " "projective heads ({n_proj_heads}) and labels ({n_labels}) do not " "match.") From 73919336fb1b003425373a07d41e5541dc5c3c46 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Wed, 7 Dec 2022 23:56:03 +0900 Subject: [PATCH 138/179] Remove spacy-sentence-segmenter from Universe (#11932) --- website/meta/universe.json | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/website/meta/universe.json b/website/meta/universe.json index 8ca657561..db533c3b2 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -1023,25 +1023,6 @@ }, "category": ["pipeline"] }, - { - "id": "spacy-sentence-segmenter", - "title": "Sentence Segmenter", - "slogan": "Custom sentence segmentation for spaCy", - "code_example": [ - "from seg.newline.segmenter import NewLineSegmenter", - "import spacy", - "", - "nlseg = NewLineSegmenter()", - "nlp = spacy.load('en')", - "nlp.add_pipe(nlseg.set_sent_starts, name='sentence_segmenter', before='parser')", - "doc = nlp(my_doc_text)" - ], - "author": "tc64", - "author_links": { - "github": "tc64" - }, - "category": ["pipeline"] - }, { "id": "spacy_cld", "title": "spaCy-CLD", From 6d2ca1ab3a545491acbe058035677a263135e52a Mon Sep 17 00:00:00 2001 From: vincent d warmerdam Date: Wed, 7 Dec 2022 16:02:09 +0100 Subject: [PATCH 139/179] Update custom solutions links (#11903) * Update custom solutions Will now point to https://explosion.ai/custom-solutions * added-sidebar * added-analysis-to-readme * update-landing-page --- README.md | 2 ++ website/meta/sidebars.json | 2 +- website/meta/site.json | 2 +- website/src/widgets/landing.js | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7595460fb..195424551 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ open-source software, released under the [MIT license](https://github.com/explos | 🛠 **[Changelog]** | Changes and version history. | | 💝 **[Contribute]** | How to contribute to the spaCy project and code base. | | spaCy Tailored Pipelines | Get a custom spaCy pipeline, tailor-made for your NLP problem by spaCy's core developers. Streamlined, production-ready, predictable and maintainable. Start by completing our 5-minute questionnaire to tell us what you need and we'll be in touch! **[Learn more →](https://explosion.ai/spacy-tailored-pipelines)** | +| spaCy Tailored Pipelines | Bespoke advice for problem solving, strategy and analysis for applied NLP projects. Services include data strategy, code reviews, pipeline design and annotation coaching. Curious? Fill in our 5-minute questionnaire to tell us what you need and we'll be in touch! **[Learn more →](https://explosion.ai/spacy-tailored-analysis)** | [spacy 101]: https://spacy.io/usage/spacy-101 [new in v3.0]: https://spacy.io/usage/v3 @@ -59,6 +60,7 @@ open-source software, released under the [MIT license](https://github.com/explos [changelog]: https://spacy.io/usage#changelog [contribute]: https://github.com/explosion/spaCy/blob/master/CONTRIBUTING.md + ## 💬 Where to ask questions The spaCy project is maintained by the [spaCy team](https://explosion.ai/about). diff --git a/website/meta/sidebars.json b/website/meta/sidebars.json index 2d8745d77..339e4085b 100644 --- a/website/meta/sidebars.json +++ b/website/meta/sidebars.json @@ -45,7 +45,7 @@ { "text": "v2.x Documentation", "url": "https://v2.spacy.io" }, { "text": "Custom Solutions", - "url": "https://explosion.ai/spacy-tailored-pipelines" + "url": "https://explosion.ai/custom-solutions" } ] } diff --git a/website/meta/site.json b/website/meta/site.json index 360a72178..fa79d3c69 100644 --- a/website/meta/site.json +++ b/website/meta/site.json @@ -51,7 +51,7 @@ { "text": "Online Course", "url": "https://course.spacy.io" }, { "text": "Custom Solutions", - "url": "https://explosion.ai/spacy-tailored-pipelines" + "url": "https://explosion.ai/custom-solutions" } ] }, diff --git a/website/src/widgets/landing.js b/website/src/widgets/landing.js index b7ae35f6e..c3aaa8a22 100644 --- a/website/src/widgets/landing.js +++ b/website/src/widgets/landing.js @@ -105,13 +105,13 @@ const Landing = ({ data }) => { - + spaCy Tailored Pipelines From f22fc7a1138545a2a75975909b5af554e8e1d616 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 9 Dec 2022 10:15:52 +0100 Subject: [PATCH 140/179] Auto-format code with black (#11955) Co-authored-by: explosion-bot --- spacy/tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py index 3104b49ff..42af08749 100644 --- a/spacy/tests/test_cli.py +++ b/spacy/tests/test_cli.py @@ -140,7 +140,7 @@ def test_issue11235(): assert os.path.exists(d / "cfg") assert os.path.exists(d / f"{lang_var}_model") assert cfg["commands"][0]["script"][0] == f"hello {lang_var}" - + def test_cli_info(): nlp = Dutch() From 8c291ace0c0978e70257906438d3585022090e9f Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Mon, 12 Dec 2022 08:38:36 +0100 Subject: [PATCH 141/179] Extend to wasabi v1.1 (#11945) * Extend to wasabi v1.1 * Temporarily run mypy and tests with newest wasabi * Temporarily skip check requirements test * Revert "Temporarily skip check requirements test" This reverts commit 44f4ce20a8e8c92e8bfc8042cc68333589a96253. * Revert "Temporarily run mypy and tests with newest wasabi" This reverts commit e677a2257ced55e696cafc3a8e89eb2f7ddfc369. --- requirements.txt | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 778c05e21..0440835f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ preshed>=3.0.2,<3.1.0 thinc>=8.1.0,<8.2.0 ml_datasets>=0.2.0,<0.3.0 murmurhash>=0.28.0,<1.1.0 -wasabi>=0.9.1,<1.1.0 +wasabi>=0.9.1,<1.2.0 srsly>=2.4.3,<3.0.0 catalogue>=2.0.6,<2.1.0 typer>=0.3.0,<0.8.0 diff --git a/setup.cfg b/setup.cfg index 5768c9d3e..cf6e6f84b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,7 +47,7 @@ install_requires = cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 thinc>=8.1.0,<8.2.0 - wasabi>=0.9.1,<1.1.0 + wasabi>=0.9.1,<1.2.0 srsly>=2.4.3,<3.0.0 catalogue>=2.0.6,<2.1.0 # Third-party dependencies From 0591e67265d7378769c0fc0df4020817f2d514ec Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Mon, 12 Dec 2022 08:45:35 +0100 Subject: [PATCH 142/179] Cast to uint64 for all array-based doc representations (#11933) * Convert all individual values explicitly to uint64 for array-based doc representations * Temporarily test with latest numpy v1.24.0rc * Remove unnecessary conversion from attr_t * Reduce number of individual casts * Convert specifically from int32 to uint64 * Revert "Temporarily test with latest numpy v1.24.0rc" This reverts commit eb0e3c5006515b9a7ff52bae59484c909b8a3f65. * Also use int32 in tests --- spacy/tests/doc/test_array.py | 4 ++-- spacy/tokens/doc.pyx | 2 ++ spacy/tokens/span.pyx | 4 ++-- spacy/training/example.pyx | 15 ++++++++------- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/spacy/tests/doc/test_array.py b/spacy/tests/doc/test_array.py index c334cc6eb..1f2d7d999 100644 --- a/spacy/tests/doc/test_array.py +++ b/spacy/tests/doc/test_array.py @@ -123,14 +123,14 @@ def test_doc_from_array_heads_in_bounds(en_vocab): # head before start arr = doc.to_array(["HEAD"]) - arr[0] = -1 + arr[0] = numpy.int32(-1).astype(numpy.uint64) doc_from_array = Doc(en_vocab, words=words) with pytest.raises(ValueError): doc_from_array.from_array(["HEAD"], arr) # head after end arr = doc.to_array(["HEAD"]) - arr[0] = 5 + arr[0] = numpy.int32(5).astype(numpy.uint64) doc_from_array = Doc(en_vocab, words=words) with pytest.raises(ValueError): doc_from_array.from_array(["HEAD"], arr) diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index f2621292c..075bc4d15 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -359,6 +359,7 @@ cdef class Doc: for annot in annotations: if annot: if annot is heads or annot is sent_starts or annot is ent_iobs: + annot = numpy.array(annot, dtype=numpy.int32).astype(numpy.uint64) for i in range(len(words)): if attrs.ndim == 1: attrs[i] = annot[i] @@ -1558,6 +1559,7 @@ cdef class Doc: for j, (attr, annot) in enumerate(token_annotations.items()): if attr is HEAD: + annot = numpy.array(annot, dtype=numpy.int32).astype(numpy.uint64) for i in range(len(words)): array[i, j] = annot[i] elif attr is MORPH: diff --git a/spacy/tokens/span.pyx b/spacy/tokens/span.pyx index c3495f497..99a5f43bd 100644 --- a/spacy/tokens/span.pyx +++ b/spacy/tokens/span.pyx @@ -299,7 +299,7 @@ cdef class Span: for ancestor in ancestors: ancestor_i = ancestor.i - self.c.start if ancestor_i in range(length): - array[i, head_col] = ancestor_i - i + array[i, head_col] = numpy.int32(ancestor_i - i).astype(numpy.uint64) # if there is no appropriate ancestor, define a new artificial root value = array[i, head_col] @@ -307,7 +307,7 @@ cdef class Span: new_root = old_to_new_root.get(ancestor_i, None) if new_root is not None: # take the same artificial root as a previous token from the same sentence - array[i, head_col] = new_root - i + array[i, head_col] = numpy.int32(new_root - i).astype(numpy.uint64) else: # set this token as the new artificial root array[i, head_col] = 0 diff --git a/spacy/training/example.pyx b/spacy/training/example.pyx index dfd337b9e..95b0f0de9 100644 --- a/spacy/training/example.pyx +++ b/spacy/training/example.pyx @@ -443,26 +443,27 @@ def _annot2array(vocab, tok_annot, doc_annot): if key not in IDS: raise ValueError(Errors.E974.format(obj="token", key=key)) elif key in ["ORTH", "SPACY"]: - pass + continue elif key == "HEAD": attrs.append(key) - values.append([h-i if h is not None else 0 for i, h in enumerate(value)]) + row = [h-i if h is not None else 0 for i, h in enumerate(value)] elif key == "DEP": attrs.append(key) - values.append([vocab.strings.add(h) if h is not None else MISSING_DEP for h in value]) + row = [vocab.strings.add(h) if h is not None else MISSING_DEP for h in value] elif key == "SENT_START": attrs.append(key) - values.append([to_ternary_int(v) for v in value]) + row = [to_ternary_int(v) for v in value] elif key == "MORPH": attrs.append(key) - values.append([vocab.morphology.add(v) for v in value]) + row = [vocab.morphology.add(v) for v in value] else: attrs.append(key) if not all(isinstance(v, str) for v in value): types = set([type(v) for v in value]) raise TypeError(Errors.E969.format(field=key, types=types)) from None - values.append([vocab.strings.add(v) for v in value]) - array = numpy.asarray(values, dtype="uint64") + row = [vocab.strings.add(v) for v in value] + values.append([numpy.array(v, dtype=numpy.int32).astype(numpy.uint64) if v < 0 else v for v in row]) + array = numpy.array(values, dtype=numpy.uint64) return attrs, array.T From e5c7f3b0776d49c4f6aab7e02b503cdb84fb2134 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Mon, 12 Dec 2022 10:13:10 +0100 Subject: [PATCH 143/179] CI: Install thinc-apple-ops through extra (#11963) --- .github/azure-steps.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml index 2f77706b8..d0db75f9a 100644 --- a/.github/azure-steps.yml +++ b/.github/azure-steps.yml @@ -107,7 +107,7 @@ steps: displayName: "Run CPU tests" - script: | - python -m pip install --pre thinc-apple-ops + python -m pip install 'spacy[apple]' python -m pytest --pyargs spacy displayName: "Run CPU tests with thinc-apple-ops" condition: and(startsWith(variables['imageName'], 'macos'), eq(variables['python.version'], '3.11')) From c9d9d6847f9685c21eeec01f4b8cd053cadf8bf5 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Thu, 15 Dec 2022 10:55:01 +0100 Subject: [PATCH 144/179] Update build constraints for python 3.11 (#11981) --- build-constraints.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build-constraints.txt b/build-constraints.txt index 956973abf..c1e82f1b0 100644 --- a/build-constraints.txt +++ b/build-constraints.txt @@ -5,4 +5,5 @@ numpy==1.17.3; python_version=='3.8' and platform_machine!='aarch64' numpy==1.19.2; python_version=='3.8' and platform_machine=='aarch64' numpy==1.19.3; python_version=='3.9' numpy==1.21.3; python_version=='3.10' -numpy; python_version>='3.11' +numpy==1.23.2; python_version=='3.11' +numpy; python_version>='3.12' From 3a2b655a29203d1c181a2c14d230b3f9cf8dd54a Mon Sep 17 00:00:00 2001 From: cfuerbachersparks <119413757+cfuerbachersparks@users.noreply.github.com> Date: Mon, 19 Dec 2022 10:33:38 +0100 Subject: [PATCH 145/179] Update lexeme.md (#11994) Change suffix_ string to end --- website/docs/api/lexeme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/api/lexeme.md b/website/docs/api/lexeme.md index eb76afa90..557d04cce 100644 --- a/website/docs/api/lexeme.md +++ b/website/docs/api/lexeme.md @@ -138,7 +138,7 @@ The L2 norm of the lexeme's vector representation. | `prefix` | Length-N substring from the start of the word. Defaults to `N=1`. ~~int~~ | | `prefix_` | Length-N substring from the start of the word. Defaults to `N=1`. ~~str~~ | | `suffix` | Length-N substring from the end of the word. Defaults to `N=3`. ~~int~~ | -| `suffix_` | Length-N substring from the start of the word. Defaults to `N=3`. ~~str~~ | +| `suffix_` | Length-N substring from the end of the word. Defaults to `N=3`. ~~str~~ | | `is_alpha` | Does the lexeme consist of alphabetic characters? Equivalent to `lexeme.text.isalpha()`. ~~bool~~ | | `is_ascii` | Does the lexeme consist of ASCII characters? Equivalent to `[any(ord(c) >= 128 for c in lexeme.text)]`. ~~bool~~ | | `is_digit` | Does the lexeme consist of digits? Equivalent to `lexeme.text.isdigit()`. ~~bool~~ | From 18ffe5bbd6a554920107ff48d1387df34c3f872a Mon Sep 17 00:00:00 2001 From: Jos Polfliet Date: Mon, 19 Dec 2022 16:17:49 +0100 Subject: [PATCH 146/179] Update stop_words.py (#11997) fix typo in "aangaande" --- spacy/lang/nl/stop_words.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/lang/nl/stop_words.py b/spacy/lang/nl/stop_words.py index a2c6198e7..cd4fdefdf 100644 --- a/spacy/lang/nl/stop_words.py +++ b/spacy/lang/nl/stop_words.py @@ -15,7 +15,7 @@ STOP_WORDS = set( """ -aan af al alle alles allebei alleen allen als altijd ander anders andere anderen aangaangde aangezien achter achterna +aan af al alle alles allebei alleen allen als altijd ander anders andere anderen aangaande aangezien achter achterna afgelopen aldus alhoewel anderzijds ben bij bijna bijvoorbeeld behalve beide beiden beneden bent bepaald beter betere betreffende binnen binnenin boven From c223cd7a86f460f3dabb9e7369eef136a653218e Mon Sep 17 00:00:00 2001 From: kadarakos Date: Tue, 20 Dec 2022 17:11:33 +0100 Subject: [PATCH 147/179] Add apply CLI (#11376) * annotate cli first try * add batch-size and n_process * rename to apply * typing fix * handle file suffixes * walk directories * support jsonl * typing fix * remove debug * make suffix optional for walk * revert unrelated * don't warn but raise * better error message * minor touch up * Update spacy/tests/test_cli.py Co-authored-by: Adriane Boyd * Update spacy/cli/apply.py Co-authored-by: Sofie Van Landeghem * Update spacy/cli/apply.py Co-authored-by: Sofie Van Landeghem * update tests and bugfix * add force_overwrite * typo * fix adding .spacy suffix * Update spacy/cli/apply.py Co-authored-by: Sofie Van Landeghem * Update spacy/cli/apply.py Co-authored-by: Sofie Van Landeghem * Update spacy/cli/apply.py Co-authored-by: Sofie Van Landeghem * store user data and rename cmd arg * include test for user attr * rename cmd arg * better help message * documentation * prettier * black * link fix * Update spacy/cli/apply.py Co-authored-by: Paul O'Leary McCann * Update website/docs/api/cli.md Co-authored-by: Paul O'Leary McCann * Update website/docs/api/cli.md Co-authored-by: Paul O'Leary McCann * Update website/docs/api/cli.md Co-authored-by: Paul O'Leary McCann * addressing reviews * dont quit but warn * prettier Co-authored-by: Adriane Boyd Co-authored-by: Sofie Van Landeghem Co-authored-by: Paul O'Leary McCann --- spacy/cli/__init__.py | 1 + spacy/cli/_util.py | 23 +++++++ spacy/cli/apply.py | 143 ++++++++++++++++++++++++++++++++++++++++ spacy/cli/convert.py | 31 +-------- spacy/tests/test_cli.py | 78 ++++++++++++++++++++++ website/docs/api/cli.md | 35 +++++++++- 6 files changed, 280 insertions(+), 31 deletions(-) create mode 100644 spacy/cli/apply.py diff --git a/spacy/cli/__init__.py b/spacy/cli/__init__.py index aab2c8d12..aabd1cfef 100644 --- a/spacy/cli/__init__.py +++ b/spacy/cli/__init__.py @@ -16,6 +16,7 @@ from .debug_config import debug_config # noqa: F401 from .debug_model import debug_model # noqa: F401 from .debug_diff import debug_diff # noqa: F401 from .evaluate import evaluate # noqa: F401 +from .apply import apply # noqa: F401 from .convert import convert # noqa: F401 from .init_pipeline import init_pipeline_cli # noqa: F401 from .init_config import init_config, fill_config # noqa: F401 diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py index 9b97a9f19..c46abffe5 100644 --- a/spacy/cli/_util.py +++ b/spacy/cli/_util.py @@ -582,6 +582,29 @@ def setup_gpu(use_gpu: int, silent=None) -> None: local_msg.info("To switch to GPU 0, use the option: --gpu-id 0") +def walk_directory(path: Path, suffix: Optional[str] = None) -> List[Path]: + if not path.is_dir(): + return [path] + paths = [path] + locs = [] + seen = set() + for path in paths: + if str(path) in seen: + continue + seen.add(str(path)) + if path.parts[-1].startswith("."): + continue + elif path.is_dir(): + paths.extend(path.iterdir()) + elif suffix is not None and not path.parts[-1].endswith(suffix): + continue + else: + locs.append(path) + # It's good to sort these, in case the ordering messes up cache. + locs.sort() + return locs + + def _format_number(number: Union[int, float], ndigits: int = 2) -> str: """Formats a number (float or int) rounding to `ndigits`, without truncating trailing 0s, as happens with `round(number, ndigits)`""" diff --git a/spacy/cli/apply.py b/spacy/cli/apply.py new file mode 100644 index 000000000..9d170bc95 --- /dev/null +++ b/spacy/cli/apply.py @@ -0,0 +1,143 @@ +import tqdm +import srsly + +from itertools import chain +from pathlib import Path +from typing import Optional, List, Iterable, cast, Union + +from wasabi import msg + +from ._util import app, Arg, Opt, setup_gpu, import_code, walk_directory + +from ..tokens import Doc, DocBin +from ..vocab import Vocab +from ..util import ensure_path, load_model + + +path_help = """Location of the documents to predict on. +Can be a single file in .spacy format or a .jsonl file. +Files with other extensions are treated as single plain text documents. +If a directory is provided it is traversed recursively to grab +all files to be processed. +The files can be a mixture of .spacy, .jsonl and text files. +If .jsonl is provided the specified field is going +to be grabbed ("text" by default).""" + +out_help = "Path to save the resulting .spacy file" +code_help = ( + "Path to Python file with additional " "code (registered functions) to be imported" +) +gold_help = "Use gold preprocessing provided in the .spacy files" +force_msg = ( + "The provided output file already exists. " + "To force overwriting the output file, set the --force or -F flag." +) + + +DocOrStrStream = Union[Iterable[str], Iterable[Doc]] + + +def _stream_docbin(path: Path, vocab: Vocab) -> Iterable[Doc]: + """ + Stream Doc objects from DocBin. + """ + docbin = DocBin().from_disk(path) + for doc in docbin.get_docs(vocab): + yield doc + + +def _stream_jsonl(path: Path, field: str) -> Iterable[str]: + """ + Stream "text" field from JSONL. If the field "text" is + not found it raises error. + """ + for entry in srsly.read_jsonl(path): + if field not in entry: + msg.fail( + f"{path} does not contain the required '{field}' field.", exits=1 + ) + else: + yield entry[field] + + +def _stream_texts(paths: Iterable[Path]) -> Iterable[str]: + """ + Yields strings from text files in paths. + """ + for path in paths: + with open(path, "r") as fin: + text = fin.read() + yield text + + +@app.command("apply") +def apply_cli( + # fmt: off + model: str = Arg(..., help="Model name or path"), + data_path: Path = Arg(..., help=path_help, exists=True), + output_file: Path = Arg(..., help=out_help, dir_okay=False), + code_path: Optional[Path] = Opt(None, "--code", "-c", help=code_help), + text_key: str = Opt("text", "--text-key", "-tk", help="Key containing text string for JSONL"), + force_overwrite: bool = Opt(False, "--force", "-F", help="Force overwriting the output file"), + use_gpu: int = Opt(-1, "--gpu-id", "-g", help="GPU ID or -1 for CPU."), + batch_size: int = Opt(1, "--batch-size", "-b", help="Batch size."), + n_process: int = Opt(1, "--n-process", "-n", help="number of processors to use.") +): + """ + Apply a trained pipeline to documents to get predictions. + Expects a loadable spaCy pipeline and path to the data, which + can be a directory or a file. + The data files can be provided in multiple formats: + 1. .spacy files + 2. .jsonl files with a specified "field" to read the text from. + 3. Files with any other extension are assumed to be containing + a single document. + DOCS: https://spacy.io/api/cli#apply + """ + data_path = ensure_path(data_path) + output_file = ensure_path(output_file) + code_path = ensure_path(code_path) + if output_file.exists() and not force_overwrite: + msg.fail(force_msg, exits=1) + if not data_path.exists(): + msg.fail(f"Couldn't find data path: {data_path}", exits=1) + import_code(code_path) + setup_gpu(use_gpu) + apply(data_path, output_file, model, text_key, batch_size, n_process) + + +def apply( + data_path: Path, + output_file: Path, + model: str, + json_field: str, + batch_size: int, + n_process: int, +): + docbin = DocBin(store_user_data=True) + paths = walk_directory(data_path) + if len(paths) == 0: + docbin.to_disk(output_file) + msg.warn("Did not find data to process," + f" {data_path} seems to be an empty directory.") + return + nlp = load_model(model) + msg.good(f"Loaded model {model}") + vocab = nlp.vocab + streams: List[DocOrStrStream] = [] + text_files = [] + for path in paths: + if path.suffix == ".spacy": + streams.append(_stream_docbin(path, vocab)) + elif path.suffix == ".jsonl": + streams.append(_stream_jsonl(path, json_field)) + else: + text_files.append(path) + if len(text_files) > 0: + streams.append(_stream_texts(text_files)) + datagen = cast(DocOrStrStream, chain(*streams)) + for doc in tqdm.tqdm(nlp.pipe(datagen, batch_size=batch_size, n_process=n_process)): + docbin.add(doc) + if output_file.suffix == "": + output_file = output_file.with_suffix(".spacy") + docbin.to_disk(output_file) diff --git a/spacy/cli/convert.py b/spacy/cli/convert.py index 04eb7078f..7f365ae2c 100644 --- a/spacy/cli/convert.py +++ b/spacy/cli/convert.py @@ -1,4 +1,4 @@ -from typing import Callable, Iterable, Mapping, Optional, Any, List, Union +from typing import Callable, Iterable, Mapping, Optional, Any, Union from enum import Enum from pathlib import Path from wasabi import Printer @@ -7,7 +7,7 @@ import re import sys import itertools -from ._util import app, Arg, Opt +from ._util import app, Arg, Opt, walk_directory from ..training import docs_to_json from ..tokens import Doc, DocBin from ..training.converters import iob_to_docs, conll_ner_to_docs, json_to_docs @@ -189,33 +189,6 @@ def autodetect_ner_format(input_data: str) -> Optional[str]: return None -def walk_directory(path: Path, converter: str) -> List[Path]: - if not path.is_dir(): - return [path] - paths = [path] - locs = [] - seen = set() - for path in paths: - if str(path) in seen: - continue - seen.add(str(path)) - if path.parts[-1].startswith("."): - continue - elif path.is_dir(): - paths.extend(path.iterdir()) - elif converter == "json" and not path.parts[-1].endswith("json"): - continue - elif converter == "conll" and not path.parts[-1].endswith("conll"): - continue - elif converter == "iob" and not path.parts[-1].endswith("iob"): - continue - else: - locs.append(path) - # It's good to sort these, in case the ordering messes up cache. - locs.sort() - return locs - - def verify_cli_args( msg: Printer, input_path: Path, diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py index 42af08749..c6768a3fd 100644 --- a/spacy/tests/test_cli.py +++ b/spacy/tests/test_cli.py @@ -5,6 +5,7 @@ from typing import Tuple, List, Dict, Any import pkg_resources import time +import spacy import numpy import pytest import srsly @@ -32,6 +33,7 @@ from spacy.cli.package import _is_permitted_package_name from spacy.cli.project.remote_storage import RemoteStorage from spacy.cli.project.run import _check_requirements from spacy.cli.validate import get_model_pkgs +from spacy.cli.apply import apply from spacy.cli.find_threshold import find_threshold from spacy.lang.en import English from spacy.lang.nl import Dutch @@ -885,6 +887,82 @@ def test_span_length_freq_dist_output_must_be_correct(): assert list(span_freqs.keys()) == [3, 1, 4, 5, 2] +def test_applycli_empty_dir(): + with make_tempdir() as data_path: + output = data_path / "test.spacy" + apply(data_path, output, "blank:en", "text", 1, 1) + + +def test_applycli_docbin(): + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + nlp = spacy.blank("en") + doc = nlp("testing apply cli.") + # test empty DocBin case + docbin = DocBin() + docbin.to_disk(data_path / "testin.spacy") + apply(data_path, output, "blank:en", "text", 1, 1) + docbin.add(doc) + docbin.to_disk(data_path / "testin.spacy") + apply(data_path, output, "blank:en", "text", 1, 1) + + +def test_applycli_jsonl(): + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + data = [{"field": "Testing apply cli.", "key": 234}] + data2 = [{"field": "234"}] + srsly.write_jsonl(data_path / "test.jsonl", data) + apply(data_path, output, "blank:en", "field", 1, 1) + srsly.write_jsonl(data_path / "test2.jsonl", data2) + apply(data_path, output, "blank:en", "field", 1, 1) + + +def test_applycli_txt(): + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + with open(data_path / "test.foo", "w") as ftest: + ftest.write("Testing apply cli.") + apply(data_path, output, "blank:en", "text", 1, 1) + + +def test_applycli_mixed(): + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + text = "Testing apply cli" + nlp = spacy.blank("en") + doc = nlp(text) + jsonl_data = [{"text": text}] + srsly.write_jsonl(data_path / "test.jsonl", jsonl_data) + docbin = DocBin() + docbin.add(doc) + docbin.to_disk(data_path / "testin.spacy") + with open(data_path / "test.txt", "w") as ftest: + ftest.write(text) + apply(data_path, output, "blank:en", "text", 1, 1) + # Check whether it worked + result = list(DocBin().from_disk(output).get_docs(nlp.vocab)) + assert len(result) == 3 + for doc in result: + assert doc.text == text + + +def test_applycli_user_data(): + Doc.set_extension("ext", default=0) + val = ("ext", 0) + with make_tempdir() as data_path: + output = data_path / "testout.spacy" + nlp = spacy.blank("en") + doc = nlp("testing apply cli.") + doc._.ext = val + docbin = DocBin(store_user_data=True) + docbin.add(doc) + docbin.to_disk(data_path / "testin.spacy") + apply(data_path, output, "blank:en", "", 1, 1) + result = list(DocBin().from_disk(output).get_docs(nlp.vocab)) + assert result[0]._.ext == val + + def test_local_remote_storage(): with make_tempdir() as d: filename = "a.txt" diff --git a/website/docs/api/cli.md b/website/docs/api/cli.md index 8823a3bd8..275e37ee0 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.md @@ -12,6 +12,7 @@ menu: - ['train', 'train'] - ['pretrain', 'pretrain'] - ['evaluate', 'evaluate'] + - ['apply', 'apply'] - ['find-threshold', 'find-threshold'] - ['assemble', 'assemble'] - ['package', 'package'] @@ -474,7 +475,7 @@ report span characteristics such as the average span length and the span (or span boundary) distinctiveness. The distinctiveness measure shows how different the tokens are with respect to the rest of the corpus using the KL-divergence of the token distributions. To learn more, you can check out Papay et al.'s work on -[*Dissecting Span Identification Tasks with Performance Prediction* (EMNLP 2020)](https://aclanthology.org/2020.emnlp-main.396/). +[_Dissecting Span Identification Tasks with Performance Prediction_ (EMNLP 2020)](https://aclanthology.org/2020.emnlp-main.396/). @@ -1162,6 +1163,37 @@ $ python -m spacy evaluate [model] [data_path] [--output] [--code] [--gold-prepr | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | Training results and optional metrics and visualizations. | +## apply {#apply new="3.5" tag="command"} + +Applies a trained pipeline to data and stores the resulting annotated documents +in a `DocBin`. The input can be a single file or a directory. The recognized +input formats are: + +1. `.spacy` +2. `.jsonl` containing a user specified `text_key` +3. Files with any other extension are assumed to be plain text files containing + a single document. + +When a directory is provided it is traversed recursively to collect all files. + +```cli +$ python -m spacy apply [model] [data-path] [output-file] [--code] [--text-key] [--force-overwrite] [--gpu-id] [--batch-size] [--n-process] +``` + +| Name | Description | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `model` | Pipeline to apply to the data. Can be a package or a path to a data directory. ~~str (positional)~~ | +| `data_path` | Location of data to be evaluated in spaCy's [binary format](/api/data-formats#training), jsonl, or plain text. ~~Path (positional)~~ | +| `output-file`, `-o` | Output `DocBin` path. ~~str (positional)~~ | +| `--code`, `-c` 3 | Path to Python file with additional code to be imported. Allows [registering custom functions](/usage/training#custom-functions) for new architectures. ~~Optional[Path] \(option)~~ | +| `--text-key`, `-tk` | The key for `.jsonl` files to use to grab the texts from. Defaults to `text`. ~~Optional[str] \(option)~~ | +| `--force-overwrite`, `-F` | If the provided `output-file` already exists, then force `apply` to overwrite it. If this is `False` (default) then quits with a warning instead. ~~bool (flag)~~ | +| `--gpu-id`, `-g` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ | +| `--batch-size`, `-b` | Batch size to use for prediction. Defaults to `1`. ~~int (option)~~ | +| `--n-process`, `-n` | Number of processes to use for prediction. Defaults to `1`. ~~int (option)~~ | +| `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | +| **CREATES** | A `DocBin` with the annotations from the `model` for all the files found in `data-path`. | + ## find-threshold {#find-threshold new="3.5" tag="command"} Runs prediction trials for a trained model with varying tresholds to maximize @@ -1187,7 +1219,6 @@ be provided. > $ python -m spacy find-threshold my_nlp data.spacy spancat threshold spans_sc_f > ``` - | Name | Description | | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `model` | Pipeline to evaluate. Can be a package or a path to a data directory. ~~str (positional)~~ | From eef3d950b4266ab9545143de8070456ce7967950 Mon Sep 17 00:00:00 2001 From: Raphael Mitsch Date: Wed, 21 Dec 2022 18:54:27 +0100 Subject: [PATCH 148/179] Fix `SpanGroup` and `Span` typing (#12009) * Correct Span.label, Span.kb_id types. Fix SpanGroup.__iter__(). * Extend test. * Rename test. Fix typo. * Add comment. * Fix types for Span.label, Span.kb_id, Span.char_span(). * Update spacy/tests/doc/test_span_group.py Co-authored-by: Adriane Boyd * Update docs. * Fix typo. * Update spacy/tokens/span_group.pyx Co-authored-by: Adriane Boyd Co-authored-by: Adriane Boyd --- spacy/tests/doc/test_span_group.py | 15 ++++++++++++++- spacy/tokens/span.pyi | 4 ++-- spacy/tokens/span_group.pyi | 1 + spacy/tokens/span_group.pyx | 10 ++++++++++ website/docs/api/spangroup.md | 17 +++++++++++++++++ 5 files changed, 44 insertions(+), 3 deletions(-) diff --git a/spacy/tests/doc/test_span_group.py b/spacy/tests/doc/test_span_group.py index 8c70a83e1..818569c64 100644 --- a/spacy/tests/doc/test_span_group.py +++ b/spacy/tests/doc/test_span_group.py @@ -1,7 +1,10 @@ +from typing import List + import pytest from random import Random from spacy.matcher import Matcher -from spacy.tokens import Span, SpanGroup +from spacy.tokens import Span, SpanGroup, Doc +from spacy.util import filter_spans @pytest.fixture @@ -240,3 +243,13 @@ def test_span_group_extend(doc): def test_span_group_dealloc(span_group): with pytest.raises(AttributeError): print(span_group.doc) + + +@pytest.mark.issue(11975) +def test_span_group_typing(doc: Doc): + """Tests whether typing of `SpanGroup` as `Iterable[Span]`-like object is accepted by mypy.""" + span_group: SpanGroup = doc.spans["SPANS"] + spans: List[Span] = list(span_group) + for i, span in enumerate(span_group): + assert span == span_group[i] == spans[i] + filter_spans(span_group) diff --git a/spacy/tokens/span.pyi b/spacy/tokens/span.pyi index 0a6f306a6..9986a90e6 100644 --- a/spacy/tokens/span.pyi +++ b/spacy/tokens/span.pyi @@ -95,8 +95,8 @@ class Span: self, start_idx: int, end_idx: int, - label: int = ..., - kb_id: int = ..., + label: Union[int, str] = ..., + kb_id: Union[int, str] = ..., vector: Optional[Floats1d] = ..., ) -> Span: ... @property diff --git a/spacy/tokens/span_group.pyi b/spacy/tokens/span_group.pyi index 21cd124ab..0b4aa83aa 100644 --- a/spacy/tokens/span_group.pyi +++ b/spacy/tokens/span_group.pyi @@ -18,6 +18,7 @@ class SpanGroup: def doc(self) -> Doc: ... @property def has_overlap(self) -> bool: ... + def __iter__(self): ... def __len__(self) -> int: ... def append(self, span: Span) -> None: ... def extend(self, spans: Iterable[Span]) -> None: ... diff --git a/spacy/tokens/span_group.pyx b/spacy/tokens/span_group.pyx index 1aa3c0bc8..608dda283 100644 --- a/spacy/tokens/span_group.pyx +++ b/spacy/tokens/span_group.pyx @@ -158,6 +158,16 @@ cdef class SpanGroup: return self._concat(other) return NotImplemented + def __iter__(self): + """ + Iterate over the spans in this SpanGroup. + YIELDS (Span): A span in this SpanGroup. + + DOCS: https://spacy.io/api/spangroup#iter + """ + for i in range(self.c.size()): + yield self[i] + def append(self, Span span): """Add a span to the group. The span must refer to the same Doc object as the span group. diff --git a/website/docs/api/spangroup.md b/website/docs/api/spangroup.md index 2d1cf73c4..bd9659acb 100644 --- a/website/docs/api/spangroup.md +++ b/website/docs/api/spangroup.md @@ -202,6 +202,23 @@ already present in the current span group. | `other` | The span group or spans to append. ~~Union[SpanGroup, Iterable[Span]]~~ | | **RETURNS** | The span group. ~~SpanGroup~~ | +## SpanGroup.\_\_iter\_\_ {#iter tag="method" new="3.5"} + +Iterate over the spans in this span group. + +> #### Example +> +> ```python +> doc = nlp("Their goi ng home") +> doc.spans["errors"] = [doc[0:1], doc[1:3]] +> for error_span in doc.spans["errors"]: +> print(error_span) +> ``` + +| Name | Description | +| ---------- | ----------------------------------- | +| **YIELDS** | A span in this span group. ~~Span~~ | + ## SpanGroup.append {#append tag="method"} Add a [`Span`](/api/span) object to the group. The span must refer to the same From 64d2d27c5dbf8e5657187975d2c9627f30e108a2 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Thu, 22 Dec 2022 10:53:16 +0100 Subject: [PATCH 149/179] Add classifier for python 3.11 (#12013) --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index cf6e6f84b..d290d706c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Topic :: Scientific/Engineering project_urls = Release notes = https://github.com/explosion/spaCy/releases From 90896504a5dba1babac04a2b88662179409ae006 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 23 Dec 2022 12:44:07 +0100 Subject: [PATCH 150/179] Auto-format code with black (#12019) Co-authored-by: explosion-bot --- spacy/cli/apply.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spacy/cli/apply.py b/spacy/cli/apply.py index 9d170bc95..f0df4e757 100644 --- a/spacy/cli/apply.py +++ b/spacy/cli/apply.py @@ -53,9 +53,7 @@ def _stream_jsonl(path: Path, field: str) -> Iterable[str]: """ for entry in srsly.read_jsonl(path): if field not in entry: - msg.fail( - f"{path} does not contain the required '{field}' field.", exits=1 - ) + msg.fail(f"{path} does not contain the required '{field}' field.", exits=1) else: yield entry[field] @@ -118,8 +116,10 @@ def apply( paths = walk_directory(data_path) if len(paths) == 0: docbin.to_disk(output_file) - msg.warn("Did not find data to process," - f" {data_path} seems to be an empty directory.") + msg.warn( + "Did not find data to process," + f" {data_path} seems to be an empty directory." + ) return nlp = load_model(model) msg.good(f"Loaded model {model}") From aa2b471a6e289d1c1bb51558df779ae028671225 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Fri, 23 Dec 2022 15:21:44 +0100 Subject: [PATCH 151/179] New console logger with expanded progress tracking (#11972) * Add `ConsoleLogger.v3` This addition expands the progress bar feature to count up the training/distillation steps to either the next evaluation pass or the maximum number of steps. * Rename progress bar types * Add defaults to docs Minor fixes * Move comment * Minor punctuation fixes * Explicitly check for `None` when validating progress bar type Co-authored-by: Paul O'Leary McCann --- spacy/errors.py | 1 + spacy/training/loggers.py | 48 ++++++++++++++++++++++++++++++++--- website/docs/api/top-level.md | 34 ++++++++++++++++++++----- 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/spacy/errors.py b/spacy/errors.py index 0e5ef91ed..cd9281e91 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -962,6 +962,7 @@ class Errors(metaclass=ErrorsWithCodes): E1046 = ("{cls_name} is an abstract class and cannot be instantiated. If you are looking for spaCy's default " "knowledge base, use `InMemoryLookupKB`.") E1047 = ("`find_threshold()` only supports components with a `scorer` attribute.") + E1048 = ("Got '{unexpected}' as console progress bar type, but expected one of the following: {expected}") # Deprecated model shortcuts, only used in errors and warnings diff --git a/spacy/training/loggers.py b/spacy/training/loggers.py index 408ea7140..7de31822e 100644 --- a/spacy/training/loggers.py +++ b/spacy/training/loggers.py @@ -26,6 +26,8 @@ def setup_table( return final_cols, final_widths, ["r" for _ in final_widths] +# We cannot rename this method as it's directly imported +# and used by external packages such as spacy-loggers. @registry.loggers("spacy.ConsoleLogger.v2") def console_logger( progress_bar: bool = False, @@ -33,7 +35,27 @@ def console_logger( output_file: Optional[Union[str, Path]] = None, ): """The ConsoleLogger.v2 prints out training logs in the console and/or saves them to a jsonl file. - progress_bar (bool): Whether the logger should print the progress bar. + progress_bar (bool): Whether the logger should print a progress bar tracking the steps till the next evaluation pass. + console_output (bool): Whether the logger should print the logs on the console. + output_file (Optional[Union[str, Path]]): The file to save the training logs to. + """ + return console_logger_v3( + progress_bar=None if progress_bar is False else "eval", + console_output=console_output, + output_file=output_file, + ) + + +@registry.loggers("spacy.ConsoleLogger.v3") +def console_logger_v3( + progress_bar: Optional[str] = None, + console_output: bool = True, + output_file: Optional[Union[str, Path]] = None, +): + """The ConsoleLogger.v3 prints out training logs in the console and/or saves them to a jsonl file. + progress_bar (Optional[str]): Type of progress bar to show in the console. Allowed values: + train - Tracks the number of steps from the beginning of training until the full training run is complete (training.max_steps is reached). + eval - Tracks the number of steps between the previous and next evaluation (training.eval_frequency is reached). console_output (bool): Whether the logger should print the logs on the console. output_file (Optional[Union[str, Path]]): The file to save the training logs to. """ @@ -70,6 +92,7 @@ def console_logger( for name, proc in nlp.pipeline if hasattr(proc, "is_trainable") and proc.is_trainable ] + max_steps = nlp.config["training"]["max_steps"] eval_frequency = nlp.config["training"]["eval_frequency"] score_weights = nlp.config["training"]["score_weights"] score_cols = [col for col, value in score_weights.items() if value is not None] @@ -84,6 +107,13 @@ def console_logger( write(msg.row(table_header, widths=table_widths, spacing=spacing)) write(msg.row(["-" * width for width in table_widths], spacing=spacing)) progress = None + expected_progress_types = ("train", "eval") + if progress_bar is not None and progress_bar not in expected_progress_types: + raise ValueError( + Errors.E1048.format( + unexpected=progress_bar, expected=expected_progress_types + ) + ) def log_step(info: Optional[Dict[str, Any]]) -> None: nonlocal progress @@ -141,11 +171,23 @@ def console_logger( ) ) if progress_bar: + if progress_bar == "train": + total = max_steps + desc = f"Last Eval Epoch: {info['epoch']}" + initial = info["step"] + else: + total = eval_frequency + desc = f"Epoch {info['epoch']+1}" + initial = 0 # Set disable=None, so that it disables on non-TTY progress = tqdm.tqdm( - total=eval_frequency, disable=None, leave=False, file=stderr + total=total, + disable=None, + leave=False, + file=stderr, + initial=initial, ) - progress.set_description(f"Epoch {info['epoch']+1}") + progress.set_description(desc) def finalize() -> None: if output_stream: diff --git a/website/docs/api/top-level.md b/website/docs/api/top-level.md index 26a5d42f4..883c5e3b9 100644 --- a/website/docs/api/top-level.md +++ b/website/docs/api/top-level.md @@ -513,7 +513,7 @@ a [Weights & Biases](https://www.wandb.com/) dashboard. Instead of using one of the built-in loggers, you can [implement your own](/usage/training#custom-logging). -#### spacy.ConsoleLogger.v2 {#ConsoleLogger tag="registered function"} +#### spacy.ConsoleLogger.v2 {tag="registered function"} > #### Example config > @@ -564,11 +564,33 @@ start decreasing across epochs. -| Name | Description | -| ---------------- | --------------------------------------------------------------------- | -| `progress_bar` | Whether the logger should print the progress bar ~~bool~~ | -| `console_output` | Whether the logger should print the logs on the console. ~~bool~~ | -| `output_file` | The file to save the training logs to. ~~Optional[Union[str, Path]]~~ | +| Name | Description | +| ---------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| `progress_bar` | Whether the logger should print a progress bar tracking the steps till the next evaluation pass (default: `False`). ~~bool~~ | +| `console_output` | Whether the logger should print the logs in the console (default: `True`). ~~bool~~ | +| `output_file` | The file to save the training logs to (default: `None`). ~~Optional[Union[str, Path]]~~ | + +#### spacy.ConsoleLogger.v3 {#ConsoleLogger tag="registered function"} + +> #### Example config +> +> ```ini +> [training.logger] +> @loggers = "spacy.ConsoleLogger.v3" +> progress_bar = "all_steps" +> console_output = true +> output_file = "training_log.jsonl" +> ``` + +Writes the results of a training step to the console in a tabular format and +optionally saves them to a `jsonl` file. + +| Name | Description | +| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `progress_bar` | Type of progress bar to show in the console: `"train"`, `"eval"` or `None`. | +| | The bar tracks the number of steps until `training.max_steps` and `training.eval_frequency` are reached respectively (default: `None`). ~~Optional[str]~~ | +| `console_output` | Whether the logger should print the logs in the console (default: `True`). ~~bool~~ | +| `output_file` | The file to save the training logs to (default: `None`). ~~Optional[Union[str, Path]]~~ | ## Readers {#readers} From 933b54ac798a7d64f9cde4d85b55556e84e44bd6 Mon Sep 17 00:00:00 2001 From: kadarakos Date: Mon, 26 Dec 2022 13:26:35 +0100 Subject: [PATCH 152/179] typo fix (#11995) --- spacy/pipeline/span_ruler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/pipeline/span_ruler.py b/spacy/pipeline/span_ruler.py index 807a4ffe5..0e7e9ebf7 100644 --- a/spacy/pipeline/span_ruler.py +++ b/spacy/pipeline/span_ruler.py @@ -170,7 +170,7 @@ def prioritize_existing_ents_filter( @registry.misc("spacy.prioritize_existing_ents_filter.v1") -def make_preverse_existing_ents_filter(): +def make_preserve_existing_ents_filter(): return prioritize_existing_ents_filter From ef9e504eacc806162666c964bd00d152fc15f9e3 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Thu, 29 Dec 2022 14:01:08 +0100 Subject: [PATCH 153/179] Rename modified textcat scorer to v2 (#11971) As a follow-up to #11696, rename the modified scorer to v2 and move the v1 scorer to `spacy-legacy`. --- requirements.txt | 2 +- setup.cfg | 2 +- spacy/pipeline/textcat.py | 4 ++-- spacy/tests/pipeline/test_textcat.py | 17 +++++++++++++++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0440835f2..5bc1c8684 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # Our libraries -spacy-legacy>=3.0.10,<3.1.0 +spacy-legacy>=3.0.11,<3.1.0 spacy-loggers>=1.0.0,<2.0.0 cymem>=2.0.2,<2.1.0 preshed>=3.0.2,<3.1.0 diff --git a/setup.cfg b/setup.cfg index d290d706c..cee8c0c33 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ setup_requires = thinc>=8.1.0,<8.2.0 install_requires = # Our libraries - spacy-legacy>=3.0.10,<3.1.0 + spacy-legacy>=3.0.11,<3.1.0 spacy-loggers>=1.0.0,<2.0.0 murmurhash>=0.28.0,<1.1.0 cymem>=2.0.2,<2.1.0 diff --git a/spacy/pipeline/textcat.py b/spacy/pipeline/textcat.py index 65121114d..650a01949 100644 --- a/spacy/pipeline/textcat.py +++ b/spacy/pipeline/textcat.py @@ -74,7 +74,7 @@ subword_features = true default_config={ "threshold": 0.0, "model": DEFAULT_SINGLE_TEXTCAT_MODEL, - "scorer": {"@scorers": "spacy.textcat_scorer.v1"}, + "scorer": {"@scorers": "spacy.textcat_scorer.v2"}, }, default_score_weights={ "cats_score": 1.0, @@ -117,7 +117,7 @@ def textcat_score(examples: Iterable[Example], **kwargs) -> Dict[str, Any]: ) -@registry.scorers("spacy.textcat_scorer.v1") +@registry.scorers("spacy.textcat_scorer.v2") def make_textcat_scorer(): return textcat_score diff --git a/spacy/tests/pipeline/test_textcat.py b/spacy/tests/pipeline/test_textcat.py index 155ce99a2..eafe4c128 100644 --- a/spacy/tests/pipeline/test_textcat.py +++ b/spacy/tests/pipeline/test_textcat.py @@ -895,3 +895,20 @@ def test_textcat_multi_threshold(): scores = nlp.evaluate(train_examples, scorer_cfg={"threshold": 0}) assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 1.0 + + +@pytest.mark.parametrize("component_name,scorer", [("textcat", "spacy.textcat_scorer.v1")]) +def test_textcat_legacy_scorers(component_name, scorer): + """Check that legacy scorers are registered and produce the expected score + keys.""" + nlp = English() + nlp.add_pipe(component_name, config={"scorer": {"@scorers": scorer}}) + + train_examples = [] + for text, annotations in TRAIN_DATA_SINGLE_LABEL: + train_examples.append(Example.from_dict(nlp.make_doc(text), annotations)) + nlp.initialize(get_examples=lambda: train_examples) + + # score the model (it's not actually trained but that doesn't matter) + scores = nlp.evaluate(train_examples) + assert 0 <= scores["cats_score"] <= 1 From abb0ab109d33d2deaa6155a61fad649a25472f9c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 2 Jan 2023 11:59:57 +0100 Subject: [PATCH 154/179] Auto-format code with black (#12035) Co-authored-by: explosion-bot --- spacy/tests/pipeline/test_textcat.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spacy/tests/pipeline/test_textcat.py b/spacy/tests/pipeline/test_textcat.py index eafe4c128..048586cec 100644 --- a/spacy/tests/pipeline/test_textcat.py +++ b/spacy/tests/pipeline/test_textcat.py @@ -897,7 +897,9 @@ def test_textcat_multi_threshold(): assert scores["cats_f_per_type"]["POSITIVE"]["r"] == 1.0 -@pytest.mark.parametrize("component_name,scorer", [("textcat", "spacy.textcat_scorer.v1")]) +@pytest.mark.parametrize( + "component_name,scorer", [("textcat", "spacy.textcat_scorer.v1")] +) def test_textcat_legacy_scorers(component_name, scorer): """Check that legacy scorers are registered and produce the expected score keys.""" From 31c1beba787446059de58a1478e6aec197fd0bbb Mon Sep 17 00:00:00 2001 From: Wannaphong Phatthiyaphaibun Date: Tue, 3 Jan 2023 15:03:59 +0700 Subject: [PATCH 155/179] Add spacy-pythainlp (#12038) * Add spacy-pythainlp * Move submission to right section * Minor cleanup * Remove extra list call * Update universe.json Co-authored-by: Paul O'Leary McCann --- website/meta/universe.json | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/website/meta/universe.json b/website/meta/universe.json index db533c3b2..99d121507 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -4062,6 +4062,33 @@ "author_links": { "github": "yasufumy" } + }, + { + "id": "spacy-pythainlp", + "title": "spaCy-PyThaiNLP", + "slogan": "PyThaiNLP for spaCy", + "description": "This package wraps the PyThaiNLP library to add support for Thai to spaCy.", + "github": "PyThaiNLP/spaCy-PyThaiNLP", + "code_example": [ + "import spacy", + "import spacy_pythainlp.core", + "", + "nlp = spacy.blank('th')", + "nlp.add_pipe('pythainlp')", + "doc = nlp('ผมเป็นคนไทย แต่มะลิอยากไปโรงเรียนส่วนผมจะไปไหน ผมอยากไปเที่ยว')", + "", + "print(list(doc.sents))", + "# output: [ผมเป็นคนไทย แต่มะลิอยากไปโรงเรียนส่วนผมจะไปไหน , ผมอยากไปเที่ยว]" + ], + "code_language": "python", + "author": "Wannaphong Phatthiyaphaibun", + "author_links": { + "twitter": "@wannaphong_p", + "github": "wannaphong", + "website": "https://iam.wannaphong.com/" + }, + "category": ["pipeline", "research"], + "tags": ["Thai"] } ], From dbd829f0ed2dba3eb6eb5b59b18396ed38e326b9 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Wed, 4 Jan 2023 12:51:40 +0900 Subject: [PATCH 156/179] Fix inconsistency in displaCy docs about page option (#12047) * Fix inconsistency in displaCy docs about page option The `page` option, which wraps the output SVG in HTML, is true by default for `serve` but not for `render`. The `render` docs were wrong though, so this updates them. * Update the same statement in more docs A few renderers used the same language --- spacy/displacy/__init__.py | 2 +- spacy/displacy/render.py | 4 ++-- website/docs/api/top-level.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spacy/displacy/__init__.py b/spacy/displacy/__init__.py index bc32001d7..2f2058b8e 100644 --- a/spacy/displacy/__init__.py +++ b/spacy/displacy/__init__.py @@ -36,7 +36,7 @@ def render( jupyter (bool): Override Jupyter auto-detection. options (dict): Visualiser-specific options, e.g. colors. manual (bool): Don't parse `Doc` and instead expect a dict/list of dicts. - RETURNS (str): Rendered HTML markup. + RETURNS (str): Rendered SVG or HTML markup. DOCS: https://spacy.io/api/top-level#displacy.render USAGE: https://spacy.io/usage/visualizers diff --git a/spacy/displacy/render.py b/spacy/displacy/render.py index 50dc3466c..f74222dc2 100644 --- a/spacy/displacy/render.py +++ b/spacy/displacy/render.py @@ -94,7 +94,7 @@ class SpanRenderer: parsed (list): Dependency parses to render. page (bool): Render parses wrapped as full HTML page. minify (bool): Minify HTML markup. - RETURNS (str): Rendered HTML markup. + RETURNS (str): Rendered SVG or HTML markup. """ rendered = [] for i, p in enumerate(parsed): @@ -510,7 +510,7 @@ class EntityRenderer: parsed (list): Dependency parses to render. page (bool): Render parses wrapped as full HTML page. minify (bool): Minify HTML markup. - RETURNS (str): Rendered HTML markup. + RETURNS (str): Rendered SVG or HTML markup. """ rendered = [] for i, p in enumerate(parsed): diff --git a/website/docs/api/top-level.md b/website/docs/api/top-level.md index 883c5e3b9..6a63e07da 100644 --- a/website/docs/api/top-level.md +++ b/website/docs/api/top-level.md @@ -266,7 +266,7 @@ Render a dependency parse tree or named entity visualization. | ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span, dict]], Doc, Span, dict]~~ | | `style` | Visualization style, `"dep"`, `"ent"` or `"span"` 3.3. Defaults to `"dep"`. ~~str~~ | -| `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ | +| `page` | Render markup as full HTML page. Defaults to `False`. ~~bool~~ | | `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ | | `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ | | `manual` | Don't parse `Doc` and instead expect a dict or list of dicts. [See here](/usage/visualizers#manual-usage) for formats and examples. Defaults to `False`. ~~bool~~ | From 7f6c638c3acd732c0b52a45a2b3ad0388cd1ae66 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Thu, 5 Jan 2023 10:21:00 +0100 Subject: [PATCH 157/179] fix processing of "auto" in convert (#12050) * fix processing of "auto" in walk_directory * add check for None * move AUTO check to convert and fix verification of args * add specific CLI test with CliRunner * cleanup * more cleanup * update docstring --- spacy/cli/_util.py | 4 ++++ spacy/cli/convert.py | 26 ++++++++++++++++---------- spacy/tests/test_cli.py | 26 +++++++++++++++++++++++++- spacy/tests/test_cli_app.py | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 spacy/tests/test_cli_app.py diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py index c46abffe5..0f4e9f599 100644 --- a/spacy/cli/_util.py +++ b/spacy/cli/_util.py @@ -583,6 +583,10 @@ def setup_gpu(use_gpu: int, silent=None) -> None: def walk_directory(path: Path, suffix: Optional[str] = None) -> List[Path]: + """Given a directory and a suffix, recursively find all files matching the suffix. + Directories or files with names beginning with a . are ignored, but hidden flags on + filesystems are not checked. + When provided with a suffix `None`, there is no suffix-based filtering.""" if not path.is_dir(): return [path] paths = [path] diff --git a/spacy/cli/convert.py b/spacy/cli/convert.py index 7f365ae2c..68d454b3e 100644 --- a/spacy/cli/convert.py +++ b/spacy/cli/convert.py @@ -28,6 +28,8 @@ CONVERTERS: Mapping[str, Callable[..., Iterable[Doc]]] = { "json": json_to_docs, } +AUTO = "auto" + # File types that can be written to stdout FILE_TYPES_STDOUT = ("json",) @@ -49,7 +51,7 @@ def convert_cli( model: Optional[str] = Opt(None, "--model", "--base", "-b", help="Trained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)"), morphology: bool = Opt(False, "--morphology", "-m", help="Enable appending morphology to tags"), merge_subtokens: bool = Opt(False, "--merge-subtokens", "-T", help="Merge CoNLL-U subtokens"), - converter: str = Opt("auto", "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"), + converter: str = Opt(AUTO, "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"), ner_map: Optional[Path] = Opt(None, "--ner-map", "-nm", help="NER tag mapping (as JSON-encoded dict of entity types)", exists=True), lang: Optional[str] = Opt(None, "--lang", "-l", help="Language (if tokenizer required)"), concatenate: bool = Opt(None, "--concatenate", "-C", help="Concatenate output to a single file"), @@ -70,8 +72,8 @@ def convert_cli( output_dir: Union[str, Path] = "-" if output_dir == Path("-") else output_dir silent = output_dir == "-" msg = Printer(no_print=silent) - verify_cli_args(msg, input_path, output_dir, file_type.value, converter, ner_map) converter = _get_converter(msg, converter, input_path) + verify_cli_args(msg, input_path, output_dir, file_type.value, converter, ner_map) convert( input_path, output_dir, @@ -100,7 +102,7 @@ def convert( model: Optional[str] = None, morphology: bool = False, merge_subtokens: bool = False, - converter: str = "auto", + converter: str, ner_map: Optional[Path] = None, lang: Optional[str] = None, concatenate: bool = False, @@ -212,18 +214,22 @@ def verify_cli_args( input_locs = walk_directory(input_path, converter) if len(input_locs) == 0: msg.fail("No input files in directory", input_path, exits=1) - file_types = list(set([loc.suffix[1:] for loc in input_locs])) - if converter == "auto" and len(file_types) >= 2: - file_types_str = ",".join(file_types) - msg.fail("All input files must be same type", file_types_str, exits=1) - if converter != "auto" and converter not in CONVERTERS: + if converter not in CONVERTERS: msg.fail(f"Can't find converter for {converter}", exits=1) def _get_converter(msg, converter, input_path: Path): if input_path.is_dir(): - input_path = walk_directory(input_path, converter)[0] - if converter == "auto": + if converter == AUTO: + input_locs = walk_directory(input_path, suffix=None) + file_types = list(set([loc.suffix[1:] for loc in input_locs])) + if len(file_types) >= 2: + file_types_str = ",".join(file_types) + msg.fail("All input files must be same type", file_types_str, exits=1) + input_path = input_locs[0] + else: + input_path = walk_directory(input_path, suffix=converter)[0] + if converter == AUTO: converter = input_path.suffix[1:] if converter == "ner" or converter == "iob": with input_path.open(encoding="utf8") as file_: diff --git a/spacy/tests/test_cli.py b/spacy/tests/test_cli.py index c6768a3fd..c88e20de2 100644 --- a/spacy/tests/test_cli.py +++ b/spacy/tests/test_cli.py @@ -4,6 +4,7 @@ from collections import Counter from typing import Tuple, List, Dict, Any import pkg_resources import time +from pathlib import Path import spacy import numpy @@ -15,7 +16,7 @@ from thinc.api import Config, ConfigValidationError from spacy import about from spacy.cli import info -from spacy.cli._util import is_subpath_of, load_project_config +from spacy.cli._util import is_subpath_of, load_project_config, walk_directory from spacy.cli._util import parse_config_overrides, string_to_list from spacy.cli._util import substitute_project_variables from spacy.cli._util import validate_project_commands @@ -1185,3 +1186,26 @@ def test_upload_download_local_file(): download_file(remote_file, local_file) with local_file.open(mode="r") as file_: assert file_.read() == content + + +def test_walk_directory(): + with make_tempdir() as d: + files = [ + "data1.iob", + "data2.iob", + "data3.json", + "data4.conll", + "data5.conll", + "data6.conll", + "data7.txt", + ] + + for f in files: + Path(d / f).touch() + + assert (len(walk_directory(d))) == 7 + assert (len(walk_directory(d, suffix=None))) == 7 + assert (len(walk_directory(d, suffix="json"))) == 1 + assert (len(walk_directory(d, suffix="iob"))) == 2 + assert (len(walk_directory(d, suffix="conll"))) == 3 + assert (len(walk_directory(d, suffix="pdf"))) == 0 diff --git a/spacy/tests/test_cli_app.py b/spacy/tests/test_cli_app.py new file mode 100644 index 000000000..873a3ff66 --- /dev/null +++ b/spacy/tests/test_cli_app.py @@ -0,0 +1,33 @@ +import os +from pathlib import Path +from typer.testing import CliRunner + +from spacy.cli._util import app +from .util import make_tempdir + + +def test_convert_auto(): + with make_tempdir() as d_in, make_tempdir() as d_out: + for f in ["data1.iob", "data2.iob", "data3.iob"]: + Path(d_in / f).touch() + + # ensure that "automatic" suffix detection works + result = CliRunner().invoke(app, ["convert", str(d_in), str(d_out)]) + assert "Generated output file" in result.stdout + out_files = os.listdir(d_out) + assert len(out_files) == 3 + assert "data1.spacy" in out_files + assert "data2.spacy" in out_files + assert "data3.spacy" in out_files + + +def test_convert_auto_conflict(): + with make_tempdir() as d_in, make_tempdir() as d_out: + for f in ["data1.iob", "data2.iob", "data3.json"]: + Path(d_in / f).touch() + + # ensure that "automatic" suffix detection warns when there are different file types + result = CliRunner().invoke(app, ["convert", str(d_in), str(d_out)]) + assert "All input files must be same type" in result.stdout + out_files = os.listdir(d_out) + assert len(out_files) == 0 From f1dcdefc8abb21345680b79e9d538f06cf62bca0 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Thu, 5 Jan 2023 11:46:04 +0100 Subject: [PATCH 158/179] Add version tag to `before_update` config key (#12059) --- website/docs/api/data-formats.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/api/data-formats.md b/website/docs/api/data-formats.md index 768844cf3..420e827a0 100644 --- a/website/docs/api/data-formats.md +++ b/website/docs/api/data-formats.md @@ -186,7 +186,7 @@ process that are used when you run [`spacy train`](/api/cli#train). | `accumulate_gradient` | Whether to divide the batch up into substeps. Defaults to `1`. ~~int~~ | | `batcher` | Callable that takes an iterator of [`Doc`](/api/doc) objects and yields batches of `Doc`s. Defaults to [`batch_by_words`](/api/top-level#batch_by_words). ~~Callable[[Iterator[Doc], Iterator[List[Doc]]]]~~ | | `before_to_disk` | Optional callback to modify `nlp` object right before it is saved to disk during and after training. Can be used to remove or reset config values or disable components. Defaults to `null`. ~~Optional[Callable[[Language], Language]]~~ | -| `before_update` | Optional callback that is invoked at the start of each training step with the `nlp` object and a `Dict` containing the following entries: `step`, `epoch`. Can be used to make deferred changes to components. Defaults to `null`. ~~Optional[Callable[[Language, Dict[str, Any]], None]]~~ | +| `before_update` 3.5 | Optional callback that is invoked at the start of each training step with the `nlp` object and a `Dict` containing the following entries: `step`, `epoch`. Can be used to make deferred changes to components. Defaults to `null`. ~~Optional[Callable[[Language, Dict[str, Any]], None]]~~ | | `dev_corpus` | Dot notation of the config location defining the dev corpus. Defaults to `corpora.dev`. ~~str~~ | | `dropout` | The dropout rate. Defaults to `0.1`. ~~float~~ | | `eval_frequency` | How often to evaluate during training (steps). Defaults to `200`. ~~int~~ | From 6d03b04901e95a71747a7e1ef0b00bc87bb2c807 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Mon, 9 Jan 2023 11:43:48 +0100 Subject: [PATCH 159/179] Improve score_cats for use with multiple textcat components (#11820) * add test for running evaluate on an nlp pipeline with two distinct textcat components * cleanup * merge dicts instead of overwrite * don't add more labels to the given set * Revert "merge dicts instead of overwrite" This reverts commit 89bee0ed7798389e6de882a0234e6075fbdaf331. * Switch tests to separate scorer keys rather than merged dicts * Revert unrelated edits * Switch textcat scorers to v2 * formatting Co-authored-by: Adriane Boyd --- spacy/pipeline/textcat_multilabel.py | 4 +- spacy/scorer.py | 6 +- spacy/tests/pipeline/test_textcat.py | 6 +- spacy/tests/test_language.py | 107 +++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 7 deletions(-) diff --git a/spacy/pipeline/textcat_multilabel.py b/spacy/pipeline/textcat_multilabel.py index 328cee723..41c0e2f63 100644 --- a/spacy/pipeline/textcat_multilabel.py +++ b/spacy/pipeline/textcat_multilabel.py @@ -74,7 +74,7 @@ subword_features = true default_config={ "threshold": 0.5, "model": DEFAULT_MULTI_TEXTCAT_MODEL, - "scorer": {"@scorers": "spacy.textcat_multilabel_scorer.v1"}, + "scorer": {"@scorers": "spacy.textcat_multilabel_scorer.v2"}, }, default_score_weights={ "cats_score": 1.0, @@ -120,7 +120,7 @@ def textcat_multilabel_score(examples: Iterable[Example], **kwargs) -> Dict[str, ) -@registry.scorers("spacy.textcat_multilabel_scorer.v1") +@registry.scorers("spacy.textcat_multilabel_scorer.v2") def make_textcat_multilabel_scorer(): return textcat_multilabel_score diff --git a/spacy/scorer.py b/spacy/scorer.py index 16fc303a0..d8c383ab8 100644 --- a/spacy/scorer.py +++ b/spacy/scorer.py @@ -476,14 +476,12 @@ class Scorer: f_per_type = {label: PRFScore() for label in labels} auc_per_type = {label: ROCAUCScore() for label in labels} labels = set(labels) - if labels: - for eg in examples: - labels.update(eg.predicted.cats.keys()) - labels.update(eg.reference.cats.keys()) for example in examples: # Through this loop, None in the gold_cats indicates missing label. pred_cats = getter(example.predicted, attr) + pred_cats = {k: v for k, v in pred_cats.items() if k in labels} gold_cats = getter(example.reference, attr) + gold_cats = {k: v for k, v in gold_cats.items() if k in labels} for label in labels: pred_score = pred_cats.get(label, 0.0) diff --git a/spacy/tests/pipeline/test_textcat.py b/spacy/tests/pipeline/test_textcat.py index 048586cec..d042f3445 100644 --- a/spacy/tests/pipeline/test_textcat.py +++ b/spacy/tests/pipeline/test_textcat.py @@ -898,7 +898,11 @@ def test_textcat_multi_threshold(): @pytest.mark.parametrize( - "component_name,scorer", [("textcat", "spacy.textcat_scorer.v1")] + "component_name,scorer", + [ + ("textcat", "spacy.textcat_scorer.v1"), + ("textcat_multilabel", "spacy.textcat_multilabel_scorer.v1"), + ], ) def test_textcat_legacy_scorers(component_name, scorer): """Check that legacy scorers are registered and produce the expected score diff --git a/spacy/tests/test_language.py b/spacy/tests/test_language.py index 03a98d32f..03790eb86 100644 --- a/spacy/tests/test_language.py +++ b/spacy/tests/test_language.py @@ -3,6 +3,7 @@ import logging from unittest import mock import pytest from spacy.language import Language +from spacy.scorer import Scorer from spacy.tokens import Doc, Span from spacy.vocab import Vocab from spacy.training import Example @@ -126,6 +127,112 @@ def test_evaluate_no_pipe(nlp): nlp.evaluate([Example.from_dict(doc, annots)]) +def test_evaluate_textcat_multilabel(en_vocab): + """Test that evaluate works with a multilabel textcat pipe.""" + nlp = Language(en_vocab) + textcat_multilabel = nlp.add_pipe("textcat_multilabel") + for label in ("FEATURE", "REQUEST", "BUG", "QUESTION"): + textcat_multilabel.add_label(label) + nlp.initialize() + + annots = {"cats": {"FEATURE": 1.0, "QUESTION": 1.0}} + doc = nlp.make_doc("hello world") + example = Example.from_dict(doc, annots) + scores = nlp.evaluate([example]) + labels = nlp.get_pipe("textcat_multilabel").labels + for label in labels: + assert scores["cats_f_per_type"].get(label) is not None + for key in example.reference.cats.keys(): + if key not in labels: + assert scores["cats_f_per_type"].get(key) is None + + +def test_evaluate_multiple_textcat_final(en_vocab): + """Test that evaluate evaluates the final textcat component in a pipeline + with more than one textcat or textcat_multilabel.""" + nlp = Language(en_vocab) + textcat = nlp.add_pipe("textcat") + for label in ("POSITIVE", "NEGATIVE"): + textcat.add_label(label) + textcat_multilabel = nlp.add_pipe("textcat_multilabel") + for label in ("FEATURE", "REQUEST", "BUG", "QUESTION"): + textcat_multilabel.add_label(label) + nlp.initialize() + + annots = { + "cats": { + "POSITIVE": 1.0, + "NEGATIVE": 0.0, + "FEATURE": 1.0, + "QUESTION": 1.0, + "POSITIVE": 1.0, + "NEGATIVE": 0.0, + } + } + doc = nlp.make_doc("hello world") + example = Example.from_dict(doc, annots) + scores = nlp.evaluate([example]) + # get the labels from the final pipe + labels = nlp.get_pipe(nlp.pipe_names[-1]).labels + for label in labels: + assert scores["cats_f_per_type"].get(label) is not None + for key in example.reference.cats.keys(): + if key not in labels: + assert scores["cats_f_per_type"].get(key) is None + + +def test_evaluate_multiple_textcat_separate(en_vocab): + """Test that evaluate can evaluate multiple textcat components separately + with custom scorers.""" + + def custom_textcat_score(examples, **kwargs): + scores = Scorer.score_cats( + examples, + "cats", + multi_label=False, + **kwargs, + ) + return {f"custom_{k}": v for k, v in scores.items()} + + @spacy.registry.scorers("test_custom_textcat_scorer") + def make_custom_textcat_scorer(): + return custom_textcat_score + + nlp = Language(en_vocab) + textcat = nlp.add_pipe( + "textcat", + config={"scorer": {"@scorers": "test_custom_textcat_scorer"}}, + ) + for label in ("POSITIVE", "NEGATIVE"): + textcat.add_label(label) + textcat_multilabel = nlp.add_pipe("textcat_multilabel") + for label in ("FEATURE", "REQUEST", "BUG", "QUESTION"): + textcat_multilabel.add_label(label) + nlp.initialize() + + annots = { + "cats": { + "POSITIVE": 1.0, + "NEGATIVE": 0.0, + "FEATURE": 1.0, + "QUESTION": 1.0, + "POSITIVE": 1.0, + "NEGATIVE": 0.0, + } + } + doc = nlp.make_doc("hello world") + example = Example.from_dict(doc, annots) + scores = nlp.evaluate([example]) + # check custom scores for the textcat pipe + assert "custom_cats_f_per_type" in scores + labels = nlp.get_pipe("textcat").labels + assert set(scores["custom_cats_f_per_type"].keys()) == set(labels) + # check default scores for the textcat_multilabel pipe + assert "cats_f_per_type" in scores + labels = nlp.get_pipe("textcat_multilabel").labels + assert set(scores["cats_f_per_type"].keys()) == set(labels) + + def vector_modification_pipe(doc): doc.vector += 1 return doc From eb8bb35c13a5f59826761065e4eeccee69d4c5a7 Mon Sep 17 00:00:00 2001 From: Zhangrp Date: Tue, 10 Jan 2023 14:52:57 +0800 Subject: [PATCH 160/179] improve ux for displacy when the serve port is in use (#11948) * check port in use and add itself * check port in use and add itself * Auto switch to nearest available port. * Use bind to check port instead of connect_ex. * Reformat. * Add auto_select_port argument. * update docs for displacy.serve * Update spacy/errors.py Co-authored-by: Paul O'Leary McCann * Update website/docs/api/top-level.md Co-authored-by: Paul O'Leary McCann * Update spacy/errors.py Co-authored-by: Paul O'Leary McCann * Add test using multiprocessing * fix argument name * Increase sleep times Want to rule this out as a cause of test failure * Don't terminate a process that isn't alive * Refactor port finding logic This moves all the port logic into its own util function, which can be tested without having to background a server directly. * Use with for the server This ensures the server is closed correctly. * Pass in the host when checking port availability * Shorten argument name * Update error codes following merge * Add types for arguments, specify docstrings. * Add typing for arguments with default value. * Update docstring to match spaCy format. * Update docstring to match spaCy format. * Fix docs Arg name changed from `auto_select_port` to just `auto_select`. * Revert "Fix docs" This reverts commit 356966fe849660c0c08b670c6aee1aa2af05c1c1. Co-authored-by: zhiiw <1302593554@qq.com> Co-authored-by: Paul O'Leary McCann Co-authored-by: Raphael Mitsch --- spacy/displacy/__init__.py | 9 ++++++- spacy/errors.py | 5 ++++ spacy/tests/test_misc.py | 15 ++++++++++- spacy/util.py | 48 +++++++++++++++++++++++++++++++++++ website/docs/api/top-level.md | 21 +++++++-------- 5 files changed, 86 insertions(+), 12 deletions(-) diff --git a/spacy/displacy/__init__.py b/spacy/displacy/__init__.py index 2f2058b8e..a3cfd96dd 100644 --- a/spacy/displacy/__init__.py +++ b/spacy/displacy/__init__.py @@ -11,6 +11,7 @@ from .render import DependencyRenderer, EntityRenderer, SpanRenderer from ..tokens import Doc, Span from ..errors import Errors, Warnings from ..util import is_in_jupyter +from ..util import find_available_port _html = {} @@ -82,6 +83,7 @@ def serve( manual: bool = False, port: int = 5000, host: str = "0.0.0.0", + auto_select_port: bool = False, ) -> None: """Serve displaCy visualisation. @@ -93,15 +95,20 @@ def serve( manual (bool): Don't parse `Doc` and instead expect a dict/list of dicts. port (int): Port to serve visualisation. host (str): Host to serve visualisation. + auto_select_port (bool): Automatically select a port if the specified port is in use. DOCS: https://spacy.io/api/top-level#displacy.serve USAGE: https://spacy.io/usage/visualizers """ from wsgiref import simple_server + port = find_available_port(port, host, auto_select_port) + if is_in_jupyter(): warnings.warn(Warnings.W011) - render(docs, style=style, page=page, minify=minify, options=options, manual=manual) + render( + docs, style=style, page=page, minify=minify, options=options, manual=manual + ) httpd = simple_server.make_server(host, port, app) print(f"\nUsing the '{style}' visualizer") print(f"Serving on http://{host}:{port} ...\n") diff --git a/spacy/errors.py b/spacy/errors.py index cd9281e91..498df0320 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -214,6 +214,7 @@ class Warnings(metaclass=ErrorsWithCodes): "is a Cython extension type.") W123 = ("Argument `enable` with value {enable} does not contain all values specified in the config option " "`enabled` ({enabled}). Be aware that this might affect other components in your pipeline.") + W124 = ("{host}:{port} is already in use, using the nearest available port {serve_port} as an alternative.") class Errors(metaclass=ErrorsWithCodes): @@ -963,6 +964,10 @@ class Errors(metaclass=ErrorsWithCodes): "knowledge base, use `InMemoryLookupKB`.") E1047 = ("`find_threshold()` only supports components with a `scorer` attribute.") E1048 = ("Got '{unexpected}' as console progress bar type, but expected one of the following: {expected}") + E1049 = ("No available port found for displaCy on host {host}. Please specify an available port " + "with `displacy.serve(doc, port)`") + E1050 = ("Port {port} is already in use. Please specify an available port with `displacy.serve(doc, port)` " + "or use `auto_switch_port=True` to pick an available port automatically.") # Deprecated model shortcuts, only used in errors and warnings diff --git a/spacy/tests/test_misc.py b/spacy/tests/test_misc.py index 1c9b045ac..618f17334 100644 --- a/spacy/tests/test_misc.py +++ b/spacy/tests/test_misc.py @@ -8,7 +8,7 @@ from spacy import prefer_gpu, require_gpu, require_cpu from spacy.ml._precomputable_affine import PrecomputableAffine from spacy.ml._precomputable_affine import _backprop_precomputable_affine_padding from spacy.util import dot_to_object, SimpleFrozenList, import_file -from spacy.util import to_ternary_int +from spacy.util import to_ternary_int, find_available_port from thinc.api import Config, Optimizer, ConfigValidationError from thinc.api import get_current_ops, set_current_ops, NumpyOps, CupyOps, MPSOps from thinc.compat import has_cupy_gpu, has_torch_mps_gpu @@ -434,3 +434,16 @@ def test_to_ternary_int(): assert to_ternary_int(-10) == -1 assert to_ternary_int("string") == -1 assert to_ternary_int([0, "string"]) == -1 + + +def test_find_available_port(): + host = "0.0.0.0" + port = 5000 + assert find_available_port(port, host) == port, "Port 5000 isn't free" + + from wsgiref.simple_server import make_server, demo_app + + with make_server(host, port, demo_app) as httpd: + with pytest.warns(UserWarning, match="already in use"): + found_port = find_available_port(port, host, auto_select=True) + assert found_port == port + 1, "Didn't find next port" diff --git a/spacy/util.py b/spacy/util.py index 8d211a9a5..8bf8fb1b0 100644 --- a/spacy/util.py +++ b/spacy/util.py @@ -31,6 +31,7 @@ import shlex import inspect import pkgutil import logging +import socket try: import cupy.random @@ -1736,3 +1737,50 @@ def all_equal(iterable): (or if the input is an empty sequence), False otherwise.""" g = itertools.groupby(iterable) return next(g, True) and not next(g, False) + + +def _is_port_in_use(port: int, host: str = "localhost") -> bool: + """Check if 'host:port' is in use. Return True if it is, False otherwise. + + port (int): the port to check + host (str): the host to check (default "localhost") + RETURNS (bool): Whether 'host:port' is in use. + """ + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + s.bind((host, port)) + return False + except socket.error: + return True + finally: + s.close() + + +def find_available_port(start: int, host: str, auto_select: bool = False) -> int: + """Given a starting port and a host, handle finding a port. + + If `auto_select` is False, a busy port will raise an error. + + If `auto_select` is True, the next free higher port will be used. + + start (int): the port to start looking from + host (str): the host to find a port on + auto_select (bool): whether to automatically select a new port if the given port is busy (default False) + RETURNS (int): The port to use. + """ + if not _is_port_in_use(start, host): + return start + + port = start + if not auto_select: + raise ValueError(Errors.E1050.format(port=port)) + + while _is_port_in_use(port, host) and port < 65535: + port += 1 + + if port == 65535 and _is_port_in_use(port, host): + raise ValueError(Errors.E1049.format(host=host)) + + # if we get here, the port changed + warnings.warn(Warnings.W124.format(host=host, port=start, serve_port=port)) + return port diff --git a/website/docs/api/top-level.md b/website/docs/api/top-level.md index 6a63e07da..9d3e463d8 100644 --- a/website/docs/api/top-level.md +++ b/website/docs/api/top-level.md @@ -237,16 +237,17 @@ browser. Will run a simple web server. > displacy.serve([doc1, doc2], style="dep") > ``` -| Name | Description | -| --------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span]], Doc, Span]~~ | -| `style` | Visualization style, `"dep"`, `"ent"` or `"span"` 3.3. Defaults to `"dep"`. ~~str~~ | -| `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ | -| `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ | -| `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ | -| `manual` | Don't parse `Doc` and instead expect a dict or list of dicts. [See here](/usage/visualizers#manual-usage) for formats and examples. Defaults to `False`. ~~bool~~ | -| `port` | Port to serve visualization. Defaults to `5000`. ~~int~~ | -| `host` | Host to serve visualization. Defaults to `"0.0.0.0"`. ~~str~~ | +| Name | Description | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `docs` | Document(s) or span(s) to visualize. ~~Union[Iterable[Union[Doc, Span]], Doc, Span]~~ | +| `style` | Visualization style, `"dep"`, `"ent"` or `"span"` 3.3. Defaults to `"dep"`. ~~str~~ | +| `page` | Render markup as full HTML page. Defaults to `True`. ~~bool~~ | +| `minify` | Minify HTML markup. Defaults to `False`. ~~bool~~ | +| `options` | [Visualizer-specific options](#displacy_options), e.g. colors. ~~Dict[str, Any]~~ | +| `manual` | Don't parse `Doc` and instead expect a dict or list of dicts. [See here](/usage/visualizers#manual-usage) for formats and examples. Defaults to `False`. ~~bool~~ | +| `port` | Port to serve visualization. Defaults to `5000`. ~~int~~ | +| `host` | Host to serve visualization. Defaults to `"0.0.0.0"`. ~~str~~ | +| `auto_select_port` | If `True`, automatically switch to a different port if the specified port is already in use. Defaults to `False`. ~~bool~~ | ### displacy.render {#displacy.render tag="method" new="2"} From 19650ebb52222cf2bc3402b7c74f68f3a9f0a4e3 Mon Sep 17 00:00:00 2001 From: Kevin Humphreys Date: Tue, 10 Jan 2023 01:36:17 -0800 Subject: [PATCH 161/179] Enable fuzzy text matching in Matcher (#11359) * enable fuzzy matching * add fuzzy param to EntityMatcher * include rapidfuzz_capi not yet used * fix type * add FUZZY predicate * add fuzzy attribute list * fix type properly * tidying * remove unnecessary dependency * handle fuzzy sets * simplify fuzzy sets * case fix * switch to FUZZYn predicates use Levenshtein distance. remove fuzzy param. remove rapidfuzz_capi. * revert changes added for fuzzy param * switch to polyleven (Python package) * enable fuzzy matching * add fuzzy param to EntityMatcher * include rapidfuzz_capi not yet used * fix type * add FUZZY predicate * add fuzzy attribute list * fix type properly * tidying * remove unnecessary dependency * handle fuzzy sets * simplify fuzzy sets * case fix * switch to FUZZYn predicates use Levenshtein distance. remove fuzzy param. remove rapidfuzz_capi. * revert changes added for fuzzy param * switch to polyleven (Python package) * fuzzy match only on oov tokens * remove polyleven * exclude whitespace tokens * don't allow more edits than characters * fix min distance * reinstate FUZZY operator with length-based distance function * handle sets inside regex operator * remove is_oov check * attempt build fix no mypy failure locally * re-attempt build fix * don't overwrite fuzzy param value * move fuzzy_match to its own Python module to allow patching * move fuzzy_match back inside Matcher simplify logic and add tests * Format tests * Parametrize fuzzyn tests * Parametrize and merge fuzzy+set tests * Format * Move fuzzy_match to a standalone method * Change regex kwarg type to bool * Add types for fuzzy_match - Refactor variable names - Add test for symmetrical behavior * Parametrize fuzzyn+set tests * Minor refactoring for fuzz/fuzzy * Make fuzzy_match a Matcher kwarg * Update type for _default_fuzzy_match * don't overwrite function param * Rename to fuzzy_compare * Update fuzzy_compare default argument declarations * allow fuzzy_compare override from EntityRuler * define new Matcher keyword arg * fix type definition * Implement fuzzy_compare config option for EntityRuler and SpanRuler * Rename _default_fuzzy_compare to fuzzy_compare, remove from reexported objects * Use simpler fuzzy_compare algorithm * Update types * Increase minimum to 2 in fuzzy_compare to allow one transposition * Fix predicate keys and matching for SetPredicate with FUZZY and REGEX * Add FUZZY6..9 * Add initial docs * Increase default fuzzy to rounded 30% of pattern length * Update docs for fuzzy_compare in components * Update EntityRuler and SpanRuler API docs * Rename EntityRuler and SpanRuler setting to matcher_fuzzy_compare To having naming similar to `phrase_matcher_attr`, rename `fuzzy_compare` setting for `EntityRuler` and `SpanRuler` to `matcher_fuzzy_compare. Organize next to `phrase_matcher_attr` in docs. * Fix schema aliases Co-authored-by: Sofie Van Landeghem * Fix typo Co-authored-by: Sofie Van Landeghem * Add FUZZY6-9 operators and update tests * Parameterize test over greedy Co-authored-by: Sofie Van Landeghem * Fix type for fuzzy_compare to remove Optional * Rename to spacy.levenshtein_compare.v1, move to spacy.matcher.levenshtein * Update docs following levenshtein_compare renaming Co-authored-by: Adriane Boyd Co-authored-by: Sofie Van Landeghem --- spacy/matcher/levenshtein.pyx | 17 +++ spacy/matcher/matcher.pxd | 1 + spacy/matcher/matcher.pyi | 3 +- spacy/matcher/matcher.pyx | 170 ++++++++++++++++----- spacy/pipeline/entityruler.py | 24 ++- spacy/pipeline/span_ruler.py | 18 ++- spacy/schemas.py | 12 +- spacy/tests/matcher/test_levenshtein.py | 29 ++++ spacy/tests/matcher/test_matcher_api.py | 173 ++++++++++++++++++++++ spacy/tests/pipeline/test_entity_ruler.py | 37 +++++ website/docs/api/entityruler.md | 53 +++---- website/docs/api/matcher.md | 31 ++-- website/docs/api/spanruler.md | 48 +++--- website/docs/usage/rule-based-matching.md | 40 +++++ 14 files changed, 554 insertions(+), 102 deletions(-) diff --git a/spacy/matcher/levenshtein.pyx b/spacy/matcher/levenshtein.pyx index 8463d913d..0e8cd26da 100644 --- a/spacy/matcher/levenshtein.pyx +++ b/spacy/matcher/levenshtein.pyx @@ -4,6 +4,8 @@ from libc.stdint cimport int64_t from typing import Optional +from ..util import registry + cdef extern from "polyleven.c": int64_t polyleven(PyObject *o1, PyObject *o2, int64_t k) @@ -13,3 +15,18 @@ cpdef int64_t levenshtein(a: str, b: str, k: Optional[int] = None): if k is None: k = -1 return polyleven(a, b, k) + + +cpdef bint levenshtein_compare(input_text: str, pattern_text: str, fuzzy: int = -1): + if fuzzy >= 0: + max_edits = fuzzy + else: + # allow at least two edits (to allow at least one transposition) and up + # to 20% of the pattern string length + max_edits = max(2, round(0.3 * len(pattern_text))) + return levenshtein(input_text, pattern_text, max_edits) <= max_edits + + +@registry.misc("spacy.levenshtein_compare.v1") +def make_levenshtein_compare(): + return levenshtein_compare diff --git a/spacy/matcher/matcher.pxd b/spacy/matcher/matcher.pxd index 455f978cc..51854d562 100644 --- a/spacy/matcher/matcher.pxd +++ b/spacy/matcher/matcher.pxd @@ -77,3 +77,4 @@ cdef class Matcher: cdef public object _extensions cdef public object _extra_predicates cdef public object _seen_attrs + cdef public object _fuzzy_compare diff --git a/spacy/matcher/matcher.pyi b/spacy/matcher/matcher.pyi index 390629ff8..77ea7b7a6 100644 --- a/spacy/matcher/matcher.pyi +++ b/spacy/matcher/matcher.pyi @@ -5,7 +5,8 @@ from ..vocab import Vocab from ..tokens import Doc, Span class Matcher: - def __init__(self, vocab: Vocab, validate: bool = ...) -> None: ... + def __init__(self, vocab: Vocab, validate: bool = ..., + fuzzy_compare: Callable[[str, str, int], bool] = ...) -> None: ... def __reduce__(self) -> Any: ... def __len__(self) -> int: ... def __contains__(self, key: str) -> bool: ... diff --git a/spacy/matcher/matcher.pyx b/spacy/matcher/matcher.pyx index c4a057ca0..ea1b4b66b 100644 --- a/spacy/matcher/matcher.pyx +++ b/spacy/matcher/matcher.pyx @@ -1,4 +1,4 @@ -# cython: infer_types=True, profile=True +# cython: binding=True, infer_types=True, profile=True from typing import List, Iterable from libcpp.vector cimport vector @@ -20,10 +20,12 @@ from ..tokens.token cimport Token from ..tokens.morphanalysis cimport MorphAnalysis from ..attrs cimport ID, attr_id_t, NULL_ATTR, ORTH, POS, TAG, DEP, LEMMA, MORPH, ENT_IOB +from .levenshtein import levenshtein_compare from ..schemas import validate_token_pattern from ..errors import Errors, MatchPatternError, Warnings from ..strings import get_string_id from ..attrs import IDS +from ..util import registry DEF PADDING = 5 @@ -36,11 +38,13 @@ cdef class Matcher: USAGE: https://spacy.io/usage/rule-based-matching """ - def __init__(self, vocab, validate=True): + def __init__(self, vocab, validate=True, *, fuzzy_compare=levenshtein_compare): """Create the Matcher. vocab (Vocab): The vocabulary object, which must be shared with the - documents the matcher will operate on. + validate (bool): Validate all patterns added to this matcher. + fuzzy_compare (Callable[[str, str, int], bool]): The comparison method + for the FUZZY operators. """ self._extra_predicates = [] self._patterns = {} @@ -51,9 +55,10 @@ cdef class Matcher: self.vocab = vocab self.mem = Pool() self.validate = validate + self._fuzzy_compare = fuzzy_compare def __reduce__(self): - data = (self.vocab, self._patterns, self._callbacks) + data = (self.vocab, self._patterns, self._callbacks, self.validate, self._fuzzy_compare) return (unpickle_matcher, data, None, None) def __len__(self): @@ -128,7 +133,7 @@ cdef class Matcher: for pattern in patterns: try: specs = _preprocess_pattern(pattern, self.vocab, - self._extensions, self._extra_predicates) + self._extensions, self._extra_predicates, self._fuzzy_compare) self.patterns.push_back(init_pattern(self.mem, key, specs)) for spec in specs: for attr, _ in spec[1]: @@ -326,8 +331,8 @@ cdef class Matcher: return key -def unpickle_matcher(vocab, patterns, callbacks): - matcher = Matcher(vocab) +def unpickle_matcher(vocab, patterns, callbacks, validate, fuzzy_compare): + matcher = Matcher(vocab, validate=validate, fuzzy_compare=fuzzy_compare) for key, pattern in patterns.items(): callback = callbacks.get(key, None) matcher.add(key, pattern, on_match=callback) @@ -754,7 +759,7 @@ cdef attr_t get_ent_id(const TokenPatternC* pattern) nogil: return id_attr.value -def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates): +def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates, fuzzy_compare): """This function interprets the pattern, converting the various bits of syntactic sugar before we compile it into a struct with init_pattern. @@ -781,7 +786,7 @@ def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates): ops = _get_operators(spec) attr_values = _get_attr_values(spec, string_store) extensions = _get_extensions(spec, string_store, extensions_table) - predicates = _get_extra_predicates(spec, extra_predicates, vocab) + predicates = _get_extra_predicates(spec, extra_predicates, vocab, fuzzy_compare) for op in ops: tokens.append((op, list(attr_values), list(extensions), list(predicates), token_idx)) return tokens @@ -826,16 +831,45 @@ def _get_attr_values(spec, string_store): # These predicate helper classes are used to match the REGEX, IN, >= etc # extensions to the matcher introduced in #3173. +class _FuzzyPredicate: + operators = ("FUZZY", "FUZZY1", "FUZZY2", "FUZZY3", "FUZZY4", "FUZZY5", + "FUZZY6", "FUZZY7", "FUZZY8", "FUZZY9") + + def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None, + regex=False, fuzzy=None, fuzzy_compare=None): + self.i = i + self.attr = attr + self.value = value + self.predicate = predicate + self.is_extension = is_extension + if self.predicate not in self.operators: + raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate)) + fuzz = self.predicate[len("FUZZY"):] # number after prefix + self.fuzzy = int(fuzz) if fuzz else -1 + self.fuzzy_compare = fuzzy_compare + self.key = (self.attr, self.fuzzy, self.predicate, srsly.json_dumps(value, sort_keys=True)) + + def __call__(self, Token token): + if self.is_extension: + value = token._.get(self.attr) + else: + value = token.vocab.strings[get_token_attr_for_matcher(token.c, self.attr)] + if self.value == value: + return True + return self.fuzzy_compare(value, self.value, self.fuzzy) + + class _RegexPredicate: operators = ("REGEX",) - def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None): + def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None, + regex=False, fuzzy=None, fuzzy_compare=None): self.i = i self.attr = attr self.value = re.compile(value) self.predicate = predicate self.is_extension = is_extension - self.key = (attr, self.predicate, srsly.json_dumps(value, sort_keys=True)) + self.key = (self.attr, self.predicate, srsly.json_dumps(value, sort_keys=True)) if self.predicate not in self.operators: raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate)) @@ -850,18 +884,28 @@ class _RegexPredicate: class _SetPredicate: operators = ("IN", "NOT_IN", "IS_SUBSET", "IS_SUPERSET", "INTERSECTS") - def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None): + def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None, + regex=False, fuzzy=None, fuzzy_compare=None): self.i = i self.attr = attr self.vocab = vocab + self.regex = regex + self.fuzzy = fuzzy + self.fuzzy_compare = fuzzy_compare if self.attr == MORPH: # normalize morph strings self.value = set(self.vocab.morphology.add(v) for v in value) else: - self.value = set(get_string_id(v) for v in value) + if self.regex: + self.value = set(re.compile(v) for v in value) + elif self.fuzzy is not None: + # add to string store + self.value = set(self.vocab.strings.add(v) for v in value) + else: + self.value = set(get_string_id(v) for v in value) self.predicate = predicate self.is_extension = is_extension - self.key = (attr, self.predicate, srsly.json_dumps(value, sort_keys=True)) + self.key = (self.attr, self.regex, self.fuzzy, self.predicate, srsly.json_dumps(value, sort_keys=True)) if self.predicate not in self.operators: raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate)) @@ -889,9 +933,29 @@ class _SetPredicate: return False if self.predicate == "IN": - return value in self.value + if self.regex: + value = self.vocab.strings[value] + return any(bool(v.search(value)) for v in self.value) + elif self.fuzzy is not None: + value = self.vocab.strings[value] + return any(self.fuzzy_compare(value, self.vocab.strings[v], self.fuzzy) + for v in self.value) + elif value in self.value: + return True + else: + return False elif self.predicate == "NOT_IN": - return value not in self.value + if self.regex: + value = self.vocab.strings[value] + return not any(bool(v.search(value)) for v in self.value) + elif self.fuzzy is not None: + value = self.vocab.strings[value] + return not any(self.fuzzy_compare(value, self.vocab.strings[v], self.fuzzy) + for v in self.value) + elif value in self.value: + return False + else: + return True elif self.predicate == "IS_SUBSET": return value <= self.value elif self.predicate == "IS_SUPERSET": @@ -906,13 +970,14 @@ class _SetPredicate: class _ComparisonPredicate: operators = ("==", "!=", ">=", "<=", ">", "<") - def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None): + def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None, + regex=False, fuzzy=None, fuzzy_compare=None): self.i = i self.attr = attr self.value = value self.predicate = predicate self.is_extension = is_extension - self.key = (attr, self.predicate, srsly.json_dumps(value, sort_keys=True)) + self.key = (self.attr, self.predicate, srsly.json_dumps(value, sort_keys=True)) if self.predicate not in self.operators: raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate)) @@ -935,7 +1000,7 @@ class _ComparisonPredicate: return value < self.value -def _get_extra_predicates(spec, extra_predicates, vocab): +def _get_extra_predicates(spec, extra_predicates, vocab, fuzzy_compare): predicate_types = { "REGEX": _RegexPredicate, "IN": _SetPredicate, @@ -949,6 +1014,16 @@ def _get_extra_predicates(spec, extra_predicates, vocab): "<=": _ComparisonPredicate, ">": _ComparisonPredicate, "<": _ComparisonPredicate, + "FUZZY": _FuzzyPredicate, + "FUZZY1": _FuzzyPredicate, + "FUZZY2": _FuzzyPredicate, + "FUZZY3": _FuzzyPredicate, + "FUZZY4": _FuzzyPredicate, + "FUZZY5": _FuzzyPredicate, + "FUZZY6": _FuzzyPredicate, + "FUZZY7": _FuzzyPredicate, + "FUZZY8": _FuzzyPredicate, + "FUZZY9": _FuzzyPredicate, } seen_predicates = {pred.key: pred.i for pred in extra_predicates} output = [] @@ -966,22 +1041,47 @@ def _get_extra_predicates(spec, extra_predicates, vocab): attr = "ORTH" attr = IDS.get(attr.upper()) if isinstance(value, dict): - processed = False - value_with_upper_keys = {k.upper(): v for k, v in value.items()} - for type_, cls in predicate_types.items(): - if type_ in value_with_upper_keys: - predicate = cls(len(extra_predicates), attr, value_with_upper_keys[type_], type_, vocab=vocab) - # Don't create a redundant predicates. - # This helps with efficiency, as we're caching the results. - if predicate.key in seen_predicates: - output.append(seen_predicates[predicate.key]) - else: - extra_predicates.append(predicate) - output.append(predicate.i) - seen_predicates[predicate.key] = predicate.i - processed = True - if not processed: - warnings.warn(Warnings.W035.format(pattern=value)) + output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types, + extra_predicates, seen_predicates, fuzzy_compare=fuzzy_compare)) + return output + + +def _get_extra_predicates_dict(attr, value_dict, vocab, predicate_types, + extra_predicates, seen_predicates, regex=False, fuzzy=None, fuzzy_compare=None): + output = [] + for type_, value in value_dict.items(): + type_ = type_.upper() + cls = predicate_types.get(type_) + if cls is None: + warnings.warn(Warnings.W035.format(pattern=value_dict)) + # ignore unrecognized predicate type + continue + elif cls == _RegexPredicate: + if isinstance(value, dict): + # add predicates inside regex operator + output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types, + extra_predicates, seen_predicates, + regex=True)) + continue + elif cls == _FuzzyPredicate: + if isinstance(value, dict): + # add predicates inside fuzzy operator + fuzz = type_[len("FUZZY"):] # number after prefix + fuzzy_val = int(fuzz) if fuzz else -1 + output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types, + extra_predicates, seen_predicates, + fuzzy=fuzzy_val, fuzzy_compare=fuzzy_compare)) + continue + predicate = cls(len(extra_predicates), attr, value, type_, vocab=vocab, + regex=regex, fuzzy=fuzzy, fuzzy_compare=fuzzy_compare) + # Don't create redundant predicates. + # This helps with efficiency, as we're caching the results. + if predicate.key in seen_predicates: + output.append(seen_predicates[predicate.key]) + else: + extra_predicates.append(predicate) + output.append(predicate.i) + seen_predicates[predicate.key] = predicate.i return output diff --git a/spacy/pipeline/entityruler.py b/spacy/pipeline/entityruler.py index 8154a077d..6a3755533 100644 --- a/spacy/pipeline/entityruler.py +++ b/spacy/pipeline/entityruler.py @@ -11,6 +11,7 @@ from ..errors import Errors, Warnings from ..util import ensure_path, to_disk, from_disk, SimpleFrozenList, registry from ..tokens import Doc, Span from ..matcher import Matcher, PhraseMatcher +from ..matcher.levenshtein import levenshtein_compare from ..scorer import get_ner_prf @@ -23,6 +24,7 @@ PatternType = Dict[str, Union[str, List[Dict[str, Any]]]] assigns=["doc.ents", "token.ent_type", "token.ent_iob"], default_config={ "phrase_matcher_attr": None, + "matcher_fuzzy_compare": {"@misc": "spacy.levenshtein_compare.v1"}, "validate": False, "overwrite_ents": False, "ent_id_sep": DEFAULT_ENT_ID_SEP, @@ -39,6 +41,7 @@ def make_entity_ruler( nlp: Language, name: str, phrase_matcher_attr: Optional[Union[int, str]], + matcher_fuzzy_compare: Callable, validate: bool, overwrite_ents: bool, ent_id_sep: str, @@ -48,6 +51,7 @@ def make_entity_ruler( nlp, name, phrase_matcher_attr=phrase_matcher_attr, + matcher_fuzzy_compare=matcher_fuzzy_compare, validate=validate, overwrite_ents=overwrite_ents, ent_id_sep=ent_id_sep, @@ -81,6 +85,7 @@ class EntityRuler(Pipe): name: str = "entity_ruler", *, phrase_matcher_attr: Optional[Union[int, str]] = None, + matcher_fuzzy_compare: Callable = levenshtein_compare, validate: bool = False, overwrite_ents: bool = False, ent_id_sep: str = DEFAULT_ENT_ID_SEP, @@ -99,7 +104,10 @@ class EntityRuler(Pipe): added. Used to disable the current entity ruler while creating phrase patterns with the nlp object. phrase_matcher_attr (int / str): Token attribute to match on, passed - to the internal PhraseMatcher as `attr` + to the internal PhraseMatcher as `attr`. + matcher_fuzzy_compare (Callable): The fuzzy comparison method for the + internal Matcher. Defaults to + spacy.matcher.levenshtein.levenshtein_compare. validate (bool): Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate` patterns (iterable): Optional patterns to load in. @@ -117,7 +125,10 @@ class EntityRuler(Pipe): self.token_patterns = defaultdict(list) # type: ignore self.phrase_patterns = defaultdict(list) # type: ignore self._validate = validate - self.matcher = Matcher(nlp.vocab, validate=validate) + self.matcher_fuzzy_compare = matcher_fuzzy_compare + self.matcher = Matcher( + nlp.vocab, validate=validate, fuzzy_compare=self.matcher_fuzzy_compare + ) self.phrase_matcher_attr = phrase_matcher_attr self.phrase_matcher = PhraseMatcher( nlp.vocab, attr=self.phrase_matcher_attr, validate=validate @@ -337,7 +348,11 @@ class EntityRuler(Pipe): self.token_patterns = defaultdict(list) self.phrase_patterns = defaultdict(list) self._ent_ids = defaultdict(tuple) - self.matcher = Matcher(self.nlp.vocab, validate=self._validate) + self.matcher = Matcher( + self.nlp.vocab, + validate=self._validate, + fuzzy_compare=self.matcher_fuzzy_compare, + ) self.phrase_matcher = PhraseMatcher( self.nlp.vocab, attr=self.phrase_matcher_attr, validate=self._validate ) @@ -431,7 +446,8 @@ class EntityRuler(Pipe): self.overwrite = cfg.get("overwrite", False) self.phrase_matcher_attr = cfg.get("phrase_matcher_attr", None) self.phrase_matcher = PhraseMatcher( - self.nlp.vocab, attr=self.phrase_matcher_attr + self.nlp.vocab, + attr=self.phrase_matcher_attr, ) self.ent_id_sep = cfg.get("ent_id_sep", DEFAULT_ENT_ID_SEP) else: diff --git a/spacy/pipeline/span_ruler.py b/spacy/pipeline/span_ruler.py index 0e7e9ebf7..b0669c0ef 100644 --- a/spacy/pipeline/span_ruler.py +++ b/spacy/pipeline/span_ruler.py @@ -13,6 +13,7 @@ from ..util import ensure_path, SimpleFrozenList, registry from ..tokens import Doc, Span from ..scorer import Scorer from ..matcher import Matcher, PhraseMatcher +from ..matcher.levenshtein import levenshtein_compare from .. import util PatternType = Dict[str, Union[str, List[Dict[str, Any]]]] @@ -28,6 +29,7 @@ DEFAULT_SPANS_KEY = "ruler" "overwrite_ents": False, "scorer": {"@scorers": "spacy.entity_ruler_scorer.v1"}, "ent_id_sep": "__unused__", + "matcher_fuzzy_compare": {"@misc": "spacy.levenshtein_compare.v1"}, }, default_score_weights={ "ents_f": 1.0, @@ -40,6 +42,7 @@ def make_entity_ruler( nlp: Language, name: str, phrase_matcher_attr: Optional[Union[int, str]], + matcher_fuzzy_compare: Callable, validate: bool, overwrite_ents: bool, scorer: Optional[Callable], @@ -57,6 +60,7 @@ def make_entity_ruler( annotate_ents=True, ents_filter=ents_filter, phrase_matcher_attr=phrase_matcher_attr, + matcher_fuzzy_compare=matcher_fuzzy_compare, validate=validate, overwrite=False, scorer=scorer, @@ -72,6 +76,7 @@ def make_entity_ruler( "annotate_ents": False, "ents_filter": {"@misc": "spacy.first_longest_spans_filter.v1"}, "phrase_matcher_attr": None, + "matcher_fuzzy_compare": {"@misc": "spacy.levenshtein_compare.v1"}, "validate": False, "overwrite": True, "scorer": { @@ -94,6 +99,7 @@ def make_span_ruler( annotate_ents: bool, ents_filter: Callable[[Iterable[Span], Iterable[Span]], Iterable[Span]], phrase_matcher_attr: Optional[Union[int, str]], + matcher_fuzzy_compare: Callable, validate: bool, overwrite: bool, scorer: Optional[Callable], @@ -106,6 +112,7 @@ def make_span_ruler( annotate_ents=annotate_ents, ents_filter=ents_filter, phrase_matcher_attr=phrase_matcher_attr, + matcher_fuzzy_compare=matcher_fuzzy_compare, validate=validate, overwrite=overwrite, scorer=scorer, @@ -216,6 +223,7 @@ class SpanRuler(Pipe): [Iterable[Span], Iterable[Span]], Iterable[Span] ] = util.filter_chain_spans, phrase_matcher_attr: Optional[Union[int, str]] = None, + matcher_fuzzy_compare: Callable = levenshtein_compare, validate: bool = False, overwrite: bool = False, scorer: Optional[Callable] = partial( @@ -246,6 +254,9 @@ class SpanRuler(Pipe): phrase_matcher_attr (Optional[Union[int, str]]): Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. + matcher_fuzzy_compare (Callable): The fuzzy comparison method for the + internal Matcher. Defaults to + spacy.matcher.levenshtein.levenshtein_compare. validate (bool): Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. overwrite (bool): Whether to remove any existing spans under this spans @@ -266,6 +277,7 @@ class SpanRuler(Pipe): self.spans_filter = spans_filter self.ents_filter = ents_filter self.scorer = scorer + self.matcher_fuzzy_compare = matcher_fuzzy_compare self._match_label_id_map: Dict[int, Dict[str, str]] = {} self.clear() @@ -451,7 +463,11 @@ class SpanRuler(Pipe): DOCS: https://spacy.io/api/spanruler#clear """ self._patterns: List[PatternType] = [] - self.matcher: Matcher = Matcher(self.nlp.vocab, validate=self.validate) + self.matcher: Matcher = Matcher( + self.nlp.vocab, + validate=self.validate, + fuzzy_compare=self.matcher_fuzzy_compare, + ) self.phrase_matcher: PhraseMatcher = PhraseMatcher( self.nlp.vocab, attr=self.phrase_matcher_attr, diff --git a/spacy/schemas.py b/spacy/schemas.py index e48fe1702..3675c12dd 100644 --- a/spacy/schemas.py +++ b/spacy/schemas.py @@ -156,12 +156,22 @@ def validate_token_pattern(obj: list) -> List[str]: class TokenPatternString(BaseModel): - REGEX: Optional[StrictStr] = Field(None, alias="regex") + REGEX: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="regex") IN: Optional[List[StrictStr]] = Field(None, alias="in") NOT_IN: Optional[List[StrictStr]] = Field(None, alias="not_in") IS_SUBSET: Optional[List[StrictStr]] = Field(None, alias="is_subset") IS_SUPERSET: Optional[List[StrictStr]] = Field(None, alias="is_superset") INTERSECTS: Optional[List[StrictStr]] = Field(None, alias="intersects") + FUZZY: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy") + FUZZY1: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy1") + FUZZY2: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy2") + FUZZY3: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy3") + FUZZY4: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy4") + FUZZY5: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy5") + FUZZY6: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy6") + FUZZY7: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy7") + FUZZY8: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy8") + FUZZY9: Optional[Union[StrictStr, "TokenPatternString"]] = Field(None, alias="fuzzy9") class Config: extra = "forbid" diff --git a/spacy/tests/matcher/test_levenshtein.py b/spacy/tests/matcher/test_levenshtein.py index d30e36132..5afb7e1fc 100644 --- a/spacy/tests/matcher/test_levenshtein.py +++ b/spacy/tests/matcher/test_levenshtein.py @@ -1,5 +1,6 @@ import pytest from spacy.matcher import levenshtein +from spacy.matcher.levenshtein import levenshtein_compare # empty string plus 10 random ASCII, 10 random unicode, and 2 random long tests @@ -42,3 +43,31 @@ from spacy.matcher import levenshtein ) def test_levenshtein(dist, a, b): assert levenshtein(a, b) == dist + + +@pytest.mark.parametrize( + "a,b,fuzzy,expected", + [ + ("a", "a", 1, True), + ("a", "a", 0, True), + ("a", "a", -1, True), + ("a", "ab", 1, True), + ("a", "ab", 0, False), + ("a", "ab", -1, True), + ("ab", "ac", 1, True), + ("ab", "ac", -1, True), + ("abc", "cde", 4, True), + ("abc", "cde", -1, False), + ("abcdef", "cdefgh", 4, True), + ("abcdef", "cdefgh", 3, False), + ("abcdef", "cdefgh", -1, False), # default (2 for length 6) + ("abcdefgh", "cdefghijk", 5, True), + ("abcdefgh", "cdefghijk", 4, False), + ("abcdefgh", "cdefghijk", -1, False), # default (2) + ("abcdefgh", "cdefghijkl", 6, True), + ("abcdefgh", "cdefghijkl", 5, False), + ("abcdefgh", "cdefghijkl", -1, False), # default (2) + ], +) +def test_levenshtein_compare(a, b, fuzzy, expected): + assert levenshtein_compare(a, b, fuzzy) == expected diff --git a/spacy/tests/matcher/test_matcher_api.py b/spacy/tests/matcher/test_matcher_api.py index ac905eeb4..09ab6c7dc 100644 --- a/spacy/tests/matcher/test_matcher_api.py +++ b/spacy/tests/matcher/test_matcher_api.py @@ -118,6 +118,155 @@ def test_matcher_match_multi(matcher): ] +@pytest.mark.parametrize( + "rules,match_locs", + [ + ( + { + "GoogleNow": [[{"ORTH": {"FUZZY": "Google"}}, {"ORTH": "Now"}]], + }, + [(2, 4)], + ), + ( + { + "Java": [[{"LOWER": {"FUZZY": "java"}}]], + }, + [(5, 6)], + ), + ( + { + "JS": [[{"ORTH": {"FUZZY": "JavaScript"}}]], + "GoogleNow": [[{"ORTH": {"FUZZY": "Google"}}, {"ORTH": "Now"}]], + "Java": [[{"LOWER": {"FUZZY": "java"}}]], + }, + [(2, 4), (5, 6), (8, 9)], + ), + # only the second pattern matches (check that predicate keys used for + # caching don't collide) + ( + { + "A": [[{"ORTH": {"FUZZY": "Javascripts"}}]], + "B": [[{"ORTH": {"FUZZY5": "Javascripts"}}]], + }, + [(8, 9)], + ), + ], +) +def test_matcher_match_fuzzy(en_vocab, rules, match_locs): + words = ["They", "like", "Goggle", "Now", "and", "Jav", "but", "not", "JvvaScrpt"] + doc = Doc(en_vocab, words=words) + + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns) + assert match_locs == [(start, end) for m_id, start, end in matcher(doc)] + + +@pytest.mark.parametrize("set_op", ["IN", "NOT_IN"]) +def test_matcher_match_fuzzy_set_op_longest(en_vocab, set_op): + rules = { + "GoogleNow": [[{"ORTH": {"FUZZY": {set_op: ["Google", "Now"]}}, "OP": "+"}]] + } + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns, greedy="LONGEST") + + words = ["They", "like", "Goggle", "Noo"] + doc = Doc(en_vocab, words=words) + assert len(matcher(doc)) == 1 + + +def test_matcher_match_fuzzy_set_multiple(en_vocab): + rules = { + "GoogleNow": [ + [ + { + "ORTH": {"FUZZY": {"IN": ["Google", "Now"]}, "NOT_IN": ["Goggle"]}, + "OP": "+", + } + ] + ] + } + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns, greedy="LONGEST") + + words = ["They", "like", "Goggle", "Noo"] + doc = Doc(matcher.vocab, words=words) + assert matcher(doc) == [ + (doc.vocab.strings["GoogleNow"], 3, 4), + ] + + +@pytest.mark.parametrize("fuzzyn", range(1, 10)) +def test_matcher_match_fuzzyn_all_insertions(en_vocab, fuzzyn): + matcher = Matcher(en_vocab) + matcher.add("GoogleNow", [[{"ORTH": {f"FUZZY{fuzzyn}": "GoogleNow"}}]]) + # words with increasing edit distance + words = ["GoogleNow" + "a" * i for i in range(0, 10)] + doc = Doc(en_vocab, words) + assert len(matcher(doc)) == fuzzyn + 1 + + +@pytest.mark.parametrize("fuzzyn", range(1, 6)) +def test_matcher_match_fuzzyn_various_edits(en_vocab, fuzzyn): + matcher = Matcher(en_vocab) + matcher.add("GoogleNow", [[{"ORTH": {f"FUZZY{fuzzyn}": "GoogleNow"}}]]) + # words with increasing edit distance of different edit types + words = [ + "GoogleNow", + "GoogleNuw", + "GoogleNuew", + "GoogleNoweee", + "GiggleNuw3", + "gouggle5New", + ] + doc = Doc(en_vocab, words) + assert len(matcher(doc)) == fuzzyn + 1 + + +@pytest.mark.parametrize("greedy", ["FIRST", "LONGEST"]) +@pytest.mark.parametrize("set_op", ["IN", "NOT_IN"]) +def test_matcher_match_fuzzyn_set_op_longest(en_vocab, greedy, set_op): + rules = { + "GoogleNow": [[{"ORTH": {"FUZZY2": {set_op: ["Google", "Now"]}}, "OP": "+"}]] + } + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns, greedy=greedy) + + words = ["They", "like", "Goggle", "Noo"] + doc = Doc(matcher.vocab, words=words) + spans = matcher(doc, as_spans=True) + assert len(spans) == 1 + if set_op == "IN": + assert spans[0].text == "Goggle Noo" + else: + assert spans[0].text == "They like" + + +def test_matcher_match_fuzzyn_set_multiple(en_vocab): + rules = { + "GoogleNow": [ + [ + { + "ORTH": {"FUZZY1": {"IN": ["Google", "Now"]}, "NOT_IN": ["Goggle"]}, + "OP": "+", + } + ] + ] + } + matcher = Matcher(en_vocab) + for key, patterns in rules.items(): + matcher.add(key, patterns, greedy="LONGEST") + + words = ["They", "like", "Goggle", "Noo"] + doc = Doc(matcher.vocab, words=words) + assert matcher(doc) == [ + (doc.vocab.strings["GoogleNow"], 3, 4), + ] + + def test_matcher_empty_dict(en_vocab): """Test matcher allows empty token specs, meaning match on any token.""" matcher = Matcher(en_vocab) @@ -437,6 +586,30 @@ def test_matcher_regex(en_vocab): assert len(matches) == 0 +def test_matcher_regex_set_in(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"ORTH": {"REGEX": {"IN": [r"(?:a)", r"(?:an)"]}}}] + matcher.add("A_OR_AN", [pattern]) + doc = Doc(en_vocab, words=["an", "a", "hi"]) + matches = matcher(doc) + assert len(matches) == 2 + doc = Doc(en_vocab, words=["bye"]) + matches = matcher(doc) + assert len(matches) == 0 + + +def test_matcher_regex_set_not_in(en_vocab): + matcher = Matcher(en_vocab) + pattern = [{"ORTH": {"REGEX": {"NOT_IN": [r"(?:a)", r"(?:an)"]}}}] + matcher.add("A_OR_AN", [pattern]) + doc = Doc(en_vocab, words=["an", "a", "hi"]) + matches = matcher(doc) + assert len(matches) == 1 + doc = Doc(en_vocab, words=["bye"]) + matches = matcher(doc) + assert len(matches) == 1 + + def test_matcher_regex_shape(en_vocab): matcher = Matcher(en_vocab) pattern = [{"SHAPE": {"REGEX": r"^[^x]+$"}}] diff --git a/spacy/tests/pipeline/test_entity_ruler.py b/spacy/tests/pipeline/test_entity_ruler.py index 6851e2a7c..417f930cb 100644 --- a/spacy/tests/pipeline/test_entity_ruler.py +++ b/spacy/tests/pipeline/test_entity_ruler.py @@ -382,6 +382,43 @@ def test_entity_ruler_overlapping_spans(nlp, entity_ruler_factory): assert doc.ents[0].label_ == "FOOBAR" +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_fuzzy_pipe(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [{"label": "HELLO", "pattern": [{"LOWER": {"FUZZY": "hello"}}]}] + ruler.add_patterns(patterns) + doc = nlp("helloo") + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "HELLO" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_fuzzy(nlp, entity_ruler_factory): + ruler = nlp.add_pipe(entity_ruler_factory, name="entity_ruler") + patterns = [{"label": "HELLO", "pattern": [{"LOWER": {"FUZZY": "hello"}}]}] + ruler.add_patterns(patterns) + doc = nlp("helloo") + assert len(doc.ents) == 1 + assert doc.ents[0].label_ == "HELLO" + + +@pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) +def test_entity_ruler_fuzzy_disabled(nlp, entity_ruler_factory): + @registry.misc("test_fuzzy_compare_disabled") + def make_test_fuzzy_compare_disabled(): + return lambda x, y, z: False + + ruler = nlp.add_pipe( + entity_ruler_factory, + name="entity_ruler", + config={"matcher_fuzzy_compare": {"@misc": "test_fuzzy_compare_disabled"}}, + ) + patterns = [{"label": "HELLO", "pattern": [{"LOWER": {"FUZZY": "hello"}}]}] + ruler.add_patterns(patterns) + doc = nlp("helloo") + assert len(doc.ents) == 0 + + @pytest.mark.parametrize("n_process", [1, 2]) @pytest.mark.parametrize("entity_ruler_factory", ENTITY_RULERS) def test_entity_ruler_multiprocessing(nlp, n_process, entity_ruler_factory): diff --git a/website/docs/api/entityruler.md b/website/docs/api/entityruler.md index c2ba33f01..f15c648ff 100644 --- a/website/docs/api/entityruler.md +++ b/website/docs/api/entityruler.md @@ -55,13 +55,14 @@ how the component should be configured. You can override its settings via the > nlp.add_pipe("entity_ruler", config=config) > ``` -| Setting | Description | -| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `phrase_matcher_attr` | Optional attribute name match on for the internal [`PhraseMatcher`](/api/phrasematcher), e.g. `LOWER` to match on the lowercase token text. Defaults to `None`. ~~Optional[Union[int, str]]~~ | -| `validate` | Whether patterns should be validated (passed to the `Matcher` and `PhraseMatcher`). Defaults to `False`. ~~bool~~ | -| `overwrite_ents` | If existing entities are present, e.g. entities added by the model, overwrite them by matches if necessary. Defaults to `False`. ~~bool~~ | -| `ent_id_sep` | Separator used internally for entity IDs. Defaults to `"\|\|"`. ~~str~~ | -| `scorer` | The scoring method. Defaults to [`spacy.scorer.get_ner_prf`](/api/scorer#get_ner_prf). ~~Optional[Callable]~~ | +| Setting | Description | +| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `phrase_matcher_attr` | Optional attribute name match on for the internal [`PhraseMatcher`](/api/phrasematcher), e.g. `LOWER` to match on the lowercase token text. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `matcher_fuzzy_compare` 3.5 | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ | +| `validate` | Whether patterns should be validated (passed to the `Matcher` and `PhraseMatcher`). Defaults to `False`. ~~bool~~ | +| `overwrite_ents` | If existing entities are present, e.g. entities added by the model, overwrite them by matches if necessary. Defaults to `False`. ~~bool~~ | +| `ent_id_sep` | Separator used internally for entity IDs. Defaults to `"\|\|"`. ~~str~~ | +| `scorer` | The scoring method. Defaults to [`spacy.scorer.get_ner_prf`](/api/scorer#get_ner_prf). ~~Optional[Callable]~~ | ```python %%GITHUB_SPACY/spacy/pipeline/entityruler.py @@ -85,23 +86,25 @@ be a token pattern (list) or a phrase pattern (string). For example: > ruler = EntityRuler(nlp, overwrite_ents=True) > ``` -| Name | Description | -| --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ | -| `name` 3 | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current entity ruler while creating phrase patterns with the nlp object. ~~str~~ | -| _keyword-only_ | | -| `phrase_matcher_attr` | Optional attribute name match on for the internal [`PhraseMatcher`](/api/phrasematcher), e.g. `LOWER` to match on the lowercase token text. Defaults to `None`. ~~Optional[Union[int, str]]~~ | -| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | -| `overwrite_ents` | If existing entities are present, e.g. entities added by the model, overwrite them by matches if necessary. Defaults to `False`. ~~bool~~ | -| `ent_id_sep` | Separator used internally for entity IDs. Defaults to `"\|\|"`. ~~str~~ | -| `patterns` | Optional patterns to load in on initialization. ~~Optional[List[Dict[str, Union[str, List[dict]]]]]~~ | +| Name | Description | +| ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ | +| `name` 3 | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current entity ruler while creating phrase patterns with the nlp object. ~~str~~ | +| _keyword-only_ | | +| `phrase_matcher_attr` | Optional attribute name match on for the internal [`PhraseMatcher`](/api/phrasematcher), e.g. `LOWER` to match on the lowercase token text. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `matcher_fuzzy_compare` 3.5 | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ | +| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | +| `overwrite_ents` | If existing entities are present, e.g. entities added by the model, overwrite them by matches if necessary. Defaults to `False`. ~~bool~~ | +| `ent_id_sep` | Separator used internally for entity IDs. Defaults to `"\|\|"`. ~~str~~ | +| `patterns` | Optional patterns to load in on initialization. ~~Optional[List[Dict[str, Union[str, List[dict]]]]]~~ | +| `scorer` | The scoring method. Defaults to [`spacy.scorer.get_ner_prf`](/api/scorer#get_ner_prf). ~~Optional[Callable]~~ | ## EntityRuler.initialize {#initialize tag="method" new="3"} Initialize the component with data and used before training to load in rules -from a [pattern file](/usage/rule-based-matching/#entityruler-files). This method -is typically called by [`Language.initialize`](/api/language#initialize) and -lets you customize arguments it receives via the +from a [pattern file](/usage/rule-based-matching/#entityruler-files). This +method is typically called by [`Language.initialize`](/api/language#initialize) +and lets you customize arguments it receives via the [`[initialize.components]`](/api/data-formats#config-initialize) block in the config. @@ -210,10 +213,10 @@ of dicts) or a phrase pattern (string). For more details, see the usage guide on | ---------- | ---------------------------------------------------------------- | | `patterns` | The patterns to add. ~~List[Dict[str, Union[str, List[dict]]]]~~ | - ## EntityRuler.remove {#remove tag="method" new="3.2.1"} -Remove a pattern by its ID from the entity ruler. A `ValueError` is raised if the ID does not exist. +Remove a pattern by its ID from the entity ruler. A `ValueError` is raised if +the ID does not exist. > #### Example > @@ -224,9 +227,9 @@ Remove a pattern by its ID from the entity ruler. A `ValueError` is raised if th > ruler.remove("apple") > ``` -| Name | Description | -| ---------- | ---------------------------------------------------------------- | -| `id` | The ID of the pattern rule. ~~str~~ | +| Name | Description | +| ---- | ----------------------------------- | +| `id` | The ID of the pattern rule. ~~str~~ | ## EntityRuler.to_disk {#to_disk tag="method"} diff --git a/website/docs/api/matcher.md b/website/docs/api/matcher.md index cd7bfa070..bd5f6ac24 100644 --- a/website/docs/api/matcher.md +++ b/website/docs/api/matcher.md @@ -86,14 +86,20 @@ it compares to another value. > ] > ``` -| Attribute | Description | -| -------------------------- | -------------------------------------------------------------------------------------------------------- | -| `IN` | Attribute value is member of a list. ~~Any~~ | -| `NOT_IN` | Attribute value is _not_ member of a list. ~~Any~~ | -| `IS_SUBSET` | Attribute value (for `MORPH` or custom list attributes) is a subset of a list. ~~Any~~ | -| `IS_SUPERSET` | Attribute value (for `MORPH` or custom list attributes) is a superset of a list. ~~Any~~ | -| `INTERSECTS` | Attribute value (for `MORPH` or custom list attribute) has a non-empty intersection with a list. ~~Any~~ | -| `==`, `>=`, `<=`, `>`, `<` | Attribute value is equal, greater or equal, smaller or equal, greater or smaller. ~~Union[int, float]~~ | +| Attribute | Description | +| -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `REGEX` | Attribute value matches the regular expression at any position in the string. ~~Any~~ | +| `FUZZY` | Attribute value matches if the `fuzzy_compare` method matches for `(value, pattern, -1)`. The default method allows a Levenshtein edit distance of at least 2 and up to 30% of the pattern string length. ~~Any~~ | +| `FUZZY1`, `FUZZY2`, ... `FUZZY9` | Attribute value matches if the `fuzzy_compare` method matches for `(value, pattern, N)`. The default method allows a Levenshtein edit distance of at most N (1-9). ~~Any~~ | +| `IN` | Attribute value is member of a list. ~~Any~~ | +| `NOT_IN` | Attribute value is _not_ member of a list. ~~Any~~ | +| `IS_SUBSET` | Attribute value (for `MORPH` or custom list attributes) is a subset of a list. ~~Any~~ | +| `IS_SUPERSET` | Attribute value (for `MORPH` or custom list attributes) is a superset of a list. ~~Any~~ | +| `INTERSECTS` | Attribute value (for `MORPH` or custom list attribute) has a non-empty intersection with a list. ~~Any~~ | +| `==`, `>=`, `<=`, `>`, `<` | Attribute value is equal, greater or equal, smaller or equal, greater or smaller. ~~Union[int, float]~~ | + +As of spaCy v3.5, `REGEX` and `FUZZY` can be used in combination with `IN` and +`NOT_IN`. ## Matcher.\_\_init\_\_ {#init tag="method"} @@ -109,10 +115,11 @@ string where an integer is expected) or unexpected property names. > matcher = Matcher(nlp.vocab) > ``` -| Name | Description | -| ---------- | ----------------------------------------------------------------------------------------------------- | -| `vocab` | The vocabulary object, which must be shared with the documents the matcher will operate on. ~~Vocab~~ | -| `validate` | Validate all patterns added to this matcher. ~~bool~~ | +| Name | Description | +| --------------- | ----------------------------------------------------------------------------------------------------- | +| `vocab` | The vocabulary object, which must be shared with the documents the matcher will operate on. ~~Vocab~~ | +| `validate` | Validate all patterns added to this matcher. ~~bool~~ | +| `fuzzy_compare` | The comparison method used for the `FUZZY` operators. ~~Callable[[str, str, int], bool]~~ | ## Matcher.\_\_call\_\_ {#call tag="method"} diff --git a/website/docs/api/spanruler.md b/website/docs/api/spanruler.md index b573f7c58..31f04ccf9 100644 --- a/website/docs/api/spanruler.md +++ b/website/docs/api/spanruler.md @@ -46,16 +46,17 @@ how the component should be configured. You can override its settings via the > nlp.add_pipe("span_ruler", config=config) > ``` -| Setting | Description | -| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | -| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | -| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | -| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | -| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | -| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | -| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | -| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | +| Setting | Description | +| ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | +| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | +| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | +| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | +| `phrase_matcher_attr` | Token attribute to match on, passed to the internal `PhraseMatcher` as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `matcher_fuzzy_compare` 3.5 | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ | +| `validate` | Whether patterns should be validated, passed to `Matcher` and `PhraseMatcher` as `validate`. Defaults to `False`. ~~bool~~ | +| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | ```python %%GITHUB_SPACY/spacy/pipeline/span_ruler.py @@ -79,19 +80,20 @@ token pattern (list) or a phrase pattern (string). For example: > ruler = SpanRuler(nlp, overwrite=True) > ``` -| Name | Description | -| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ | -| `name` | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current span ruler while creating phrase patterns with the nlp object. ~~str~~ | -| _keyword-only_ | | -| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | -| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | -| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | -| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | -| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | -| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | -| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | -| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | +| Name | Description | +| ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `nlp` | The shared nlp object to pass the vocab to the matchers and process phrase patterns. ~~Language~~ | +| `name` | Instance name of the current pipeline component. Typically passed in automatically from the factory when the component is added. Used to disable the current span ruler while creating phrase patterns with the nlp object. ~~str~~ | +| _keyword-only_ | | +| `spans_key` | The spans key to save the spans under. If `None`, no spans are saved. Defaults to `"ruler"`. ~~Optional[str]~~ | +| `spans_filter` | The optional method to filter spans before they are assigned to doc.spans. Defaults to `None`. ~~Optional[Callable[[Iterable[Span], Iterable[Span]], List[Span]]]~~ | +| `annotate_ents` | Whether to save spans to doc.ents. Defaults to `False`. ~~bool~~ | +| `ents_filter` | The method to filter spans before they are assigned to doc.ents. Defaults to `util.filter_chain_spans`. ~~Callable[[Iterable[Span], Iterable[Span]], List[Span]]~~ | +| `phrase_matcher_attr` | Token attribute to match on, passed to the internal PhraseMatcher as `attr`. Defaults to `None`. ~~Optional[Union[int, str]]~~ | +| `matcher_fuzzy_compare` 3.5 | The fuzzy comparison method, passed on to the internal `Matcher`. Defaults to `spacy.matcher.levenshtein.levenshtein_compare`. ~~Callable~~ | +| `validate` | Whether patterns should be validated, passed to Matcher and PhraseMatcher as `validate`. Defaults to `False`. ~~bool~~ | +| `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | +| `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | ## SpanRuler.initialize {#initialize tag="method"} diff --git a/website/docs/usage/rule-based-matching.md b/website/docs/usage/rule-based-matching.md index ad8ea27f3..3e15fca36 100644 --- a/website/docs/usage/rule-based-matching.md +++ b/website/docs/usage/rule-based-matching.md @@ -364,6 +364,46 @@ else: +#### Fuzzy matching {#fuzzy new="3.5"} + +Fuzzy matching allows you to match tokens with alternate spellings, typos, etc. +without specifying every possible variant. + +```python +# Matches "favourite", "favorites", "gavorite", "theatre", "theatr", ... +pattern = [{"TEXT": {"FUZZY": "favorite"}}, + {"TEXT": {"FUZZY": "theater"}}] +``` + +The `FUZZY` attribute allows fuzzy matches for any attribute string value, +including custom attributes. Just like `REGEX`, it always needs to be applied to +an attribute like `TEXT` or `LOWER`. By default `FUZZY` allows a Levenshtein +edit distance of at least 2 and up to 30% of the pattern string length. Using +the more specific attributes `FUZZY1`..`FUZZY9` you can specify the maximum +allowed edit distance directly. + +```python +# Match lowercase with fuzzy matching (allows 2 edits) +pattern = [{"LOWER": {"FUZZY": "definitely"}}] + +# Match custom attribute values with fuzzy matching (allows 2 edits) +pattern = [{"_": {"country": {"FUZZY": "Kyrgyzstan"}}}] + +# Match with exact Levenshtein edit distance limits (allows 3 edits) +pattern = [{"_": {"country": {"FUZZY3": "Kyrgyzstan"}}}] +``` + +#### Regex and fuzzy matching with lists {#regex-fuzzy-lists new="3.5"} + +Starting in spaCy v3.5, both `REGEX` and `FUZZY` can be combined with the +attributes `IN` and `NOT_IN`: + +```python +pattern = [{"TEXT": {"FUZZY": {"IN": ["awesome", "cool", "wonderful"]}}}] + +pattern = [{"TEXT": {"REGEX": {"NOT_IN": ["^awe(some)?$", "^wonder(ful)?"]}}}] +``` + --- #### Operators and quantifiers {#quantifiers} From 9e0322de1abfb21c4d87d1e58a9ef886f5e20603 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 11 Jan 2023 08:01:47 +0100 Subject: [PATCH 162/179] Restore v2 token_acc score implementation (#12073) In the v3 scorer refactoring, `token_acc` was implemented incorrectly. It should use `precision` instead of `fscore` for the measure of correctly aligned tokens / number of predicted tokens. Fix the docs to reflect that the measure uses the number of predicted tokens rather than the number of gold tokens. --- spacy/scorer.py | 2 +- spacy/tests/test_scorer.py | 2 +- website/docs/api/scorer.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spacy/scorer.py b/spacy/scorer.py index d8c383ab8..de4f52be6 100644 --- a/spacy/scorer.py +++ b/spacy/scorer.py @@ -174,7 +174,7 @@ class Scorer: prf_score.score_set(pred_spans, gold_spans) if len(acc_score) > 0: return { - "token_acc": acc_score.fscore, + "token_acc": acc_score.precision, "token_p": prf_score.precision, "token_r": prf_score.recall, "token_f": prf_score.fscore, diff --git a/spacy/tests/test_scorer.py b/spacy/tests/test_scorer.py index b903f1669..dbb47b423 100644 --- a/spacy/tests/test_scorer.py +++ b/spacy/tests/test_scorer.py @@ -110,7 +110,7 @@ def test_tokenization(sented_doc): ) example.predicted[1].is_sent_start = False scores = scorer.score([example]) - assert scores["token_acc"] == approx(0.66666666) + assert scores["token_acc"] == 0.5 assert scores["token_p"] == 0.5 assert scores["token_r"] == approx(0.33333333) assert scores["token_f"] == 0.4 diff --git a/website/docs/api/scorer.md b/website/docs/api/scorer.md index 9ef36e6fc..86e61da1e 100644 --- a/website/docs/api/scorer.md +++ b/website/docs/api/scorer.md @@ -76,7 +76,7 @@ core pipeline components, the individual score names start with the `Token` or Scores the tokenization: -- `token_acc`: number of correct tokens / number of gold tokens +- `token_acc`: number of correct tokens / number of predicted tokens - `token_p`, `token_r`, `token_f`: precision, recall and F-score for token character spans From e0168ccce940251351711ac0196d8560cb77547e Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 11 Jan 2023 13:54:58 +0100 Subject: [PATCH 163/179] Allow spacy-transformers v1.2.x in transformers extra (#12092) --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index cee8c0c33..79dff9e30 100644 --- a/setup.cfg +++ b/setup.cfg @@ -74,7 +74,7 @@ console_scripts = lookups = spacy_lookups_data>=1.0.3,<1.1.0 transformers = - spacy_transformers>=1.1.2,<1.2.0 + spacy_transformers>=1.1.2,<1.3.0 ray = spacy_ray>=0.1.0,<1.0.0 cuda = From 554df9ef20184bf439495873a7454ec8f28cf94e Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Wed, 11 Jan 2023 17:30:07 +0100 Subject: [PATCH 164/179] Website migration from Gatsby to Next (#12058) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Rename all MDX file to `.mdx` * Lock current node version (#11885) * Apply Prettier (#11996) * Minor website fixes (#11974) [ci skip] * fix table * Migrate to Next WEB-17 (#12005) * Initial commit * Run `npx create-next-app@13 next-blog` * Install MDX packages Following: https://github.com/vercel/next.js/blob/77b5f79a4dff453abb62346bf75b14d859539b81/packages/next-mdx/readme.md * Add MDX to Next * Allow Next to handle `.md` and `.mdx` files. * Add VSCode extension recommendation * Disabled TypeScript strict mode for now * Add prettier * Apply Prettier to all files * Make sure to use correct Node version * Add basic implementation for `MDXRemote` * Add experimental Rust MDX parser * Add `/public` * Add SASS support * Remove default pages and styling * Convert to module This allows to use `import/export` syntax * Add import for custom components * Add ability to load plugins * Extract function This will make the next commit easier to read * Allow to handle directories for page creation * Refactoring * Allow to parse subfolders for pages * Extract logic * Redirect `index.mdx` to parent directory * Disabled ESLint during builds * Disabled typescript during build * Remove Gatsby from `README.md` * Rephrase Docker part of `README.md` * Update project structure in `README.md` * Move and rename plugins * Update plugin for wrapping sections * Add dependencies for plugin * Use plugin * Rename wrapper type * Simplify unnessary adding of id to sections The slugified section ids are useless, because they can not be referenced anywhere anyway. The navigation only works if the section has the same id as the heading. * Add plugin for custom attributes on Markdown elements * Add plugin to readd support for tables * Add plugin to fix problem with wrapped images For more details see this issue: https://github.com/mdx-js/mdx/issues/1798 * Add necessary meta data to pages * Install necessary dependencies * Remove outdated MDX handling * Remove reliance on `InlineList` * Use existing Remark components * Remove unallowed heading Before `h1` components where not overwritten and would never have worked and they aren't used anywhere either. * Add missing components to MDX * Add correct styling * Fix broken list * Fix broken CSS classes * Implement layout * Fix links * Fix broken images * Fix pattern image * Fix heading attributes * Rename heading attribute `new` was causing some weird issue, so renaming it to `version` * Update comment syntax in MDX * Merge imports * Fix markdown rendering inside components * Add model pages * Simplify anchors * Fix default value for theme * Add Universe index page * Add Universe categories * Add Universe projects * Fix Next problem with copy Next complains when the server renders something different then the client, therfor we move the differing logic to `useEffect` * Fix improper component nesting Next doesn't allow block elements inside a `

` * Replace landing page MDX with page component * Remove inlined iframe content * Remove ability to inline HTML content in iFrames * Remove MDX imports * Fix problem with image inside link in MDX * Escape character for MDX * Fix unescaped characters in MDX * Fix headings with logo * Allow to export static HTML pages * Add prebuild script This command is automatically run by Next * Replace `svg-loader` with `react-inlinesvg` `svg-loader` is no longer maintained * Fix ESLint `react-hooks/exhaustive-deps` * Fix dropdowns * Change code language from `cli` to `bash` * Remove unnessary language `none` * Fix invalid code language `markdown_` with an underscore was used to basically turn of syntax highlighting, but using unknown languages know throws an error. * Enable code blocks plugin * Readd `InlineCode` component MDX2 removed the `inlineCode` component > The special component name `inlineCode` was removed, we recommend to use `pre` for the block version of code, and code for both the block and inline versions Source: https://mdxjs.com/migrating/v2/#update-mdx-content * Remove unused code * Extract function to own file * Fix code syntax highlighting * Update syntax for code block meta data * Remove unused prop * Fix internal link recognition There is a problem with regex between Node and browser, and since Next runs the component on both, this create an error. `Prop `rel` did not match. Server: "null" Client: "noopener nofollow noreferrer"` This simplifies the implementation and fixes the above error. * Replace `react-helmet` with `next/head` * Fix `className` problem for JSX component * Fix broken bold markdown * Convert file to `.mjs` to be used by Node process * Add plugin to replace strings * Fix custom table row styling * Fix problem with `span` inside inline `code` React doesn't allow a `span` inside an inline `code` element and throws an error in dev mode. * Add `_document` to be able to customize `` and `` * Add `lang="en"` * Store Netlify settings in file This way we don't need to update via Netlify UI, which can be tricky if changing build settings. * Add sitemap * Add Smartypants * Add PWA support * Add `manifest.webmanifest` * Fix bug with anchor links after reloading There was no need for the previous implementation, since the browser handles this nativly. Additional the manual scrolling into view was actually broken, because the heading would disappear behind the menu bar. * Rename custom event I was googeling for ages to find out what kind of event `inview` is, only to figure out it was a custom event with a name that sounds pretty much like a native one. 🫠 * Fix missing comment syntax highlighting * Refactor Quickstart component The previous implementation was hidding the irrelevant lines via data-props and dynamically generated CSS. This created problems with Next and was also hard to follow. CSS was used to do what React is supposed to handle. The new implementation simplfy filters the list of children (React elements) via their props. * Fix syntax highlighting for Training Quickstart * Unify code rendering * Improve error logging in Juniper * Fix Juniper component * Automatically generate "Read Next" link * Add Plausible * Use recent DocSearch component and adjust styling * Fix images * Turn of image optimization > Image Optimization using Next.js' default loader is not compatible with `next export`. We currently deploy to Netlify via `next export` * Dont build pages starting with `_` * Remove unused files * Add Next plugin to Netlify * Fix button layout MDX automatically adds `p` tags around text on a new line and Prettier wants to put the text on a new line. Hacking with JSX string. * Add 404 page * Apply Prettier * Update Prettier for `package.json` Next sometimes wants to patch `package-lock.json`. The old Prettier setting indended with 4 spaces, but Next always indends with 2 spaces. Since `npm install` automatically uses the indendation from `package.json` for `package-lock.json` and to avoid the format switching back and forth, both files are now set to 2 spaces. * Apply Next patch to `package-lock.json` When starting the dev server Next would warn `warn - Found lockfile missing swc dependencies, patching...` and update the `package-lock.json`. These are the patched changes. * fix link Co-authored-by: Sofie Van Landeghem * small backslash fixes * adjust to new style Co-authored-by: Marcus Blättermann --- website/.eslintrc.json | 3 + website/.gitignore | 44 + website/.nvmrc | 1 + website/.prettierignore | 1 + website/.prettierrc | 5 +- website/.vscode/extensions.json | 8 + website/README.md | 28 +- website/UNIVERSE.md | 78 +- .../{architectures.md => architectures.mdx} | 78 +- .../{attributeruler.md => attributeruler.mdx} | 30 +- .../api/{attributes.md => attributes.mdx} | 7 +- website/docs/api/{cli.md => cli.mdx} | 177 +- website/docs/api/{coref.md => coref.mdx} | 32 +- website/docs/api/{corpus.md => corpus.mdx} | 17 +- .../{cython-classes.md => cython-classes.mdx} | 32 +- .../{cython-structs.md => cython-structs.mdx} | 22 +- website/docs/api/{cython.md => cython.mdx} | 4 +- .../api/{data-formats.md => data-formats.mdx} | 44 +- ...ndencymatcher.md => dependencymatcher.mdx} | 21 +- ...pendencyparser.md => dependencyparser.mdx} | 42 +- website/docs/api/{doc.md => doc.mdx} | 72 +- website/docs/api/{docbin.md => docbin.mdx} | 23 +- ...eelemmatizer.md => edittreelemmatizer.mdx} | 40 +- .../api/{entitylinker.md => entitylinker.mdx} | 36 +- ...tityrecognizer.md => entityrecognizer.mdx} | 42 +- .../api/{entityruler.md => entityruler.mdx} | 36 +- website/docs/api/{example.md => example.mdx} | 40 +- website/docs/api/{index.md => index.mdx} | 2 - website/docs/api/{kb.md => kb.mdx} | 28 +- .../api/{kb_in_memory.md => kb_in_memory.mdx} | 36 +- .../docs/api/{language.md => language.mdx} | 88 +- website/docs/api/{legacy.md => legacy.mdx} | 32 +- .../api/{lemmatizer.md => lemmatizer.mdx} | 32 +- website/docs/api/{lexeme.md => lexeme.mdx} | 16 +- website/docs/api/{lookups.md => lookups.mdx} | 40 +- website/docs/api/{matcher.md => matcher.mdx} | 16 +- .../{morphologizer.md => morphologizer.mdx} | 42 +- .../api/{morphology.md => morphology.mdx} | 40 +- .../{phrasematcher.md => phrasematcher.mdx} | 14 +- website/docs/api/{pipe.md => pipe.mdx} | 50 +- ...ne-functions.md => pipeline-functions.mdx} | 12 +- website/docs/api/{scorer.md => scorer.mdx} | 24 +- ...cerecognizer.md => sentencerecognizer.mdx} | 38 +- .../api/{sentencizer.md => sentencizer.mdx} | 23 +- .../{span-resolver.md => span-resolver.mdx} | 32 +- website/docs/api/{span.md => span.mdx} | 56 +- ...spancategorizer.md => spancategorizer.mdx} | 50 +- .../docs/api/{spangroup.md => spangroup.mdx} | 33 +- .../docs/api/{spanruler.md => spanruler.mdx} | 40 +- .../api/{stringstore.md => stringstore.mdx} | 24 +- website/docs/api/{tagger.md => tagger.mdx} | 42 +- ...textcategorizer.md => textcategorizer.mdx} | 46 +- website/docs/api/{tok2vec.md => tok2vec.mdx} | 32 +- website/docs/api/{token.md => token.mdx} | 48 +- .../docs/api/{tokenizer.md => tokenizer.mdx} | 28 +- .../docs/api/{top-level.md => top-level.mdx} | 177 +- .../api/{transformer.md => transformer.mdx} | 54 +- website/docs/api/{vectors.md => vectors.mdx} | 50 +- website/docs/api/{vocab.md => vocab.mdx} | 36 +- website/docs/images/displacy-dep-founded.html | 58 - website/docs/images/displacy-ent-custom.html | 33 - website/docs/images/displacy-ent-snek.html | 26 - website/docs/images/displacy-ent1.html | 37 - website/docs/images/displacy-ent2.html | 39 - website/docs/images/displacy-long2.html | 84 - website/docs/images/displacy-span-custom.html | 31 - website/docs/images/displacy-span.html | 41 - website/docs/index.md | 6 - website/docs/models/{index.md => index.mdx} | 22 +- .../docs/{styleguide.md => styleguide.mdx} | 165 +- .../{_architecture.md => _architecture.mdx} | 12 +- .../{_language-data.md => _language-data.mdx} | 0 ..._named-entities.md => _named-entities.mdx} | 20 +- .../101/{_pipelines.md => _pipelines.mdx} | 4 +- .../usage/101/{_pos-deps.md => _pos-deps.mdx} | 12 +- .../{_serialization.md => _serialization.mdx} | 0 .../{_tokenization.md => _tokenization.mdx} | 5 +- .../usage/101/{_training.md => _training.mdx} | 8 +- ...-similarity.md => _vectors-similarity.mdx} | 18 +- ...marks-models.md => _benchmarks-models.mdx} | 10 +- ...formers.md => embeddings-transformers.mdx} | 99 +- .../{facts-figures.md => facts-figures.mdx} | 24 +- website/docs/usage/{index.md => index.mdx} | 61 +- ...hitectures.md => layers-architectures.mdx} | 138 +- ...ic-features.md => linguistic-features.mdx} | 249 +- website/docs/usage/{models.md => models.mdx} | 72 +- ...-pipelines.md => processing-pipelines.mdx} | 133 +- .../docs/usage/{projects.md => projects.mdx} | 219 +- ...ed-matching.md => rule-based-matching.mdx} | 200 +- .../{saving-loading.md => saving-loading.mdx} | 87 +- .../usage/{spacy-101.md => spacy-101.mdx} | 102 +- .../docs/usage/{training.md => training.mdx} | 256 +- website/docs/usage/{v2-1.md => v2-1.mdx} | 24 +- website/docs/usage/{v2-2.md => v2-2.mdx} | 17 +- website/docs/usage/{v2-3.md => v2-3.mdx} | 12 +- website/docs/usage/{v2.md => v2.mdx} | 44 +- website/docs/usage/{v3-1.md => v3-1.mdx} | 48 +- website/docs/usage/{v3-2.md => v3-2.mdx} | 32 +- website/docs/usage/{v3-3.md => v3-3.mdx} | 28 +- website/docs/usage/{v3-4.md => v3-4.mdx} | 22 +- website/docs/usage/{v3.md => v3.mdx} | 123 +- .../usage/{visualizers.md => visualizers.mdx} | 115 +- website/gatsby-browser.js | 50 - website/gatsby-config.js | 200 - website/gatsby-node.js | 290 - website/meta/dynamicMeta.mjs | 14 + website/meta/languageSorted.tsx | 5 + website/meta/languages.json | 152 +- website/meta/recordLanguages.tsx | 7 + website/meta/recordSections.tsx | 5 + website/meta/recordUniverse.tsx | 9 + website/meta/sidebarFlat.tsx | 5 + website/meta/site.json | 1 - website/meta/universe.json | 62 +- website/netlify.toml | 18 + website/next-sitemap.config.mjs | 10 + website/next.config.mjs | 38 + website/package-lock.json | 50124 +++++++--------- website/package.json | 142 +- website/pages/404.js | 32 + website/pages/[...listPathPage].tsx | 150 + website/pages/_app.tsx | 33 + website/pages/_document.tsx | 13 + .../widgets/landing.js => pages/index.tsx} | 198 +- website/pages/models/[slug].tsx | 66 + website/pages/universe/category/[slug].tsx | 43 + website/pages/universe/index.tsx | 17 + website/pages/universe/project/[slug].tsx | 41 + website/plugins/getProps.mjs | 39 + website/plugins/index.mjs | 20 + .../remarkCodeBlocks.mjs} | 45 +- website/plugins/remarkCustomAttrs.mjs | 38 + website/plugins/remarkFindAndReplace.mjs | 42 + .../remarkWrapSections.mjs} | 19 +- website/public/favicon.ico | Bin 0 -> 25931 bytes website/public/icons/icon-192x192.png | Bin 0 -> 12396 bytes website/public/icons/icon-256x256.png | Bin 0 -> 11554 bytes website/public/icons/icon-384x384.png | Bin 0 -> 28487 bytes website/public/icons/icon-512x512.png | Bin 0 -> 24892 bytes .../{docs => public}/images/architecture.svg | 0 .../images/cli_init_fill-config_diff.jpg | Bin website/{docs => public}/images/course.jpg | Bin .../images/dep-match-diagram.svg | 0 .../images/displacy-compact.svg | 0 .../images/displacy-custom-parser.svg | 0 .../public/images/displacy-dep-founded.html | 155 + .../public/images/displacy-ent-custom.html | 80 + website/public/images/displacy-ent-snek.html | 59 + website/public/images/displacy-ent1.html | 84 + website/public/images/displacy-ent2.html | 86 + .../images/displacy-long.html | 8 +- website/public/images/displacy-long2.html | 212 + .../images/displacy-model-rules.svg | 0 .../images/displacy-model-rules2.svg | 0 .../images/displacy-small.svg | 0 .../public/images/displacy-span-custom.html | 84 + website/public/images/displacy-span.html | 123 + website/{docs => public}/images/displacy.svg | 0 .../images/displacy_jupyter.jpg | Bin .../images/huggingface_hub.jpg | Bin website/{docs => public}/images/lifecycle.svg | 0 .../{docs => public}/images/matcher-demo.jpg | Bin .../images/pipeline-design.svg | 0 website/{docs => public}/images/pipeline.svg | 0 .../images/pipeline_transformer.svg | 0 website/{docs => public}/images/prodigy.jpg | Bin .../images/prodigy_overview.jpg | Bin .../images/prodigy_spans-manual.jpg | Bin .../images/prodigy_train_curve.jpg | Bin .../images/project_document.jpg | Bin website/{docs => public}/images/projects.png | Bin website/{docs => public}/images/projects.svg | 0 website/{docs => public}/images/sense2vec.jpg | Bin website/{docs => public}/images/spacy-ray.svg | 0 .../images/spacy-streamlit.png | Bin .../images/spacy-tailored-pipelines_wide.png | Bin .../{docs => public}/images/thinc_mypy.jpg | Bin .../images/tok2vec-listener.svg | 0 website/{docs => public}/images/tok2vec.svg | 0 .../{docs => public}/images/tokenization.svg | 0 .../images/trainable_component.svg | 0 website/{docs => public}/images/training.svg | 0 .../images/vocab_stringstore.svg | 0 website/{docs => public}/images/wandb1.jpg | Bin website/{docs => public}/images/wandb2.jpg | Bin website/public/manifest.webmanifest | 31 + website/public/vercel.svg | 4 + website/runtime.txt | 2 +- website/setup/setup.sh | 2 +- website/src/components/accordion.js | 4 +- website/src/components/card.js | 1 + website/src/components/code.js | 436 +- website/src/components/copy.js | 8 +- website/src/components/dropdown.js | 5 +- website/src/components/embed.js | 37 +- website/src/components/footer.js | 122 +- website/src/components/github.js | 20 +- website/src/components/icon.js | 98 +- website/src/components/infobox.js | 2 +- website/src/components/juniper.js | 181 +- website/src/components/landing.js | 57 +- website/src/components/link.js | 33 +- website/src/components/list.js | 6 +- website/src/components/main.js | 12 +- website/src/components/navigation.js | 9 +- website/src/components/quickstart.js | 133 +- website/src/components/search.js | 41 +- website/src/components/section.js | 2 +- website/src/components/seo.js | 173 +- website/src/components/sidebar.js | 38 +- website/src/components/table.js | 8 +- website/src/components/title.js | 6 +- website/src/components/typography.js | 16 +- website/src/components/util.js | 39 +- website/src/html.js | 43 - website/src/pages/404.js | 49 - website/src/plugins/remark-custom-attrs.js | 52 - website/src/remark.js | 113 + website/src/styles/aside.module.sass | 7 +- website/src/styles/code.module.sass | 2 +- website/src/styles/embed.module.sass | 13 +- website/src/styles/grid.module.sass | 3 +- website/src/styles/layout.sass | 76 +- website/src/styles/quickstart.module.sass | 7 +- website/src/styles/search.module.sass | 58 - website/src/styles/search.sass | 27 + website/src/styles/sidebar.module.sass | 9 +- website/src/templates/docs.js | 205 +- website/src/templates/index.js | 157 +- website/src/templates/mdx-renderer.js | 21 - website/src/templates/models.js | 123 +- website/src/templates/universe.js | 123 +- website/src/widgets/changelog.js | 14 +- website/src/widgets/features.js | 125 +- website/src/widgets/integration.js | 36 +- website/src/widgets/languages.js | 122 +- website/src/widgets/quickstart-install.js | 397 +- website/src/widgets/quickstart-models.js | 125 +- website/src/widgets/quickstart-training.js | 106 +- website/src/widgets/styleguide.js | 15 +- website/tsconfig.json | 20 + 241 files changed, 26957 insertions(+), 34416 deletions(-) create mode 100644 website/.eslintrc.json create mode 100644 website/.gitignore create mode 100644 website/.nvmrc create mode 100644 website/.prettierignore create mode 100644 website/.vscode/extensions.json rename website/docs/api/{architectures.md => architectures.mdx} (96%) rename website/docs/api/{attributeruler.md => attributeruler.mdx} (94%) rename website/docs/api/{attributes.md => attributes.mdx} (98%) rename website/docs/api/{cli.md => cli.mdx} (97%) rename website/docs/api/{coref.md => coref.mdx} (94%) rename website/docs/api/{corpus.md => corpus.mdx} (96%) rename website/docs/api/{cython-classes.md => cython-classes.mdx} (91%) rename website/docs/api/{cython-structs.md => cython-structs.mdx} (94%) rename website/docs/api/{cython.md => cython.mdx} (99%) rename website/docs/api/{data-formats.md => data-formats.mdx} (98%) rename website/docs/api/{dependencymatcher.md => dependencymatcher.mdx} (96%) rename website/docs/api/{dependencyparser.md => dependencyparser.mdx} (95%) rename website/docs/api/{doc.md => doc.mdx} (95%) rename website/docs/api/{docbin.md => docbin.mdx} (93%) rename website/docs/api/{edittreelemmatizer.md => edittreelemmatizer.mdx} (95%) rename website/docs/api/{entitylinker.md => entitylinker.mdx} (96%) rename website/docs/api/{entityrecognizer.md => entityrecognizer.mdx} (95%) rename website/docs/api/{entityruler.md => entityruler.mdx} (94%) rename website/docs/api/{example.md => example.mdx} (92%) rename website/docs/api/{index.md => index.mdx} (58%) rename website/docs/api/{kb.md => kb.mdx} (92%) rename website/docs/api/{kb_in_memory.md => kb_in_memory.mdx} (90%) rename website/docs/api/{language.md => language.mdx} (96%) rename website/docs/api/{legacy.md => legacy.mdx} (95%) rename website/docs/api/{lemmatizer.md => lemmatizer.mdx} (95%) rename website/docs/api/{lexeme.md => lexeme.mdx} (97%) rename website/docs/api/{lookups.md => lookups.mdx} (89%) rename website/docs/api/{matcher.md => matcher.mdx} (97%) rename website/docs/api/{morphologizer.md => morphologizer.mdx} (95%) rename website/docs/api/{morphology.md => morphology.mdx} (89%) rename website/docs/api/{phrasematcher.md => phrasematcher.mdx} (96%) rename website/docs/api/{pipe.md => pipe.mdx} (93%) rename website/docs/api/{pipeline-functions.md => pipeline-functions.mdx} (95%) rename website/docs/api/{scorer.md => scorer.mdx} (96%) rename website/docs/api/{sentencerecognizer.md => sentencerecognizer.mdx} (94%) rename website/docs/api/{sentencizer.md => sentencizer.mdx} (94%) rename website/docs/api/{span-resolver.md => span-resolver.mdx} (94%) rename website/docs/api/{span.md => span.mdx} (93%) rename website/docs/api/{spancategorizer.md => spancategorizer.mdx} (94%) rename website/docs/api/{spangroup.md => spangroup.mdx} (92%) rename website/docs/api/{spanruler.md => spanruler.mdx} (94%) rename website/docs/api/{stringstore.md => stringstore.mdx} (89%) rename website/docs/api/{tagger.md => tagger.mdx} (95%) rename website/docs/api/{textcategorizer.md => textcategorizer.mdx} (94%) rename website/docs/api/{tok2vec.md => tok2vec.mdx} (94%) rename website/docs/api/{token.md => token.mdx} (96%) rename website/docs/api/{tokenizer.md => tokenizer.mdx} (95%) rename website/docs/api/{top-level.md => top-level.mdx} (93%) rename website/docs/api/{transformer.md => transformer.mdx} (95%) rename website/docs/api/{vectors.md => vectors.mdx} (94%) rename website/docs/api/{vocab.md => vocab.mdx} (94%) delete mode 100644 website/docs/images/displacy-dep-founded.html delete mode 100644 website/docs/images/displacy-ent-custom.html delete mode 100644 website/docs/images/displacy-ent-snek.html delete mode 100644 website/docs/images/displacy-ent1.html delete mode 100644 website/docs/images/displacy-ent2.html delete mode 100644 website/docs/images/displacy-long2.html delete mode 100644 website/docs/images/displacy-span-custom.html delete mode 100644 website/docs/images/displacy-span.html delete mode 100644 website/docs/index.md rename website/docs/models/{index.md => index.mdx} (95%) rename website/docs/{styleguide.md => styleguide.mdx} (86%) rename website/docs/usage/101/{_architecture.md => _architecture.mdx} (96%) rename website/docs/usage/101/{_language-data.md => _language-data.mdx} (100%) rename website/docs/usage/101/{_named-entities.md => _named-entities.mdx} (75%) rename website/docs/usage/101/{_pipelines.md => _pipelines.mdx} (98%) rename website/docs/usage/101/{_pos-deps.md => _pos-deps.mdx} (92%) rename website/docs/usage/101/{_serialization.md => _serialization.mdx} (100%) rename website/docs/usage/101/{_tokenization.md => _tokenization.mdx} (95%) rename website/docs/usage/101/{_training.md => _training.mdx} (91%) rename website/docs/usage/101/{_vectors-similarity.md => _vectors-similarity.mdx} (96%) rename website/docs/usage/{_benchmarks-models.md => _benchmarks-models.mdx} (86%) rename website/docs/usage/{embeddings-transformers.md => embeddings-transformers.mdx} (94%) rename website/docs/usage/{facts-figures.md => facts-figures.mdx} (92%) rename website/docs/usage/{index.md => index.mdx} (93%) rename website/docs/usage/{layers-architectures.md => layers-architectures.mdx} (91%) rename website/docs/usage/{linguistic-features.md => linguistic-features.mdx} (94%) rename website/docs/usage/{models.md => models.mdx} (93%) rename website/docs/usage/{processing-pipelines.md => processing-pipelines.mdx} (96%) rename website/docs/usage/{projects.md => projects.mdx} (92%) rename website/docs/usage/{rule-based-matching.md => rule-based-matching.mdx} (95%) rename website/docs/usage/{saving-loading.md => saving-loading.mdx} (95%) rename website/docs/usage/{spacy-101.md => spacy-101.mdx} (91%) rename website/docs/usage/{training.md => training.mdx} (91%) rename website/docs/usage/{v2-1.md => v2-1.mdx} (94%) rename website/docs/usage/{v2-2.md => v2-2.mdx} (97%) rename website/docs/usage/{v2-3.md => v2-3.mdx} (98%) rename website/docs/usage/{v2.md => v2.mdx} (95%) rename website/docs/usage/{v3-1.md => v3-1.mdx} (91%) rename website/docs/usage/{v3-2.md => v3-2.mdx} (92%) rename website/docs/usage/{v3-3.md => v3-3.mdx} (95%) rename website/docs/usage/{v3-4.md => v3-4.mdx} (90%) rename website/docs/usage/{v3.md => v3.mdx} (95%) rename website/docs/usage/{visualizers.md => visualizers.mdx} (87%) delete mode 100644 website/gatsby-browser.js delete mode 100644 website/gatsby-config.js delete mode 100644 website/gatsby-node.js create mode 100644 website/meta/dynamicMeta.mjs create mode 100644 website/meta/languageSorted.tsx create mode 100644 website/meta/recordLanguages.tsx create mode 100644 website/meta/recordSections.tsx create mode 100644 website/meta/recordUniverse.tsx create mode 100644 website/meta/sidebarFlat.tsx create mode 100644 website/netlify.toml create mode 100644 website/next-sitemap.config.mjs create mode 100644 website/next.config.mjs create mode 100644 website/pages/404.js create mode 100644 website/pages/[...listPathPage].tsx create mode 100644 website/pages/_app.tsx create mode 100644 website/pages/_document.tsx rename website/{src/widgets/landing.js => pages/index.tsx} (62%) create mode 100644 website/pages/models/[slug].tsx create mode 100644 website/pages/universe/category/[slug].tsx create mode 100644 website/pages/universe/index.tsx create mode 100644 website/pages/universe/project/[slug].tsx create mode 100644 website/plugins/getProps.mjs create mode 100644 website/plugins/index.mjs rename website/{src/plugins/remark-code-blocks.js => plugins/remarkCodeBlocks.mjs} (67%) create mode 100644 website/plugins/remarkCustomAttrs.mjs create mode 100644 website/plugins/remarkFindAndReplace.mjs rename website/{src/plugins/remark-wrap-section.js => plugins/remarkWrapSections.mjs} (80%) create mode 100644 website/public/favicon.ico create mode 100644 website/public/icons/icon-192x192.png create mode 100644 website/public/icons/icon-256x256.png create mode 100644 website/public/icons/icon-384x384.png create mode 100644 website/public/icons/icon-512x512.png rename website/{docs => public}/images/architecture.svg (100%) rename website/{docs => public}/images/cli_init_fill-config_diff.jpg (100%) rename website/{docs => public}/images/course.jpg (100%) rename website/{docs => public}/images/dep-match-diagram.svg (100%) rename website/{docs => public}/images/displacy-compact.svg (100%) rename website/{docs => public}/images/displacy-custom-parser.svg (100%) create mode 100644 website/public/images/displacy-dep-founded.html create mode 100644 website/public/images/displacy-ent-custom.html create mode 100644 website/public/images/displacy-ent-snek.html create mode 100644 website/public/images/displacy-ent1.html create mode 100644 website/public/images/displacy-ent2.html rename website/{docs => public}/images/displacy-long.html (98%) create mode 100644 website/public/images/displacy-long2.html rename website/{docs => public}/images/displacy-model-rules.svg (100%) rename website/{docs => public}/images/displacy-model-rules2.svg (100%) rename website/{docs => public}/images/displacy-small.svg (100%) create mode 100644 website/public/images/displacy-span-custom.html create mode 100644 website/public/images/displacy-span.html rename website/{docs => public}/images/displacy.svg (100%) rename website/{docs => public}/images/displacy_jupyter.jpg (100%) rename website/{docs => public}/images/huggingface_hub.jpg (100%) rename website/{docs => public}/images/lifecycle.svg (100%) rename website/{docs => public}/images/matcher-demo.jpg (100%) rename website/{docs => public}/images/pipeline-design.svg (100%) rename website/{docs => public}/images/pipeline.svg (100%) rename website/{docs => public}/images/pipeline_transformer.svg (100%) rename website/{docs => public}/images/prodigy.jpg (100%) rename website/{docs => public}/images/prodigy_overview.jpg (100%) rename website/{docs => public}/images/prodigy_spans-manual.jpg (100%) rename website/{docs => public}/images/prodigy_train_curve.jpg (100%) rename website/{docs => public}/images/project_document.jpg (100%) rename website/{docs => public}/images/projects.png (100%) rename website/{docs => public}/images/projects.svg (100%) rename website/{docs => public}/images/sense2vec.jpg (100%) rename website/{docs => public}/images/spacy-ray.svg (100%) rename website/{docs => public}/images/spacy-streamlit.png (100%) rename website/{docs => public}/images/spacy-tailored-pipelines_wide.png (100%) rename website/{docs => public}/images/thinc_mypy.jpg (100%) rename website/{docs => public}/images/tok2vec-listener.svg (100%) rename website/{docs => public}/images/tok2vec.svg (100%) rename website/{docs => public}/images/tokenization.svg (100%) rename website/{docs => public}/images/trainable_component.svg (100%) rename website/{docs => public}/images/training.svg (100%) rename website/{docs => public}/images/vocab_stringstore.svg (100%) rename website/{docs => public}/images/wandb1.jpg (100%) rename website/{docs => public}/images/wandb2.jpg (100%) create mode 100644 website/public/manifest.webmanifest create mode 100644 website/public/vercel.svg delete mode 100644 website/src/html.js delete mode 100644 website/src/pages/404.js delete mode 100644 website/src/plugins/remark-custom-attrs.js create mode 100644 website/src/remark.js delete mode 100644 website/src/styles/search.module.sass create mode 100644 website/src/styles/search.sass delete mode 100644 website/src/templates/mdx-renderer.js create mode 100644 website/tsconfig.json diff --git a/website/.eslintrc.json b/website/.eslintrc.json new file mode 100644 index 000000000..1c2aa65d7 --- /dev/null +++ b/website/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": "next/core-web-vitals" +} diff --git a/website/.gitignore b/website/.gitignore new file mode 100644 index 000000000..70ef99fa5 --- /dev/null +++ b/website/.gitignore @@ -0,0 +1,44 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.js + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# local env files +.env*.local + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts + +!.vscode/extensions.json +!public + +public/robots.txt +public/sitemap* +public/sw.js* +public/workbox* \ No newline at end of file diff --git a/website/.nvmrc b/website/.nvmrc new file mode 100644 index 000000000..3c032078a --- /dev/null +++ b/website/.nvmrc @@ -0,0 +1 @@ +18 diff --git a/website/.prettierignore b/website/.prettierignore new file mode 100644 index 000000000..d0d878e40 --- /dev/null +++ b/website/.prettierignore @@ -0,0 +1 @@ +.next \ No newline at end of file diff --git a/website/.prettierrc b/website/.prettierrc index 7555c734a..03904b1c4 100644 --- a/website/.prettierrc +++ b/website/.prettierrc @@ -20,12 +20,11 @@ } }, { - "files": "*.md", + "files": ["package.json", "package-lock.json"], "options": { "tabWidth": 2, "printWidth": 80, - "proseWrap": "always", - "htmlWhitespaceSensitivity": "strict" + "proseWrap": "always" } }, { diff --git a/website/.vscode/extensions.json b/website/.vscode/extensions.json new file mode 100644 index 000000000..4b533827a --- /dev/null +++ b/website/.vscode/extensions.json @@ -0,0 +1,8 @@ +{ + "recommendations": [ + "dbaeumer.vscode-eslint", + "unifiedjs.vscode-mdx", + "esbenp.prettier-vscode", + "syler.sass-indented" + ] +} diff --git a/website/README.md b/website/README.md index 890a48ef9..e9d7aec26 100644 --- a/website/README.md +++ b/website/README.md @@ -7,17 +7,16 @@ The styleguide for the spaCy website is available at ## Setup and installation -Before running the setup, make sure your versions of -[Node](https://nodejs.org/en/) and [npm](https://www.npmjs.com/) are up to date. -Node v10.15 or later is required. - ```bash # Clone the repository git clone https://github.com/explosion/spaCy cd spaCy/website -# Install Gatsby's command-line tool -npm install --global gatsby-cli +# Switch to the correct Node version +# +# If you don't have NVM and don't want to use it, you can manually switch to the Node version +# stated in /.nvmrc and skip this step +nvm use # Install the dependencies npm install @@ -36,8 +35,7 @@ file in the root defines the settings used in this codebase. ## Building & developing the site with Docker -Sometimes it's hard to get a local environment working due to rapid updates to -node dependencies, so it may be easier to use docker for building the docs. +While it shouldn't be necessary and is not recommended you can run this site in a Docker container. If you'd like to do this, **be sure you do _not_ include your local `node_modules` folder**, since there are some dependencies that need to be built @@ -76,12 +74,14 @@ bit of time. ```yaml ├── docs # the actual markdown content ├── meta # JSON-formatted site metadata +| ├── dynamicMeta.js # At build time generated meta data | ├── languages.json # supported languages and statistical models | ├── sidebars.json # sidebar navigations for different sections | ├── site.json # general site metadata | ├── type-annotations.json # Type annotations | └── universe.json # data for the spaCy universe section -├── public # compiled site +├── pages # Next router pages +├── public # static images and other assets ├── setup # Jinja setup ├── src # source | ├── components # React components @@ -96,9 +96,11 @@ bit of time. | | └── universe.js # layout templates for universe | └── widgets # non-reusable components with content, e.g. changelog ├── .eslintrc.json # ESLint config file +├── .nvmrc # NVM config file +| # (to support "nvm use" to switch to correct Node version) +| ├── .prettierrc # Prettier config file -├── gatsby-browser.js # browser-specific hooks for Gatsby -├── gatsby-config.js # Gatsby configuration -├── gatsby-node.js # Node-specific hooks for Gatsby -└── package.json # package settings and dependencies +├── next.config.mjs # Next config file +├── package.json # package settings and dependencies +└── tsconfig.json # TypeScript config file ``` diff --git a/website/UNIVERSE.md b/website/UNIVERSE.md index 770bbde13..ac4e2e684 100644 --- a/website/UNIVERSE.md +++ b/website/UNIVERSE.md @@ -2,42 +2,52 @@ # spaCy Universe -The [spaCy Universe](https://spacy.io/universe) collects the many great resources developed with or for spaCy. It -includes standalone packages, plugins, extensions, educational materials, -operational utilities and bindings for other languages. +The [spaCy Universe](https://spacy.io/universe) collects the many great +resources developed with or for spaCy. It includes standalone packages, plugins, +extensions, educational materials, operational utilities and bindings for other +languages. If you have a project that you want the spaCy community to make use of, you can suggest it by submitting a pull request to this repository. The Universe database is open-source and collected in a simple JSON file. Looking for inspiration for your own spaCy plugin or extension? Check out the -[`project ideas`](https://github.com/explosion/spaCy/discussions?discussions_q=category%3A%22New+Features+%26+Project+Ideas%22) +[`project ideas`](https://github.com/explosion/spaCy/discussions?discussions_q=category%3A%22New+Features+%26+Project+Ideas%22) discussion forum. ## Checklist ### Projects -✅ Libraries and packages should be **open-source** (with a user-friendly license) and at least somewhat **documented** (e.g. a simple `README` with usage instructions). +✅ Libraries and packages should be **open-source** (with a user-friendly +license) and at least somewhat **documented** (e.g. a simple `README` with usage +instructions). -✅ We're happy to include work in progress and prereleases, but we'd like to keep the emphasis on projects that should be useful to the community **right away**. +✅ We're happy to include work in progress and prereleases, but we'd like to +keep the emphasis on projects that should be useful to the community **right +away**. ✅ Demos and visualizers should be available via a **public URL**. ### Educational Materials -✅ Books should be **available for purchase or download** (not just pre-order). Ebooks and self-published books are fine, too, if they include enough substantial content. +✅ Books should be **available for purchase or download** (not just pre-order). +Ebooks and self-published books are fine, too, if they include enough +substantial content. -✅ The `"url"` of book entries should either point to the publisher's website or a reseller of your choice (ideally one that ships worldwide or as close as possible). +✅ The `"url"` of book entries should either point to the publisher's website or +a reseller of your choice (ideally one that ships worldwide or as close as +possible). -✅ If an online course is only available behind a paywall, it should at least have a **free excerpt** or chapter available, so users know what to expect. +✅ If an online course is only available behind a paywall, it should at least +have a **free excerpt** or chapter available, so users know what to expect. ## JSON format -To add a project, fork this repository, edit the [`universe.json`](meta/universe.json) -and add an object of the following format to the list of `"resources"`. Before -you submit your pull request, make sure to use a linter to verify that your -markup is correct. +To add a project, fork this repository, edit the +[`universe.json`](meta/universe.json) and add an object of the following format +to the list of `"resources"`. Before you submit your pull request, make sure to +use a linter to verify that your markup is correct. ```json { @@ -69,26 +79,26 @@ markup is correct. } ``` -| Field | Type | Description | -| --- | --- | --- | -| `id` | string | Unique ID of the project. | -| `title` | string | Project title. If not set, the `id` will be used as the display title. | -| `slogan` | string | A short description of the project. Displayed in the overview and under the title. | -| `description` | string | A longer description of the project. Markdown is allowed, but should be limited to basic formatting like bold, italics, code or links. | -| `github` | string | Associated GitHub repo in the format `user/repo`. Will be displayed as a link and used for release, license and star badges. | -| `pip` | string | Package name on pip. If available, the installation command will be displayed. | -| `cran` | string | For R packages: package name on CRAN. If available, the installation command will be displayed. | -| `code_example` | array | Short example that shows how to use the project. Formatted as an array with one string per line. | -| `code_language` | string | Defaults to `'python'`. Optional code language used for syntax highlighting with [Prism](http://prismjs.com/). | -| `url` | string | Optional project link to display as button. | -| `thumb` | string | Optional URL to project thumbnail to display in overview and project header. Recommended size is 100x100px. | -| `image` | string | Optional URL to project image to display with description. | -| `author` | string | Name(s) of project author(s). | -| `author_links` | object | Usernames and links to display as icons to author info. Currently supports `twitter` and `github` usernames, as well as `website` link. | -| `category` | list | One or more categories to assign to project. Must be one of the available options. | -| `tags` | list | Still experimental and not used for filtering: one or more tags to assign to project. | +| Field | Type | Description | +| --------------- | ------ | --------------------------------------------------------------------------------------------------------------------------------------- | +| `id` | string | Unique ID of the project. | +| `title` | string | Project title. If not set, the `id` will be used as the display title. | +| `slogan` | string | A short description of the project. Displayed in the overview and under the title. | +| `description` | string | A longer description of the project. Markdown is allowed, but should be limited to basic formatting like bold, italics, code or links. | +| `github` | string | Associated GitHub repo in the format `user/repo`. Will be displayed as a link and used for release, license and star badges. | +| `pip` | string | Package name on pip. If available, the installation command will be displayed. | +| `cran` | string | For R packages: package name on CRAN. If available, the installation command will be displayed. | +| `code_example` | array | Short example that shows how to use the project. Formatted as an array with one string per line. | +| `code_language` | string | Defaults to `'python'`. Optional code language used for syntax highlighting with [Prism](http://prismjs.com/). | +| `url` | string | Optional project link to display as button. | +| `thumb` | string | Optional URL to project thumbnail to display in overview and project header. Recommended size is 100x100px. | +| `image` | string | Optional URL to project image to display with description. | +| `author` | string | Name(s) of project author(s). | +| `author_links` | object | Usernames and links to display as icons to author info. Currently supports `twitter` and `github` usernames, as well as `website` link. | +| `category` | list | One or more categories to assign to project. Must be one of the available options. | +| `tags` | list | Still experimental and not used for filtering: one or more tags to assign to project. | To separate them from the projects, educational materials also specify -`"type": "education`. Books can also set a `"cover"` field containing a URL -to a cover image. If available, it's used in the overview and displayed on -the individual book page. +`"type": "education`. Books can also set a `"cover"` field containing a URL to a +cover image. If available, it's used in the overview and displayed on the +individual book page. diff --git a/website/docs/api/architectures.md b/website/docs/api/architectures.mdx similarity index 96% rename from website/docs/api/architectures.md rename to website/docs/api/architectures.mdx index 4c5447f75..2a1bc4380 100644 --- a/website/docs/api/architectures.md +++ b/website/docs/api/architectures.mdx @@ -26,9 +26,9 @@ part of the [training config](/usage/training#custom-functions). Also see the usage documentation on [layers and model architectures](/usage/layers-architectures). -## Tok2Vec architectures {#tok2vec-arch source="spacy/ml/models/tok2vec.py"} +## Tok2Vec architectures {id="tok2vec-arch",source="spacy/ml/models/tok2vec.py"} -### spacy.Tok2Vec.v2 {#Tok2Vec} +### spacy.Tok2Vec.v2 {id="Tok2Vec"} > #### Example config > @@ -56,7 +56,7 @@ blog post for background. | `encode` | Encode context into the embeddings, using an architecture such as a CNN, BiLSTM or transformer. For example, [MaxoutWindowEncoder](/api/architectures#MaxoutWindowEncoder). ~~Model[List[Floats2d], List[Floats2d]]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy.HashEmbedCNN.v2 {#HashEmbedCNN} +### spacy.HashEmbedCNN.v2 {id="HashEmbedCNN"} > #### Example Config > @@ -89,7 +89,7 @@ consisting of a CNN and a layer-normalized maxout activation function. | `pretrained_vectors` | Whether to also use static vectors. ~~bool~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy.Tok2VecListener.v1 {#Tok2VecListener} +### spacy.Tok2VecListener.v1 {id="Tok2VecListener"} > #### Example config > @@ -139,7 +139,7 @@ the `Tok2Vec` component. | `upstream` | A string to identify the "upstream" `Tok2Vec` component to communicate with. By default, the upstream name is the wildcard string `"*"`, but you could also specify the name of the `Tok2Vec` component. You'll almost never have multiple upstream `Tok2Vec` components, so the wildcard string will almost always be fine. ~~str~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy.MultiHashEmbed.v2 {#MultiHashEmbed} +### spacy.MultiHashEmbed.v2 {id="MultiHashEmbed"} > #### Example config > @@ -170,7 +170,7 @@ updated). | `include_static_vectors` | Whether to also use static word vectors. Requires a vectors table to be loaded in the [`Doc`](/api/doc) objects' vocab. ~~bool~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy.CharacterEmbed.v2 {#CharacterEmbed} +### spacy.CharacterEmbed.v2 {id="CharacterEmbed"} > #### Example config > @@ -207,7 +207,7 @@ network to construct a single vector to represent the information. | `nC` | The number of UTF-8 bytes to embed per word. Recommended values are between `3` and `8`, although it may depend on the length of words in the language. ~~int~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy.MaxoutWindowEncoder.v2 {#MaxoutWindowEncoder} +### spacy.MaxoutWindowEncoder.v2 {id="MaxoutWindowEncoder"} > #### Example config > @@ -231,7 +231,7 @@ and residual connections. | `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ | | **CREATES** | The model using the architecture. ~~Model[List[Floats2d], List[Floats2d]]~~ | -### spacy.MishWindowEncoder.v2 {#MishWindowEncoder} +### spacy.MishWindowEncoder.v2 {id="MishWindowEncoder"} > #### Example config > @@ -254,7 +254,7 @@ and residual connections. | `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ | | **CREATES** | The model using the architecture. ~~Model[List[Floats2d], List[Floats2d]]~~ | -### spacy.TorchBiLSTMEncoder.v1 {#TorchBiLSTMEncoder} +### spacy.TorchBiLSTMEncoder.v1 {id="TorchBiLSTMEncoder"} > #### Example config > @@ -276,7 +276,7 @@ Encode context using bidirectional LSTM layers. Requires | `dropout` | Creates a Dropout layer on the outputs of each LSTM layer except the last layer. Set to 0.0 to disable this functionality. ~~float~~ | | **CREATES** | The model using the architecture. ~~Model[List[Floats2d], List[Floats2d]]~~ | -### spacy.StaticVectors.v2 {#StaticVectors} +### spacy.StaticVectors.v2 {id="StaticVectors"} > #### Example config > @@ -306,7 +306,7 @@ mapped to a zero vector. See the documentation on | `key_attr` | Defaults to `"ORTH"`. ~~str~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Ragged]~~ | -### spacy.FeatureExtractor.v1 {#FeatureExtractor} +### spacy.FeatureExtractor.v1 {id="FeatureExtractor"} > #### Example config > @@ -324,7 +324,7 @@ of feature names to extract, which should refer to token attributes. | `columns` | The token attributes to extract. ~~List[Union[int, str]]~~ | | **CREATES** | The created feature extraction layer. ~~Model[List[Doc], List[Ints2d]]~~ | -## Transformer architectures {#transformers source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/architectures.py"} +## Transformer architectures {id="transformers",source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/architectures.py"} The following architectures are provided by the package [`spacy-transformers`](https://github.com/explosion/spacy-transformers). See the @@ -341,7 +341,7 @@ for details and system requirements. -### spacy-transformers.TransformerModel.v3 {#TransformerModel} +### spacy-transformers.TransformerModel.v3 {id="TransformerModel"} > #### Example Config > @@ -390,7 +390,7 @@ in other components, see | | | -Mixed-precision support is currently an experimental feature. + Mixed-precision support is currently an experimental feature. @@ -404,7 +404,7 @@ The other arguments are shared between all versions. -### spacy-transformers.TransformerListener.v1 {#TransformerListener} +### spacy-transformers.TransformerListener.v1 {id="TransformerListener"} > #### Example Config > @@ -434,7 +434,7 @@ a single token vector given zero or more wordpiece vectors. | `upstream` | A string to identify the "upstream" `Transformer` component to communicate with. By default, the upstream name is the wildcard string `"*"`, but you could also specify the name of the `Transformer` component. You'll almost never have multiple upstream `Transformer` components, so the wildcard string will almost always be fine. ~~str~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy-transformers.Tok2VecTransformer.v3 {#Tok2VecTransformer} +### spacy-transformers.Tok2VecTransformer.v3 {id="Tok2VecTransformer"} > #### Example Config > @@ -467,7 +467,7 @@ one component. | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -Mixed-precision support is currently an experimental feature. + Mixed-precision support is currently an experimental feature. @@ -481,7 +481,7 @@ The other arguments are shared between all versions. -## Pretraining architectures {#pretrain source="spacy/ml/models/multi_task.py"} +## Pretraining architectures {id="pretrain",source="spacy/ml/models/multi_task.py"} The spacy `pretrain` command lets you initialize a `Tok2Vec` layer in your pipeline with information from raw text. To this end, additional layers are @@ -494,7 +494,7 @@ BERT. For more information, see the section on [pretraining](/usage/embeddings-transformers#pretraining). -### spacy.PretrainVectors.v1 {#pretrain_vectors} +### spacy.PretrainVectors.v1 {id="pretrain_vectors"} > #### Example config > @@ -525,7 +525,7 @@ vectors. | `loss` | The loss function can be either "cosine" or "L2". We typically recommend to use "cosine". ~~~str~~ | | **CREATES** | A callable function that can create the Model, given the `vocab` of the pipeline and the `tok2vec` layer to pretrain. ~~Callable[[Vocab, Model], Model]~~ | -### spacy.PretrainCharacters.v1 {#pretrain_chars} +### spacy.PretrainCharacters.v1 {id="pretrain_chars"} > #### Example config > @@ -551,9 +551,9 @@ for a Tok2Vec layer. | `n_characters` | The window of characters - e.g. if `n_characters = 2`, the model will try to predict the first two and last two characters of the word. ~~int~~ | | **CREATES** | A callable function that can create the Model, given the `vocab` of the pipeline and the `tok2vec` layer to pretrain. ~~Callable[[Vocab, Model], Model]~~ | -## Parser & NER architectures {#parser} +## Parser & NER architectures {id="parser"} -### spacy.TransitionBasedParser.v2 {#TransitionBasedParser source="spacy/ml/models/parser.py"} +### spacy.TransitionBasedParser.v2 {id="TransitionBasedParser",source="spacy/ml/models/parser.py"} > #### Example Config > @@ -612,9 +612,9 @@ same signature, but the `use_upper` argument was `True` by default. -## Tagging architectures {#tagger source="spacy/ml/models/tagger.py"} +## Tagging architectures {id="tagger",source="spacy/ml/models/tagger.py"} -### spacy.Tagger.v2 {#Tagger} +### spacy.Tagger.v2 {id="Tagger"} > #### Example Config > @@ -648,7 +648,7 @@ The other arguments are shared between all versions. -## Text classification architectures {#textcat source="spacy/ml/models/textcat.py"} +## Text classification architectures {id="textcat",source="spacy/ml/models/textcat.py"} A text classification architecture needs to take a [`Doc`](/api/doc) as input, and produce a score for each potential label class. Textcat challenges can be @@ -672,7 +672,7 @@ single-label use-cases where `exclusive_classes = true`, while the -### spacy.TextCatEnsemble.v2 {#TextCatEnsemble} +### spacy.TextCatEnsemble.v2 {id="TextCatEnsemble"} > #### Example Config > @@ -737,7 +737,7 @@ but used an internal `tok2vec` instead of taking it as argument: -### spacy.TextCatCNN.v2 {#TextCatCNN} +### spacy.TextCatCNN.v2 {id="TextCatCNN"} > #### Example Config > @@ -777,7 +777,7 @@ after training. -### spacy.TextCatBOW.v2 {#TextCatBOW} +### spacy.TextCatBOW.v2 {id="TextCatBOW"} > #### Example Config > @@ -809,9 +809,9 @@ after training. -## Span classification architectures {#spancat source="spacy/ml/models/spancat.py"} +## Span classification architectures {id="spancat",source="spacy/ml/models/spancat.py"} -### spacy.SpanCategorizer.v1 {#SpanCategorizer} +### spacy.SpanCategorizer.v1 {id="SpanCategorizer"} > #### Example Config > @@ -848,7 +848,7 @@ single vector, and a scorer model to map the vectors to probabilities. | `scorer` | The scorer model. ~~Model[Floats2d, Floats2d]~~ | | **CREATES** | The model using the architecture. ~~Model[Tuple[List[Doc], Ragged], Floats2d]~~ | -### spacy.mean_max_reducer.v1 {#mean_max_reducer} +### spacy.mean_max_reducer.v1 {id="mean_max_reducer"} Reduce sequences by concatenating their mean and max pooled vectors, and then combine the concatenated vectors with a hidden layer. @@ -857,7 +857,7 @@ combine the concatenated vectors with a hidden layer. | ------------- | ------------------------------------- | | `hidden_size` | The size of the hidden layer. ~~int~~ | -## Entity linking architectures {#entitylinker source="spacy/ml/models/entity_linker.py"} +## Entity linking architectures {id="entitylinker",source="spacy/ml/models/entity_linker.py"} An [`EntityLinker`](/api/entitylinker) component disambiguates textual mentions (tagged as named entities) to unique identifiers, grounding the named entities @@ -870,7 +870,7 @@ into the "real world". This requires 3 main components: - A machine learning [`Model`](https://thinc.ai/docs/api-model) that picks the most plausible ID from the set of candidates. -### spacy.EntityLinker.v2 {#EntityLinker} +### spacy.EntityLinker.v2 {id="EntityLinker"} > #### Example Config > @@ -899,7 +899,7 @@ The `EntityLinker` model architecture is a Thinc `Model` with a | `nO` | Output dimension, determined by the length of the vectors encoding each entity in the KB. If the `nO` dimension is not set, the entity linking component will set it when `initialize` is called. ~~Optional[int]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | -### spacy.EmptyKB.v1 {#EmptyKB} +### spacy.EmptyKB.v1 {id="EmptyKB"} A function that creates an empty `KnowledgeBase` from a [`Vocab`](/api/vocab) instance. This is the default when a new entity linker component is created. @@ -908,7 +908,7 @@ instance. This is the default when a new entity linker component is created. | ---------------------- | ----------------------------------------------------------------------------------- | | `entity_vector_length` | The length of the vectors encoding each entity in the KB. Defaults to `64`. ~~int~~ | -### spacy.KBFromFile.v1 {#KBFromFile} +### spacy.KBFromFile.v1 {id="KBFromFile"} A function that reads an existing `KnowledgeBase` from file. @@ -916,7 +916,7 @@ A function that reads an existing `KnowledgeBase` from file. | --------- | -------------------------------------------------------- | | `kb_path` | The location of the KB that was stored to file. ~~Path~~ | -### spacy.CandidateGenerator.v1 {#CandidateGenerator} +### spacy.CandidateGenerator.v1 {id="CandidateGenerator"} A function that takes as input a [`KnowledgeBase`](/api/kb) and a [`Span`](/api/span) object denoting a named entity, and returns a list of @@ -924,7 +924,7 @@ plausible [`Candidate`](/api/kb/#candidate) objects. The default `CandidateGenerator` uses the text of a mention to find its potential aliases in the `KnowledgeBase`. Note that this function is case-dependent. -## Coreference {#coref-architectures tag="experimental"} +## Coreference {id="coref-architectures",tag="experimental"} A [`CoreferenceResolver`](/api/coref) component identifies tokens that refer to the same entity. A [`SpanResolver`](/api/span-resolver) component infers spans @@ -932,7 +932,7 @@ from single tokens. Together these components can be used to reproduce traditional coreference models. You can also omit the `SpanResolver` if working with only token-level clusters is acceptable. -### spacy-experimental.Coref.v1 {#Coref tag="experimental"} +### spacy-experimental.Coref.v1 {id="Coref",tag="experimental"} > #### Example Config > @@ -967,7 +967,7 @@ The `Coref` model architecture is a Thinc `Model`. | `antecedent_batch_size` | Internal batch size. ~~int~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | -### spacy-experimental.SpanResolver.v1 {#SpanResolver tag="experimental"} +### spacy-experimental.SpanResolver.v1 {id="SpanResolver",tag="experimental"} > #### Example Config > diff --git a/website/docs/api/attributeruler.md b/website/docs/api/attributeruler.mdx similarity index 94% rename from website/docs/api/attributeruler.md rename to website/docs/api/attributeruler.mdx index 965bffbcc..c18319187 100644 --- a/website/docs/api/attributeruler.md +++ b/website/docs/api/attributeruler.mdx @@ -2,7 +2,7 @@ title: AttributeRuler tag: class source: spacy/pipeline/attributeruler.py -new: 3 +version: 3 teaser: 'Pipeline component for rule-based token attribute assignment' api_string_name: attribute_ruler api_trainable: false @@ -15,7 +15,7 @@ between attributes such as mapping fine-grained POS tags to coarse-grained POS tags. See the [usage guide](/usage/linguistic-features/#mappings-exceptions) for examples. -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -37,7 +37,7 @@ how the component should be configured. You can override its settings via the %%GITHUB_SPACY/spacy/pipeline/attributeruler.py ``` -## AttributeRuler.\_\_init\_\_ {#init tag="method"} +## AttributeRuler.\_\_init\_\_ {id="init",tag="method"} Initialize the attribute ruler. @@ -56,7 +56,7 @@ Initialize the attribute ruler. | `validate` | Whether patterns should be validated (passed to the [`Matcher`](/api/matcher#init)). Defaults to `False`. ~~bool~~ | | `scorer` | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attributes `"tag`", `"pos"`, `"morph"` and `"lemma"` and [`Scorer.score_token_attr_per_feat`](/api/scorer#score_token_attr_per_feat) for the attribute `"morph"`. ~~Optional[Callable]~~ | -## AttributeRuler.\_\_call\_\_ {#call tag="method"} +## AttributeRuler.\_\_call\_\_ {id="call",tag="method"} Apply the attribute ruler to a `Doc`, setting token attributes for tokens matched by the provided patterns. @@ -66,7 +66,7 @@ matched by the provided patterns. | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## AttributeRuler.add {#add tag="method"} +## AttributeRuler.add {id="add",tag="method"} Add patterns to the attribute ruler. The patterns are a list of `Matcher` patterns and the attributes are a dict of attributes to set on the matched @@ -89,7 +89,7 @@ may be negative to index from the end of the span. | `attrs` | The attributes to assign to the target token in the matched span. ~~Dict[str, Any]~~ | | `index` | The index of the token in the matched span to modify. May be negative to index from the end of the span. Defaults to `0`. ~~int~~ | -## AttributeRuler.add_patterns {#add_patterns tag="method"} +## AttributeRuler.add_patterns {id="add_patterns",tag="method"} > #### Example > @@ -116,7 +116,7 @@ keys `"patterns"`, `"attrs"` and `"index"`, which match the arguments of | ---------- | -------------------------------------------------------------------------- | | `patterns` | The patterns to add. ~~Iterable[Dict[str, Union[List[dict], dict, int]]]~~ | -## AttributeRuler.patterns {#patterns tag="property"} +## AttributeRuler.patterns {id="patterns",tag="property"} Get all patterns that have been added to the attribute ruler in the `patterns_dict` format accepted by @@ -126,7 +126,7 @@ Get all patterns that have been added to the attribute ruler in the | ----------- | -------------------------------------------------------------------------------------------- | | **RETURNS** | The patterns added to the attribute ruler. ~~List[Dict[str, Union[List[dict], dict, int]]]~~ | -## AttributeRuler.initialize {#initialize tag="method"} +## AttributeRuler.initialize {id="initialize",tag="method"} Initialize the component with data and used before training to load in rules from a file. This method is typically called by @@ -160,7 +160,7 @@ config. | `tag_map` | The tag map that maps fine-grained tags to coarse-grained tags and morphological features. Defaults to `None`. ~~Optional[Dict[str, Dict[Union[int, str], Union[int, str]]]]~~ | | `morph_rules` | The morph rules that map token text and fine-grained tags to coarse-grained tags, lemmas and morphological features. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Dict[Union[int, str], Union[int, str]]]]]~~ | -## AttributeRuler.load_from_tag_map {#load_from_tag_map tag="method"} +## AttributeRuler.load_from_tag_map {id="load_from_tag_map",tag="method"} Load attribute ruler patterns from a tag map. @@ -168,7 +168,7 @@ Load attribute ruler patterns from a tag map. | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | | `tag_map` | The tag map that maps fine-grained tags to coarse-grained tags and morphological features. ~~Dict[str, Dict[Union[int, str], Union[int, str]]]~~ | -## AttributeRuler.load_from_morph_rules {#load_from_morph_rules tag="method"} +## AttributeRuler.load_from_morph_rules {id="load_from_morph_rules",tag="method"} Load attribute ruler patterns from morph rules. @@ -176,7 +176,7 @@ Load attribute ruler patterns from morph rules. | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `morph_rules` | The morph rules that map token text and fine-grained tags to coarse-grained tags, lemmas and morphological features. ~~Dict[str, Dict[str, Dict[Union[int, str], Union[int, str]]]]~~ | -## AttributeRuler.to_disk {#to_disk tag="method"} +## AttributeRuler.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -193,7 +193,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## AttributeRuler.from_disk {#from_disk tag="method"} +## AttributeRuler.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -211,7 +211,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `AttributeRuler` object. ~~AttributeRuler~~ | -## AttributeRuler.to_bytes {#to_bytes tag="method"} +## AttributeRuler.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -228,7 +228,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `AttributeRuler` object. ~~bytes~~ | -## AttributeRuler.from_bytes {#from_bytes tag="method"} +## AttributeRuler.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -247,7 +247,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `AttributeRuler` object. ~~AttributeRuler~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/attributes.md b/website/docs/api/attributes.mdx similarity index 98% rename from website/docs/api/attributes.md rename to website/docs/api/attributes.mdx index adacd3898..3142b741d 100644 --- a/website/docs/api/attributes.md +++ b/website/docs/api/attributes.mdx @@ -41,10 +41,9 @@ from string attribute names to internal attribute IDs is stored in The corresponding [`Token` object attributes](/api/token#attributes) can be accessed using the same names in lowercase, e.g. `token.orth` or `token.length`. -For attributes that represent string values, the internal integer ID is -accessed as `Token.attr`, e.g. `token.dep`, while the string value can be -retrieved by appending `_` as in `token.dep_`. - +For attributes that represent string values, the internal integer ID is accessed +as `Token.attr`, e.g. `token.dep`, while the string value can be retrieved by +appending `_` as in `token.dep_`. | Attribute | Description | | ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/website/docs/api/cli.md b/website/docs/api/cli.mdx similarity index 97% rename from website/docs/api/cli.md rename to website/docs/api/cli.mdx index 275e37ee0..8b84a02ff 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.mdx @@ -26,7 +26,7 @@ a list of available commands, you can type `python -m spacy --help`. You can also add the `--help` flag to any command or subcommand to see the description, available arguments and usage. -## download {#download tag="command"} +## download {id="download",tag="command"} Download [trained pipelines](/usage/models) for spaCy. The downloader finds the best-matching compatible version and uses `pip install` to download the Python @@ -44,7 +44,7 @@ pipeline name to be specified with its version (e.g. `en_core_web_sm-3.0.0`). > will also allow you to add it as a versioned package dependency to your > project. -```cli +```bash $ python -m spacy download [model] [--direct] [--sdist] [pip_args] ``` @@ -57,24 +57,24 @@ $ python -m spacy download [model] [--direct] [--sdist] [pip_args] | pip args | Additional installation options to be passed to `pip install` when installing the pipeline package. For example, `--user` to install to the user home directory or `--no-deps` to not install package dependencies. ~~Any (option/flag)~~ | | **CREATES** | The installed pipeline package in your `site-packages` directory. | -## info {#info tag="command"} +## info {id="info",tag="command"} Print information about your spaCy installation, trained pipelines and local setup, and generate [Markdown](https://en.wikipedia.org/wiki/Markdown)-formatted markup to copy-paste into [GitHub issues](https://github.com/explosion/spaCy/issues). -```cli +```bash $ python -m spacy info [--markdown] [--silent] [--exclude] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy info en_core_web_lg --markdown > ``` -```cli +```bash $ python -m spacy info [model] [--markdown] [--silent] [--exclude] ``` @@ -88,7 +88,7 @@ $ python -m spacy info [model] [--markdown] [--silent] [--exclude] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **PRINTS** | Information about your spaCy installation. | -## validate {#validate new="2" tag="command"} +## validate {id="validate",version="2",tag="command"} Find all trained pipeline packages installed in the current environment and check whether they are compatible with the currently installed version of spaCy. @@ -103,7 +103,7 @@ compatible versions and command for updating are shown. > suite, to ensure all packages are up to date before proceeding. If > incompatible packages are found, it will return `1`. -```cli +```bash $ python -m spacy validate ``` @@ -111,12 +111,12 @@ $ python -m spacy validate | ---------- | -------------------------------------------------------------------- | | **PRINTS** | Details about the compatibility of your installed pipeline packages. | -## init {#init new="3"} +## init {id="init",version="3"} The `spacy init` CLI includes helpful commands for initializing training config files and pipeline directories. -### init config {#init-config new="3" tag="command"} +### init config {id="init-config",version="3",tag="command"} Initialize and save a [`config.cfg` file](/usage/training#config) using the **recommended settings** for your use case. It works just like the @@ -128,11 +128,11 @@ customize those settings in your config file later. > #### Example > -> ```cli +> ```bash > $ python -m spacy init config config.cfg --lang en --pipeline ner,textcat --optimize accuracy > ``` -```cli +```bash $ python -m spacy init config [output_file] [--lang] [--pipeline] [--optimize] [--gpu] [--pretraining] [--force] ``` @@ -148,7 +148,7 @@ $ python -m spacy init config [output_file] [--lang] [--pipeline] [--optimize] [ | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | The config file for training. | -### init fill-config {#init-fill-config new="3"} +### init fill-config {id="init-fill-config",version="3"} Auto-fill a partial [.cfg file](/usage/training#config) with **all default values**, e.g. a config generated with the @@ -162,15 +162,15 @@ validation error with more details. > #### Example > -> ```cli +> ```bash > $ python -m spacy init fill-config base.cfg config.cfg --diff > ``` > > #### Example diff > -> ![Screenshot of visual diff in terminal](../images/cli_init_fill-config_diff.jpg) +> ![Screenshot of visual diff in terminal](/images/cli_init_fill-config_diff.jpg) -```cli +```bash $ python -m spacy init fill-config [base_path] [output_file] [--diff] ``` @@ -184,7 +184,7 @@ $ python -m spacy init fill-config [base_path] [output_file] [--diff] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | Complete and auto-filled config file for training. | -### init vectors {#init-vectors new="3" tag="command"} +### init vectors {id="init-vectors",version="3",tag="command"} Convert [word vectors](/usage/linguistic-features#vectors-similarity) for use with spaCy. Will export an `nlp` object that you can use in the @@ -199,7 +199,7 @@ This functionality was previously available as part of the command `init-model`. -```cli +```bash $ python -m spacy init vectors [lang] [vectors_loc] [output_dir] [--prune] [--truncate] [--name] [--verbose] ``` @@ -216,7 +216,7 @@ $ python -m spacy init vectors [lang] [vectors_loc] [output_dir] [--prune] [--tr | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | A spaCy pipeline directory containing the vocab and vectors. | -### init labels {#init-labels new="3" tag="command"} +### init labels {id="init-labels",version="3",tag="command"} Generate JSON files for the labels in the data. This helps speed up the training process, since spaCy won't have to preprocess the data to extract the labels. @@ -234,7 +234,7 @@ After generating the labels, you can provide them to components that accept a > path = "corpus/labels/ner.json > ``` -```cli +```bash $ python -m spacy init labels [config_path] [output_path] [--code] [--verbose] [--gpu-id] [overrides] ``` @@ -249,7 +249,7 @@ $ python -m spacy init labels [config_path] [output_path] [--code] [--verbose] [ | overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | | **CREATES** | The label files. | -## convert {#convert tag="command"} +## convert {id="convert",tag="command"} Convert files into spaCy's [binary training data format](/api/data-formats#binary-training), a serialized @@ -257,7 +257,7 @@ Convert files into spaCy's management functions. The converter can be specified on the command line, or chosen based on the file extension of the input file. -```cli +```bash $ python -m spacy convert [input_file] [output_dir] [--converter] [--file-type] [--n-sents] [--seg-sents] [--base] [--morphology] [--merge-subtokens] [--ner-map] [--lang] ``` @@ -278,7 +278,7 @@ $ python -m spacy convert [input_file] [output_dir] [--converter] [--file-type] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | Binary [`DocBin`](/api/docbin) training data that can be used with [`spacy train`](/api/cli#train). | -### Converters {#converters} +### Converters {id="converters"} | ID | Description | | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -288,12 +288,12 @@ $ python -m spacy convert [input_file] [output_dir] [--converter] [--file-type] | `ner` / `conll` | NER with IOB/IOB2/BILUO tags, one token per line with columns separated by whitespace. The first column is the token and the final column is the NER tag. Sentences are separated by blank lines and documents are separated by the line `-DOCSTART- -X- O O`. Supports CoNLL 2003 NER format. See [sample data](%%GITHUB_SPACY/extra/example_data/ner_example_data). | | `iob` | NER with IOB/IOB2/BILUO tags, one sentence per line with tokens separated by whitespace and annotation separated by `\|`, either `word\|B-ENT`or`word\|POS\|B-ENT`. See [sample data](%%GITHUB_SPACY/extra/example_data/ner_example_data). | -## debug {#debug new="3"} +## debug {id="debug",version="3"} The `spacy debug` CLI includes helpful commands for debugging and profiling your configs, data and implementations. -### debug config {#debug-config new="3" tag="command"} +### debug config {id="debug-config",version="3",tag="command"} Debug a [`config.cfg` file](/usage/training#config) and show validation errors. The command will create all objects in the tree and validate them. Note that @@ -303,13 +303,13 @@ errors at once and some issues are only shown once previous errors have been fixed. To auto-fill a partial config and save the result, you can use the [`init fill-config`](/api/cli#init-fill-config) command. -```cli +```bash $ python -m spacy debug config [config_path] [--code] [--show-functions] [--show-variables] [overrides] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy debug config config.cfg > ``` @@ -333,7 +333,7 @@ python -m spacy init fill-config tmp/starter-config_invalid.cfg tmp/starter-conf -```cli +```bash $ python -m spacy debug config ./config.cfg --show-functions --show-variables ``` @@ -453,7 +453,7 @@ File /path/to/thinc/thinc/schedules.py (line 91) | overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | | **PRINTS** | Config validation errors, if available. | -### debug data {#debug-data tag="command"} +### debug data {id="debug-data",tag="command"} Analyze, debug and validate your training and development data. Get useful stats, and find problems like invalid entity annotations, cyclic dependencies, @@ -479,13 +479,13 @@ the token distributions. To learn more, you can check out Papay et al.'s work on -```cli +```bash $ python -m spacy debug data [config_path] [--code] [--ignore-warnings] [--verbose] [--no-format] [overrides] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy debug data ./config.cfg > ``` @@ -639,7 +639,7 @@ will not be available. | overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | | **PRINTS** | Debugging information. | -### debug diff-config {#debug-diff tag="command"} +### debug diff-config {id="debug-diff",tag="command"} Show a diff of a config file with respect to spaCy's defaults or another config file. If additional settings were used in the creation of the config file, then @@ -647,13 +647,13 @@ you must supply these as extra parameters to the command when comparing to the default settings. The generated diff can also be used when posting to the discussion forum to provide more information for the maintainers. -```cli +```bash $ python -m spacy debug diff-config [config_path] [--compare-to] [--optimize] [--gpu] [--pretraining] [--markdown] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy debug diff-config ./config.cfg > ``` @@ -868,7 +868,7 @@ after_init = null | `markdown`, `-md` | Generate Markdown for Github issues. Defaults to `False`. ~~bool (flag)~~ | | **PRINTS** | Diff between the two config files. | -### debug profile {#debug-profile tag="command"} +### debug profile {id="debug-profile",tag="command"} Profile which functions take the most time in a spaCy pipeline. Input should be formatted as one JSON object per line with a key `"text"`. It can either be @@ -882,7 +882,7 @@ The `profile` command is now available as a subcommand of `spacy debug`. -```cli +```bash $ python -m spacy debug profile [model] [inputs] [--n-texts] ``` @@ -894,12 +894,12 @@ $ python -m spacy debug profile [model] [inputs] [--n-texts] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **PRINTS** | Profiling information for the pipeline. | -### debug model {#debug-model new="3" tag="command"} +### debug model {id="debug-model",version="3",tag="command"} Debug a Thinc [`Model`](https://thinc.ai/docs/api-model) by running it on a sample text and checking how it updates its internal weights and parameters. -```cli +```bash $ python -m spacy debug model [config_path] [component] [--layers] [--dimensions] [--parameters] [--gradients] [--attributes] [--print-step0] [--print-step1] [--print-step2] [--print-step3] [--gpu-id] ``` @@ -910,7 +910,7 @@ model ("Step 0"), which helps us to understand the internal structure of the Neural Network, and to focus on specific layers that we want to inspect further (see next example). -```cli +```bash $ python -m spacy debug model ./config.cfg tagger -P0 ``` @@ -956,7 +956,7 @@ an all-zero matrix determined by the `nO` and `nI` dimensions. After a first training step (Step 2), this matrix has clearly updated its values through the training feedback loop. -```cli +```bash $ python -m spacy debug model ./config.cfg tagger -l "5,15" -DIM -PAR -P0 -P1 -P2 ``` @@ -1017,7 +1017,7 @@ $ python -m spacy debug model ./config.cfg tagger -l "5,15" -DIM -PAR -P0 -P1 -P | overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | | **PRINTS** | Debugging information. | -## train {#train tag="command"} +## train {id="train",tag="command"} Train a pipeline. Expects data in spaCy's [binary format](/api/data-formats#training) and a @@ -1043,11 +1043,11 @@ in the section `[paths]`. > #### Example > -> ```cli +> ```bash > $ python -m spacy train config.cfg --output ./output --paths.train ./train --paths.dev ./dev > ``` -```cli +```bash $ python -m spacy train [config_path] [--output] [--code] [--verbose] [--gpu-id] [overrides] ``` @@ -1062,7 +1062,7 @@ $ python -m spacy train [config_path] [--output] [--code] [--verbose] [--gpu-id] | overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.train ./train.spacy`. ~~Any (option/flag)~~ | | **CREATES** | The final trained pipeline and the best trained pipeline. | -### Calling the training function from Python {#train-function new="3.2"} +### Calling the training function from Python {id="train-function",version="3.2"} The training CLI exposes a `train` helper function that lets you run the training just like `spacy train`. Usually it's easier to use the command line @@ -1085,7 +1085,7 @@ directly, but if you need to kick off training from code this is how to do it. | `use_gpu` | Which GPU to use. Defaults to -1 for no GPU. ~~int~~ | | `overrides` | Values to override config settings. ~~Dict[str, Any]~~ | -## pretrain {#pretrain new="2.1" tag="command,experimental"} +## pretrain {id="pretrain",version="2.1",tag="command,experimental"} Pretrain the "token to vector" ([`Tok2vec`](/api/tok2vec)) layer of pipeline components on raw text, using an approximate language-modeling objective. @@ -1113,11 +1113,11 @@ auto-generated by setting `--pretraining` on > #### Example > -> ```cli +> ```bash > $ python -m spacy pretrain config.cfg ./output_pretrain --paths.raw_text ./data.jsonl > ``` -```cli +```bash $ python -m spacy pretrain [config_path] [output_dir] [--code] [--resume-path] [--epoch-resume] [--gpu-id] [overrides] ``` @@ -1133,7 +1133,7 @@ $ python -m spacy pretrain [config_path] [output_dir] [--code] [--resume-path] [ | overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--training.dropout 0.2`. ~~Any (option/flag)~~ | | **CREATES** | The pretrained weights that can be used to initialize `spacy train`. | -## evaluate {#evaluate new="2" tag="command"} +## evaluate {id="evaluate",version="2",tag="command"} Evaluate a trained pipeline. Expects a loadable spaCy pipeline (package name or path) and evaluation data in the @@ -1146,7 +1146,7 @@ skew. To render a sample of dependency parses in a HTML file using the [displaCy visualizations](/usage/visualizers), set as output directory as the `--displacy-path` argument. -```cli +```bash $ python -m spacy evaluate [model] [data_path] [--output] [--code] [--gold-preproc] [--gpu-id] [--displacy-path] [--displacy-limit] ``` @@ -1163,7 +1163,7 @@ $ python -m spacy evaluate [model] [data_path] [--output] [--code] [--gold-prepr | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | Training results and optional metrics and visualizations. | -## apply {#apply new="3.5" tag="command"} +## apply {id="apply", version="3.5", tag="command"} Applies a trained pipeline to data and stores the resulting annotated documents in a `DocBin`. The input can be a single file or a directory. The recognized @@ -1194,7 +1194,8 @@ $ python -m spacy apply [model] [data-path] [output-file] [--code] [--text-key] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | A `DocBin` with the annotations from the `model` for all the files found in `data-path`. | -## find-threshold {#find-threshold new="3.5" tag="command"} + +## find-threshold {id="find-threshold",version="3.5",tag="command"} Runs prediction trials for a trained model with varying tresholds to maximize the specified metric. The search space for the threshold is traversed linearly @@ -1209,12 +1210,12 @@ be provided. > #### Examples > -> ```cli +> ```bash > # For textcat_multilabel: > $ python -m spacy find-threshold my_nlp data.spacy textcat_multilabel threshold cats_macro_f > ``` > -> ```cli +> ```bash > # For spancat: > $ python -m spacy find-threshold my_nlp data.spacy spancat threshold spans_sc_f > ``` @@ -1233,7 +1234,7 @@ be provided. | `--silent`, `-V`, `-VV` | GPU to use, if any. Defaults to `-1` for CPU. ~~int (option)~~ | | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | -## assemble {#assemble tag="command"} +## assemble {id="assemble",tag="command"} Assemble a pipeline from a config file without additional training. Expects a [config file](/api/data-formats#config) with all settings and hyperparameters. @@ -1243,11 +1244,11 @@ config. > #### Example > -> ```cli +> ```bash > $ python -m spacy assemble config.cfg ./output > ``` -```cli +```bash $ python -m spacy assemble [config_path] [output_dir] [--code] [--verbose] [overrides] ``` @@ -1261,7 +1262,7 @@ $ python -m spacy assemble [config_path] [output_dir] [--code] [--verbose] [over | overrides | Config parameters to override. Should be options starting with `--` that correspond to the config section and value to override, e.g. `--paths.data ./data`. ~~Any (option/flag)~~ | | **CREATES** | The final assembled pipeline. | -## package {#package tag="command"} +## package {id="package",tag="command"} Generate an installable [Python package](/usage/training#models-generating) from an existing pipeline data directory. All data files are copied over. If @@ -1287,13 +1288,13 @@ the sdist and wheel by setting `--build sdist,wheel`. -```cli +```bash $ python -m spacy package [input_dir] [output_dir] [--code] [--meta-path] [--create-meta] [--build] [--name] [--version] [--force] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy package /input /output > $ cd /output/en_pipeline-0.0.0 > $ pip install dist/en_pipeline-0.0.0.tar.gz @@ -1313,13 +1314,13 @@ $ python -m spacy package [input_dir] [output_dir] [--code] [--meta-path] [--cre | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | A Python package containing the spaCy pipeline. | -## project {#project new="3"} +## project {id="project",version="3"} The `spacy project` CLI includes subcommands for working with [spaCy projects](/usage/projects), end-to-end workflows for building and deploying custom spaCy pipelines. -### project clone {#project-clone tag="command"} +### project clone {id="project-clone",tag="command"} Clone a project template from a Git repository. Calls into `git` under the hood and can use the sparse checkout feature if available, so you're only downloading @@ -1328,19 +1329,19 @@ what you need. By default, spaCy's can provide any other repo (public or private) that you have access to using the `--repo` option. -```cli +```bash $ python -m spacy project clone [name] [dest] [--repo] [--branch] [--sparse] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy project clone pipelines/ner_wikiner > ``` > > Clone from custom repo: > -> ```cli +> ```bash > $ python -m spacy project clone template --repo https://github.com/your_org/your_repo > ``` @@ -1354,7 +1355,7 @@ $ python -m spacy project clone [name] [dest] [--repo] [--branch] [--sparse] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | The cloned [project directory](/usage/projects#project-files). | -### project assets {#project-assets tag="command"} +### project assets {id="project-assets",tag="command"} Fetch project assets like datasets and pretrained weights. Assets are defined in the `assets` section of the [`project.yml`](/usage/projects#project-yml). If a @@ -1365,13 +1366,13 @@ considered "private" and you have to take care of putting them into the destination directory yourself. If a local path is provided, the asset is copied into the current project. -```cli +```bash $ python -m spacy project assets [project_dir] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy project assets [--sparse] > ``` @@ -1382,7 +1383,7 @@ $ python -m spacy project assets [project_dir] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | Downloaded or copied assets defined in the `project.yml`. | -### project run {#project-run tag="command"} +### project run {id="project-run",tag="command"} Run a named command or workflow defined in the [`project.yml`](/usage/projects#project-yml). If a workflow name is specified, @@ -1391,13 +1392,13 @@ all commands in the workflow are run, in order. If commands define re-run if state has changed. For example, if the input dataset changes, a preprocessing command that depends on those files will be re-run. -```cli +```bash $ python -m spacy project run [subcommand] [project_dir] [--force] [--dry] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy project run train > ``` @@ -1410,7 +1411,7 @@ $ python -m spacy project run [subcommand] [project_dir] [--force] [--dry] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **EXECUTES** | The command defined in the `project.yml`. | -### project push {#project-push tag="command"} +### project push {id="project-push",tag="command"} Upload all available files or directories listed as in the `outputs` section of commands to a remote storage. Outputs are archived and compressed prior to @@ -1430,13 +1431,13 @@ remote storages, so you can use any protocol that `Pathy` supports, including filesystem, although you may need to install extra dependencies to use certain protocols. -```cli +```bash $ python -m spacy project push [remote] [project_dir] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy project push my_bucket > ``` > @@ -1453,7 +1454,7 @@ $ python -m spacy project push [remote] [project_dir] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **UPLOADS** | All project outputs that exist and are not already stored in the remote. | -### project pull {#project-pull tag="command"} +### project pull {id="project-pull",tag="command"} Download all files or directories listed as `outputs` for commands, unless they are not already present locally. When searching for files in the remote, `pull` @@ -1475,13 +1476,13 @@ remote storages, so you can use any protocol that `Pathy` supports, including filesystem, although you may need to install extra dependencies to use certain protocols. -```cli +```bash $ python -m spacy project pull [remote] [project_dir] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy project pull my_bucket > ``` > @@ -1498,7 +1499,7 @@ $ python -m spacy project pull [remote] [project_dir] | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **DOWNLOADS** | All project outputs that do not exist locally and can be found in the remote. | -### project document {#project-document tag="command"} +### project document {id="project-document",tag="command"} Auto-generate a pretty Markdown-formatted `README` for your project, based on its [`project.yml`](/usage/projects#project-yml). Will create sections that @@ -1507,13 +1508,13 @@ content will be placed between two hidden markers, so you can add your own custom content before or after the auto-generated documentation. When you re-run the `project document` command, only the auto-generated part is replaced. -```cli +```bash $ python -m spacy project document [project_dir] [--output] [--no-emoji] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy project document --output README.md > ``` @@ -1522,7 +1523,7 @@ $ python -m spacy project document [project_dir] [--output] [--no-emoji] For more examples, see the templates in our [`projects`](https://github.com/explosion/projects) repo. -![Screenshot of auto-generated Markdown Readme](../images/project_document.jpg) +![Screenshot of auto-generated Markdown Readme](/images/project_document.jpg) @@ -1533,7 +1534,7 @@ For more examples, see the templates in our | `--no-emoji`, `-NE` | Don't use emoji in the titles. ~~bool (flag)~~ | | **CREATES** | The Markdown-formatted project documentation. | -### project dvc {#project-dvc tag="command"} +### project dvc {id="project-dvc",tag="command"} Auto-generate [Data Version Control](https://dvc.org) (DVC) config file. Calls [`dvc run`](https://dvc.org/doc/command-reference/run) with `--no-exec` under @@ -1553,13 +1554,13 @@ You'll also need to add the assets you want to track with -```cli +```bash $ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] [--quiet] ``` > #### Example > -> ```cli +> ```bash > $ git init > $ dvc init > $ python -m spacy project dvc all @@ -1575,14 +1576,14 @@ $ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] [-- | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | A `dvc.yaml` file in the project directory, based on the steps defined in the given workflow. | -## huggingface-hub {#huggingface-hub new="3.1"} +## huggingface-hub {id="huggingface-hub",version="3.1"} The `spacy huggingface-cli` CLI includes commands for uploading your trained spaCy pipelines to the [Hugging Face Hub](https://huggingface.co/). > #### Installation > -> ```cli +> ```bash > $ pip install spacy-huggingface-hub > $ huggingface-cli login > ``` @@ -1596,19 +1597,19 @@ package installed. Installing the package will automatically add the -### huggingface-hub push {#huggingface-hub-push tag="command"} +### huggingface-hub push {id="huggingface-hub-push",tag="command"} Push a spaCy pipeline to the Hugging Face Hub. Expects a `.whl` file packaged with [`spacy package`](/api/cli#package) and `--build wheel`. For more details, see the spaCy project [integration](/usage/projects#huggingface_hub). -```cli +```bash $ python -m spacy huggingface-hub push [whl_path] [--org] [--msg] [--local-repo] [--verbose] ``` > #### Example > -> ```cli +> ```bash > $ python -m spacy huggingface-hub push en_ner_fashion-0.0.0-py3-none-any.whl > ``` diff --git a/website/docs/api/coref.md b/website/docs/api/coref.mdx similarity index 94% rename from website/docs/api/coref.md rename to website/docs/api/coref.mdx index 8f54422d6..8647f35d1 100644 --- a/website/docs/api/coref.md +++ b/website/docs/api/coref.mdx @@ -34,7 +34,7 @@ same thing. Clusters are represented as SpanGroups that start with a prefix A `CoreferenceResolver` component can be paired with a [`SpanResolver`](/api/span-resolver) to expand single tokens to spans. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions will be saved to `Doc.spans` as a [`SpanGroup`](/api/spangroup). The span key will be a prefix plus a serial number referring to the coreference @@ -47,7 +47,7 @@ parameter. | ------------------------------------------ | ------------------------------------------------------------------------------------------------------- | | `Doc.spans[prefix + "_" + cluster_number]` | One coreference cluster, represented as single-token spans. Cluster numbers start from 1. ~~SpanGroup~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -73,7 +73,7 @@ details on the architectures and their arguments and hyperparameters. | `model` | The [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. Defaults to [Coref](/api/architectures#Coref). ~~Model~~ | | `span_cluster_prefix` | The prefix for the keys for clusters saved to `doc.spans`. Defaults to `coref_clusters`. ~~str~~ | -## CoreferenceResolver.\_\_init\_\_ {#init tag="method"} +## CoreferenceResolver.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -102,7 +102,7 @@ shortcut for this and instantiate the component using its string name and | _keyword-only_ | | | `span_cluster_prefix` | The prefix for the key for saving clusters of spans. ~~bool~~ | -## CoreferenceResolver.\_\_call\_\_ {#call tag="method"} +## CoreferenceResolver.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -125,7 +125,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## CoreferenceResolver.pipe {#pipe tag="method"} +## CoreferenceResolver.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -148,7 +148,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/coref#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## CoreferenceResolver.initialize {#initialize tag="method"} +## CoreferenceResolver.initialize {id="initialize",tag="method"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -172,7 +172,7 @@ by [`Language.initialize`](/api/language#initialize). | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | -## CoreferenceResolver.predict {#predict tag="method"} +## CoreferenceResolver.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. Clusters are returned as a list of `MentionClusters`, one for @@ -192,7 +192,7 @@ to token indices. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The predicted coreference clusters for the `docs`. ~~List[MentionClusters]~~ | -## CoreferenceResolver.set_annotations {#set_annotations tag="method"} +## CoreferenceResolver.set_annotations {id="set_annotations",tag="method"} Modify a batch of documents, saving coreference clusters in `Doc.spans`. @@ -209,7 +209,7 @@ Modify a batch of documents, saving coreference clusters in `Doc.spans`. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `clusters` | The predicted coreference clusters for the `docs`. ~~List[MentionClusters]~~ | -## CoreferenceResolver.update {#update tag="method"} +## CoreferenceResolver.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects. Delegates to [`predict`](/api/coref#predict). @@ -231,7 +231,7 @@ Learn from a batch of [`Example`](/api/example) objects. Delegates to | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## CoreferenceResolver.create_optimizer {#create_optimizer tag="method"} +## CoreferenceResolver.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -246,7 +246,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## CoreferenceResolver.use_params {#use_params tag="method, contextmanager"} +## CoreferenceResolver.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -263,7 +263,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## CoreferenceResolver.to_disk {#to_disk tag="method"} +## CoreferenceResolver.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -280,7 +280,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## CoreferenceResolver.from_disk {#from_disk tag="method"} +## CoreferenceResolver.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -298,7 +298,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `CoreferenceResolver` object. ~~CoreferenceResolver~~ | -## CoreferenceResolver.to_bytes {#to_bytes tag="method"} +## CoreferenceResolver.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -315,7 +315,7 @@ Serialize the pipe to a bytestring, including the `KnowledgeBase`. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `CoreferenceResolver` object. ~~bytes~~ | -## CoreferenceResolver.from_bytes {#from_bytes tag="method"} +## CoreferenceResolver.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -334,7 +334,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `CoreferenceResolver` object. ~~CoreferenceResolver~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/corpus.md b/website/docs/api/corpus.mdx similarity index 96% rename from website/docs/api/corpus.md rename to website/docs/api/corpus.mdx index 88c4befd7..c58723e82 100644 --- a/website/docs/api/corpus.md +++ b/website/docs/api/corpus.mdx @@ -3,7 +3,7 @@ title: Corpus teaser: An annotated corpus tag: class source: spacy/training/corpus.py -new: 3 +version: 3 --- This class manages annotated corpora and can be used for training and @@ -13,7 +13,7 @@ customize the data loading during training, you can register your own see the usage guide on [data utilities](/usage/training#data) for more details and examples. -## Config and implementation {#config} +## Config and implementation {id="config"} `spacy.Corpus.v1` is a registered function that creates a `Corpus` of training or evaluation data. It takes the same arguments as the `Corpus` class and @@ -49,7 +49,7 @@ streaming. %%GITHUB_SPACY/spacy/training/corpus.py ``` -## Corpus.\_\_init\_\_ {#init tag="method"} +## Corpus.\_\_init\_\_ {id="init",tag="method"} Create a `Corpus` for iterating [Example](/api/example) objects from a file or directory of [`.spacy` data files](/api/data-formats#binary-training). The @@ -81,7 +81,7 @@ train/test skew. | `augmenter` | Optional data augmentation callback. ~~Callable[[Language, Example], Iterable[Example]]~~ | | `shuffle` | Whether to shuffle the examples. Defaults to `False`. ~~bool~~ | -## Corpus.\_\_call\_\_ {#call tag="method"} +## Corpus.\_\_call\_\_ {id="call",tag="method"} Yield examples from the data. @@ -101,7 +101,7 @@ Yield examples from the data. | `nlp` | The current `nlp` object. ~~Language~~ | | **YIELDS** | The examples. ~~Example~~ | -## JsonlCorpus {#jsonlcorpus tag="class"} +## JsonlCorpus {id="jsonlcorpus",tag="class"} Iterate Doc objects from a file or directory of JSONL (newline-delimited JSON) formatted raw text files. Can be used to read the raw text corpus for language @@ -120,14 +120,13 @@ file. > srsly.write_jsonl("/path/to/text.jsonl", data) > ``` -```json -### Example +```json {title="Example"} {"text": "Can I ask where you work now and what you do, and if you enjoy it?"} {"text": "They may just pull out of the Seattle market completely, at least until they have autonomous vehicles."} {"text": "My cynical view on this is that it will never be free to the public. Reason: what would be the draw of joining the military? Right now their selling point is free Healthcare and Education. Ironically both are run horribly and most, that I've talked to, come out wishing they never went in."} ``` -### JsonlCorpus.\_\init\_\_ {#jsonlcorpus tag="method"} +### JsonlCorpus.\_\_init\_\_ {id="jsonlcorpus",tag="method"} Initialize the reader. @@ -157,7 +156,7 @@ Initialize the reader. | `max_length` | Maximum document length (in tokens). Longer documents will be skipped. Defaults to `0`, which indicates no limit. ~~int~~ | | `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | -### JsonlCorpus.\_\_call\_\_ {#jsonlcorpus-call tag="method"} +### JsonlCorpus.\_\_call\_\_ {id="jsonlcorpus-call",tag="method"} Yield examples from the data. diff --git a/website/docs/api/cython-classes.md b/website/docs/api/cython-classes.mdx similarity index 91% rename from website/docs/api/cython-classes.md rename to website/docs/api/cython-classes.mdx index a4ecf294a..ce7c03940 100644 --- a/website/docs/api/cython-classes.md +++ b/website/docs/api/cython-classes.mdx @@ -9,7 +9,7 @@ menu: - ['StringStore', 'stringstore'] --- -## Doc {#doc tag="cdef class" source="spacy/tokens/doc.pxd"} +## Doc {id="doc",tag="cdef class",source="spacy/tokens/doc.pxd"} The `Doc` object holds an array of [`TokenC`](/api/cython-structs#tokenc) structs. @@ -21,7 +21,7 @@ accessed from Python. For the Python documentation, see [`Doc`](/api/doc). -### Attributes {#doc_attributes} +### Attributes {id="doc_attributes"} | Name | Description | | ------------ | -------------------------------------------------------------------------------------------------------- | @@ -31,7 +31,7 @@ accessed from Python. For the Python documentation, see [`Doc`](/api/doc). | `length` | The number of tokens in the document. ~~int~~ | | `max_length` | The underlying size of the `Doc.c` array. ~~int~~ | -### Doc.push_back {#doc_push_back tag="method"} +### Doc.push_back {id="doc_push_back",tag="method"} Append a token to the `Doc`. The token can be provided as a [`LexemeC`](/api/cython-structs#lexemec) or @@ -55,7 +55,7 @@ Append a token to the `Doc`. The token can be provided as a | `lex_or_tok` | The word to append to the `Doc`. ~~LexemeOrToken~~ | | `has_space` | Whether the word has trailing whitespace. ~~bint~~ | -## Token {#token tag="cdef class" source="spacy/tokens/token.pxd"} +## Token {id="token",tag="cdef class",source="spacy/tokens/token.pxd"} A Cython class providing access and methods for a [`TokenC`](/api/cython-structs#tokenc) struct. Note that the `Token` object does @@ -68,7 +68,7 @@ accessed from Python. For the Python documentation, see [`Token`](/api/token). -### Attributes {#token_attributes} +### Attributes {id="token_attributes"} | Name | Description | | ------- | -------------------------------------------------------------------------- | @@ -77,7 +77,7 @@ accessed from Python. For the Python documentation, see [`Token`](/api/token). | `i` | The offset of the token within the document. ~~int~~ | | `doc` | The parent document. ~~Doc~~ | -### Token.cinit {#token_cinit tag="method"} +### Token.cinit {id="token_cinit",tag="method"} Create a `Token` object from a `TokenC*` pointer. @@ -94,7 +94,7 @@ Create a `Token` object from a `TokenC*` pointer. | `offset` | The offset of the token within the document. ~~int~~ | | `doc` | The parent document. ~~int~~ | -## Span {#span tag="cdef class" source="spacy/tokens/span.pxd"} +## Span {id="span",tag="cdef class",source="spacy/tokens/span.pxd"} A Cython class providing access and methods for a slice of a `Doc` object. @@ -105,7 +105,7 @@ accessed from Python. For the Python documentation, see [`Span`](/api/span). -### Attributes {#span_attributes} +### Attributes {id="span_attributes"} | Name | Description | | ------------ | ----------------------------------------------------------------------------- | @@ -116,7 +116,7 @@ accessed from Python. For the Python documentation, see [`Span`](/api/span). | `end_char` | The index of the last character of the span. ~~int~~ | | `label` | A label to attach to the span, e.g. for named entities. ~~attr_t (uint64_t)~~ | -## Lexeme {#lexeme tag="cdef class" source="spacy/lexeme.pxd"} +## Lexeme {id="lexeme",tag="cdef class",source="spacy/lexeme.pxd"} A Cython class providing access and methods for an entry in the vocabulary. @@ -127,7 +127,7 @@ accessed from Python. For the Python documentation, see [`Lexeme`](/api/lexeme). -### Attributes {#lexeme_attributes} +### Attributes {id="lexeme_attributes"} | Name | Description | | ------- | ----------------------------------------------------------------------------- | @@ -135,7 +135,7 @@ accessed from Python. For the Python documentation, see [`Lexeme`](/api/lexeme). | `vocab` | A reference to the shared `Vocab` object. ~~Vocab~~ | | `orth` | ID of the verbatim text content. ~~attr_t (uint64_t)~~ | -## Vocab {#vocab tag="cdef class" source="spacy/vocab.pxd"} +## Vocab {id="vocab",tag="cdef class",source="spacy/vocab.pxd"} A Cython class providing access and methods for a vocabulary and other data shared across a language. @@ -147,7 +147,7 @@ accessed from Python. For the Python documentation, see [`Vocab`](/api/vocab). -### Attributes {#vocab_attributes} +### Attributes {id="vocab_attributes"} | Name | Description | | --------- | ---------------------------------------------------------------------------------------------------------- | @@ -155,7 +155,7 @@ accessed from Python. For the Python documentation, see [`Vocab`](/api/vocab). | `strings` | A `StringStore` that maps string to hash values and vice versa. ~~StringStore~~ | | `length` | The number of entries in the vocabulary. ~~int~~ | -### Vocab.get {#vocab_get tag="method"} +### Vocab.get {id="vocab_get",tag="method"} Retrieve a [`LexemeC*`](/api/cython-structs#lexemec) pointer from the vocabulary. @@ -172,7 +172,7 @@ vocabulary. | `string` | The string of the word to look up. ~~str~~ | | **RETURNS** | The lexeme in the vocabulary. ~~const LexemeC\*~~ | -### Vocab.get_by_orth {#vocab_get_by_orth tag="method"} +### Vocab.get_by_orth {id="vocab_get_by_orth",tag="method"} Retrieve a [`LexemeC*`](/api/cython-structs#lexemec) pointer from the vocabulary. @@ -189,7 +189,7 @@ vocabulary. | `orth` | ID of the verbatim text content. ~~attr_t (uint64_t)~~ | | **RETURNS** | The lexeme in the vocabulary. ~~const LexemeC\*~~ | -## StringStore {#stringstore tag="cdef class" source="spacy/strings.pxd"} +## StringStore {id="stringstore",tag="cdef class",source="spacy/strings.pxd"} A lookup table to retrieve strings by 64-bit hashes. @@ -201,7 +201,7 @@ accessed from Python. For the Python documentation, see -### Attributes {#stringstore_attributes} +### Attributes {id="stringstore_attributes"} | Name | Description | | ------ | ---------------------------------------------------------------------------------------------------------------- | diff --git a/website/docs/api/cython-structs.md b/website/docs/api/cython-structs.mdx similarity index 94% rename from website/docs/api/cython-structs.md rename to website/docs/api/cython-structs.mdx index 4c8514b64..106a27e90 100644 --- a/website/docs/api/cython-structs.md +++ b/website/docs/api/cython-structs.mdx @@ -7,7 +7,7 @@ menu: - ['LexemeC', 'lexemec'] --- -## TokenC {#tokenc tag="C struct" source="spacy/structs.pxd"} +## TokenC {id="tokenc",tag="C struct",source="spacy/structs.pxd"} Cython data container for the `Token` object. @@ -39,7 +39,7 @@ Cython data container for the `Token` object. | `ent_type` | Named entity type. ~~attr_t (uint64_t)~~ | | `ent_id` | ID of the entity the token is an instance of, if any. Currently not used, but potentially for coreference resolution. ~~attr_t (uint64_t)~~ | -### Token.get_struct_attr {#token_get_struct_attr tag="staticmethod, nogil" source="spacy/tokens/token.pxd"} +### Token.get_struct_attr {id="token_get_struct_attr",tag="staticmethod, nogil",source="spacy/tokens/token.pxd"} Get the value of an attribute from the `TokenC` struct by attribute ID. @@ -58,7 +58,7 @@ Get the value of an attribute from the `TokenC` struct by attribute ID. | `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | | **RETURNS** | The value of the attribute. ~~attr_t (uint64_t)~~ | -### Token.set_struct_attr {#token_set_struct_attr tag="staticmethod, nogil" source="spacy/tokens/token.pxd"} +### Token.set_struct_attr {id="token_set_struct_attr",tag="staticmethod, nogil",source="spacy/tokens/token.pxd"} Set the value of an attribute of the `TokenC` struct by attribute ID. @@ -78,7 +78,7 @@ Set the value of an attribute of the `TokenC` struct by attribute ID. | `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | | `value` | The value to set. ~~attr_t (uint64_t)~~ | -### token_by_start {#token_by_start tag="function" source="spacy/tokens/doc.pxd"} +### token_by_start {id="token_by_start",tag="function",source="spacy/tokens/doc.pxd"} Find a token in a `TokenC*` array by the offset of its first character. @@ -100,7 +100,7 @@ Find a token in a `TokenC*` array by the offset of its first character. | `start_char` | The start index to search for. ~~int~~ | | **RETURNS** | The index of the token in the array or `-1` if not found. ~~int~~ | -### token_by_end {#token_by_end tag="function" source="spacy/tokens/doc.pxd"} +### token_by_end {id="token_by_end",tag="function",source="spacy/tokens/doc.pxd"} Find a token in a `TokenC*` array by the offset of its final character. @@ -122,7 +122,7 @@ Find a token in a `TokenC*` array by the offset of its final character. | `end_char` | The end index to search for. ~~int~~ | | **RETURNS** | The index of the token in the array or `-1` if not found. ~~int~~ | -### set_children_from_heads {#set_children_from_heads tag="function" source="spacy/tokens/doc.pxd"} +### set_children_from_heads {id="set_children_from_heads",tag="function",source="spacy/tokens/doc.pxd"} Set attributes that allow lookup of syntactic children on a `TokenC*` array. This function must be called after making changes to the `TokenC.head` @@ -148,7 +148,7 @@ attribute, in order to make the parse tree navigation consistent. | `tokens` | A `TokenC*` array. ~~const TokenC\*~~ | | `length` | The number of tokens in the array. ~~int~~ | -## LexemeC {#lexemec tag="C struct" source="spacy/structs.pxd"} +## LexemeC {id="lexemec",tag="C struct",source="spacy/structs.pxd"} Struct holding information about a lexical type. `LexemeC` structs are usually owned by the `Vocab`, and accessed through a read-only pointer on the `TokenC` @@ -172,7 +172,7 @@ struct. | `prefix` | Length-N substring from the start of the lexeme. Defaults to `N=1`. ~~attr_t (uint64_t)~~ | | `suffix` | Length-N substring from the end of the lexeme. Defaults to `N=3`. ~~attr_t (uint64_t)~~ | -### Lexeme.get_struct_attr {#lexeme_get_struct_attr tag="staticmethod, nogil" source="spacy/lexeme.pxd"} +### Lexeme.get_struct_attr {id="lexeme_get_struct_attr",tag="staticmethod, nogil",source="spacy/lexeme.pxd"} Get the value of an attribute from the `LexemeC` struct by attribute ID. @@ -192,7 +192,7 @@ Get the value of an attribute from the `LexemeC` struct by attribute ID. | `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | | **RETURNS** | The value of the attribute. ~~attr_t (uint64_t)~~ | -### Lexeme.set_struct_attr {#lexeme_set_struct_attr tag="staticmethod, nogil" source="spacy/lexeme.pxd"} +### Lexeme.set_struct_attr {id="lexeme_set_struct_attr",tag="staticmethod, nogil",source="spacy/lexeme.pxd"} Set the value of an attribute of the `LexemeC` struct by attribute ID. @@ -212,7 +212,7 @@ Set the value of an attribute of the `LexemeC` struct by attribute ID. | `feat_name` | The ID of the attribute to look up. The attributes are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | | `value` | The value to set. ~~attr_t (uint64_t)~~ | -### Lexeme.c_check_flag {#lexeme_c_check_flag tag="staticmethod, nogil" source="spacy/lexeme.pxd"} +### Lexeme.c_check_flag {id="lexeme_c_check_flag",tag="staticmethod, nogil",source="spacy/lexeme.pxd"} Check the value of a binary flag attribute. @@ -232,7 +232,7 @@ Check the value of a binary flag attribute. | `flag_id` | The ID of the flag to look up. The flag IDs are enumerated in `spacy.typedefs`. ~~attr_id_t~~ | | **RETURNS** | The boolean value of the flag. ~~bint~~ | -### Lexeme.c_set_flag {#lexeme_c_set_flag tag="staticmethod, nogil" source="spacy/lexeme.pxd"} +### Lexeme.c_set_flag {id="lexeme_c_set_flag",tag="staticmethod, nogil",source="spacy/lexeme.pxd"} Set the value of a binary flag attribute. diff --git a/website/docs/api/cython.md b/website/docs/api/cython.mdx similarity index 99% rename from website/docs/api/cython.md rename to website/docs/api/cython.mdx index 16b11cead..764ff10f4 100644 --- a/website/docs/api/cython.md +++ b/website/docs/api/cython.mdx @@ -6,7 +6,7 @@ menu: - ['Conventions', 'conventions'] --- -## Overview {#overview hidden="true"} +## Overview {id="overview",hidden="true"} > #### What's Cython? > @@ -37,7 +37,7 @@ class holds a [`LexemeC`](/api/cython-structs#lexemec) struct, at `Lexeme.c`. This lets you shed the Python container, and pass a pointer to the underlying data into C-level functions. -## Conventions {#conventions} +## Conventions {id="conventions"} spaCy's core data structures are implemented as [Cython](http://cython.org/) `cdef` classes. Memory is managed through the diff --git a/website/docs/api/data-formats.md b/website/docs/api/data-formats.mdx similarity index 98% rename from website/docs/api/data-formats.md rename to website/docs/api/data-formats.mdx index 420e827a0..c9d88f87c 100644 --- a/website/docs/api/data-formats.md +++ b/website/docs/api/data-formats.mdx @@ -14,7 +14,7 @@ vocabulary data. For an overview of label schemes used by the models, see the [models directory](/models). Each trained pipeline documents the label schemes used in its components, depending on the data it was trained on. -## Training config {#config new="3"} +## Training config {id="config",version="3"} Config files define the training process and pipeline and can be passed to [`spacy train`](/api/cli#train). They use @@ -52,7 +52,7 @@ your config and check that it's valid, you can run the -### nlp {#config-nlp tag="section"} +### nlp {id="config-nlp",tag="section"} > #### Example > @@ -83,7 +83,7 @@ Defines the `nlp` object, its tokenizer and | `tokenizer` | The tokenizer to use. Defaults to [`Tokenizer`](/api/tokenizer). ~~Callable[[str], Doc]~~ | | `batch_size` | Default batch size for [`Language.pipe`](/api/language#pipe) and [`Language.evaluate`](/api/language#evaluate). ~~int~~ | -### components {#config-components tag="section"} +### components {id="config-components",tag="section"} > #### Example > @@ -106,7 +106,7 @@ function to use to create component) or a `source` (name of path of trained pipeline to copy components from). See the docs on [defining pipeline components](/usage/training#config-components) for details. -### paths, system {#config-variables tag="variables"} +### paths, system {id="config-variables",tag="variables"} These sections define variables that can be referenced across the other sections as variables. For example `${paths.train}` uses the value of `train` defined in @@ -116,11 +116,11 @@ need paths, you can define them here. All config values can also be [`spacy train`](/api/cli#train), which is especially relevant for data paths that you don't want to hard-code in your config file. -```cli +```bash $ python -m spacy train config.cfg --paths.train ./corpus/train.spacy ``` -### corpora {#config-corpora tag="section"} +### corpora {id="config-corpora",tag="section"} > #### Example > @@ -176,7 +176,7 @@ single corpus once and then divide it up into `train` and `dev` partitions. | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `corpora` | A dictionary keyed by string names, mapped to corpus functions that receive the current `nlp` object and return an iterator of [`Example`](/api/example) objects. ~~Dict[str, Callable[[Language], Iterator[Example]]]~~ | -### training {#config-training tag="section"} +### training {id="config-training",tag="section"} This section defines settings and controls for the training and evaluation process that are used when you run [`spacy train`](/api/cli#train). @@ -202,7 +202,7 @@ process that are used when you run [`spacy train`](/api/cli#train). | `seed` | The random seed. Defaults to variable `${system.seed}`. ~~int~~ | | `train_corpus` | Dot notation of the config location defining the train corpus. Defaults to `corpora.train`. ~~str~~ | -### pretraining {#config-pretraining tag="section,optional"} +### pretraining {id="config-pretraining",tag="section,optional"} This section is optional and defines settings and controls for [language model pretraining](/usage/embeddings-transformers#pretraining). It's @@ -220,7 +220,7 @@ used when you run [`spacy pretrain`](/api/cli#pretrain). | `component` | Component name to identify the layer with the model to pretrain. Defaults to `"tok2vec"`. ~~str~~ | | `layer` | The specific layer of the model to pretrain. If empty, the whole model will be used. ~~str~~ | -### initialize {#config-initialize tag="section"} +### initialize {id="config-initialize",tag="section"} This config block lets you define resources for **initializing the pipeline**. It's used by [`Language.initialize`](/api/language#initialize) and typically @@ -255,9 +255,9 @@ Also see the usage guides on the | `vectors` | Name or path of pipeline containing pretrained word vectors to use, e.g. created with [`init vectors`](/api/cli#init-vectors). Defaults to `null`. ~~Optional[str]~~ | | `vocab_data` | Path to JSONL-formatted [vocabulary file](/api/data-formats#vocab-jsonl) to initialize vocabulary. ~~Optional[str]~~ | -## Training data {#training} +## Training data {id="training"} -### Binary training format {#binary-training new="3"} +### Binary training format {id="binary-training",version="3"} > #### Example > @@ -288,7 +288,7 @@ Note that while this is the format used to save training data, you do not have to understand the internal details to use it or create training data. See the section on [preparing training data](/usage/training#training-data). -### JSON training format {#json-input tag="deprecated"} +### JSON training format {id="json-input",tag="deprecated"} @@ -300,7 +300,7 @@ objects to JSON, you can now serialize them directly using the [`spacy convert`](/api/cli) lets you convert your JSON data to the new `.spacy` format: -```cli +```bash $ python -m spacy convert ./data.json . ``` @@ -317,8 +317,7 @@ $ python -m spacy convert ./data.json . > [`offsets_to_biluo_tags`](/api/top-level#offsets_to_biluo_tags) function can > help you convert entity offsets to the right format. -```python -### Example structure +```python {title="Example structure"} [{ "id": int, # ID of the document within the corpus "paragraphs": [{ # list of paragraphs in the corpus @@ -357,7 +356,7 @@ https://github.com/explosion/spaCy/blob/v2.3.x/examples/training/training-data.j -### Annotation format for creating training examples {#dict-input} +### Annotation format for creating training examples {id="dict-input"} An [`Example`](/api/example) object holds the information for one training instance. It stores two [`Doc`](/api/doc) objects: one for holding the @@ -436,8 +435,7 @@ file to keep track of your settings and hyperparameters and your own -```python -### Examples +```python {title="Examples"} # Training data for a part-of-speech tagger doc = Doc(vocab, words=["I", "like", "stuff"]) gold_dict = {"tags": ["NOUN", "VERB", "NOUN"]} @@ -466,7 +464,7 @@ gold_dict = {"entities": [(0, 12, "PERSON")], example = Example.from_dict(doc, gold_dict) ``` -## Lexical data for vocabulary {#vocab-jsonl new="2"} +## Lexical data for vocabulary {id="vocab-jsonl",version="2"} This data file can be provided via the `vocab_data` setting in the `[initialize]` block of the training config to pre-define the lexical data to @@ -483,13 +481,11 @@ spaCy's [`Lexeme`](/api/lexeme#attributes) object. > vocab_data = "/path/to/vocab-data.jsonl" > ``` -```python -### First line +```python {title="First line"} {"lang": "en", "settings": {"oov_prob": -20.502029418945312}} ``` -```python -### Entry structure +```python {title="Entry structure"} { "orth": string, # the word text "id": int, # can correspond to row in vectors table @@ -526,7 +522,7 @@ Here's an example of the 20 most frequent lexemes in the English training data: %%GITHUB_SPACY/extra/example_data/vocab-data.jsonl ``` -## Pipeline meta {#meta} +## Pipeline meta {id="meta"} The pipeline meta is available as the file `meta.json` and exported automatically when you save an `nlp` object to disk. Its contents are available diff --git a/website/docs/api/dependencymatcher.md b/website/docs/api/dependencymatcher.mdx similarity index 96% rename from website/docs/api/dependencymatcher.md rename to website/docs/api/dependencymatcher.mdx index cae4221bf..390034a6c 100644 --- a/website/docs/api/dependencymatcher.md +++ b/website/docs/api/dependencymatcher.mdx @@ -2,7 +2,7 @@ title: DependencyMatcher teaser: Match subtrees within a dependency parse tag: class -new: 3 +version: 3 source: spacy/matcher/dependencymatcher.pyx --- @@ -14,7 +14,7 @@ It requires a pretrained [`DependencyParser`](/api/parser) or other component that sets the `Token.dep` and `Token.head` attributes. See the [usage guide](/usage/rule-based-matching#dependencymatcher) for examples. -## Pattern format {#patterns} +## Pattern format {id="patterns"} > ```python > ### Example @@ -62,7 +62,7 @@ of relations, see the usage guide on -### Operators {#operators} +### Operators {id="operators"} The following operators are supported by the `DependencyMatcher`, most of which come directly from @@ -87,8 +87,7 @@ come directly from | `A <++ B` | `B` is a right parent of `A`, i.e. `A` is a child of `B` and `A.i < B.i` _(not in Semgrex)_. | | `A <-- B` | `B` is a left parent of `A`, i.e. `A` is a child of `B` and `A.i > B.i` _(not in Semgrex)_. | - -## DependencyMatcher.\_\_init\_\_ {#init tag="method"} +## DependencyMatcher.\_\_init\_\_ {id="init",tag="method"} Create a `DependencyMatcher`. @@ -105,7 +104,7 @@ Create a `DependencyMatcher`. | _keyword-only_ | | | `validate` | Validate all patterns added to this matcher. ~~bool~~ | -## DependencyMatcher.\_\call\_\_ {#call tag="method"} +## DependencyMatcher.\_\_call\_\_ {id="call",tag="method"} Find all tokens matching the supplied patterns on the `Doc` or `Span`. @@ -127,7 +126,7 @@ Find all tokens matching the supplied patterns on the `Doc` or `Span`. | `doclike` | The `Doc` or `Span` to match over. ~~Union[Doc, Span]~~ | | **RETURNS** | A list of `(match_id, token_ids)` tuples, describing the matches. The `match_id` is the ID of the match pattern and `token_ids` is a list of token indices matched by the pattern, where the position of each token in the list corresponds to the position of the node specification in the pattern. ~~List[Tuple[int, List[int]]]~~ | -## DependencyMatcher.\_\_len\_\_ {#len tag="method"} +## DependencyMatcher.\_\_len\_\_ {id="len",tag="method"} Get the number of rules added to the dependency matcher. Note that this only returns the number of rules (identical with the number of IDs), not the number @@ -148,7 +147,7 @@ of individual patterns. | ----------- | ---------------------------- | | **RETURNS** | The number of rules. ~~int~~ | -## DependencyMatcher.\_\_contains\_\_ {#contains tag="method"} +## DependencyMatcher.\_\_contains\_\_ {id="contains",tag="method"} Check whether the matcher contains rules for a match ID. @@ -166,7 +165,7 @@ Check whether the matcher contains rules for a match ID. | `key` | The match ID. ~~str~~ | | **RETURNS** | Whether the matcher contains rules for this match ID. ~~bool~~ | -## DependencyMatcher.add {#add tag="method"} +## DependencyMatcher.add {id="add",tag="method"} Add a rule to the matcher, consisting of an ID key, one or more patterns, and an optional callback function to act on the matches. The callback function will @@ -191,7 +190,7 @@ will be overwritten. | _keyword-only_ | | | `on_match` | Callback function to act on matches. Takes the arguments `matcher`, `doc`, `i` and `matches`. ~~Optional[Callable[[DependencyMatcher, Doc, int, List[Tuple], Any]]~~ | -## DependencyMatcher.get {#get tag="method"} +## DependencyMatcher.get {id="get",tag="method"} Retrieve the pattern stored for a key. Returns the rule as an `(on_match, patterns)` tuple containing the callback and available patterns. @@ -208,7 +207,7 @@ Retrieve the pattern stored for a key. Returns the rule as an | `key` | The ID of the match rule. ~~str~~ | | **RETURNS** | The rule, as an `(on_match, patterns)` tuple. ~~Tuple[Optional[Callable], List[List[Union[Dict, Tuple]]]]~~ | -## DependencyMatcher.remove {#remove tag="method"} +## DependencyMatcher.remove {id="remove",tag="method"} Remove a rule from the dependency matcher. A `KeyError` is raised if the match ID does not exist. diff --git a/website/docs/api/dependencyparser.md b/website/docs/api/dependencyparser.mdx similarity index 95% rename from website/docs/api/dependencyparser.md rename to website/docs/api/dependencyparser.mdx index 27e315592..a6bc48cdf 100644 --- a/website/docs/api/dependencyparser.md +++ b/website/docs/api/dependencyparser.mdx @@ -25,7 +25,7 @@ current state. The weights are updated such that the scores assigned to the set of optimal actions is increased, while scores assigned to other actions are decreased. Note that more than one action may be optimal for a given state. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Dependency predictions are assigned to the `Token.dep` and `Token.head` fields. Beside the dependencies themselves, the parser decides sentence boundaries, @@ -39,7 +39,7 @@ which are saved in `Token.is_sent_start` and accessible via `Doc.sents`. | `Token.is_sent_start` | A boolean value indicating whether the token starts a sentence. After the parser runs this will be `True` or `False` for all tokens. ~~bool~~ | | `Doc.sents` | An iterator over sentences in the `Doc`, determined by `Token.is_sent_start` values. ~~Iterator[Span]~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -74,7 +74,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/dep_parser.pyx ``` -## DependencyParser.\_\_init\_\_ {#init tag="method"} +## DependencyParser.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -107,7 +107,7 @@ shortcut for this and instantiate the component using its string name and | `min_action_freq` | The minimum frequency of labelled actions to retain. Rarer labelled actions have their label backed-off to "dep". While this primarily affects the label accuracy, it can also affect the attachment structure, as the labels are used to represent the pseudo-projectivity transformation. ~~int~~ | | `scorer` | The scoring method. Defaults to [`Scorer.score_deps`](/api/scorer#score_deps) for the attribute `"dep"` ignoring the labels `p` and `punct` and [`Scorer.score_spans`](/api/scorer/#score_spans) for the attribute `"sents"`. ~~Optional[Callable]~~ | -## DependencyParser.\_\_call\_\_ {#call tag="method"} +## DependencyParser.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -131,7 +131,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## DependencyParser.pipe {#pipe tag="method"} +## DependencyParser.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -155,7 +155,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/dependencyparser#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## DependencyParser.initialize {#initialize tag="method" new="3"} +## DependencyParser.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -198,7 +198,7 @@ This method was previously called `begin_training`. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Dict[str, Dict[str, int]]]~~ | -## DependencyParser.predict {#predict tag="method"} +## DependencyParser.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -215,7 +215,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | A helper class for the parse state (internal). ~~StateClass~~ | -## DependencyParser.set_annotations {#set_annotations tag="method"} +## DependencyParser.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -232,7 +232,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `DependencyParser.predict`. Returns an internal helper class for the parse state. ~~List[StateClass]~~ | -## DependencyParser.update {#update tag="method"} +## DependencyParser.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects, updating the pipe's model. Delegates to [`predict`](/api/dependencyparser#predict) and @@ -255,7 +255,7 @@ model. Delegates to [`predict`](/api/dependencyparser#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## DependencyParser.get_loss {#get_loss tag="method"} +## DependencyParser.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -274,7 +274,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. ~~StateClass~~ | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## DependencyParser.create_optimizer {#create_optimizer tag="method"} +## DependencyParser.create_optimizer {id="create_optimizer",tag="method"} Create an [`Optimizer`](https://thinc.ai/docs/api-optimizers) for the pipeline component. @@ -290,7 +290,7 @@ component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## DependencyParser.use_params {#use_params tag="method, contextmanager"} +## DependencyParser.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -307,7 +307,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## DependencyParser.add_label {#add_label tag="method"} +## DependencyParser.add_label {id="add_label",tag="method"} Add a new label to the pipe. Note that you don't have to call this method if you provide a **representative data sample** to the [`initialize`](#initialize) @@ -327,7 +327,7 @@ to the model, and the output dimension will be | `label` | The label to add. ~~str~~ | | **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | -## DependencyParser.set_output {#set_output tag="method"} +## DependencyParser.set_output {id="set_output",tag="method"} Change the output dimension of the component's model by calling the model's attribute `resize_output`. This is a function that takes the original model and @@ -346,7 +346,7 @@ forgetting" problem. | ---- | --------------------------------- | | `nO` | The new output dimension. ~~int~~ | -## DependencyParser.to_disk {#to_disk tag="method"} +## DependencyParser.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -363,7 +363,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## DependencyParser.from_disk {#from_disk tag="method"} +## DependencyParser.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -381,7 +381,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `DependencyParser` object. ~~DependencyParser~~ | -## DependencyParser.to_bytes {#to_bytes tag="method"} +## DependencyParser.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -398,7 +398,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `DependencyParser` object. ~~bytes~~ | -## DependencyParser.from_bytes {#from_bytes tag="method"} +## DependencyParser.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -417,7 +417,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `DependencyParser` object. ~~DependencyParser~~ | -## DependencyParser.labels {#labels tag="property"} +## DependencyParser.labels {id="labels",tag="property"} The labels currently added to the component. @@ -432,7 +432,7 @@ The labels currently added to the component. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## DependencyParser.label_data {#label_data tag="property" new="3"} +## DependencyParser.label_data {id="label_data",tag="property",version="3"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -450,7 +450,7 @@ the model with a pre-defined label set. | ----------- | ------------------------------------------------------------------------------- | | **RETURNS** | The label data added to the component. ~~Dict[str, Dict[str, Dict[str, int]]]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/doc.md b/website/docs/api/doc.mdx similarity index 95% rename from website/docs/api/doc.md rename to website/docs/api/doc.mdx index 090489d83..a5f3de6be 100644 --- a/website/docs/api/doc.md +++ b/website/docs/api/doc.mdx @@ -12,7 +12,7 @@ compressed binary strings. The `Doc` object holds an array of [`Span`](/api/span) objects are views of this array, i.e. they don't own the data themselves. -## Doc.\_\_init\_\_ {#init tag="method"} +## Doc.\_\_init\_\_ {id="init",tag="method"} Construct a `Doc` object. The most common way to get a `Doc` object is via the `nlp` object. @@ -47,7 +47,7 @@ Construct a `Doc` object. The most common way to get a `Doc` object is via the | `sent_starts` 3 | A list of values, of the same length as `words`, to assign as `token.is_sent_start`. Will be overridden by heads if `heads` is provided. Defaults to `None`. ~~Optional[List[Union[bool, int, None]]]~~ | | `ents` 3 | A list of strings, of the same length of `words`, to assign the token-based IOB tag. Defaults to `None`. ~~Optional[List[str]]~~ | -## Doc.\_\_getitem\_\_ {#getitem tag="method"} +## Doc.\_\_getitem\_\_ {id="getitem",tag="method"} Get a [`Token`](/api/token) object at position `i`, where `i` is an integer. Negative indexing is supported, and follows the usual Python semantics, i.e. @@ -80,7 +80,7 @@ semantics. | `start_end` | The slice of the document to get. ~~Tuple[int, int]~~ | | **RETURNS** | The span at `doc[start:end]`. ~~Span~~ | -## Doc.\_\_iter\_\_ {#iter tag="method"} +## Doc.\_\_iter\_\_ {id="iter",tag="method"} Iterate over `Token` objects, from which the annotations can be easily accessed. @@ -100,7 +100,7 @@ underlying C data directly from Cython. | ---------- | --------------------------- | | **YIELDS** | A `Token` object. ~~Token~~ | -## Doc.\_\_len\_\_ {#len tag="method"} +## Doc.\_\_len\_\_ {id="len",tag="method"} Get the number of tokens in the document. @@ -115,7 +115,7 @@ Get the number of tokens in the document. | ----------- | --------------------------------------------- | | **RETURNS** | The number of tokens in the document. ~~int~~ | -## Doc.set_extension {#set_extension tag="classmethod" new="2"} +## Doc.set_extension {id="set_extension",tag="classmethod",version="2"} Define a custom attribute on the `Doc` which becomes available via `Doc._`. For details, see the documentation on @@ -140,7 +140,7 @@ details, see the documentation on | `setter` | Setter function that takes the `Doc` and a value, and modifies the object. Is called when the user writes to the `Doc._` attribute. ~~Optional[Callable[[Doc, Any], None]]~~ | | `force` | Force overwriting existing attribute. ~~bool~~ | -## Doc.get_extension {#get_extension tag="classmethod" new="2"} +## Doc.get_extension {id="get_extension",tag="classmethod",version="2"} Look up a previously registered extension by name. Returns a 4-tuple `(default, method, getter, setter)` if the extension is registered. Raises a @@ -160,7 +160,7 @@ Look up a previously registered extension by name. Returns a 4-tuple | `name` | Name of the extension. ~~str~~ | | **RETURNS** | A `(default, method, getter, setter)` tuple of the extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | -## Doc.has_extension {#has_extension tag="classmethod" new="2"} +## Doc.has_extension {id="has_extension",tag="classmethod",version="2"} Check whether an extension has been registered on the `Doc` class. @@ -177,7 +177,7 @@ Check whether an extension has been registered on the `Doc` class. | `name` | Name of the extension to check. ~~str~~ | | **RETURNS** | Whether the extension has been registered. ~~bool~~ | -## Doc.remove_extension {#remove_extension tag="classmethod" new="2.0.12"} +## Doc.remove_extension {id="remove_extension",tag="classmethod",version="2.0.12"} Remove a previously registered extension. @@ -195,7 +195,7 @@ Remove a previously registered extension. | `name` | Name of the extension. ~~str~~ | | **RETURNS** | A `(default, method, getter, setter)` tuple of the removed extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | -## Doc.char_span {#char_span tag="method" new="2"} +## Doc.char_span {id="char_span",tag="method",version="2"} Create a `Span` object from the slice `doc.text[start_idx:end_idx]`. Returns `None` if the character indices don't map to a valid span using the default @@ -219,7 +219,7 @@ alignment mode `"strict". | `alignment_mode` | How character indices snap to token boundaries. Options: `"strict"` (no snapping), `"contract"` (span of all tokens completely within the character span), `"expand"` (span of all tokens at least partially covered by the character span). Defaults to `"strict"`. ~~str~~ | | **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ | -## Doc.set_ents {#set_ents tag="method" new="3"} +## Doc.set_ents {id="set_ents",tag="method",version="3"} Set the named entities in the document. @@ -243,7 +243,7 @@ Set the named entities in the document. | `outside` | Spans outside of entities (O in IOB). ~~Optional[List[Span]]~~ | | `default` | How to set entity annotation for tokens outside of any provided spans. Options: `"blocked"`, `"missing"`, `"outside"` and `"unmodified"` (preserve current state). Defaults to `"outside"`. ~~str~~ | -## Doc.similarity {#similarity tag="method" model="vectors"} +## Doc.similarity {id="similarity",tag="method",model="vectors"} Make a semantic similarity estimate. The default estimate is cosine similarity using an average of word vectors. @@ -263,7 +263,7 @@ using an average of word vectors. | `other` | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ | | **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ | -## Doc.count_by {#count_by tag="method"} +## Doc.count_by {id="count_by",tag="method"} Count the frequencies of a given attribute. Produces a dict of `{attr (int): count (ints)}` frequencies, keyed by the values of the given @@ -284,7 +284,7 @@ attribute ID. | `attr_id` | The attribute ID. ~~int~~ | | **RETURNS** | A dictionary mapping attributes to integer counts. ~~Dict[int, int]~~ | -## Doc.get_lca_matrix {#get_lca_matrix tag="method"} +## Doc.get_lca_matrix {id="get_lca_matrix",tag="method"} Calculates the lowest common ancestor matrix for a given `Doc`. Returns LCA matrix containing the integer index of the ancestor, or `-1` if no common @@ -302,7 +302,7 @@ ancestor is found, e.g. if span excludes a necessary ancestor. | ----------- | -------------------------------------------------------------------------------------- | | **RETURNS** | The lowest common ancestor matrix of the `Doc`. ~~numpy.ndarray[ndim=2, dtype=int32]~~ | -## Doc.has_annotation {#has_annotation tag="method"} +## Doc.has_annotation {id="has_annotation",tag="method"} Check whether the doc contains annotation on a [`Token` attribute](/api/token#attributes). @@ -327,7 +327,7 @@ doc = nlp("This is a text") | `require_complete` | Whether to check that the attribute is set on every token in the doc. Defaults to `False`. ~~bool~~ | | **RETURNS** | Whether specified annotation is present in the doc. ~~bool~~ | -## Doc.to_array {#to_array tag="method"} +## Doc.to_array {id="to_array",tag="method"} Export given token attributes to a numpy `ndarray`. If `attr_ids` is a sequence of `M` attributes, the output array will be of shape `(N, M)`, where `N` is the @@ -355,7 +355,7 @@ Returns a 2D array with one row per token and one column per attribute (when | `attr_ids` | A list of attributes (int IDs or string names) or a single attribute (int ID or string name). ~~Union[int, str, List[Union[int, str]]]~~ | | **RETURNS** | The exported attributes as a numpy array. ~~Union[numpy.ndarray[ndim=2, dtype=uint64], numpy.ndarray[ndim=1, dtype=uint64]]~~ | -## Doc.from_array {#from_array tag="method"} +## Doc.from_array {id="from_array",tag="method"} Load attributes from a numpy array. Write to a `Doc` object, from an `(M, N)` array of attributes. @@ -379,7 +379,7 @@ array of attributes. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Doc` itself. ~~Doc~~ | -## Doc.from_docs {#from_docs tag="staticmethod" new="3"} +## Doc.from_docs {id="from_docs",tag="staticmethod",version="3"} Concatenate multiple `Doc` objects to form a new one. Raises an error if the `Doc` objects do not all share the same `Vocab`. @@ -408,7 +408,7 @@ Concatenate multiple `Doc` objects to form a new one. Raises an error if the | `exclude` 3.3 | String names of Doc attributes to exclude. Supported: `spans`, `tensor`, `user_data`. ~~Iterable[str]~~ | | **RETURNS** | The new `Doc` object that is containing the other docs or `None`, if `docs` is empty or `None`. ~~Optional[Doc]~~ | -## Doc.to_disk {#to_disk tag="method" new="2"} +## Doc.to_disk {id="to_disk",tag="method",version="2"} Save the current state to a directory. @@ -424,7 +424,7 @@ Save the current state to a directory. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Doc.from_disk {#from_disk tag="method" new="2"} +## Doc.from_disk {id="from_disk",tag="method",version="2"} Loads state from a directory. Modifies the object in place and returns it. @@ -443,7 +443,7 @@ Loads state from a directory. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Doc` object. ~~Doc~~ | -## Doc.to_bytes {#to_bytes tag="method"} +## Doc.to_bytes {id="to_bytes",tag="method"} Serialize, i.e. export the document contents to a binary string. @@ -460,7 +460,7 @@ Serialize, i.e. export the document contents to a binary string. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | A losslessly serialized copy of the `Doc`, including all annotations. ~~bytes~~ | -## Doc.from_bytes {#from_bytes tag="method"} +## Doc.from_bytes {id="from_bytes",tag="method"} Deserialize, i.e. import the document contents from a binary string. @@ -481,7 +481,7 @@ Deserialize, i.e. import the document contents from a binary string. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Doc` object. ~~Doc~~ | -## Doc.to_json {#to_json tag="method"} +## Doc.to_json {id="to_json",tag="method"} Serializes a document to JSON. Note that this is format differs from the deprecated [`JSON training format`](/api/data-formats#json-input). @@ -498,7 +498,7 @@ deprecated [`JSON training format`](/api/data-formats#json-input). | `underscore` | Optional list of string names of custom `Doc` attributes. Attribute values need to be JSON-serializable. Values will be added to an `"_"` key in the data, e.g. `"_": {"foo": "bar"}`. ~~Optional[List[str]]~~ | | **RETURNS** | The data in JSON format. ~~Dict[str, Any]~~ | -## Doc.from_json {#from_json tag="method" new="3.3.1"} +## Doc.from_json {id="from_json",tag="method",version="3.3.1"} Deserializes a document from JSON, i.e. generates a document from the provided JSON data as generated by [`Doc.to_json()`](/api/doc#to_json). @@ -520,7 +520,7 @@ JSON data as generated by [`Doc.to_json()`](/api/doc#to_json). | `validate` | Whether to validate the JSON input against the expected schema for detailed debugging. Defaults to `False`. ~~bool~~ | | **RETURNS** | A `Doc` corresponding to the provided JSON. ~~Doc~~ | -## Doc.retokenize {#retokenize tag="contextmanager" new="2.1"} +## Doc.retokenize {id="retokenize",tag="contextmanager",version="2.1"} Context manager to handle retokenization of the `Doc`. Modifications to the `Doc`'s tokenization are stored, and then made all at once when the context @@ -540,7 +540,7 @@ invalidated, although they may accidentally continue to work. | ----------- | -------------------------------- | | **RETURNS** | The retokenizer. ~~Retokenizer~~ | -### Retokenizer.merge {#retokenizer.merge tag="method"} +### Retokenizer.merge {id="retokenizer.merge",tag="method"} Mark a span for merging. The `attrs` will be applied to the resulting token (if they're context-dependent token attributes like `LEMMA` or `DEP`) or to the @@ -563,7 +563,7 @@ values. | `span` | The span to merge. ~~Span~~ | | `attrs` | Attributes to set on the merged token. ~~Dict[Union[str, int], Any]~~ | -### Retokenizer.split {#retokenizer.split tag="method"} +### Retokenizer.split {id="retokenizer.split",tag="method"} Mark a token for splitting, into the specified `orths`. The `heads` are required to specify how the new subtokens should be integrated into the dependency tree. @@ -599,7 +599,7 @@ underlying lexeme (if they're context-independent lexical attributes like | `heads` | List of `token` or `(token, subtoken)` tuples specifying the tokens to attach the newly split subtokens to. ~~List[Union[Token, Tuple[Token, int]]]~~ | | `attrs` | Attributes to set on all split tokens. Attribute names mapped to list of per-token attribute values. ~~Dict[Union[str, int], List[Any]]~~ | -## Doc.ents {#ents tag="property" model="NER"} +## Doc.ents {id="ents",tag="property",model="NER"} The named entities in the document. Returns a tuple of named entity `Span` objects, if the entity recognizer has been applied. @@ -617,7 +617,7 @@ objects, if the entity recognizer has been applied. | ----------- | ---------------------------------------------------------------- | | **RETURNS** | Entities in the document, one `Span` per entity. ~~Tuple[Span]~~ | -## Doc.spans {#spans tag="property"} +## Doc.spans {id="spans",tag="property"} A dictionary of named span groups, to store and access additional span annotations. You can write to it by assigning a list of [`Span`](/api/span) @@ -634,7 +634,7 @@ objects or a [`SpanGroup`](/api/spangroup) to a given key. | ----------- | ------------------------------------------------------------------ | | **RETURNS** | The span groups assigned to the document. ~~Dict[str, SpanGroup]~~ | -## Doc.cats {#cats tag="property" model="text classifier"} +## Doc.cats {id="cats",tag="property",model="text classifier"} Maps a label to a score for categories applied to the document. Typically set by the [`TextCategorizer`](/api/textcategorizer). @@ -650,7 +650,7 @@ the [`TextCategorizer`](/api/textcategorizer). | ----------- | ---------------------------------------------------------- | | **RETURNS** | The text categories mapped to scores. ~~Dict[str, float]~~ | -## Doc.noun_chunks {#noun_chunks tag="property" model="parser"} +## Doc.noun_chunks {id="noun_chunks",tag="property",model="parser"} Iterate over the base noun phrases in the document. Yields base noun-phrase `Span` objects, if the document has been syntactically parsed. A base noun @@ -677,7 +677,7 @@ implemented for the given language, a `NotImplementedError` is raised. | ---------- | ------------------------------------- | | **YIELDS** | Noun chunks in the document. ~~Span~~ | -## Doc.sents {#sents tag="property" model="sentences"} +## Doc.sents {id="sents",tag="property",model="sentences"} Iterate over the sentences in the document. Sentence spans have no label. @@ -699,7 +699,7 @@ will raise an error otherwise. | ---------- | ----------------------------------- | | **YIELDS** | Sentences in the document. ~~Span~~ | -## Doc.has_vector {#has_vector tag="property" model="vectors"} +## Doc.has_vector {id="has_vector",tag="property",model="vectors"} A boolean value indicating whether a word vector is associated with the object. @@ -714,7 +714,7 @@ A boolean value indicating whether a word vector is associated with the object. | ----------- | --------------------------------------------------------- | | **RETURNS** | Whether the document has a vector data attached. ~~bool~~ | -## Doc.vector {#vector tag="property" model="vectors"} +## Doc.vector {id="vector",tag="property",model="vectors"} A real-valued meaning representation. Defaults to an average of the token vectors. @@ -731,7 +731,7 @@ vectors. | ----------- | -------------------------------------------------------------------------------------------------- | | **RETURNS** | A 1-dimensional array representing the document's vector. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Doc.vector_norm {#vector_norm tag="property" model="vectors"} +## Doc.vector_norm {id="vector_norm",tag="property",model="vectors"} The L2 norm of the document's vector representation. @@ -749,7 +749,7 @@ The L2 norm of the document's vector representation. | ----------- | --------------------------------------------------- | | **RETURNS** | The L2 norm of the vector representation. ~~float~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | @@ -768,7 +768,7 @@ The L2 norm of the document's vector representation. | `has_unknown_spaces` | Whether the document was constructed without known spacing between tokens (typically when created from gold tokenization). ~~bool~~ | | `_` | User space for adding custom [attribute extensions](/usage/processing-pipelines#custom-components-attributes). ~~Underscore~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/docbin.md b/website/docs/api/docbin.mdx similarity index 93% rename from website/docs/api/docbin.md rename to website/docs/api/docbin.mdx index b1d1798ba..b5cf29df7 100644 --- a/website/docs/api/docbin.md +++ b/website/docs/api/docbin.mdx @@ -1,7 +1,7 @@ --- title: DocBin tag: class -new: 2.2 +version: 2.2 teaser: Pack Doc objects for binary serialization source: spacy/tokens/_serialize.py --- @@ -15,8 +15,7 @@ notable downside to this format is that you can't easily extract just one document from the `DocBin`. The serialization format is gzipped msgpack, where the msgpack object has the following structure: -```python -### msgpack object structure +```python {title="msgpack object structure"} { "version": str, # DocBin version number "attrs": List[uint64], # e.g. [TAG, HEAD, ENT_IOB, ENT_TYPE] @@ -33,7 +32,7 @@ object. This means the storage is more efficient if you pack more documents together, because you have less duplication in the strings. For usage examples, see the docs on [serializing `Doc` objects](/usage/saving-loading#docs). -## DocBin.\_\_init\_\_ {#init tag="method"} +## DocBin.\_\_init\_\_ {id="init",tag="method"} Create a `DocBin` object to hold serialized annotations. @@ -50,7 +49,7 @@ Create a `DocBin` object to hold serialized annotations. | `store_user_data` | Whether to write the `Doc.user_data` and the values of custom extension attributes to file/bytes. Defaults to `False`. ~~bool~~ | | `docs` | `Doc` objects to add on initialization. ~~Iterable[Doc]~~ | -## DocBin.\_\len\_\_ {#len tag="method"} +## DocBin.\_\_len\_\_ {id="len",tag="method"} Get the number of `Doc` objects that were added to the `DocBin`. @@ -67,7 +66,7 @@ Get the number of `Doc` objects that were added to the `DocBin`. | ----------- | --------------------------------------------------- | | **RETURNS** | The number of `Doc`s added to the `DocBin`. ~~int~~ | -## DocBin.add {#add tag="method"} +## DocBin.add {id="add",tag="method"} Add a `Doc`'s annotations to the `DocBin` for serialization. @@ -83,7 +82,7 @@ Add a `Doc`'s annotations to the `DocBin` for serialization. | -------- | -------------------------------- | | `doc` | The `Doc` object to add. ~~Doc~~ | -## DocBin.get_docs {#get_docs tag="method"} +## DocBin.get_docs {id="get_docs",tag="method"} Recover `Doc` objects from the annotations, using the given vocab. @@ -98,7 +97,7 @@ Recover `Doc` objects from the annotations, using the given vocab. | `vocab` | The shared vocab. ~~Vocab~~ | | **YIELDS** | The `Doc` objects. ~~Doc~~ | -## DocBin.merge {#merge tag="method"} +## DocBin.merge {id="merge",tag="method"} Extend the annotations of this `DocBin` with the annotations from another. Will raise an error if the pre-defined `attrs` of the two `DocBin`s don't match. @@ -118,7 +117,7 @@ raise an error if the pre-defined `attrs` of the two `DocBin`s don't match. | -------- | ------------------------------------------------------ | | `other` | The `DocBin` to merge into the current bin. ~~DocBin~~ | -## DocBin.to_bytes {#to_bytes tag="method"} +## DocBin.to_bytes {id="to_bytes",tag="method"} Serialize the `DocBin`'s annotations to a bytestring. @@ -134,7 +133,7 @@ Serialize the `DocBin`'s annotations to a bytestring. | ----------- | ---------------------------------- | | **RETURNS** | The serialized `DocBin`. ~~bytes~~ | -## DocBin.from_bytes {#from_bytes tag="method"} +## DocBin.from_bytes {id="from_bytes",tag="method"} Deserialize the `DocBin`'s annotations from a bytestring. @@ -150,7 +149,7 @@ Deserialize the `DocBin`'s annotations from a bytestring. | `bytes_data` | The data to load from. ~~bytes~~ | | **RETURNS** | The loaded `DocBin`. ~~DocBin~~ | -## DocBin.to_disk {#to_disk tag="method" new="3"} +## DocBin.to_disk {id="to_disk",tag="method",version="3"} Save the serialized `DocBin` to a file. Typically uses the `.spacy` extension and the result can be used as the input data for @@ -168,7 +167,7 @@ and the result can be used as the input data for | -------- | -------------------------------------------------------------------------- | | `path` | The file path, typically with the `.spacy` extension. ~~Union[str, Path]~~ | -## DocBin.from_disk {#from_disk tag="method" new="3"} +## DocBin.from_disk {id="from_disk",tag="method",version="3"} Load a serialized `DocBin` from a file. Typically uses the `.spacy` extension. diff --git a/website/docs/api/edittreelemmatizer.md b/website/docs/api/edittreelemmatizer.mdx similarity index 95% rename from website/docs/api/edittreelemmatizer.md rename to website/docs/api/edittreelemmatizer.mdx index 63e4bf910..82967482c 100644 --- a/website/docs/api/edittreelemmatizer.md +++ b/website/docs/api/edittreelemmatizer.mdx @@ -2,7 +2,7 @@ title: EditTreeLemmatizer tag: class source: spacy/pipeline/edit_tree_lemmatizer.py -new: 3.3 +version: 3.3 teaser: 'Pipeline component for lemmatization' api_base_class: /api/pipe api_string_name: trainable_lemmatizer @@ -18,7 +18,7 @@ and construction method used by this lemmatizer were proposed in For a lookup and rule-based lemmatizer, see [`Lemmatizer`](/api/lemmatizer). -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions are assigned to `Token.lemma`. @@ -27,7 +27,7 @@ Predictions are assigned to `Token.lemma`. | `Token.lemma` | The lemma (hash). ~~int~~ | | `Token.lemma_` | The lemma. ~~str~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -57,7 +57,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/edit_tree_lemmatizer.py ``` -## EditTreeLemmatizer.\_\_init\_\_ {#init tag="method"} +## EditTreeLemmatizer.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -90,7 +90,7 @@ shortcut for this and instantiate the component using its string name and | `top_k` | The number of most probable edit trees to try before resorting to `backoff`. Defaults to `1`. ~~int~~ | | `scorer` | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attribute `"lemma"`. ~~Optional[Callable]~~ | -## EditTreeLemmatizer.\_\_call\_\_ {#call tag="method"} +## EditTreeLemmatizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -114,7 +114,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## EditTreeLemmatizer.pipe {#pipe tag="method"} +## EditTreeLemmatizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -138,7 +138,7 @@ and [`pipe`](/api/edittreelemmatizer#pipe) delegate to the | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## EditTreeLemmatizer.initialize {#initialize tag="method" new="3"} +## EditTreeLemmatizer.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -175,7 +175,7 @@ config. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | -## EditTreeLemmatizer.predict {#predict tag="method"} +## EditTreeLemmatizer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -192,7 +192,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## EditTreeLemmatizer.set_annotations {#set_annotations tag="method"} +## EditTreeLemmatizer.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed tree identifiers. @@ -210,7 +210,7 @@ identifiers. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `tree_ids` | The identifiers of the edit trees to apply, produced by `EditTreeLemmatizer.predict`. | -## EditTreeLemmatizer.update {#update tag="method"} +## EditTreeLemmatizer.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -234,7 +234,7 @@ Delegates to [`predict`](/api/edittreelemmatizer#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## EditTreeLemmatizer.get_loss {#get_loss tag="method"} +## EditTreeLemmatizer.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -253,7 +253,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## EditTreeLemmatizer.create_optimizer {#create_optimizer tag="method"} +## EditTreeLemmatizer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -268,7 +268,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## EditTreeLemmatizer.use_params {#use_params tag="method, contextmanager"} +## EditTreeLemmatizer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -285,7 +285,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## EditTreeLemmatizer.to_disk {#to_disk tag="method"} +## EditTreeLemmatizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -302,7 +302,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## EditTreeLemmatizer.from_disk {#from_disk tag="method"} +## EditTreeLemmatizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -320,7 +320,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `EditTreeLemmatizer` object. ~~EditTreeLemmatizer~~ | -## EditTreeLemmatizer.to_bytes {#to_bytes tag="method"} +## EditTreeLemmatizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -337,7 +337,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `EditTreeLemmatizer` object. ~~bytes~~ | -## EditTreeLemmatizer.from_bytes {#from_bytes tag="method"} +## EditTreeLemmatizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -356,7 +356,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `EditTreeLemmatizer` object. ~~EditTreeLemmatizer~~ | -## EditTreeLemmatizer.labels {#labels tag="property"} +## EditTreeLemmatizer.labels {id="labels",tag="property"} The labels currently added to the component. @@ -371,7 +371,7 @@ identifiers of edit trees. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## EditTreeLemmatizer.label_data {#label_data tag="property" new="3"} +## EditTreeLemmatizer.label_data {id="label_data",tag="property",version="3"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -389,7 +389,7 @@ initialize the model with a pre-defined label set. | ----------- | ---------------------------------------------------------- | | **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/entitylinker.md b/website/docs/api/entitylinker.mdx similarity index 96% rename from website/docs/api/entitylinker.md rename to website/docs/api/entitylinker.mdx index 40ec8afb5..5c30d252e 100644 --- a/website/docs/api/entitylinker.md +++ b/website/docs/api/entitylinker.mdx @@ -2,7 +2,7 @@ title: EntityLinker tag: class source: spacy/pipeline/entity_linker.py -new: 2.2 +version: 2.2 teaser: 'Pipeline component for named entity linking and disambiguation' api_base_class: /api/pipe api_string_name: entity_linker @@ -17,7 +17,7 @@ and a machine learning model to pick the right candidate, given the local context of the mention. `EntityLinker` defaults to using the [`InMemoryLookupKB`](/api/kb_in_memory) implementation. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions, in the form of knowledge base IDs, will be assigned to `Token.ent_kb_id_`. @@ -27,7 +27,7 @@ Predictions, in the form of knowledge base IDs, will be assigned to | `Token.ent_kb_id` | Knowledge base ID (hash). ~~int~~ | | `Token.ent_kb_id_` | Knowledge base ID. ~~str~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -71,7 +71,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/entity_linker.py ``` -## EntityLinker.\_\_init\_\_ {#init tag="method"} +## EntityLinker.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -114,7 +114,7 @@ custom knowledge base, you should either call | `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_links`](/api/scorer#score_links). ~~Optional[Callable]~~ | | `threshold` 3.4 | Confidence threshold for entity predictions. The default of `None` implies that all predictions are accepted, otherwise those with a score beneath the treshold are discarded. If there are no predictions with scores above the threshold, the linked entity is `NIL`. ~~Optional[float]~~ | -## EntityLinker.\_\_call\_\_ {#call tag="method"} +## EntityLinker.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -137,7 +137,7 @@ delegate to the [`predict`](/api/entitylinker#predict) and | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## EntityLinker.pipe {#pipe tag="method"} +## EntityLinker.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -161,7 +161,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/entitylinker#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## EntityLinker.set_kb {#set_kb tag="method" new="3"} +## EntityLinker.set_kb {id="set_kb",tag="method",version="3"} The `kb_loader` should be a function that takes a `Vocab` instance and creates the `KnowledgeBase`, ensuring that the strings of the knowledge base are synced @@ -183,7 +183,7 @@ with the current vocab. | ----------- | ---------------------------------------------------------------------------------------------------------------- | | `kb_loader` | Function that creates a [`KnowledgeBase`](/api/kb) from a `Vocab` instance. ~~Callable[[Vocab], KnowledgeBase]~~ | -## EntityLinker.initialize {#initialize tag="method" new="3"} +## EntityLinker.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -219,7 +219,7 @@ This method was previously called `begin_training`. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `kb_loader` | Function that creates a [`KnowledgeBase`](/api/kb) from a `Vocab` instance. ~~Callable[[Vocab], KnowledgeBase]~~ | -## EntityLinker.predict {#predict tag="method"} +## EntityLinker.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. Returns the KB IDs for each entity in each doc, including `NIL` @@ -237,7 +237,7 @@ if there is no prediction. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The predicted KB identifiers for the entities in the `docs`. ~~List[str]~~ | -## EntityLinker.set_annotations {#set_annotations tag="method"} +## EntityLinker.set_annotations {id="set_annotations",tag="method"} Modify a batch of documents, using pre-computed entity IDs for a list of named entities. @@ -255,7 +255,7 @@ entities. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `kb_ids` | The knowledge base identifiers for the entities in the docs, predicted by `EntityLinker.predict`. ~~List[str]~~ | -## EntityLinker.update {#update tag="method"} +## EntityLinker.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects, updating both the pipe's entity linking model and context encoder. Delegates to @@ -278,7 +278,7 @@ pipe's entity linking model and context encoder. Delegates to | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## EntityLinker.create_optimizer {#create_optimizer tag="method"} +## EntityLinker.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -293,7 +293,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## EntityLinker.use_params {#use_params tag="method, contextmanager"} +## EntityLinker.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -310,7 +310,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## EntityLinker.to_disk {#to_disk tag="method"} +## EntityLinker.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -327,7 +327,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## EntityLinker.from_disk {#from_disk tag="method"} +## EntityLinker.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -345,7 +345,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `EntityLinker` object. ~~EntityLinker~~ | -## EntityLinker.to_bytes {#to_bytes tag="method"} +## EntityLinker.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -362,7 +362,7 @@ Serialize the pipe to a bytestring, including the `KnowledgeBase`. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `EntityLinker` object. ~~bytes~~ | -## EntityLinker.from_bytes {#from_bytes tag="method"} +## EntityLinker.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -381,7 +381,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `EntityLinker` object. ~~EntityLinker~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/entityrecognizer.md b/website/docs/api/entityrecognizer.mdx similarity index 95% rename from website/docs/api/entityrecognizer.md rename to website/docs/api/entityrecognizer.mdx index a535e8316..c80406a5b 100644 --- a/website/docs/api/entityrecognizer.md +++ b/website/docs/api/entityrecognizer.mdx @@ -20,7 +20,7 @@ your entities will be close to their initial tokens. If your entities are long and characterized by tokens in their middle, the component will likely not be a good fit for your task. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions will be saved to `Doc.ents` as a tuple. Each label will also be reflected to each underlying token, where it is saved in the `Token.ent_type` @@ -38,7 +38,7 @@ non-overlapping, or an error will be thrown. | `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ | | `Token.ent_type_` | The label part of the named entity tag. ~~str~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -72,7 +72,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/ner.pyx ``` -## EntityRecognizer.\_\_init\_\_ {#init tag="method"} +## EntityRecognizer.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -103,7 +103,7 @@ shortcut for this and instantiate the component using its string name and | `update_with_oracle_cut_size` | During training, cut long sequences into shorter segments by creating intermediate states based on the gold-standard history. The model is not very sensitive to this parameter, so you usually won't need to change it. Defaults to `100`. ~~int~~ | | `incorrect_spans_key` | Identifies spans that are known to be incorrect entity annotations. The incorrect entity annotations can be stored in the span group in [`Doc.spans`](/api/doc#spans), under this key. Defaults to `None`. ~~Optional[str]~~ | -## EntityRecognizer.\_\_call\_\_ {#call tag="method"} +## EntityRecognizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -127,7 +127,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## EntityRecognizer.pipe {#pipe tag="method"} +## EntityRecognizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -151,7 +151,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/entityrecognizer#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## EntityRecognizer.initialize {#initialize tag="method" new="3"} +## EntityRecognizer.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -194,7 +194,7 @@ This method was previously called `begin_training`. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Dict[str, Dict[str, int]]]~~ | -## EntityRecognizer.predict {#predict tag="method"} +## EntityRecognizer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -211,7 +211,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | A helper class for the parse state (internal). ~~StateClass~~ | -## EntityRecognizer.set_annotations {#set_annotations tag="method"} +## EntityRecognizer.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -228,7 +228,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `EntityRecognizer.predict`. Returns an internal helper class for the parse state. ~~List[StateClass]~~ | -## EntityRecognizer.update {#update tag="method"} +## EntityRecognizer.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects, updating the pipe's model. Delegates to [`predict`](/api/entityrecognizer#predict) and @@ -251,7 +251,7 @@ model. Delegates to [`predict`](/api/entityrecognizer#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## EntityRecognizer.get_loss {#get_loss tag="method"} +## EntityRecognizer.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -270,7 +270,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. ~~StateClass~~ | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## EntityRecognizer.create_optimizer {#create_optimizer tag="method"} +## EntityRecognizer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -285,7 +285,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## EntityRecognizer.use_params {#use_params tag="method, contextmanager"} +## EntityRecognizer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -302,7 +302,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## EntityRecognizer.add_label {#add_label tag="method"} +## EntityRecognizer.add_label {id="add_label",tag="method"} Add a new label to the pipe. Note that you don't have to call this method if you provide a **representative data sample** to the [`initialize`](#initialize) @@ -322,7 +322,7 @@ to the model, and the output dimension will be | `label` | The label to add. ~~str~~ | | **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | -## EntityRecognizer.set_output {#set_output tag="method"} +## EntityRecognizer.set_output {id="set_output",tag="method"} Change the output dimension of the component's model by calling the model's attribute `resize_output`. This is a function that takes the original model and @@ -341,7 +341,7 @@ forgetting" problem. | ---- | --------------------------------- | | `nO` | The new output dimension. ~~int~~ | -## EntityRecognizer.to_disk {#to_disk tag="method"} +## EntityRecognizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -358,7 +358,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## EntityRecognizer.from_disk {#from_disk tag="method"} +## EntityRecognizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -376,7 +376,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `EntityRecognizer` object. ~~EntityRecognizer~~ | -## EntityRecognizer.to_bytes {#to_bytes tag="method"} +## EntityRecognizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -393,7 +393,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `EntityRecognizer` object. ~~bytes~~ | -## EntityRecognizer.from_bytes {#from_bytes tag="method"} +## EntityRecognizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -412,7 +412,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `EntityRecognizer` object. ~~EntityRecognizer~~ | -## EntityRecognizer.labels {#labels tag="property"} +## EntityRecognizer.labels {id="labels",tag="property"} The labels currently added to the component. @@ -427,7 +427,7 @@ The labels currently added to the component. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## EntityRecognizer.label_data {#label_data tag="property" new="3"} +## EntityRecognizer.label_data {id="label_data",tag="property",version="3"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -445,7 +445,7 @@ the model with a pre-defined label set. | ----------- | ------------------------------------------------------------------------------- | | **RETURNS** | The label data added to the component. ~~Dict[str, Dict[str, Dict[str, int]]]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/entityruler.md b/website/docs/api/entityruler.mdx similarity index 94% rename from website/docs/api/entityruler.md rename to website/docs/api/entityruler.mdx index f15c648ff..27624398e 100644 --- a/website/docs/api/entityruler.md +++ b/website/docs/api/entityruler.mdx @@ -2,7 +2,7 @@ title: EntityRuler tag: class source: spacy/pipeline/entityruler.py -new: 2.1 +version: 2.1 teaser: 'Pipeline component for rule-based named entity recognition' api_string_name: entity_ruler api_trainable: false @@ -15,7 +15,7 @@ used on its own to implement a purely rule-based entity recognition system. For usage examples, see the docs on [rule-based entity recognition](/usage/rule-based-matching#entityruler). -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} This component assigns predictions basically the same way as the [`EntityRecognizer`](/api/entityrecognizer). @@ -36,7 +36,7 @@ non-overlapping, or an error will be thrown. | `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ | | `Token.ent_type_` | The label part of the named entity tag. ~~str~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -68,7 +68,7 @@ how the component should be configured. You can override its settings via the %%GITHUB_SPACY/spacy/pipeline/entityruler.py ``` -## EntityRuler.\_\_init\_\_ {#init tag="method"} +## EntityRuler.\_\_init\_\_ {id="init",tag="method"} Initialize the entity ruler. If patterns are supplied here, they need to be a list of dictionaries with a `"label"` and `"pattern"` key. A pattern can either @@ -99,7 +99,7 @@ be a token pattern (list) or a phrase pattern (string). For example: | `patterns` | Optional patterns to load in on initialization. ~~Optional[List[Dict[str, Union[str, List[dict]]]]]~~ | | `scorer` | The scoring method. Defaults to [`spacy.scorer.get_ner_prf`](/api/scorer#get_ner_prf). ~~Optional[Callable]~~ | -## EntityRuler.initialize {#initialize tag="method" new="3"} +## EntityRuler.initialize {id="initialize",tag="method",version="3"} Initialize the component with data and used before training to load in rules from a [pattern file](/usage/rule-based-matching/#entityruler-files). This @@ -131,7 +131,7 @@ config. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `patterns` | The list of patterns. Defaults to `None`. ~~Optional[Sequence[Dict[str, Union[str, List[Dict[str, Any]]]]]]~~ | -## EntityRuler.\_\len\_\_ {#len tag="method"} +## EntityRuler.\_\_len\_\_ {id="len",tag="method"} The number of all patterns added to the entity ruler. @@ -148,7 +148,7 @@ The number of all patterns added to the entity ruler. | ----------- | ------------------------------- | | **RETURNS** | The number of patterns. ~~int~~ | -## EntityRuler.\_\_contains\_\_ {#contains tag="method"} +## EntityRuler.\_\_contains\_\_ {id="contains",tag="method"} Whether a label is present in the patterns. @@ -166,7 +166,7 @@ Whether a label is present in the patterns. | `label` | The label to check. ~~str~~ | | **RETURNS** | Whether the entity ruler contains the label. ~~bool~~ | -## EntityRuler.\_\_call\_\_ {#call tag="method"} +## EntityRuler.\_\_call\_\_ {id="call",tag="method"} Find matches in the `Doc` and add them to the `doc.ents`. Typically, this happens automatically after the component has been added to the pipeline using @@ -192,7 +192,7 @@ is chosen. | `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | | **RETURNS** | The modified `Doc` with added entities, if available. ~~Doc~~ | -## EntityRuler.add_patterns {#add_patterns tag="method"} +## EntityRuler.add_patterns {id="add_patterns",tag="method"} Add patterns to the entity ruler. A pattern can either be a token pattern (list of dicts) or a phrase pattern (string). For more details, see the usage guide on @@ -213,7 +213,7 @@ of dicts) or a phrase pattern (string). For more details, see the usage guide on | ---------- | ---------------------------------------------------------------- | | `patterns` | The patterns to add. ~~List[Dict[str, Union[str, List[dict]]]]~~ | -## EntityRuler.remove {#remove tag="method" new="3.2.1"} +## EntityRuler.remove {id="remove",tag="method",version="3.2.1"} Remove a pattern by its ID from the entity ruler. A `ValueError` is raised if the ID does not exist. @@ -231,7 +231,7 @@ the ID does not exist. | ---- | ----------------------------------- | | `id` | The ID of the pattern rule. ~~str~~ | -## EntityRuler.to_disk {#to_disk tag="method"} +## EntityRuler.to_disk {id="to_disk",tag="method"} Save the entity ruler patterns to a directory. The patterns will be saved as newline-delimited JSON (JSONL). If a file with the suffix `.jsonl` is provided, @@ -250,7 +250,7 @@ only the patterns are saved as JSONL. If a directory name is provided, a | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------- | | `path` | A path to a JSONL file or directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | -## EntityRuler.from_disk {#from_disk tag="method"} +## EntityRuler.from_disk {id="from_disk",tag="method"} Load the entity ruler from a path. Expects either a file containing newline-delimited JSON (JSONL) with one entry per line, or a directory @@ -270,7 +270,7 @@ configuration. | `path` | A path to a JSONL file or directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | **RETURNS** | The modified `EntityRuler` object. ~~EntityRuler~~ | -## EntityRuler.to_bytes {#to_bytes tag="method"} +## EntityRuler.to_bytes {id="to_bytes",tag="method"} Serialize the entity ruler patterns to a bytestring. @@ -285,7 +285,7 @@ Serialize the entity ruler patterns to a bytestring. | ----------- | ---------------------------------- | | **RETURNS** | The serialized patterns. ~~bytes~~ | -## EntityRuler.from_bytes {#from_bytes tag="method"} +## EntityRuler.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -302,7 +302,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `bytes_data` | The bytestring to load. ~~bytes~~ | | **RETURNS** | The modified `EntityRuler` object. ~~EntityRuler~~ | -## EntityRuler.labels {#labels tag="property"} +## EntityRuler.labels {id="labels",tag="property"} All labels present in the match patterns. @@ -310,7 +310,7 @@ All labels present in the match patterns. | ----------- | -------------------------------------- | | **RETURNS** | The string labels. ~~Tuple[str, ...]~~ | -## EntityRuler.ent_ids {#ent_ids tag="property" new="2.2.2"} +## EntityRuler.ent_ids {id="ent_ids",tag="property",version="2.2.2"} All entity IDs present in the `id` properties of the match patterns. @@ -318,7 +318,7 @@ All entity IDs present in the `id` properties of the match patterns. | ----------- | ----------------------------------- | | **RETURNS** | The string IDs. ~~Tuple[str, ...]~~ | -## EntityRuler.patterns {#patterns tag="property"} +## EntityRuler.patterns {id="patterns",tag="property"} Get all patterns that were added to the entity ruler. @@ -326,7 +326,7 @@ Get all patterns that were added to the entity ruler. | ----------- | ---------------------------------------------------------------------------------------- | | **RETURNS** | The original patterns, one dictionary per pattern. ~~List[Dict[str, Union[str, dict]]]~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | ----------------- | --------------------------------------------------------------------------------------------------------------------- | diff --git a/website/docs/api/example.md b/website/docs/api/example.mdx similarity index 92% rename from website/docs/api/example.md rename to website/docs/api/example.mdx index 63768d58f..a29d5a7e0 100644 --- a/website/docs/api/example.md +++ b/website/docs/api/example.mdx @@ -3,7 +3,7 @@ title: Example teaser: A training instance tag: class source: spacy/training/example.pyx -new: 3.0 +version: 3.0 --- An `Example` holds the information for one training instance. It stores two @@ -12,7 +12,7 @@ holding the predictions of the pipeline. An [`Alignment`](/api/example#alignment-object) object stores the alignment between these two documents, as they can differ in tokenization. -## Example.\_\_init\_\_ {#init tag="method"} +## Example.\_\_init\_\_ {id="init",tag="method"} Construct an `Example` object from the `predicted` document and the `reference` document. If `alignment` is `None`, it will be initialized from the words in @@ -40,7 +40,7 @@ both documents. | _keyword-only_ | | | `alignment` | An object holding the alignment between the tokens of the `predicted` and `reference` documents. ~~Optional[Alignment]~~ | -## Example.from_dict {#from_dict tag="classmethod"} +## Example.from_dict {id="from_dict",tag="classmethod"} Construct an `Example` object from the `predicted` document and the reference annotations provided as a dictionary. For more details on the required format, @@ -64,7 +64,7 @@ see the [training format documentation](/api/data-formats#dict-input). | `example_dict` | The gold-standard annotations as a dictionary. Cannot be `None`. ~~Dict[str, Any]~~ | | **RETURNS** | The newly constructed object. ~~Example~~ | -## Example.text {#text tag="property"} +## Example.text {id="text",tag="property"} The text of the `predicted` document in this `Example`. @@ -78,7 +78,7 @@ The text of the `predicted` document in this `Example`. | ----------- | --------------------------------------------- | | **RETURNS** | The text of the `predicted` document. ~~str~~ | -## Example.predicted {#predicted tag="property"} +## Example.predicted {id="predicted",tag="property"} The `Doc` holding the predictions. Occasionally also referred to as `example.x`. @@ -94,7 +94,7 @@ The `Doc` holding the predictions. Occasionally also referred to as `example.x`. | ----------- | ------------------------------------------------------ | | **RETURNS** | The document containing (partial) predictions. ~~Doc~~ | -## Example.reference {#reference tag="property"} +## Example.reference {id="reference",tag="property"} The `Doc` holding the gold-standard annotations. Occasionally also referred to as `example.y`. @@ -111,7 +111,7 @@ as `example.y`. | ----------- | ---------------------------------------------------------- | | **RETURNS** | The document containing gold-standard annotations. ~~Doc~~ | -## Example.alignment {#alignment tag="property"} +## Example.alignment {id="alignment",tag="property"} The [`Alignment`](/api/example#alignment-object) object mapping the tokens of the `predicted` document to those of the `reference` document. @@ -131,7 +131,7 @@ the `predicted` document to those of the `reference` document. | ----------- | ---------------------------------------------------------------- | | **RETURNS** | The document containing gold-standard annotations. ~~Alignment~~ | -## Example.get_aligned {#get_aligned tag="method"} +## Example.get_aligned {id="get_aligned",tag="method"} Get the aligned view of a certain token attribute, denoted by its int ID or string name. @@ -152,7 +152,7 @@ string name. | `as_string` | Whether or not to return the list of values as strings. Defaults to `False`. ~~bool~~ | | **RETURNS** | List of integer values, or string values if `as_string` is `True`. ~~Union[List[int], List[str]]~~ | -## Example.get_aligned_parse {#get_aligned_parse tag="method"} +## Example.get_aligned_parse {id="get_aligned_parse",tag="method"} Get the aligned view of the dependency parse. If `projectivize` is set to `True`, non-projective dependency trees are made projective through the @@ -172,7 +172,7 @@ Pseudo-Projective Dependency Parsing algorithm by Nivre and Nilsson (2005). | `projectivize` | Whether or not to projectivize the dependency trees. Defaults to `True`. ~~bool~~ | | **RETURNS** | List of integer values, or string values if `as_string` is `True`. ~~Union[List[int], List[str]]~~ | -## Example.get_aligned_ner {#get_aligned_ner tag="method"} +## Example.get_aligned_ner {id="get_aligned_ner",tag="method"} Get the aligned view of the NER [BILUO](/usage/linguistic-features#accessing-ner) tags. @@ -193,7 +193,7 @@ Get the aligned view of the NER | ----------- | ------------------------------------------------------------------------------------------------- | | **RETURNS** | List of BILUO values, denoting whether tokens are part of an NER annotation or not. ~~List[str]~~ | -## Example.get_aligned_spans_y2x {#get_aligned_spans_y2x tag="method"} +## Example.get_aligned_spans_y2x {id="get_aligned_spans_y2x",tag="method"} Get the aligned view of any set of [`Span`](/api/span) objects defined over [`Example.reference`](/api/example#reference). The resulting span indices will @@ -219,7 +219,7 @@ align to the tokenization in [`Example.predicted`](/api/example#predicted). | `allow_overlap` | Whether the resulting `Span` objects may overlap or not. Set to `False` by default. ~~bool~~ | | **RETURNS** | `Span` objects aligned to the tokenization of `predicted`. ~~List[Span]~~ | -## Example.get_aligned_spans_x2y {#get_aligned_spans_x2y tag="method"} +## Example.get_aligned_spans_x2y {id="get_aligned_spans_x2y",tag="method"} Get the aligned view of any set of [`Span`](/api/span) objects defined over [`Example.predicted`](/api/example#predicted). The resulting span indices will @@ -247,7 +247,7 @@ against the original gold-standard annotation. | `allow_overlap` | Whether the resulting `Span` objects may overlap or not. Set to `False` by default. ~~bool~~ | | **RETURNS** | `Span` objects aligned to the tokenization of `reference`. ~~List[Span]~~ | -## Example.to_dict {#to_dict tag="method"} +## Example.to_dict {id="to_dict",tag="method"} Return a [dictionary representation](/api/data-formats#dict-input) of the reference annotation contained in this `Example`. @@ -262,7 +262,7 @@ reference annotation contained in this `Example`. | ----------- | ------------------------------------------------------------------------- | | **RETURNS** | Dictionary representation of the reference annotation. ~~Dict[str, Any]~~ | -## Example.split_sents {#split_sents tag="method"} +## Example.split_sents {id="split_sents",tag="method"} Split one `Example` into multiple `Example` objects, one for each sentence. @@ -282,15 +282,15 @@ Split one `Example` into multiple `Example` objects, one for each sentence. | ----------- | ---------------------------------------------------------------------------- | | **RETURNS** | List of `Example` objects, one for each original sentence. ~~List[Example]~~ | -## Alignment {#alignment-object new="3"} +## Alignment {id="alignment-object",version="3"} Calculate alignment tables between two tokenizations. -### Alignment attributes {#alignment-attributes"} +### Alignment attributes {id="alignment-attributes"} -Alignment attributes are managed using `AlignmentArray`, which is a -simplified version of Thinc's [Ragged](https://thinc.ai/docs/api-types#ragged) -type that only supports the `data` and `length` attributes. +Alignment attributes are managed using `AlignmentArray`, which is a simplified +version of Thinc's [Ragged](https://thinc.ai/docs/api-types#ragged) type that +only supports the `data` and `length` attributes. | Name | Description | | ----- | ------------------------------------------------------------------------------------- | @@ -321,7 +321,7 @@ tokenizations add up to the same string. For example, you'll be able to align > If `a2b.data[1] == a2b.data[2] == 1`, that means that `A[1]` (`"'"`) and > `A[2]` (`"s"`) both align to `B[1]` (`"'s"`). -### Alignment.from_strings {#classmethod tag="function"} +### Alignment.from_strings {id="classmethod",tag="function"} | Name | Description | | ----------- | ------------------------------------------------------------- | diff --git a/website/docs/api/index.md b/website/docs/api/index.mdx similarity index 58% rename from website/docs/api/index.md rename to website/docs/api/index.mdx index a9dc408f6..6c6e1fff4 100644 --- a/website/docs/api/index.md +++ b/website/docs/api/index.mdx @@ -3,6 +3,4 @@ title: Library Architecture next: /api/architectures --- -import Architecture101 from 'usage/101/\_architecture.md' - diff --git a/website/docs/api/kb.md b/website/docs/api/kb.mdx similarity index 92% rename from website/docs/api/kb.md rename to website/docs/api/kb.mdx index b217a1678..887b7fe97 100644 --- a/website/docs/api/kb.md +++ b/website/docs/api/kb.mdx @@ -5,7 +5,7 @@ teaser: (ontology) tag: class source: spacy/kb/kb.pyx -new: 2.2 +version: 2.2 --- The `KnowledgeBase` object is an abstract class providing a method to generate @@ -26,7 +26,7 @@ onwards. -## KnowledgeBase.\_\_init\_\_ {#init tag="method"} +## KnowledgeBase.\_\_init\_\_ {id="init",tag="method"} `KnowledgeBase` is an abstract class and cannot be instantiated. Its child classes should call `__init__()` to set up some necessary attributes. @@ -50,7 +50,7 @@ classes should call `__init__()` to set up some necessary attributes. | `vocab` | The shared vocabulary. ~~Vocab~~ | | `entity_vector_length` | Length of the fixed-size entity vectors. ~~int~~ | -## KnowledgeBase.entity_vector_length {#entity_vector_length tag="property"} +## KnowledgeBase.entity_vector_length {id="entity_vector_length",tag="property"} The length of the fixed-size entity vectors in the knowledge base. @@ -58,7 +58,7 @@ The length of the fixed-size entity vectors in the knowledge base. | ----------- | ------------------------------------------------ | | **RETURNS** | Length of the fixed-size entity vectors. ~~int~~ | -## KnowledgeBase.get_candidates {#get_candidates tag="method"} +## KnowledgeBase.get_candidates {id="get_candidates",tag="method"} Given a certain textual mention as input, retrieve a list of candidate entities of type [`Candidate`](/api/kb#candidate). @@ -77,7 +77,7 @@ of type [`Candidate`](/api/kb#candidate). | `mention` | The textual mention or alias. ~~Span~~ | | **RETURNS** | An iterable of relevant `Candidate` objects. ~~Iterable[Candidate]~~ | -## KnowledgeBase.get_candidates_batch {#get_candidates_batch tag="method"} +## KnowledgeBase.get_candidates_batch {id="get_candidates_batch",tag="method"} Same as [`get_candidates()`](/api/kb#get_candidates), but for an arbitrary number of mentions. The [`EntityLinker`](/api/entitylinker) component will call @@ -103,10 +103,10 @@ to you. | `mentions` | The textual mention or alias. ~~Iterable[Span]~~ | | **RETURNS** | An iterable of iterable with relevant `Candidate` objects. ~~Iterable[Iterable[Candidate]]~~ | -## KnowledgeBase.get_alias_candidates {#get_alias_candidates tag="method"} +## KnowledgeBase.get_alias_candidates {id="get_alias_candidates",tag="method"} -This method is _not_ available from spaCy 3.5 onwards. + This method is _not_ available from spaCy 3.5 onwards. From spaCy 3.5 on `KnowledgeBase` is an abstract class (with @@ -119,7 +119,7 @@ Note: [`InMemoryLookupKB.get_candidates()`](/api/kb_in_memory#get_candidates) defaults to [`InMemoryLookupKB.get_alias_candidates()`](/api/kb_in_memory#get_alias_candidates). -## KnowledgeBase.get_vector {#get_vector tag="method"} +## KnowledgeBase.get_vector {id="get_vector",tag="method"} Given a certain entity ID, retrieve its pretrained entity vector. @@ -134,7 +134,7 @@ Given a certain entity ID, retrieve its pretrained entity vector. | `entity` | The entity ID. ~~str~~ | | **RETURNS** | The entity vector. ~~Iterable[float]~~ | -## KnowledgeBase.get_vectors {#get_vectors tag="method"} +## KnowledgeBase.get_vectors {id="get_vectors",tag="method"} Same as [`get_vector()`](/api/kb#get_vector), but for an arbitrary number of entity IDs. @@ -154,7 +154,7 @@ entities at once, if performance is of concern to you. | `entities` | The entity IDs. ~~Iterable[str]~~ | | **RETURNS** | The entity vectors. ~~Iterable[Iterable[numpy.ndarray]]~~ | -## KnowledgeBase.to_disk {#to_disk tag="method"} +## KnowledgeBase.to_disk {id="to_disk",tag="method"} Save the current state of the knowledge base to a directory. @@ -169,7 +169,7 @@ Save the current state of the knowledge base to a directory. | `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | `exclude` | List of components to exclude. ~~Iterable[str]~~ | -## KnowledgeBase.from_disk {#from_disk tag="method"} +## KnowledgeBase.from_disk {id="from_disk",tag="method"} Restore the state of the knowledge base from a given directory. Note that the [`Vocab`](/api/vocab) should also be the same as the one used to create the KB. @@ -189,7 +189,7 @@ Restore the state of the knowledge base from a given directory. Note that the | `exclude` | List of components to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `KnowledgeBase` object. ~~KnowledgeBase~~ | -## Candidate {#candidate tag="class"} +## Candidate {id="candidate",tag="class"} A `Candidate` object refers to a textual mention (alias) that may or may not be resolved to a specific entity from a `KnowledgeBase`. This will be used as input @@ -197,7 +197,7 @@ for the entity linking algorithm which will disambiguate the various candidates to the correct one. Each candidate `(alias, entity)` pair is assigned to a certain prior probability. -### Candidate.\_\_init\_\_ {#candidate-init tag="method"} +### Candidate.\_\_init\_\_ {id="candidate-init",tag="method"} Construct a `Candidate` object. Usually this constructor is not called directly, but instead these objects are returned by the `get_candidates` method of the @@ -218,7 +218,7 @@ but instead these objects are returned by the `get_candidates` method of the | `alias_hash` | The hash of the textual mention or alias. ~~int~~ | | `prior_prob` | The prior probability of the `alias` referring to the `entity`. ~~float~~ | -## Candidate attributes {#candidate-attributes} +## Candidate attributes {id="candidate-attributes"} | Name | Description | | --------------- | ------------------------------------------------------------------------ | diff --git a/website/docs/api/kb_in_memory.md b/website/docs/api/kb_in_memory.mdx similarity index 90% rename from website/docs/api/kb_in_memory.md rename to website/docs/api/kb_in_memory.mdx index 9e3279e6a..e85b63c45 100644 --- a/website/docs/api/kb_in_memory.md +++ b/website/docs/api/kb_in_memory.mdx @@ -5,7 +5,7 @@ teaser: information in-memory. tag: class source: spacy/kb/kb_in_memory.pyx -new: 3.5 +version: 3.5 --- The `InMemoryLookupKB` class inherits from [`KnowledgeBase`](/api/kb) and @@ -14,7 +14,7 @@ implements all of its methods. It stores all KB data in-memory and generates entity names. It's highly optimized for both a low memory footprint and speed of retrieval. -## InMemoryLookupKB.\_\_init\_\_ {#init tag="method"} +## InMemoryLookupKB.\_\_init\_\_ {id="init",tag="method"} Create the knowledge base. @@ -31,7 +31,7 @@ Create the knowledge base. | `vocab` | The shared vocabulary. ~~Vocab~~ | | `entity_vector_length` | Length of the fixed-size entity vectors. ~~int~~ | -## InMemoryLookupKB.entity_vector_length {#entity_vector_length tag="property"} +## InMemoryLookupKB.entity_vector_length {id="entity_vector_length",tag="property"} The length of the fixed-size entity vectors in the knowledge base. @@ -39,7 +39,7 @@ The length of the fixed-size entity vectors in the knowledge base. | ----------- | ------------------------------------------------ | | **RETURNS** | Length of the fixed-size entity vectors. ~~int~~ | -## InMemoryLookupKB.add_entity {#add_entity tag="method"} +## InMemoryLookupKB.add_entity {id="add_entity",tag="method"} Add an entity to the knowledge base, specifying its corpus frequency and entity vector, which should be of length @@ -58,7 +58,7 @@ vector, which should be of length | `freq` | The frequency of the entity in a typical corpus. ~~float~~ | | `entity_vector` | The pretrained vector of the entity. ~~numpy.ndarray~~ | -## InMemoryLookupKB.set_entities {#set_entities tag="method"} +## InMemoryLookupKB.set_entities {id="set_entities",tag="method"} Define the full list of entities in the knowledge base, specifying the corpus frequency and entity vector for each entity. @@ -75,7 +75,7 @@ frequency and entity vector for each entity. | `freq_list` | List of entity frequencies. ~~Iterable[int]~~ | | `vector_list` | List of entity vectors. ~~Iterable[numpy.ndarray]~~ | -## InMemoryLookupKB.add_alias {#add_alias tag="method"} +## InMemoryLookupKB.add_alias {id="add_alias",tag="method"} Add an alias or mention to the knowledge base, specifying its potential KB identifiers and their prior probabilities. The entity identifiers should refer @@ -96,7 +96,7 @@ alias. | `entities` | The potential entities that the alias may refer to. ~~Iterable[Union[str, int]]~~ | | `probabilities` | The prior probabilities of each entity. ~~Iterable[float]~~ | -## InMemoryLookupKB.\_\_len\_\_ {#len tag="method"} +## InMemoryLookupKB.\_\_len\_\_ {id="len",tag="method"} Get the total number of entities in the knowledge base. @@ -110,7 +110,7 @@ Get the total number of entities in the knowledge base. | ----------- | ----------------------------------------------------- | | **RETURNS** | The number of entities in the knowledge base. ~~int~~ | -## InMemoryLookupKB.get_entity_strings {#get_entity_strings tag="method"} +## InMemoryLookupKB.get_entity_strings {id="get_entity_strings",tag="method"} Get a list of all entity IDs in the knowledge base. @@ -124,7 +124,7 @@ Get a list of all entity IDs in the knowledge base. | ----------- | --------------------------------------------------------- | | **RETURNS** | The list of entities in the knowledge base. ~~List[str]~~ | -## InMemoryLookupKB.get_size_aliases {#get_size_aliases tag="method"} +## InMemoryLookupKB.get_size_aliases {id="get_size_aliases",tag="method"} Get the total number of aliases in the knowledge base. @@ -138,7 +138,7 @@ Get the total number of aliases in the knowledge base. | ----------- | ---------------------------------------------------- | | **RETURNS** | The number of aliases in the knowledge base. ~~int~~ | -## InMemoryLookupKB.get_alias_strings {#get_alias_strings tag="method"} +## InMemoryLookupKB.get_alias_strings {id="get_alias_strings",tag="method"} Get a list of all aliases in the knowledge base. @@ -152,7 +152,7 @@ Get a list of all aliases in the knowledge base. | ----------- | -------------------------------------------------------- | | **RETURNS** | The list of aliases in the knowledge base. ~~List[str]~~ | -## InMemoryLookupKB.get_candidates {#get_candidates tag="method"} +## InMemoryLookupKB.get_candidates {id="get_candidates",tag="method"} Given a certain textual mention as input, retrieve a list of candidate entities of type [`Candidate`](/api/kb#candidate). Wraps @@ -172,7 +172,7 @@ of type [`Candidate`](/api/kb#candidate). Wraps | `mention` | The textual mention or alias. ~~Span~~ | | **RETURNS** | An iterable of relevant `Candidate` objects. ~~Iterable[Candidate]~~ | -## InMemoryLookupKB.get_candidates_batch {#get_candidates_batch tag="method"} +## InMemoryLookupKB.get_candidates_batch {id="get_candidates_batch",tag="method"} Same as [`get_candidates()`](/api/kb_in_memory#get_candidates), but for an arbitrary number of mentions. The [`EntityLinker`](/api/entitylinker) component @@ -198,7 +198,7 @@ to you. | `mentions` | The textual mention or alias. ~~Iterable[Span]~~ | | **RETURNS** | An iterable of iterable with relevant `Candidate` objects. ~~Iterable[Iterable[Candidate]]~~ | -## InMemoryLookupKB.get_alias_candidates {#get_alias_candidates tag="method"} +## InMemoryLookupKB.get_alias_candidates {id="get_alias_candidates",tag="method"} Given a certain textual mention as input, retrieve a list of candidate entities of type [`Candidate`](/api/kb#candidate). @@ -214,7 +214,7 @@ of type [`Candidate`](/api/kb#candidate). | `alias` | The textual mention or alias. ~~str~~ | | **RETURNS** | The list of relevant `Candidate` objects. ~~List[Candidate]~~ | -## InMemoryLookupKB.get_vector {#get_vector tag="method"} +## InMemoryLookupKB.get_vector {id="get_vector",tag="method"} Given a certain entity ID, retrieve its pretrained entity vector. @@ -229,7 +229,7 @@ Given a certain entity ID, retrieve its pretrained entity vector. | `entity` | The entity ID. ~~str~~ | | **RETURNS** | The entity vector. ~~numpy.ndarray~~ | -## InMemoryLookupKB.get_vectors {#get_vectors tag="method"} +## InMemoryLookupKB.get_vectors {id="get_vectors",tag="method"} Same as [`get_vector()`](/api/kb_in_memory#get_vector), but for an arbitrary number of entity IDs. @@ -249,7 +249,7 @@ entities at once, if performance is of concern to you. | `entities` | The entity IDs. ~~Iterable[str]~~ | | **RETURNS** | The entity vectors. ~~Iterable[Iterable[numpy.ndarray]]~~ | -## InMemoryLookupKB.get_prior_prob {#get_prior_prob tag="method"} +## InMemoryLookupKB.get_prior_prob {id="get_prior_prob",tag="method"} Given a certain entity ID and a certain textual mention, retrieve the prior probability of the fact that the mention links to the entity ID. @@ -266,7 +266,7 @@ probability of the fact that the mention links to the entity ID. | `alias` | The textual mention or alias. ~~str~~ | | **RETURNS** | The prior probability of the `alias` referring to the `entity`. ~~float~~ | -## InMemoryLookupKB.to_disk {#to_disk tag="method"} +## InMemoryLookupKB.to_disk {id="to_disk",tag="method"} Save the current state of the knowledge base to a directory. @@ -281,7 +281,7 @@ Save the current state of the knowledge base to a directory. | `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | `exclude` | List of components to exclude. ~~Iterable[str]~~ | -## InMemoryLookupKB.from_disk {#from_disk tag="method"} +## InMemoryLookupKB.from_disk {id="from_disk",tag="method"} Restore the state of the knowledge base from a given directory. Note that the [`Vocab`](/api/vocab) should also be the same as the one used to create the KB. diff --git a/website/docs/api/language.md b/website/docs/api/language.mdx similarity index 96% rename from website/docs/api/language.md rename to website/docs/api/language.mdx index ad0ac2a46..93ddd79a2 100644 --- a/website/docs/api/language.md +++ b/website/docs/api/language.mdx @@ -15,7 +15,7 @@ the tagger or parser that are called on a document in order. You can also add your own processing pipeline components that take a `Doc` object, modify it and return it. -## Language.\_\_init\_\_ {#init tag="method"} +## Language.\_\_init\_\_ {id="init",tag="method"} Initialize a `Language` object. Note that the `meta` is only used for meta information in [`Language.meta`](/api/language#meta) and not to configure the @@ -44,7 +44,7 @@ information in [`Language.meta`](/api/language#meta) and not to configure the | `create_tokenizer` | Optional function that receives the `nlp` object and returns a tokenizer. ~~Callable[[Language], Callable[[str], Doc]]~~ | | `batch_size` | Default batch size for [`pipe`](#pipe) and [`evaluate`](#evaluate). Defaults to `1000`. ~~int~~ | -## Language.from_config {#from_config tag="classmethod" new="3"} +## Language.from_config {id="from_config",tag="classmethod",version="3"} Create a `Language` object from a loaded config. Will set up the tokenizer and language data, add pipeline components based on the pipeline and add pipeline @@ -76,7 +76,7 @@ spaCy loads a model under the hood based on its | `validate` | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | | **RETURNS** | The initialized object. ~~Language~~ | -## Language.component {#component tag="classmethod" new="3"} +## Language.component {id="component",tag="classmethod",version="3"} Register a custom pipeline component under a given name. This allows initializing the component by name using @@ -112,7 +112,7 @@ decorator. For more details and examples, see the | `retokenizes` | Whether the component changes tokenization. Used for [pipe analysis](/usage/processing-pipelines#analysis). ~~bool~~ | | `func` | Optional function if not used as a decorator. ~~Optional[Callable[[Doc], Doc]]~~ | -## Language.factory {#factory tag="classmethod"} +## Language.factory {id="factory",tag="classmethod"} Register a custom pipeline component factory under a given name. This allows initializing the component by name using @@ -159,7 +159,7 @@ examples, see the | `default_score_weights` | The scores to report during training, and their default weight towards the final score used to select the best model. Weights should sum to `1.0` per component and will be combined and normalized for the whole pipeline. If a weight is set to `None`, the score will not be logged or weighted. ~~Dict[str, Optional[float]]~~ | | `func` | Optional function if not used as a decorator. ~~Optional[Callable[[...], Callable[[Doc], Doc]]]~~ | -## Language.\_\_call\_\_ {#call tag="method"} +## Language.\_\_call\_\_ {id="call",tag="method"} Apply the pipeline to some text. The text can span multiple sentences, and can contain arbitrary whitespace. Alignment into the original string is preserved. @@ -182,7 +182,7 @@ skipped, but the rest of the pipeline is run. | `component_cfg` | Optional dictionary of keyword arguments for components, keyed by component names. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Any]]]~~ | | **RETURNS** | A container for accessing the annotations. ~~Doc~~ | -## Language.pipe {#pipe tag="method"} +## Language.pipe {id="pipe",tag="method"} Process texts as a stream, and yield `Doc` objects in order. This is usually more efficient than processing texts one-by-one. @@ -209,7 +209,7 @@ tokenization is skipped but the rest of the pipeline is run. | `n_process` | Number of processors to use. Defaults to `1`. ~~int~~ | | **YIELDS** | Documents in the order of the original text. ~~Doc~~ | -## Language.set_error_handler {#set_error_handler tag="method" new="3"} +## Language.set_error_handler {id="set_error_handler",tag="method",version="3"} Define a callback that will be invoked when an error is thrown during processing of one or more documents. Specifically, this function will call @@ -231,7 +231,7 @@ being processed, and the original error. | --------------- | -------------------------------------------------------------------------------------------------------------- | | `error_handler` | A function that performs custom error handling. ~~Callable[[str, Callable[[Doc], Doc], List[Doc], Exception]~~ | -## Language.initialize {#initialize tag="method" new="3"} +## Language.initialize {id="initialize",tag="method",version="3"} Initialize the pipeline for training and return an [`Optimizer`](https://thinc.ai/docs/api-optimizers). Under the hood, it uses the @@ -282,7 +282,7 @@ objects. | `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## Language.resume_training {#resume_training tag="method,experimental" new="3"} +## Language.resume_training {id="resume_training",tag="method,experimental",version="3"} Continue training a trained pipeline. Create and return an optimizer, and initialize "rehearsal" for any pipeline component that has a `rehearse` method. @@ -304,7 +304,7 @@ a batch of [Example](/api/example) objects. | `sgd` | An optimizer. Will be created via [`create_optimizer`](#create_optimizer) if not set. ~~Optional[Optimizer]~~ | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## Language.update {#update tag="method"} +## Language.update {id="update",tag="method"} Update the models in the pipeline. @@ -342,7 +342,7 @@ and custom registered functions if needed. See the | `component_cfg` | Optional dictionary of keyword arguments for components, keyed by component names. Defaults to `None`. ~~Optional[Dict[str, Dict[str, Any]]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Language.rehearse {#rehearse tag="method,experimental" new="3"} +## Language.rehearse {id="rehearse",tag="method,experimental",version="3"} Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the current model to make predictions similar to an initial model, to try to address @@ -364,7 +364,7 @@ the "catastrophic forgetting" problem. This feature is experimental. | `losses` | Dictionary to update with the loss, keyed by pipeline component. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Language.evaluate {#evaluate tag="method"} +## Language.evaluate {id="evaluate",tag="method"} Evaluate a pipeline's components. @@ -392,7 +392,7 @@ objects instead of tuples of `Doc` and `GoldParse` objects. | `scorer_cfg` | Optional dictionary of keyword arguments for the `Scorer`. Defaults to `None`. ~~Optional[Dict[str, Any]]~~ | | **RETURNS** | A dictionary of evaluation scores. ~~Dict[str, Union[float, Dict[str, float]]]~~ | -## Language.use_params {#use_params tag="contextmanager, method"} +## Language.use_params {id="use_params",tag="contextmanager, method"} Replace weights of models in the pipeline with those provided in the params dictionary. Can be used as a context manager, in which case, models go back to @@ -409,7 +409,7 @@ their original weights after the block. | -------- | ------------------------------------------------------ | | `params` | A dictionary of parameters keyed by model ID. ~~dict~~ | -## Language.add_pipe {#add_pipe tag="method" new="2"} +## Language.add_pipe {id="add_pipe",tag="method",version="2"} Add a component to the processing pipeline. Expects a name that maps to a component factory registered using @@ -458,7 +458,7 @@ component, adds it to the pipeline and returns it. | `validate` 3 | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | | **RETURNS** | The pipeline component. ~~Callable[[Doc], Doc]~~ | -## Language.create_pipe {#create_pipe tag="method" new="2"} +## Language.create_pipe {id="create_pipe",tag="method",version="2"} Create a pipeline component from a factory. @@ -487,7 +487,7 @@ To create a component and add it to the pipeline, you should always use | `validate` 3 | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | | **RETURNS** | The pipeline component. ~~Callable[[Doc], Doc]~~ | -## Language.has_factory {#has_factory tag="classmethod" new="3"} +## Language.has_factory {id="has_factory",tag="classmethod",version="3"} Check whether a factory name is registered on the `Language` class or subclass. Will check for @@ -514,7 +514,7 @@ the `Language` base class, available to all subclasses. | `name` | Name of the pipeline factory to check. ~~str~~ | | **RETURNS** | Whether a factory of that name is registered on the class. ~~bool~~ | -## Language.has_pipe {#has_pipe tag="method" new="2"} +## Language.has_pipe {id="has_pipe",tag="method",version="2"} Check whether a component is present in the pipeline. Equivalent to `name in nlp.pipe_names`. @@ -536,7 +536,7 @@ Check whether a component is present in the pipeline. Equivalent to | `name` | Name of the pipeline component to check. ~~str~~ | | **RETURNS** | Whether a component of that name exists in the pipeline. ~~bool~~ | -## Language.get_pipe {#get_pipe tag="method" new="2"} +## Language.get_pipe {id="get_pipe",tag="method",version="2"} Get a pipeline component for a given component name. @@ -552,7 +552,7 @@ Get a pipeline component for a given component name. | `name` | Name of the pipeline component to get. ~~str~~ | | **RETURNS** | The pipeline component. ~~Callable[[Doc], Doc]~~ | -## Language.replace_pipe {#replace_pipe tag="method" new="2"} +## Language.replace_pipe {id="replace_pipe",tag="method",version="2"} Replace a component in the pipeline and return the new component. @@ -580,7 +580,7 @@ and instead expects the **name of a component factory** registered using | `validate` 3 | Whether to validate the component config and arguments against the types expected by the factory. Defaults to `True`. ~~bool~~ | | **RETURNS** | The new pipeline component. ~~Callable[[Doc], Doc]~~ | -## Language.rename_pipe {#rename_pipe tag="method" new="2"} +## Language.rename_pipe {id="rename_pipe",tag="method",version="2"} Rename a component in the pipeline. Useful to create custom names for pre-defined and pre-loaded components. To change the default name of a component @@ -598,7 +598,7 @@ added to the pipeline, you can also use the `name` argument on | `old_name` | Name of the component to rename. ~~str~~ | | `new_name` | New name of the component. ~~str~~ | -## Language.remove_pipe {#remove_pipe tag="method" new="2"} +## Language.remove_pipe {id="remove_pipe",tag="method",version="2"} Remove a component from the pipeline. Returns the removed component name and component function. @@ -615,7 +615,7 @@ component function. | `name` | Name of the component to remove. ~~str~~ | | **RETURNS** | A `(name, component)` tuple of the removed component. ~~Tuple[str, Callable[[Doc], Doc]]~~ | -## Language.disable_pipe {#disable_pipe tag="method" new="3"} +## Language.disable_pipe {id="disable_pipe",tag="method",version="3"} Temporarily disable a pipeline component so it's not run as part of the pipeline. Disabled components are listed in @@ -641,7 +641,7 @@ does nothing. | ------ | ----------------------------------------- | | `name` | Name of the component to disable. ~~str~~ | -## Language.enable_pipe {#enable_pipe tag="method" new="3"} +## Language.enable_pipe {id="enable_pipe",tag="method",version="3"} Enable a previously disabled component (e.g. via [`Language.disable_pipes`](/api/language#disable_pipes)) so it's run as part of @@ -663,7 +663,7 @@ already enabled, this method does nothing. | ------ | ---------------------------------------- | | `name` | Name of the component to enable. ~~str~~ | -## Language.select_pipes {#select_pipes tag="contextmanager, method" new="3"} +## Language.select_pipes {id="select_pipes",tag="contextmanager, method",version="3"} Disable one or more pipeline components. If used as a context manager, the pipeline will be restored to the initial state at the end of the block. @@ -706,7 +706,7 @@ As of spaCy v3.0, the `disable_pipes` method has been renamed to `select_pipes`: | `enable` | Name(s) of pipeline component(s) that will not be disabled. ~~Optional[Union[str, Iterable[str]]]~~ | | **RETURNS** | The disabled pipes that can be restored by calling the object's `.restore()` method. ~~DisabledPipes~~ | -## Language.get_factory_meta {#get_factory_meta tag="classmethod" new="3"} +## Language.get_factory_meta {id="get_factory_meta",tag="classmethod",version="3"} Get the factory meta information for a given pipeline component name. Expects the name of the component **factory**. The factory meta is an instance of the @@ -728,7 +728,7 @@ information about the component and its default provided by the | `name` | The factory name. ~~str~~ | | **RETURNS** | The factory meta. ~~FactoryMeta~~ | -## Language.get_pipe_meta {#get_pipe_meta tag="method" new="3"} +## Language.get_pipe_meta {id="get_pipe_meta",tag="method",version="3"} Get the factory meta information for a given pipeline component name. Expects the name of the component **instance** in the pipeline. The factory meta is an @@ -751,7 +751,7 @@ contains the information about the component and its default provided by the | `name` | The pipeline component name. ~~str~~ | | **RETURNS** | The factory meta. ~~FactoryMeta~~ | -## Language.analyze_pipes {#analyze_pipes tag="method" new="3"} +## Language.analyze_pipes {id="analyze_pipes",tag="method",version="3"} Analyze the current pipeline components and show a summary of the attributes they assign and require, and the scores they set. The data is based on the @@ -780,8 +780,7 @@ doesn't, the pipeline analysis won't catch that. -```json -### Structured +```json {title="Structured"} { "summary": { "tagger": { @@ -799,7 +798,12 @@ doesn't, the pipeline analysis won't catch that. }, "problems": { "tagger": [], - "entity_linker": ["doc.ents", "doc.sents", "token.ent_iob", "token.ent_type"] + "entity_linker": [ + "doc.ents", + "doc.sents", + "token.ent_iob", + "token.ent_type" + ] }, "attrs": { "token.ent_iob": { "assigns": [], "requires": ["entity_linker"] }, @@ -840,7 +844,7 @@ token.ent_iob, token.ent_type | `pretty` | Pretty-print the results as a table. Defaults to `False`. ~~bool~~ | | **RETURNS** | Dictionary containing the pipe analysis, keyed by `"summary"` (component meta by pipe), `"problems"` (attribute names by pipe) and `"attrs"` (pipes that assign and require an attribute, keyed by attribute). ~~Optional[Dict[str, Any]]~~ | -## Language.replace_listeners {#replace_listeners tag="method" new="3"} +## Language.replace_listeners {id="replace_listeners",tag="method",version="3"} Find [listener layers](/usage/embeddings-transformers#embedding-layers) (connecting to a shared token-to-vector embedding component) of a given pipeline @@ -885,7 +889,7 @@ when loading a config with | `pipe_name` | Name of pipeline component to replace listeners for. ~~str~~ | | `listeners` | The paths to the listeners, relative to the component config, e.g. `["model.tok2vec"]`. Typically, implementations will only connect to one tok2vec component, `model.tok2vec`, but in theory, custom models can use multiple listeners. The value here can either be an empty list to not replace any listeners, or a _complete_ list of the paths to all listener layers used by the model that should be replaced.~~Iterable[str]~~ | -## Language.meta {#meta tag="property"} +## Language.meta {id="meta",tag="property"} Meta data for the `Language` class, including name, version, data sources, license, author information and more. If a trained pipeline is loaded, this @@ -911,7 +915,7 @@ information is expressed in the [`config.cfg`](/api/data-formats#config). | ----------- | --------------------------------- | | **RETURNS** | The meta data. ~~Dict[str, Any]~~ | -## Language.config {#config tag="property" new="3"} +## Language.config {id="config",tag="property",version="3"} Export a trainable [`config.cfg`](/api/data-formats#config) for the current `nlp` object. Includes the current pipeline, all configs used to create the @@ -932,7 +936,7 @@ subclass of the built-in `dict`. It supports the additional methods `to_disk` | ----------- | ---------------------- | | **RETURNS** | The config. ~~Config~~ | -## Language.to_disk {#to_disk tag="method" new="2"} +## Language.to_disk {id="to_disk",tag="method",version="2"} Save the current state to a directory. Under the hood, this method delegates to the `to_disk` methods of the individual pipeline components, if available. This @@ -951,7 +955,7 @@ will be saved to disk. | _keyword-only_ | | | `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Language.from_disk {#from_disk tag="method" new="2"} +## Language.from_disk {id="from_disk",tag="method",version="2"} Loads state from a directory, including all data that was saved with the `Language` object. Modifies the object in place and returns it. @@ -984,7 +988,7 @@ you want to load a serialized pipeline from a directory, you should use | `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Language` object. ~~Language~~ | -## Language.to_bytes {#to_bytes tag="method"} +## Language.to_bytes {id="to_bytes",tag="method"} Serialize the current state to a binary string. @@ -1000,7 +1004,7 @@ Serialize the current state to a binary string. | `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~iterable~~ | | **RETURNS** | The serialized form of the `Language` object. ~~bytes~~ | -## Language.from_bytes {#from_bytes tag="method"} +## Language.from_bytes {id="from_bytes",tag="method"} Load state from a binary string. Note that this method is commonly used via the subclasses like `English` or `German` to make language-specific functionality @@ -1028,7 +1032,7 @@ details. | `exclude` | Names of pipeline components or [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Language` object. ~~Language~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | -------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | @@ -1046,7 +1050,7 @@ details. | `disabled` 3 | Names of components that are currently disabled and don't run as part of the pipeline. ~~List[str]~~ | | `path` | Path to the pipeline data directory, if a pipeline is loaded from a path or package. Otherwise `None`. ~~Optional[Path]~~ | -## Class attributes {#class-attributes} +## Class attributes {id="class-attributes"} | Name | Description | | ---------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -1054,7 +1058,7 @@ details. | `lang` | [IETF language tag](https://www.w3.org/International/articles/language-tags/), such as 'en' for English. ~~str~~ | | `default_config` | Base [config](/usage/training#config) to use for [Language.config](/api/language#config). Defaults to [`default_config.cfg`](%%GITHUB_SPACY/spacy/default_config.cfg). ~~Config~~ | -## Defaults {#defaults} +## Defaults {id="defaults"} The following attributes can be set on the `Language.Defaults` class to customize the default language data: @@ -1097,7 +1101,7 @@ customize the default language data: | `writing_system` | Information about the language's writing system, available via `Vocab.writing_system`. Defaults to: `{"direction": "ltr", "has_case": True, "has_letters": True}.`.
**Example:** [`zh/__init__.py`](%%GITHUB_SPACY/spacy/lang/zh/__init__.py) ~~Dict[str, Any]~~ | | `config` | Default [config](/usage/training#config) added to `nlp.config`. This can include references to custom tokenizers or lemmatizers.
**Example:** [`zh/__init__.py`](%%GITHUB_SPACY/spacy/lang/zh/__init__.py) ~~Config~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from @@ -1117,7 +1121,7 @@ serialization by passing in the string names via the `exclude` argument. | `meta` | The meta data, available as [`Language.meta`](/api/language#meta). | | ... | String names of pipeline components, e.g. `"ner"`. | -## FactoryMeta {#factorymeta new="3" tag="dataclass"} +## FactoryMeta {id="factorymeta",version="3",tag="dataclass"} The `FactoryMeta` contains the information about the component and its default provided by the [`@Language.component`](/api/language#component) or diff --git a/website/docs/api/legacy.md b/website/docs/api/legacy.mdx similarity index 95% rename from website/docs/api/legacy.md rename to website/docs/api/legacy.mdx index d9167c76f..ea6d3a899 100644 --- a/website/docs/api/legacy.md +++ b/website/docs/api/legacy.mdx @@ -12,11 +12,11 @@ functions that may still be used in projects. You can find the detailed documentation of each such legacy function on this page. -## Architectures {#architectures} +## Architectures {id="architectures"} These functions are available from `@spacy.registry.architectures`. -### spacy.Tok2Vec.v1 {#Tok2Vec_v1} +### spacy.Tok2Vec.v1 {id="Tok2Vec_v1"} The `spacy.Tok2Vec.v1` architecture was expecting an `encode` model of type `Model[Floats2D, Floats2D]` such as `spacy.MaxoutWindowEncoder.v1` or @@ -48,7 +48,7 @@ blog post for background. | `encode` | Encode context into the embeddings, using an architecture such as a CNN, BiLSTM or transformer. For example, [MaxoutWindowEncoder.v1](/api/legacy#MaxoutWindowEncoder_v1). ~~Model[Floats2d, Floats2d]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], List[Floats2d]]~~ | -### spacy.MaxoutWindowEncoder.v1 {#MaxoutWindowEncoder_v1} +### spacy.MaxoutWindowEncoder.v1 {id="MaxoutWindowEncoder_v1"} The `spacy.MaxoutWindowEncoder.v1` architecture was producing a model of type `Model[Floats2D, Floats2D]`. Since `spacy.MaxoutWindowEncoder.v2`, this has been @@ -76,7 +76,7 @@ and residual connections. | `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ | | **CREATES** | The model using the architecture. ~~Model[Floats2d, Floats2d]~~ | -### spacy.MishWindowEncoder.v1 {#MishWindowEncoder_v1} +### spacy.MishWindowEncoder.v1 {id="MishWindowEncoder_v1"} The `spacy.MishWindowEncoder.v1` architecture was producing a model of type `Model[Floats2D, Floats2D]`. Since `spacy.MishWindowEncoder.v2`, this has been @@ -103,24 +103,24 @@ and residual connections. | `depth` | The number of convolutional layers. Recommended value is `4`. ~~int~~ | | **CREATES** | The model using the architecture. ~~Model[Floats2d, Floats2d]~~ | -### spacy.HashEmbedCNN.v1 {#HashEmbedCNN_v1} +### spacy.HashEmbedCNN.v1 {id="HashEmbedCNN_v1"} Identical to [`spacy.HashEmbedCNN.v2`](/api/architectures#HashEmbedCNN) except using [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are included. -### spacy.MultiHashEmbed.v1 {#MultiHashEmbed_v1} +### spacy.MultiHashEmbed.v1 {id="MultiHashEmbed_v1"} Identical to [`spacy.MultiHashEmbed.v2`](/api/architectures#MultiHashEmbed) except with [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are included. -### spacy.CharacterEmbed.v1 {#CharacterEmbed_v1} +### spacy.CharacterEmbed.v1 {id="CharacterEmbed_v1"} Identical to [`spacy.CharacterEmbed.v2`](/api/architectures#CharacterEmbed) except using [`spacy.StaticVectors.v1`](#StaticVectors_v1) if vectors are included. -### spacy.TextCatEnsemble.v1 {#TextCatEnsemble_v1} +### spacy.TextCatEnsemble.v1 {id="TextCatEnsemble_v1"} The `spacy.TextCatEnsemble.v1` architecture built an internal `tok2vec` and `linear_model`. Since `spacy.TextCatEnsemble.v2`, this has been refactored so @@ -158,7 +158,7 @@ network has an internal CNN Tok2Vec layer and uses attention. | `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | -### spacy.TextCatCNN.v1 {#TextCatCNN_v1} +### spacy.TextCatCNN.v1 {id="TextCatCNN_v1"} Since `spacy.TextCatCNN.v2`, this architecture has become resizable, which means that you can add labels to a previously trained textcat. `TextCatCNN` v1 did not @@ -194,7 +194,7 @@ architecture is usually less accurate than the ensemble, but runs faster. | `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | -### spacy.TextCatBOW.v1 {#TextCatBOW_v1} +### spacy.TextCatBOW.v1 {id="TextCatBOW_v1"} Since `spacy.TextCatBOW.v2`, this architecture has become resizable, which means that you can add labels to a previously trained textcat. `TextCatBOW` v1 did not @@ -222,17 +222,17 @@ the others, but may not be as accurate, especially if texts are short. | `nO` | Output dimension, determined by the number of different labels. If not set, the [`TextCategorizer`](/api/textcategorizer) component will set it when `initialize` is called. ~~Optional[int]~~ | | **CREATES** | The model using the architecture. ~~Model[List[Doc], Floats2d]~~ | -### spacy.TransitionBasedParser.v1 {#TransitionBasedParser_v1} +### spacy.TransitionBasedParser.v1 {id="TransitionBasedParser_v1"} Identical to [`spacy.TransitionBasedParser.v2`](/api/architectures#TransitionBasedParser) except the `use_upper` was set to `True` by default. -## Layers {#layers} +## Layers {id="layers"} These functions are available from `@spacy.registry.layers`. -### spacy.StaticVectors.v1 {#StaticVectors_v1} +### spacy.StaticVectors.v1 {id="StaticVectors_v1"} Identical to [`spacy.StaticVectors.v2`](/api/architectures#StaticVectors) except for the handling of tokens without vectors. @@ -246,11 +246,11 @@ added to an existing vectors table. See more details in -## Loggers {#loggers} +## Loggers {id="loggers"} These functions are available from `@spacy.registry.loggers`. -### spacy.ConsoleLogger.v1 {#ConsoleLogger_v1} +### spacy.ConsoleLogger.v1 {id="ConsoleLogger_v1"} > #### Example config > @@ -264,7 +264,7 @@ Writes the results of a training step to the console in a tabular format. -```cli +```bash $ python -m spacy train config.cfg ``` diff --git a/website/docs/api/lemmatizer.md b/website/docs/api/lemmatizer.mdx similarity index 95% rename from website/docs/api/lemmatizer.md rename to website/docs/api/lemmatizer.mdx index 905096338..f6657dbf4 100644 --- a/website/docs/api/lemmatizer.md +++ b/website/docs/api/lemmatizer.mdx @@ -2,7 +2,7 @@ title: Lemmatizer tag: class source: spacy/pipeline/lemmatizer.py -new: 3 +version: 3 teaser: 'Pipeline component for lemmatization' api_string_name: lemmatizer api_trainable: false @@ -32,7 +32,7 @@ available in the pipeline and runs _before_ the lemmatizer. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Lemmas generated by rules or predicted will be saved to `Token.lemma`. @@ -94,7 +94,7 @@ libraries (`pymorphy3`). %%GITHUB_SPACY/spacy/pipeline/lemmatizer.py ``` -## Lemmatizer.\_\_init\_\_ {#init tag="method"} +## Lemmatizer.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -120,7 +120,7 @@ shortcut for this and instantiate the component using its string name and | mode | The lemmatizer mode, e.g. `"lookup"` or `"rule"`. Defaults to `"lookup"`. ~~str~~ | | overwrite | Whether to overwrite existing lemmas. ~~bool~~ | -## Lemmatizer.\_\_call\_\_ {#call tag="method"} +## Lemmatizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -140,7 +140,7 @@ and all pipeline components are applied to the `Doc` in order. | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## Lemmatizer.pipe {#pipe tag="method"} +## Lemmatizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -161,7 +161,7 @@ applied to the `Doc` in order. | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## Lemmatizer.initialize {#initialize tag="method"} +## Lemmatizer.initialize {id="initialize",tag="method"} Initialize the lemmatizer and load any data resources. This method is typically called by [`Language.initialize`](/api/language#initialize) and lets you @@ -192,7 +192,7 @@ training. At runtime, all data is loaded from disk. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `lookups` | The lookups object containing the tables such as `"lemma_rules"`, `"lemma_index"`, `"lemma_exc"` and `"lemma_lookup"`. If `None`, default tables are loaded from [`spacy-lookups-data`](https://github.com/explosion/spacy-lookups-data). Defaults to `None`. ~~Optional[Lookups]~~ | -## Lemmatizer.lookup_lemmatize {#lookup_lemmatize tag="method"} +## Lemmatizer.lookup_lemmatize {id="lookup_lemmatize",tag="method"} Lemmatize a token using a lookup-based approach. If no lemma is found, the original string is returned. @@ -202,7 +202,7 @@ original string is returned. | `token` | The token to lemmatize. ~~Token~~ | | **RETURNS** | A list containing one or more lemmas. ~~List[str]~~ | -## Lemmatizer.rule_lemmatize {#rule_lemmatize tag="method"} +## Lemmatizer.rule_lemmatize {id="rule_lemmatize",tag="method"} Lemmatize a token using a rule-based approach. Typically relies on POS tags. @@ -211,7 +211,7 @@ Lemmatize a token using a rule-based approach. Typically relies on POS tags. | `token` | The token to lemmatize. ~~Token~~ | | **RETURNS** | A list containing one or more lemmas. ~~List[str]~~ | -## Lemmatizer.is_base_form {#is_base_form tag="method"} +## Lemmatizer.is_base_form {id="is_base_form",tag="method"} Check whether we're dealing with an uninflected paradigm, so we can avoid lemmatization entirely. @@ -221,7 +221,7 @@ lemmatization entirely. | `token` | The token to analyze. ~~Token~~ | | **RETURNS** | Whether the token's attributes (e.g., part-of-speech tag, morphological features) describe a base form. ~~bool~~ | -## Lemmatizer.get_lookups_config {#get_lookups_config tag="classmethod"} +## Lemmatizer.get_lookups_config {id="get_lookups_config",tag="classmethod"} Returns the lookups configuration settings for a given mode for use in [`Lemmatizer.load_lookups`](/api/lemmatizer#load_lookups). @@ -231,7 +231,7 @@ Returns the lookups configuration settings for a given mode for use in | `mode` | The lemmatizer mode. ~~str~~ | | **RETURNS** | The required table names and the optional table names. ~~Tuple[List[str], List[str]]~~ | -## Lemmatizer.to_disk {#to_disk tag="method"} +## Lemmatizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -248,7 +248,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Lemmatizer.from_disk {#from_disk tag="method"} +## Lemmatizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -266,7 +266,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Lemmatizer` object. ~~Lemmatizer~~ | -## Lemmatizer.to_bytes {#to_bytes tag="method"} +## Lemmatizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -283,7 +283,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Lemmatizer` object. ~~bytes~~ | -## Lemmatizer.from_bytes {#from_bytes tag="method"} +## Lemmatizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -302,7 +302,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Lemmatizer` object. ~~Lemmatizer~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | --------- | ------------------------------------------- | @@ -310,7 +310,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `lookups` | The lookups object. ~~Lookups~~ | | `mode` | The lemmatizer mode. ~~str~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/lexeme.md b/website/docs/api/lexeme.mdx similarity index 97% rename from website/docs/api/lexeme.md rename to website/docs/api/lexeme.mdx index 557d04cce..539f502f0 100644 --- a/website/docs/api/lexeme.md +++ b/website/docs/api/lexeme.mdx @@ -9,7 +9,7 @@ A `Lexeme` has no string context – it's a word type, as opposed to a word toke It therefore has no part-of-speech tag, dependency parse, or lemma (if lemmatization depends on the part-of-speech tag). -## Lexeme.\_\_init\_\_ {#init tag="method"} +## Lexeme.\_\_init\_\_ {id="init",tag="method"} Create a `Lexeme` object. @@ -18,7 +18,7 @@ Create a `Lexeme` object. | `vocab` | The parent vocabulary. ~~Vocab~~ | | `orth` | The orth id of the lexeme. ~~int~~ | -## Lexeme.set_flag {#set_flag tag="method"} +## Lexeme.set_flag {id="set_flag",tag="method"} Change the value of a boolean flag. @@ -34,7 +34,7 @@ Change the value of a boolean flag. | `flag_id` | The attribute ID of the flag to set. ~~int~~ | | `value` | The new value of the flag. ~~bool~~ | -## Lexeme.check_flag {#check_flag tag="method"} +## Lexeme.check_flag {id="check_flag",tag="method"} Check the value of a boolean flag. @@ -51,7 +51,7 @@ Check the value of a boolean flag. | `flag_id` | The attribute ID of the flag to query. ~~int~~ | | **RETURNS** | The value of the flag. ~~bool~~ | -## Lexeme.similarity {#similarity tag="method" model="vectors"} +## Lexeme.similarity {id="similarity",tag="method",model="vectors"} Compute a semantic similarity estimate. Defaults to cosine over vectors. @@ -70,7 +70,7 @@ Compute a semantic similarity estimate. Defaults to cosine over vectors. | other | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ | | **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ | -## Lexeme.has_vector {#has_vector tag="property" model="vectors"} +## Lexeme.has_vector {id="has_vector",tag="property",model="vectors"} A boolean value indicating whether a word vector is associated with the lexeme. @@ -85,7 +85,7 @@ A boolean value indicating whether a word vector is associated with the lexeme. | ----------- | ------------------------------------------------------- | | **RETURNS** | Whether the lexeme has a vector data attached. ~~bool~~ | -## Lexeme.vector {#vector tag="property" model="vectors"} +## Lexeme.vector {id="vector",tag="property",model="vectors"} A real-valued meaning representation. @@ -101,7 +101,7 @@ A real-valued meaning representation. | ----------- | ------------------------------------------------------------------------------------------------ | | **RETURNS** | A 1-dimensional array representing the lexeme's vector. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Lexeme.vector_norm {#vector_norm tag="property" model="vectors"} +## Lexeme.vector_norm {id="vector_norm",tag="property",model="vectors"} The L2 norm of the lexeme's vector representation. @@ -119,7 +119,7 @@ The L2 norm of the lexeme's vector representation. | ----------- | --------------------------------------------------- | | **RETURNS** | The L2 norm of the vector representation. ~~float~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | ---------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/website/docs/api/lookups.md b/website/docs/api/lookups.mdx similarity index 89% rename from website/docs/api/lookups.md rename to website/docs/api/lookups.mdx index 9565e478f..71a857c60 100644 --- a/website/docs/api/lookups.md +++ b/website/docs/api/lookups.mdx @@ -3,7 +3,7 @@ title: Lookups teaser: A container for large lookup tables and dictionaries tag: class source: spacy/lookups.py -new: 2.2 +version: 2.2 --- This class allows convenient access to large lookup tables and dictionaries, @@ -13,7 +13,7 @@ can be accessed before the pipeline components are applied (e.g. in the tokenizer and lemmatizer), as well as within the pipeline components via `doc.vocab.lookups`. -## Lookups.\_\_init\_\_ {#init tag="method"} +## Lookups.\_\_init\_\_ {id="init",tag="method"} Create a `Lookups` object. @@ -24,7 +24,7 @@ Create a `Lookups` object. > lookups = Lookups() > ``` -## Lookups.\_\_len\_\_ {#len tag="method"} +## Lookups.\_\_len\_\_ {id="len",tag="method"} Get the current number of tables in the lookups. @@ -39,7 +39,7 @@ Get the current number of tables in the lookups. | ----------- | -------------------------------------------- | | **RETURNS** | The number of tables in the lookups. ~~int~~ | -## Lookups.\_\contains\_\_ {#contains tag="method"} +## Lookups.\_\_contains\_\_ {id="contains",tag="method"} Check if the lookups contain a table of a given name. Delegates to [`Lookups.has_table`](/api/lookups#has_table). @@ -57,7 +57,7 @@ Check if the lookups contain a table of a given name. Delegates to | `name` | Name of the table. ~~str~~ | | **RETURNS** | Whether a table of that name is in the lookups. ~~bool~~ | -## Lookups.tables {#tables tag="property"} +## Lookups.tables {id="tables",tag="property"} Get the names of all tables in the lookups. @@ -73,7 +73,7 @@ Get the names of all tables in the lookups. | ----------- | ------------------------------------------------- | | **RETURNS** | Names of the tables in the lookups. ~~List[str]~~ | -## Lookups.add_table {#add_table tag="method"} +## Lookups.add_table {id="add_table",tag="method"} Add a new table with optional data to the lookups. Raises an error if the table exists. @@ -91,7 +91,7 @@ exists. | `data` | Optional data to add to the table. ~~dict~~ | | **RETURNS** | The newly added table. ~~Table~~ | -## Lookups.get_table {#get_table tag="method"} +## Lookups.get_table {id="get_table",tag="method"} Get a table from the lookups. Raises an error if the table doesn't exist. @@ -109,7 +109,7 @@ Get a table from the lookups. Raises an error if the table doesn't exist. | `name` | Name of the table. ~~str~~ | | **RETURNS** | The table. ~~Table~~ | -## Lookups.remove_table {#remove_table tag="method"} +## Lookups.remove_table {id="remove_table",tag="method"} Remove a table from the lookups. Raises an error if the table doesn't exist. @@ -127,7 +127,7 @@ Remove a table from the lookups. Raises an error if the table doesn't exist. | `name` | Name of the table to remove. ~~str~~ | | **RETURNS** | The removed table. ~~Table~~ | -## Lookups.has_table {#has_table tag="method"} +## Lookups.has_table {id="has_table",tag="method"} Check if the lookups contain a table of a given name. Equivalent to [`Lookups.__contains__`](/api/lookups#contains). @@ -145,7 +145,7 @@ Check if the lookups contain a table of a given name. Equivalent to | `name` | Name of the table. ~~str~~ | | **RETURNS** | Whether a table of that name is in the lookups. ~~bool~~ | -## Lookups.to_bytes {#to_bytes tag="method"} +## Lookups.to_bytes {id="to_bytes",tag="method"} Serialize the lookups to a bytestring. @@ -159,7 +159,7 @@ Serialize the lookups to a bytestring. | ----------- | --------------------------------- | | **RETURNS** | The serialized lookups. ~~bytes~~ | -## Lookups.from_bytes {#from_bytes tag="method"} +## Lookups.from_bytes {id="from_bytes",tag="method"} Load the lookups from a bytestring. @@ -176,7 +176,7 @@ Load the lookups from a bytestring. | `bytes_data` | The data to load from. ~~bytes~~ | | **RETURNS** | The loaded lookups. ~~Lookups~~ | -## Lookups.to_disk {#to_disk tag="method"} +## Lookups.to_disk {id="to_disk",tag="method"} Save the lookups to a directory as `lookups.bin`. Expects a path to a directory, which will be created if it doesn't exist. @@ -191,7 +191,7 @@ which will be created if it doesn't exist. | ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | | `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | -## Lookups.from_disk {#from_disk tag="method"} +## Lookups.from_disk {id="from_disk",tag="method"} Load lookups from a directory containing a `lookups.bin`. Will skip loading if the file doesn't exist. @@ -209,7 +209,7 @@ the file doesn't exist. | `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | **RETURNS** | The loaded lookups. ~~Lookups~~ | -## Table {#table tag="class, ordererddict"} +## Table {id="table",tag="class, ordererddict"} A table in the lookups. Subclass of `OrderedDict` that implements a slightly more consistent and unified API and includes a Bloom filter to speed up missed @@ -218,7 +218,7 @@ lookups. Supports **all other methods and attributes** of `OrderedDict` / accept both integers and strings (which will be hashed before being added to the table). -### Table.\_\_init\_\_ {#table.init tag="method"} +### Table.\_\_init\_\_ {id="table.init",tag="method"} Initialize a new table. @@ -236,7 +236,7 @@ Initialize a new table. | ------ | ------------------------------------------ | | `name` | Optional table name for reference. ~~str~~ | -### Table.from_dict {#table.from_dict tag="classmethod"} +### Table.from_dict {id="table.from_dict",tag="classmethod"} Initialize a new table from a dict. @@ -254,7 +254,7 @@ Initialize a new table from a dict. | `name` | Optional table name for reference. ~~str~~ | | **RETURNS** | The newly constructed object. ~~Table~~ | -### Table.set {#table.set tag="method"} +### Table.set {id="table.set",tag="method"} Set a new key / value pair. String keys will be hashed. Same as `table[key] = value`. @@ -273,7 +273,7 @@ Set a new key / value pair. String keys will be hashed. Same as | `key` | The key. ~~Union[str, int]~~ | | `value` | The value. | -### Table.to_bytes {#table.to_bytes tag="method"} +### Table.to_bytes {id="table.to_bytes",tag="method"} Serialize the table to a bytestring. @@ -287,7 +287,7 @@ Serialize the table to a bytestring. | ----------- | ------------------------------- | | **RETURNS** | The serialized table. ~~bytes~~ | -### Table.from_bytes {#table.from_bytes tag="method"} +### Table.from_bytes {id="table.from_bytes",tag="method"} Load a table from a bytestring. @@ -304,7 +304,7 @@ Load a table from a bytestring. | `bytes_data` | The data to load. ~~bytes~~ | | **RETURNS** | The loaded table. ~~Table~~ | -### Attributes {#table-attributes} +### Attributes {id="table-attributes"} | Name | Description | | -------------- | ------------------------------------------------------------- | diff --git a/website/docs/api/matcher.md b/website/docs/api/matcher.mdx similarity index 97% rename from website/docs/api/matcher.md rename to website/docs/api/matcher.mdx index bd5f6ac24..c66579da8 100644 --- a/website/docs/api/matcher.md +++ b/website/docs/api/matcher.mdx @@ -13,7 +13,7 @@ tokens in context. For in-depth examples and workflows for combining rules and statistical models, see the [usage guide](/usage/rule-based-matching) on rule-based matching. -## Pattern format {#patterns} +## Pattern format {id="patterns"} > ```json > ### Example @@ -101,7 +101,7 @@ it compares to another value. As of spaCy v3.5, `REGEX` and `FUZZY` can be used in combination with `IN` and `NOT_IN`. -## Matcher.\_\_init\_\_ {#init tag="method"} +## Matcher.\_\_init\_\_ {id="init",tag="method"} Create the rule-based `Matcher`. If `validate=True` is set, all patterns added to the matcher will be validated against a JSON schema and a `MatchPatternError` @@ -121,7 +121,7 @@ string where an integer is expected) or unexpected property names. | `validate` | Validate all patterns added to this matcher. ~~bool~~ | | `fuzzy_compare` | The comparison method used for the `FUZZY` operators. ~~Callable[[str, str, int], bool]~~ | -## Matcher.\_\_call\_\_ {#call tag="method"} +## Matcher.\_\_call\_\_ {id="call",tag="method"} Find all token sequences matching the supplied patterns on the `Doc` or `Span`. @@ -150,7 +150,7 @@ the match. | `with_alignments` 3.0.6 | Return match alignment information as part of the match tuple as `List[int]` with the same length as the matched span. Each entry denotes the corresponding index of the token in the pattern. If `as_spans` is set to `True`, this setting is ignored. Defaults to `False`. ~~bool~~ | | **RETURNS** | A list of `(match_id, start, end)` tuples, describing the matches. A match tuple describes a span `doc[start:end`]. The `match_id` is the ID of the added match pattern. If `as_spans` is set to `True`, a list of `Span` objects is returned instead. ~~Union[List[Tuple[int, int, int]], List[Span]]~~ | -## Matcher.\_\_len\_\_ {#len tag="method" new="2"} +## Matcher.\_\_len\_\_ {id="len",tag="method",version="2"} Get the number of rules added to the matcher. Note that this only returns the number of rules (identical with the number of IDs), not the number of individual @@ -169,7 +169,7 @@ patterns. | ----------- | ---------------------------- | | **RETURNS** | The number of rules. ~~int~~ | -## Matcher.\_\_contains\_\_ {#contains tag="method" new="2"} +## Matcher.\_\_contains\_\_ {id="contains",tag="method",version="2"} Check whether the matcher contains rules for a match ID. @@ -187,7 +187,7 @@ Check whether the matcher contains rules for a match ID. | `key` | The match ID. ~~str~~ | | **RETURNS** | Whether the matcher contains rules for this match ID. ~~bool~~ | -## Matcher.add {#add tag="method" new="2"} +## Matcher.add {id="add",tag="method",version="2"} Add a rule to the matcher, consisting of an ID key, one or more patterns, and an optional callback function to act on the matches. The callback function will @@ -233,7 +233,7 @@ patterns = [[{"TEXT": "Google"}, {"TEXT": "Now"}], [{"TEXT": "GoogleNow"}]] | `on_match` | Callback function to act on matches. Takes the arguments `matcher`, `doc`, `i` and `matches`. ~~Optional[Callable[[Matcher, Doc, int, List[tuple], Any]]~~ | | `greedy` 3 | Optional filter for greedy matches. Can either be `"FIRST"` or `"LONGEST"`. ~~Optional[str]~~ | -## Matcher.remove {#remove tag="method" new="2"} +## Matcher.remove {id="remove",tag="method",version="2"} Remove a rule from the matcher. A `KeyError` is raised if the match ID does not exist. @@ -251,7 +251,7 @@ exist. | ----- | --------------------------------- | | `key` | The ID of the match rule. ~~str~~ | -## Matcher.get {#get tag="method" new="2"} +## Matcher.get {id="get",tag="method",version="2"} Retrieve the pattern stored for a key. Returns the rule as an `(on_match, patterns)` tuple containing the callback and available patterns. diff --git a/website/docs/api/morphologizer.md b/website/docs/api/morphologizer.mdx similarity index 95% rename from website/docs/api/morphologizer.md rename to website/docs/api/morphologizer.mdx index f874e8bea..f097f2ae3 100644 --- a/website/docs/api/morphologizer.md +++ b/website/docs/api/morphologizer.mdx @@ -2,7 +2,7 @@ title: Morphologizer tag: class source: spacy/pipeline/morphologizer.pyx -new: 3 +version: 3 teaser: 'Pipeline component for predicting morphological features' api_base_class: /api/tagger api_string_name: morphologizer @@ -15,7 +15,7 @@ coarse-grained POS tags following the Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) annotation guidelines. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions are saved to `Token.morph` and `Token.pos`. @@ -25,7 +25,7 @@ Predictions are saved to `Token.morph` and `Token.pos`. | `Token.pos_` | The UPOS part of speech. ~~str~~ | | `Token.morph` | Morphological features. ~~MorphAnalysis~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -53,7 +53,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/morphologizer.pyx ``` -## Morphologizer.\_\_init\_\_ {#init tag="method"} +## Morphologizer.\_\_init\_\_ {id="init",tag="method"} Create a new pipeline instance. In your application, you would normally use a shortcut for this and instantiate the component using its string name and @@ -97,7 +97,7 @@ annotation `C=E|X=Y`): | `extend` 3.2 | Whether existing feature types (whose values may or may not be overwritten depending on `overwrite`) are preserved. Defaults to `False`. ~~bool~~ | | `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attributes `"pos"` and `"morph"` and [`Scorer.score_token_attr_per_feat`](/api/scorer#score_token_attr_per_feat) for the attribute `"morph"`. ~~Optional[Callable]~~ | -## Morphologizer.\_\_call\_\_ {#call tag="method"} +## Morphologizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -120,7 +120,7 @@ delegate to the [`predict`](/api/morphologizer#predict) and | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## Morphologizer.pipe {#pipe tag="method"} +## Morphologizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -144,7 +144,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/morphologizer#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## Morphologizer.initialize {#initialize tag="method"} +## Morphologizer.initialize {id="initialize",tag="method"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -181,7 +181,7 @@ config. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[dict]~~ | -## Morphologizer.predict {#predict tag="method"} +## Morphologizer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -198,7 +198,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## Morphologizer.set_annotations {#set_annotations tag="method"} +## Morphologizer.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -215,7 +215,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `Morphologizer.predict`. | -## Morphologizer.update {#update tag="method"} +## Morphologizer.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -239,7 +239,7 @@ Delegates to [`predict`](/api/morphologizer#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Morphologizer.get_loss {#get_loss tag="method"} +## Morphologizer.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -258,7 +258,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## Morphologizer.create_optimizer {#create_optimizer tag="method"} +## Morphologizer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -273,7 +273,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## Morphologizer.use_params {#use_params tag="method, contextmanager"} +## Morphologizer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -290,7 +290,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## Morphologizer.add_label {#add_label tag="method"} +## Morphologizer.add_label {id="add_label",tag="method"} Add a new label to the pipe. If the `Morphologizer` should set annotations for both `pos` and `morph`, the label should include the UPOS as the feature `POS`. @@ -313,7 +313,7 @@ will be automatically added to the model, and the output dimension will be | `label` | The label to add. ~~str~~ | | **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | -## Morphologizer.to_disk {#to_disk tag="method"} +## Morphologizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -330,7 +330,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Morphologizer.from_disk {#from_disk tag="method"} +## Morphologizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -348,7 +348,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Morphologizer` object. ~~Morphologizer~~ | -## Morphologizer.to_bytes {#to_bytes tag="method"} +## Morphologizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -365,7 +365,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Morphologizer` object. ~~bytes~~ | -## Morphologizer.from_bytes {#from_bytes tag="method"} +## Morphologizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -384,7 +384,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Morphologizer` object. ~~Morphologizer~~ | -## Morphologizer.labels {#labels tag="property"} +## Morphologizer.labels {id="labels",tag="property"} The labels currently added to the component in the Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) @@ -403,7 +403,7 @@ coarse-grained POS as the feature `POS`. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## Morphologizer.label_data {#label_data tag="property" new="3"} +## Morphologizer.label_data {id="label_data",tag="property",version="3"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -421,7 +421,7 @@ model with a pre-defined label set. | ----------- | ----------------------------------------------- | | **RETURNS** | The label data added to the component. ~~dict~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/morphology.md b/website/docs/api/morphology.mdx similarity index 89% rename from website/docs/api/morphology.md rename to website/docs/api/morphology.mdx index 20fcd1a40..68d80b814 100644 --- a/website/docs/api/morphology.md +++ b/website/docs/api/morphology.mdx @@ -10,7 +10,7 @@ morphological analysis, so queries of morphological attributes are delegated to this class. See [`MorphAnalysis`](/api/morphology#morphanalysis) for the container storing a single morphological analysis. -## Morphology.\_\_init\_\_ {#init tag="method"} +## Morphology.\_\_init\_\_ {id="init",tag="method"} Create a `Morphology` object. @@ -26,7 +26,7 @@ Create a `Morphology` object. | --------- | --------------------------------- | | `strings` | The string store. ~~StringStore~~ | -## Morphology.add {#add tag="method"} +## Morphology.add {id="add",tag="method"} Insert a morphological analysis in the morphology table, if not already present. The morphological analysis may be provided in the Universal Dependencies @@ -46,7 +46,7 @@ new analysis. | ---------- | ------------------------------------------------ | | `features` | The morphological features. ~~Union[Dict, str]~~ | -## Morphology.get {#get tag="method"} +## Morphology.get {id="get",tag="method"} > #### Example > @@ -64,7 +64,7 @@ string for the hash of the morphological analysis. | ------- | ----------------------------------------------- | | `morph` | The hash of the morphological analysis. ~~int~~ | -## Morphology.feats_to_dict {#feats_to_dict tag="staticmethod"} +## Morphology.feats_to_dict {id="feats_to_dict",tag="staticmethod"} Convert a string [FEATS](https://universaldependencies.org/format.html#morphological-annotation) @@ -84,7 +84,7 @@ tag map. | `feats` | The morphological features in Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) format. ~~str~~ | | **RETURNS** | The morphological features as a dictionary. ~~Dict[str, str]~~ | -## Morphology.dict_to_feats {#dict_to_feats tag="staticmethod"} +## Morphology.dict_to_feats {id="dict_to_feats",tag="staticmethod"} Convert a dictionary of features and values to a string [FEATS](https://universaldependencies.org/format.html#morphological-annotation) @@ -103,19 +103,19 @@ representation. | `feats_dict` | The morphological features as a dictionary. ~~Dict[str, str]~~ | | **RETURNS** | The morphological features in Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) format. ~~str~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} -| Name | Description | -| ------------- | ------------------------------------------------------------------------------------------------------------------------------ | -| `FEATURE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) feature separator. Default is `|`. ~~str~~ | -| `FIELD_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) field separator. Default is `=`. ~~str~~ | -| `VALUE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) value separator. Default is `,`. ~~str~~ | +| Name | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `FEATURE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) feature separator. Default is `\|`. ~~str~~ | +| `FIELD_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) field separator. Default is `=`. ~~str~~ | +| `VALUE_SEP` | The [FEATS](https://universaldependencies.org/format.html#morphological-annotation) value separator. Default is `,`. ~~str~~ | -## MorphAnalysis {#morphanalysis tag="class" source="spacy/tokens/morphanalysis.pyx"} +## MorphAnalysis {id="morphanalysis",tag="class",source="spacy/tokens/morphanalysis.pyx"} Stores a single morphological analysis. -### MorphAnalysis.\_\_init\_\_ {#morphanalysis-init tag="method"} +### MorphAnalysis.\_\_init\_\_ {id="morphanalysis-init",tag="method"} Initialize a MorphAnalysis object from a Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) @@ -135,7 +135,7 @@ string or a dictionary of morphological features. | `vocab` | The vocab. ~~Vocab~~ | | `features` | The morphological features. ~~Union[Dict[str, str], str]~~ | -### MorphAnalysis.\_\_contains\_\_ {#morphanalysis-contains tag="method"} +### MorphAnalysis.\_\_contains\_\_ {id="morphanalysis-contains",tag="method"} Whether a feature/value pair is in the analysis. @@ -151,7 +151,7 @@ Whether a feature/value pair is in the analysis. | ----------- | --------------------------------------------- | | **RETURNS** | A feature/value pair in the analysis. ~~str~~ | -### MorphAnalysis.\_\_iter\_\_ {#morphanalysis-iter tag="method"} +### MorphAnalysis.\_\_iter\_\_ {id="morphanalysis-iter",tag="method"} Iterate over the feature/value pairs in the analysis. @@ -167,7 +167,7 @@ Iterate over the feature/value pairs in the analysis. | ---------- | --------------------------------------------- | | **YIELDS** | A feature/value pair in the analysis. ~~str~~ | -### MorphAnalysis.\_\_len\_\_ {#morphanalysis-len tag="method"} +### MorphAnalysis.\_\_len\_\_ {id="morphanalysis-len",tag="method"} Returns the number of features in the analysis. @@ -183,7 +183,7 @@ Returns the number of features in the analysis. | ----------- | ----------------------------------------------- | | **RETURNS** | The number of features in the analysis. ~~int~~ | -### MorphAnalysis.\_\_str\_\_ {#morphanalysis-str tag="method"} +### MorphAnalysis.\_\_str\_\_ {id="morphanalysis-str",tag="method"} Returns the morphological analysis in the Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) @@ -201,7 +201,7 @@ string format. | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------ | | **RETURNS** | The analysis in the Universal Dependencies [FEATS](https://universaldependencies.org/format.html#morphological-annotation) format. ~~str~~ | -### MorphAnalysis.get {#morphanalysis-get tag="method"} +### MorphAnalysis.get {id="morphanalysis-get",tag="method"} Retrieve values for a feature by field. @@ -218,7 +218,7 @@ Retrieve values for a feature by field. | `field` | The field to retrieve. ~~str~~ | | **RETURNS** | A list of the individual features. ~~List[str]~~ | -### MorphAnalysis.to_dict {#morphanalysis-to_dict tag="method"} +### MorphAnalysis.to_dict {id="morphanalysis-to_dict",tag="method"} Produce a dict representation of the analysis, in the same format as the tag map. @@ -235,7 +235,7 @@ map. | ----------- | ----------------------------------------------------------- | | **RETURNS** | The dict representation of the analysis. ~~Dict[str, str]~~ | -### MorphAnalysis.from_id {#morphanalysis-from_id tag="classmethod"} +### MorphAnalysis.from_id {id="morphanalysis-from_id",tag="classmethod"} Create a morphological analysis from a given hash ID. diff --git a/website/docs/api/phrasematcher.md b/website/docs/api/phrasematcher.mdx similarity index 96% rename from website/docs/api/phrasematcher.md rename to website/docs/api/phrasematcher.mdx index cd419ae5c..14ccefb77 100644 --- a/website/docs/api/phrasematcher.md +++ b/website/docs/api/phrasematcher.mdx @@ -3,7 +3,7 @@ title: PhraseMatcher teaser: Match sequences of tokens, based on documents tag: class source: spacy/matcher/phrasematcher.pyx -new: 2 +version: 2 --- The `PhraseMatcher` lets you efficiently match large terminology lists. While @@ -12,7 +12,7 @@ descriptions, the `PhraseMatcher` accepts match patterns in the form of `Doc` objects. See the [usage guide](/usage/rule-based-matching#phrasematcher) for examples. -## PhraseMatcher.\_\_init\_\_ {#init tag="method"} +## PhraseMatcher.\_\_init\_\_ {id="init",tag="method"} Create the rule-based `PhraseMatcher`. Setting a different `attr` to match on will change the token attributes that will be compared to determine a match. By @@ -42,7 +42,7 @@ be shown. | `attr` | The token attribute to match on. Defaults to `ORTH`, i.e. the verbatim token text. ~~Union[int, str]~~ | | `validate` | Validate patterns added to the matcher. ~~bool~~ | -## PhraseMatcher.\_\_call\_\_ {#call tag="method"} +## PhraseMatcher.\_\_call\_\_ {id="call",tag="method"} Find all token sequences matching the supplied patterns on the `Doc` or `Span`. @@ -76,7 +76,7 @@ match_id_string = nlp.vocab.strings[match_id] -## PhraseMatcher.\_\_len\_\_ {#len tag="method"} +## PhraseMatcher.\_\_len\_\_ {id="len",tag="method"} Get the number of rules added to the matcher. Note that this only returns the number of rules (identical with the number of IDs), not the number of individual @@ -95,7 +95,7 @@ patterns. | ----------- | ---------------------------- | | **RETURNS** | The number of rules. ~~int~~ | -## PhraseMatcher.\_\_contains\_\_ {#contains tag="method"} +## PhraseMatcher.\_\_contains\_\_ {id="contains",tag="method"} Check whether the matcher contains rules for a match ID. @@ -113,7 +113,7 @@ Check whether the matcher contains rules for a match ID. | `key` | The match ID. ~~str~~ | | **RETURNS** | Whether the matcher contains rules for this match ID. ~~bool~~ | -## PhraseMatcher.add {#add tag="method"} +## PhraseMatcher.add {id="add",tag="method"} Add a rule to the matcher, consisting of an ID key, one or more patterns, and a callback function to act on the matches. The callback function will receive the @@ -155,7 +155,7 @@ patterns = [nlp("health care reform"), nlp("healthcare reform")] | _keyword-only_ | | | `on_match` | Callback function to act on matches. Takes the arguments `matcher`, `doc`, `i` and `matches`. ~~Optional[Callable[[Matcher, Doc, int, List[tuple], Any]]~~ | -## PhraseMatcher.remove {#remove tag="method" new="2.2"} +## PhraseMatcher.remove {id="remove",tag="method",version="2.2"} Remove a rule from the matcher by match ID. A `KeyError` is raised if the key does not exist. diff --git a/website/docs/api/pipe.md b/website/docs/api/pipe.mdx similarity index 93% rename from website/docs/api/pipe.md rename to website/docs/api/pipe.mdx index 263942e3e..c2777edf0 100644 --- a/website/docs/api/pipe.md +++ b/website/docs/api/pipe.mdx @@ -12,7 +12,7 @@ spaCy pipeline. See the docs on [writing trainable components](/usage/processing-pipelines#trainable-components) for how to use the `TrainablePipe` base class to implement custom components. - +{/* TODO: Pipe vs TrainablePipe, check methods below (all renamed to TrainablePipe for now) */} > #### Why is it implemented in Cython? > @@ -27,7 +27,7 @@ for how to use the `TrainablePipe` base class to implement custom components. %%GITHUB_SPACY/spacy/pipeline/trainable_pipe.pyx ``` -## TrainablePipe.\_\_init\_\_ {#init tag="method"} +## TrainablePipe.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -54,7 +54,7 @@ shortcut for this and instantiate the component using its string name and | `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | | `**cfg` | Additional config parameters and settings. Will be available as the dictionary `cfg` and is serialized with the component. | -## TrainablePipe.\_\_call\_\_ {#call tag="method"} +## TrainablePipe.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -77,7 +77,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## TrainablePipe.pipe {#pipe tag="method"} +## TrainablePipe.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -100,7 +100,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/pipe#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## TrainablePipe.set_error_handler {#set_error_handler tag="method" new="3"} +## TrainablePipe.set_error_handler {id="set_error_handler",tag="method",version="3"} Define a callback that will be invoked when an error is thrown during processing of one or more documents with either [`__call__`](/api/pipe#call) or @@ -122,7 +122,7 @@ processed, and the original error. | --------------- | -------------------------------------------------------------------------------------------------------------- | | `error_handler` | A function that performs custom error handling. ~~Callable[[str, Callable[[Doc], Doc], List[Doc], Exception]~~ | -## TrainablePipe.get_error_handler {#get_error_handler tag="method" new="3"} +## TrainablePipe.get_error_handler {id="get_error_handler",tag="method",version="3"} Retrieve the callback that performs error handling for this component's [`__call__`](/api/pipe#call) and [`pipe`](/api/pipe#pipe) methods. If no custom @@ -141,7 +141,7 @@ returned that simply reraises the exception. | ----------- | ---------------------------------------------------------------------------------------------------------------- | | **RETURNS** | The function that performs custom error handling. ~~Callable[[str, Callable[[Doc], Doc], List[Doc], Exception]~~ | -## TrainablePipe.initialize {#initialize tag="method" new="3"} +## TrainablePipe.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. The data examples are @@ -171,7 +171,7 @@ This method was previously called `begin_training`. | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | -## TrainablePipe.predict {#predict tag="method"} +## TrainablePipe.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -194,7 +194,7 @@ This method needs to be overwritten with your own custom `predict` method. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## TrainablePipe.set_annotations {#set_annotations tag="method"} +## TrainablePipe.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -218,7 +218,7 @@ method. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `Tagger.predict`. | -## TrainablePipe.update {#update tag="method"} +## TrainablePipe.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -240,7 +240,7 @@ predictions and gold-standard annotations, and update the component's model. | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## TrainablePipe.rehearse {#rehearse tag="method,experimental" new="3"} +## TrainablePipe.rehearse {id="rehearse",tag="method,experimental",version="3"} Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the current model to make predictions similar to an initial model, to try to address @@ -262,7 +262,7 @@ the "catastrophic forgetting" problem. This feature is experimental. | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## TrainablePipe.get_loss {#get_loss tag="method"} +## TrainablePipe.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -287,7 +287,7 @@ This method needs to be overwritten with your own custom `get_loss` method. | `scores` | Scores representing the model's predictions. | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## TrainablePipe.score {#score tag="method" new="3"} +## TrainablePipe.score {id="score",tag="method",version="3"} Score a batch of examples. @@ -304,7 +304,7 @@ Score a batch of examples. | `\*\*kwargs` | Any additional settings to pass on to the scorer. ~~Any~~ | | **RETURNS** | The scores, e.g. produced by the [`Scorer`](/api/scorer). ~~Dict[str, Union[float, Dict[str, float]]]~~ | -## TrainablePipe.create_optimizer {#create_optimizer tag="method"} +## TrainablePipe.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. Defaults to [`Adam`](https://thinc.ai/docs/api-optimizers#adam) with default settings. @@ -320,7 +320,7 @@ Create an optimizer for the pipeline component. Defaults to | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## TrainablePipe.use_params {#use_params tag="method, contextmanager"} +## TrainablePipe.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -337,7 +337,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## TrainablePipe.finish_update {#finish_update tag="method"} +## TrainablePipe.finish_update {id="finish_update",tag="method"} Update parameters using the current parameter gradients. Defaults to calling [`self.model.finish_update`](https://thinc.ai/docs/api-model#finish_update). @@ -355,7 +355,7 @@ Update parameters using the current parameter gradients. Defaults to calling | ----- | ------------------------------------- | | `sgd` | An optimizer. ~~Optional[Optimizer]~~ | -## TrainablePipe.add_label {#add_label tag="method"} +## TrainablePipe.add_label {id="add_label",tag="method"} > #### Example > @@ -390,7 +390,7 @@ case, all labels found in the sample will be automatically added to the model, and the output dimension will be [inferred](/usage/layers-architectures#thinc-shape-inference) automatically. -## TrainablePipe.is_resizable {#is_resizable tag="property"} +## TrainablePipe.is_resizable {id="is_resizable",tag="property"} > #### Example > @@ -421,7 +421,7 @@ as an attribute to the component's model. | ----------- | ---------------------------------------------------------------------------------------------- | | **RETURNS** | Whether or not the output dimension of the model can be changed after initialization. ~~bool~~ | -## TrainablePipe.set_output {#set_output tag="method"} +## TrainablePipe.set_output {id="set_output",tag="method"} Change the output dimension of the component's model. If the component is not [resizable](#is_resizable), this method will raise a `NotImplementedError`. If a @@ -441,7 +441,7 @@ care should be taken to avoid the "catastrophic forgetting" problem. | ---- | --------------------------------- | | `nO` | The new output dimension. ~~int~~ | -## TrainablePipe.to_disk {#to_disk tag="method"} +## TrainablePipe.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -458,7 +458,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## TrainablePipe.from_disk {#from_disk tag="method"} +## TrainablePipe.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -476,7 +476,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified pipe. ~~TrainablePipe~~ | -## TrainablePipe.to_bytes {#to_bytes tag="method"} +## TrainablePipe.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -493,7 +493,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the pipe. ~~bytes~~ | -## TrainablePipe.from_bytes {#from_bytes tag="method"} +## TrainablePipe.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -512,7 +512,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The pipe. ~~TrainablePipe~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | ------- | --------------------------------------------------------------------------------------------------------------------------------- | @@ -521,7 +521,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `name` | The name of the component instance in the pipeline. Can be used in the losses. ~~str~~ | | `cfg` | Keyword arguments passed to [`TrainablePipe.__init__`](/api/pipe#init). Will be serialized with the component. ~~Dict[str, Any]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/pipeline-functions.md b/website/docs/api/pipeline-functions.mdx similarity index 95% rename from website/docs/api/pipeline-functions.md rename to website/docs/api/pipeline-functions.mdx index 070292782..545ace2f2 100644 --- a/website/docs/api/pipeline-functions.md +++ b/website/docs/api/pipeline-functions.mdx @@ -10,7 +10,7 @@ menu: - ['doc_cleaner', 'doc_cleaner'] --- -## merge_noun_chunks {#merge_noun_chunks tag="function"} +## merge_noun_chunks {id="merge_noun_chunks",tag="function"} Merge noun chunks into a single token. Also available via the string name `"merge_noun_chunks"`. @@ -40,7 +40,7 @@ all other components. | `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | | **RETURNS** | The modified `Doc` with merged noun chunks. ~~Doc~~ | -## merge_entities {#merge_entities tag="function"} +## merge_entities {id="merge_entities",tag="function"} Merge named entities into a single token. Also available via the string name `"merge_entities"`. @@ -70,7 +70,7 @@ components to the end of the pipeline and after all other components. | `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | | **RETURNS** | The modified `Doc` with merged entities. ~~Doc~~ | -## merge_subtokens {#merge_subtokens tag="function" new="2.1"} +## merge_subtokens {id="merge_subtokens",tag="function",version="2.1"} Merge subtokens into a single token. Also available via the string name `"merge_subtokens"`. As of v2.1, the parser is able to predict "subtokens" that @@ -110,7 +110,7 @@ end of the pipeline and after all other components. | `label` | The subtoken dependency label. Defaults to `"subtok"`. ~~str~~ | | **RETURNS** | The modified `Doc` with merged subtokens. ~~Doc~~ | -## token_splitter {#token_splitter tag="function" new="3.0"} +## token_splitter {id="token_splitter",tag="function",version="3.0"} Split tokens longer than a minimum length into shorter tokens. Intended for use with transformer pipelines where long spaCy tokens lead to input text that @@ -132,7 +132,7 @@ exceed the transformer model max length. | `split_length` | The length of the split tokens. Defaults to `5`. ~~int~~ | | **RETURNS** | The modified `Doc` with the split tokens. ~~Doc~~ | -## doc_cleaner {#doc_cleaner tag="function" new="3.2.1"} +## doc_cleaner {id="doc_cleaner",tag="function",version="3.2.1"} Clean up `Doc` attributes. Intended for use at the end of pipelines with `tok2vec` or `transformer` pipeline components that store tensors and other @@ -154,7 +154,7 @@ whole pipeline has run. | `silent` | If `False`, show warnings if attributes aren't found or can't be set. Defaults to `True`. ~~bool~~ | | **RETURNS** | The modified `Doc` with the modified attributes. ~~Doc~~ | -## span_cleaner {#span_cleaner tag="function,experimental"} +## span_cleaner {id="span_cleaner",tag="function,experimental"} Remove `SpanGroup`s from `doc.spans` based on a key prefix. This is used to clean up after the [`CoreferenceResolver`](/api/coref) when it's paired with a diff --git a/website/docs/api/scorer.md b/website/docs/api/scorer.mdx similarity index 96% rename from website/docs/api/scorer.md rename to website/docs/api/scorer.mdx index 86e61da1e..6f0c95f6f 100644 --- a/website/docs/api/scorer.md +++ b/website/docs/api/scorer.mdx @@ -10,7 +10,7 @@ The `Scorer` computes evaluation scores. It's typically created by provides a number of evaluation methods for evaluating [`Token`](/api/token) and [`Doc`](/api/doc) attributes. -## Scorer.\_\_init\_\_ {#init tag="method"} +## Scorer.\_\_init\_\_ {id="init",tag="method"} Create a new `Scorer`. @@ -35,7 +35,7 @@ Create a new `Scorer`. | _keyword-only_ | | | `\*\*kwargs` | Any additional settings to pass on to the individual scoring methods. ~~Any~~ | -## Scorer.score {#score tag="method"} +## Scorer.score {id="score",tag="method"} Calculate the scores for a list of [`Example`](/api/example) objects using the scoring methods provided by the components in the pipeline. @@ -72,7 +72,7 @@ core pipeline components, the individual score names start with the `Token` or | `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | | **RETURNS** | A dictionary of scores. ~~Dict[str, Union[float, Dict[str, float]]]~~ | -## Scorer.score_tokenization {#score_tokenization tag="staticmethod" new="3"} +## Scorer.score_tokenization {id="score_tokenization",tag="staticmethod",version="3"} Scores the tokenization: @@ -93,7 +93,7 @@ Docs with `has_unknown_spaces` are skipped during scoring. | `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | | **RETURNS** | `Dict` | A dictionary containing the scores `token_acc`, `token_p`, `token_r`, `token_f`. ~~Dict[str, float]]~~ | -## Scorer.score_token_attr {#score_token_attr tag="staticmethod" new="3"} +## Scorer.score_token_attr {id="score_token_attr",tag="staticmethod",version="3"} Scores a single token attribute. Tokens with missing values in the reference doc are skipped during scoring. @@ -114,7 +114,7 @@ are skipped during scoring. | `missing_values` | Attribute values to treat as missing annotation in the reference annotation. Defaults to `{0, None, ""}`. ~~Set[Any]~~ | | **RETURNS** | A dictionary containing the score `{attr}_acc`. ~~Dict[str, float]~~ | -## Scorer.score_token_attr_per_feat {#score_token_attr_per_feat tag="staticmethod" new="3"} +## Scorer.score_token_attr_per_feat {id="score_token_attr_per_feat",tag="staticmethod",version="3"} Scores a single token attribute per feature for a token attribute in the Universal Dependencies @@ -138,7 +138,7 @@ scoring. | `missing_values` | Attribute values to treat as missing annotation in the reference annotation. Defaults to `{0, None, ""}`. ~~Set[Any]~~ | | **RETURNS** | A dictionary containing the micro PRF scores under the key `{attr}_micro_p/r/f` and the per-feature PRF scores under `{attr}_per_feat`. ~~Dict[str, Dict[str, float]]~~ | -## Scorer.score_spans {#score_spans tag="staticmethod" new="3"} +## Scorer.score_spans {id="score_spans",tag="staticmethod",version="3"} Returns PRF scores for labeled or unlabeled spans. @@ -160,7 +160,7 @@ Returns PRF scores for labeled or unlabeled spans. | `allow_overlap` | Defaults to `False`. Whether or not to allow overlapping spans. If set to `False`, the alignment will automatically resolve conflicts. ~~bool~~ | | **RETURNS** | A dictionary containing the PRF scores under the keys `{attr}_p`, `{attr}_r`, `{attr}_f` and the per-type PRF scores under `{attr}_per_type`. ~~Dict[str, Union[float, Dict[str, float]]]~~ | -## Scorer.score_deps {#score_deps tag="staticmethod" new="3"} +## Scorer.score_deps {id="score_deps",tag="staticmethod",version="3"} Calculate the UAS, LAS, and LAS per type scores for dependency parses. Tokens with missing values for the `attr` (typically `dep`) are skipped during scoring. @@ -194,7 +194,7 @@ with missing values for the `attr` (typically `dep`) are skipped during scoring. | `missing_values` | Attribute values to treat as missing annotation in the reference annotation. Defaults to `{0, None, ""}`. ~~Set[Any]~~ | | **RETURNS** | A dictionary containing the scores: `{attr}_uas`, `{attr}_las`, and `{attr}_las_per_type`. ~~Dict[str, Union[float, Dict[str, float]]]~~ | -## Scorer.score_cats {#score_cats tag="staticmethod" new="3"} +## Scorer.score_cats {id="score_cats",tag="staticmethod",version="3"} Calculate PRF and ROC AUC scores for a doc-level attribute that is a dict containing scores for each label like `Doc.cats`. The returned dictionary @@ -241,7 +241,7 @@ The reported `{attr}_score` depends on the classification properties: | `threshold` | Cutoff to consider a prediction "positive". Defaults to `0.5` for multi-label, and `0.0` (i.e. whatever's highest scoring) otherwise. ~~float~~ | | **RETURNS** | A dictionary containing the scores, with inapplicable scores as `None`. ~~Dict[str, Optional[float]]~~ | -## Scorer.score_links {#score_links tag="staticmethod" new="3"} +## Scorer.score_links {id="score_links",tag="staticmethod",version="3"} Returns PRF for predicted links on the entity level. To disentangle the performance of the NEL from the NER, this method only evaluates NEL links for @@ -264,7 +264,7 @@ entities that overlap between the gold reference and the predictions. | `negative_labels` | The string values that refer to no annotation (e.g. "NIL"). ~~Iterable[str]~~ | | **RETURNS** | A dictionary containing the scores. ~~Dict[str, Optional[float]]~~ | -## get_ner_prf {#get_ner_prf new="3"} +## get_ner_prf {id="get_ner_prf",version="3"} Compute micro-PRF and per-entity PRF scores. @@ -272,7 +272,7 @@ Compute micro-PRF and per-entity PRF scores. | ---------- | ------------------------------------------------------------------------------------------------------------------- | | `examples` | The `Example` objects holding both the predictions and the correct gold-standard annotations. ~~Iterable[Example]~~ | -## score_coref_clusters {#score_coref_clusters tag="experimental"} +## score_coref_clusters {id="score_coref_clusters",tag="experimental"} Returns LEA ([Moosavi and Strube, 2016](https://aclanthology.org/P16-1060/)) PRF scores for coreference clusters. @@ -301,7 +301,7 @@ the [CoreferenceResolver](/api/coref) docs. | `span_cluster_prefix` | The prefix used for spans representing coreference clusters. ~~str~~ | | **RETURNS** | A dictionary containing the scores. ~~Dict[str, Optional[float]]~~ | -## score_span_predictions {#score_span_predictions tag="experimental"} +## score_span_predictions {id="score_span_predictions",tag="experimental"} Return accuracy for reconstructions of spans from single tokens. Only exactly correct predictions are counted as correct, there is no partial credit for near diff --git a/website/docs/api/sentencerecognizer.md b/website/docs/api/sentencerecognizer.mdx similarity index 94% rename from website/docs/api/sentencerecognizer.md rename to website/docs/api/sentencerecognizer.mdx index 2f50350ae..5435399f9 100644 --- a/website/docs/api/sentencerecognizer.md +++ b/website/docs/api/sentencerecognizer.mdx @@ -2,7 +2,7 @@ title: SentenceRecognizer tag: class source: spacy/pipeline/senter.pyx -new: 3 +version: 3 teaser: 'Pipeline component for sentence segmentation' api_base_class: /api/tagger api_string_name: senter @@ -12,7 +12,7 @@ api_trainable: true A trainable pipeline component for sentence segmentation. For a simpler, rule-based strategy, see the [`Sentencizer`](/api/sentencizer). -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predicted values will be assigned to `Token.is_sent_start`. The resulting sentences can be accessed using `Doc.sents`. @@ -22,7 +22,7 @@ sentences can be accessed using `Doc.sents`. | `Token.is_sent_start` | A boolean value indicating whether the token starts a sentence. This will be either `True` or `False` for all tokens. ~~bool~~ | | `Doc.sents` | An iterator over sentences in the `Doc`, determined by `Token.is_sent_start` values. ~~Iterator[Span]~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -49,7 +49,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/senter.pyx ``` -## SentenceRecognizer.\_\_init\_\_ {#init tag="method"} +## SentenceRecognizer.\_\_init\_\_ {id="init",tag="method"} Initialize the sentence recognizer. @@ -81,7 +81,7 @@ shortcut for this and instantiate the component using its string name and | `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | | `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for the attribute `"sents"`. ~~Optional[Callable]~~ | -## SentenceRecognizer.\_\_call\_\_ {#call tag="method"} +## SentenceRecognizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -105,7 +105,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## SentenceRecognizer.pipe {#pipe tag="method"} +## SentenceRecognizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -129,7 +129,7 @@ and [`pipe`](/api/sentencerecognizer#pipe) delegate to the | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## SentenceRecognizer.initialize {#initialize tag="method"} +## SentenceRecognizer.initialize {id="initialize",tag="method"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -153,7 +153,7 @@ by [`Language.initialize`](/api/language#initialize). | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | -## SentenceRecognizer.predict {#predict tag="method"} +## SentenceRecognizer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -170,7 +170,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## SentenceRecognizer.set_annotations {#set_annotations tag="method"} +## SentenceRecognizer.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -187,7 +187,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `SentenceRecognizer.predict`. | -## SentenceRecognizer.update {#update tag="method"} +## SentenceRecognizer.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -211,7 +211,7 @@ Delegates to [`predict`](/api/sentencerecognizer#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## SentenceRecognizer.rehearse {#rehearse tag="method,experimental" new="3"} +## SentenceRecognizer.rehearse {id="rehearse",tag="method,experimental",version="3"} Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the current model to make predictions similar to an initial model to try to address @@ -234,7 +234,7 @@ the "catastrophic forgetting" problem. This feature is experimental. | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## SentenceRecognizer.get_loss {#get_loss tag="method"} +## SentenceRecognizer.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -253,7 +253,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## SentenceRecognizer.create_optimizer {#create_optimizer tag="method"} +## SentenceRecognizer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -268,7 +268,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## SentenceRecognizer.use_params {#use_params tag="method, contextmanager"} +## SentenceRecognizer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -285,7 +285,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## SentenceRecognizer.to_disk {#to_disk tag="method"} +## SentenceRecognizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -302,7 +302,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## SentenceRecognizer.from_disk {#from_disk tag="method"} +## SentenceRecognizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -320,7 +320,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `SentenceRecognizer` object. ~~SentenceRecognizer~~ | -## SentenceRecognizer.to_bytes {#to_bytes tag="method"} +## SentenceRecognizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -337,7 +337,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `SentenceRecognizer` object. ~~bytes~~ | -## SentenceRecognizer.from_bytes {#from_bytes tag="method"} +## SentenceRecognizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -356,7 +356,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `SentenceRecognizer` object. ~~SentenceRecognizer~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/sentencizer.md b/website/docs/api/sentencizer.mdx similarity index 94% rename from website/docs/api/sentencizer.md rename to website/docs/api/sentencizer.mdx index b75c7a2f1..9fb5ea71f 100644 --- a/website/docs/api/sentencizer.md +++ b/website/docs/api/sentencizer.mdx @@ -13,7 +13,7 @@ performed by the [`DependencyParser`](/api/dependencyparser), so the `Sentencizer` lets you implement a simpler, rule-based strategy that doesn't require a statistical model to be loaded. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Calculated values will be assigned to `Token.is_sent_start`. The resulting sentences can be accessed using `Doc.sents`. @@ -23,7 +23,7 @@ sentences can be accessed using `Doc.sents`. | `Token.is_sent_start` | A boolean value indicating whether the token starts a sentence. This will be either `True` or `False` for all tokens. ~~bool~~ | | `Doc.sents` | An iterator over sentences in the `Doc`, determined by `Token.is_sent_start` values. ~~Iterator[Span]~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -39,7 +39,7 @@ how the component should be configured. You can override its settings via the | Setting | Description | | ---------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `punct_chars` | Optional custom list of punctuation characters that mark sentence ends. See below for defaults if not set. Defaults to `None`. ~~Optional[List[str]]~~ | `None` | +| `punct_chars` | Optional custom list of punctuation characters that mark sentence ends. See below for defaults if not set. Defaults to `None`. ~~Optional[List[str]]~~ | | `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | | `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for the attribute `"sents"` ~~Optional[Callable]~~ | @@ -47,7 +47,7 @@ how the component should be configured. You can override its settings via the %%GITHUB_SPACY/spacy/pipeline/sentencizer.pyx ``` -## Sentencizer.\_\_init\_\_ {#init tag="method"} +## Sentencizer.\_\_init\_\_ {id="init",tag="method"} Initialize the sentencizer. @@ -69,8 +69,7 @@ Initialize the sentencizer. | `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | | `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for the attribute `"sents"` ~~Optional[Callable]~~ | -```python -### punct_chars defaults +```python {title="punct_chars defaults"} ['!', '.', '?', '։', '؟', '۔', '܀', '܁', '܂', '߹', '।', '॥', '၊', '။', '።', '፧', '፨', '᙮', '᜵', '᜶', '᠃', '᠉', '᥄', '᥅', '᪨', '᪩', '᪪', '᪫', '᭚', '᭛', '᭞', '᭟', '᰻', '᰼', '᱾', '᱿', '‼', '‽', '⁇', '⁈', '⁉', @@ -83,7 +82,7 @@ Initialize the sentencizer. '𑪜', '𑱁', '𑱂', '𖩮', '𖩯', '𖫵', '𖬷', '𖬸', '𖭄', '𛲟', '𝪈', '｡', '。'] ``` -## Sentencizer.\_\_call\_\_ {#call tag="method"} +## Sentencizer.\_\_call\_\_ {id="call",tag="method"} Apply the sentencizer on a `Doc`. Typically, this happens automatically after the component has been added to the pipeline using @@ -105,7 +104,7 @@ the component has been added to the pipeline using | `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | | **RETURNS** | The modified `Doc` with added sentence boundaries. ~~Doc~~ | -## Sentencizer.pipe {#pipe tag="method"} +## Sentencizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -126,7 +125,7 @@ applied to the `Doc` in order. | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## Sentencizer.to_disk {#to_disk tag="method"} +## Sentencizer.to_disk {id="to_disk",tag="method"} Save the sentencizer settings (punctuation characters) to a directory. Will create a file `sentencizer.json`. This also happens automatically when you save @@ -144,7 +143,7 @@ an `nlp` object with a sentencizer added to its pipeline. | ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | | `path` | A path to a JSON file, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | -## Sentencizer.from_disk {#from_disk tag="method"} +## Sentencizer.from_disk {id="from_disk",tag="method"} Load the sentencizer settings from a file. Expects a JSON file. This also happens automatically when you load an `nlp` object or model with a sentencizer @@ -162,7 +161,7 @@ added to its pipeline. | `path` | A path to a JSON file. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | **RETURNS** | The modified `Sentencizer` object. ~~Sentencizer~~ | -## Sentencizer.to_bytes {#to_bytes tag="method"} +## Sentencizer.to_bytes {id="to_bytes",tag="method"} Serialize the sentencizer settings to a bytestring. @@ -178,7 +177,7 @@ Serialize the sentencizer settings to a bytestring. | ----------- | ------------------------------ | | **RETURNS** | The serialized data. ~~bytes~~ | -## Sentencizer.from_bytes {#from_bytes tag="method"} +## Sentencizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. diff --git a/website/docs/api/span-resolver.md b/website/docs/api/span-resolver.mdx similarity index 94% rename from website/docs/api/span-resolver.md rename to website/docs/api/span-resolver.mdx index 3e992cd03..f061d8df3 100644 --- a/website/docs/api/span-resolver.md +++ b/website/docs/api/span-resolver.mdx @@ -33,7 +33,7 @@ use case is as a post-processing step on word-level [coreference resolution](/api/coref). The input and output keys used to store `Span` objects are configurable. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions will be saved to `Doc.spans` as [`SpanGroup`s](/api/spangroup). @@ -46,7 +46,7 @@ prefixes are configurable. | ------------------------------------------------- | ------------------------------------------------------------------------- | | `Doc.spans[output_prefix + "_" + cluster_number]` | One group of predicted spans. Cluster number starts from 1. ~~SpanGroup~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -74,7 +74,7 @@ details on the architectures and their arguments and hyperparameters. | `input_prefix` | The prefix to use for input `SpanGroup`s. Defaults to `coref_head_clusters`. ~~str~~ | | `output_prefix` | The prefix for predicted `SpanGroup`s. Defaults to `coref_clusters`. ~~str~~ | -## SpanResolver.\_\_init\_\_ {#init tag="method"} +## SpanResolver.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -104,7 +104,7 @@ shortcut for this and instantiate the component using its string name and | `input_prefix` | The prefix to use for input `SpanGroup`s. Defaults to `coref_head_clusters`. ~~str~~ | | `output_prefix` | The prefix for predicted `SpanGroup`s. Defaults to `coref_clusters`. ~~str~~ | -## SpanResolver.\_\_call\_\_ {#call tag="method"} +## SpanResolver.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -126,7 +126,7 @@ and [`set_annotations`](#set_annotations) methods. | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## SpanResolver.pipe {#pipe tag="method"} +## SpanResolver.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -150,7 +150,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/span-resolver#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## SpanResolver.initialize {#initialize tag="method"} +## SpanResolver.initialize {id="initialize",tag="method"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -174,7 +174,7 @@ by [`Language.initialize`](/api/language#initialize). | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | -## SpanResolver.predict {#predict tag="method"} +## SpanResolver.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. Predictions are returned as a list of `MentionClusters`, one for @@ -194,7 +194,7 @@ correspond to token indices. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The predicted spans for the `Doc`s. ~~List[MentionClusters]~~ | -## SpanResolver.set_annotations {#set_annotations tag="method"} +## SpanResolver.set_annotations {id="set_annotations",tag="method"} Modify a batch of documents, saving predictions using the output prefix in `Doc.spans`. @@ -212,7 +212,7 @@ Modify a batch of documents, saving predictions using the output prefix in | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `spans` | The predicted spans for the `docs`. ~~List[MentionClusters]~~ | -## SpanResolver.update {#update tag="method"} +## SpanResolver.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects. Delegates to [`predict`](/api/span-resolver#predict). @@ -234,7 +234,7 @@ Learn from a batch of [`Example`](/api/example) objects. Delegates to | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## SpanResolver.create_optimizer {#create_optimizer tag="method"} +## SpanResolver.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -249,7 +249,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## SpanResolver.use_params {#use_params tag="method, contextmanager"} +## SpanResolver.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -266,7 +266,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## SpanResolver.to_disk {#to_disk tag="method"} +## SpanResolver.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -283,7 +283,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## SpanResolver.from_disk {#from_disk tag="method"} +## SpanResolver.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -301,7 +301,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `SpanResolver` object. ~~SpanResolver~~ | -## SpanResolver.to_bytes {#to_bytes tag="method"} +## SpanResolver.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -318,7 +318,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `SpanResolver` object. ~~bytes~~ | -## SpanResolver.from_bytes {#from_bytes tag="method"} +## SpanResolver.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -337,7 +337,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `SpanResolver` object. ~~SpanResolver~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/span.md b/website/docs/api/span.mdx similarity index 93% rename from website/docs/api/span.md rename to website/docs/api/span.mdx index 69bbe8db1..bd7794edc 100644 --- a/website/docs/api/span.md +++ b/website/docs/api/span.mdx @@ -6,7 +6,7 @@ source: spacy/tokens/span.pyx A slice from a [`Doc`](/api/doc) object. -## Span.\_\_init\_\_ {#init tag="method"} +## Span.\_\_init\_\_ {id="init",tag="method"} Create a `Span` object from the slice `doc[start : end]`. @@ -29,7 +29,7 @@ Create a `Span` object from the slice `doc[start : end]`. | `kb_id` | A knowledge base ID to attach to the span, e.g. for named entities. ~~Union[str, int]~~ | | `span_id` | An ID to associate with the span. ~~Union[str, int]~~ | -## Span.\_\_getitem\_\_ {#getitem tag="method"} +## Span.\_\_getitem\_\_ {id="getitem",tag="method"} Get a `Token` object. @@ -61,7 +61,7 @@ Get a `Span` object. | `start_end` | The slice of the span to get. ~~Tuple[int, int]~~ | | **RETURNS** | The span at `span[start : end]`. ~~Span~~ | -## Span.\_\_iter\_\_ {#iter tag="method"} +## Span.\_\_iter\_\_ {id="iter",tag="method"} Iterate over `Token` objects. @@ -77,7 +77,7 @@ Iterate over `Token` objects. | ---------- | --------------------------- | | **YIELDS** | A `Token` object. ~~Token~~ | -## Span.\_\_len\_\_ {#len tag="method"} +## Span.\_\_len\_\_ {id="len",tag="method"} Get the number of tokens in the span. @@ -93,7 +93,7 @@ Get the number of tokens in the span. | ----------- | ----------------------------------------- | | **RETURNS** | The number of tokens in the span. ~~int~~ | -## Span.set_extension {#set_extension tag="classmethod" new="2"} +## Span.set_extension {id="set_extension",tag="classmethod",version="2"} Define a custom attribute on the `Span` which becomes available via `Span._`. For details, see the documentation on @@ -118,7 +118,7 @@ For details, see the documentation on | `setter` | Setter function that takes the `Span` and a value, and modifies the object. Is called when the user writes to the `Span._` attribute. ~~Optional[Callable[[Span, Any], None]]~~ | | `force` | Force overwriting existing attribute. ~~bool~~ | -## Span.get_extension {#get_extension tag="classmethod" new="2"} +## Span.get_extension {id="get_extension",tag="classmethod",version="2"} Look up a previously registered extension by name. Returns a 4-tuple `(default, method, getter, setter)` if the extension is registered. Raises a @@ -138,7 +138,7 @@ Look up a previously registered extension by name. Returns a 4-tuple | `name` | Name of the extension. ~~str~~ | | **RETURNS** | A `(default, method, getter, setter)` tuple of the extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | -## Span.has_extension {#has_extension tag="classmethod" new="2"} +## Span.has_extension {id="has_extension",tag="classmethod",version="2"} Check whether an extension has been registered on the `Span` class. @@ -155,7 +155,7 @@ Check whether an extension has been registered on the `Span` class. | `name` | Name of the extension to check. ~~str~~ | | **RETURNS** | Whether the extension has been registered. ~~bool~~ | -## Span.remove_extension {#remove_extension tag="classmethod" new="2.0.12"} +## Span.remove_extension {id="remove_extension",tag="classmethod",version="2.0.12"} Remove a previously registered extension. @@ -173,7 +173,7 @@ Remove a previously registered extension. | `name` | Name of the extension. ~~str~~ | | **RETURNS** | A `(default, method, getter, setter)` tuple of the removed extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | -## Span.char_span {#char_span tag="method" new="2.2.4"} +## Span.char_span {id="char_span",tag="method",version="2.2.4"} Create a `Span` object from the slice `span.text[start:end]`. Returns `None` if the character indices don't map to a valid span. @@ -195,7 +195,7 @@ the character indices don't map to a valid span. | `vector` | A meaning representation of the span. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | | **RETURNS** | The newly constructed object or `None`. ~~Optional[Span]~~ | -## Span.similarity {#similarity tag="method" model="vectors"} +## Span.similarity {id="similarity",tag="method",model="vectors"} Make a semantic similarity estimate. The default estimate is cosine similarity using an average of word vectors. @@ -216,7 +216,7 @@ using an average of word vectors. | `other` | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ | | **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ | -## Span.get_lca_matrix {#get_lca_matrix tag="method"} +## Span.get_lca_matrix {id="get_lca_matrix",tag="method"} Calculates the lowest common ancestor matrix for a given `Span`. Returns LCA matrix containing the integer index of the ancestor, or `-1` if no common @@ -235,7 +235,7 @@ ancestor is found, e.g. if span excludes a necessary ancestor. | ----------- | --------------------------------------------------------------------------------------- | | **RETURNS** | The lowest common ancestor matrix of the `Span`. ~~numpy.ndarray[ndim=2, dtype=int32]~~ | -## Span.to_array {#to_array tag="method" new="2"} +## Span.to_array {id="to_array",tag="method",version="2"} Given a list of `M` attribute IDs, export the tokens to a numpy `ndarray` of shape `(N, M)`, where `N` is the length of the document. The values will be @@ -256,7 +256,7 @@ shape `(N, M)`, where `N` is the length of the document. The values will be | `attr_ids` | A list of attributes (int IDs or string names) or a single attribute (int ID or string name). ~~Union[int, str, List[Union[int, str]]]~~ | | **RETURNS** | The exported attributes as a numpy array. ~~Union[numpy.ndarray[ndim=2, dtype=uint64], numpy.ndarray[ndim=1, dtype=uint64]]~~ | -## Span.ents {#ents tag="property" new="2.0.13" model="ner"} +## Span.ents {id="ents",tag="property",version="2.0.13",model="ner"} The named entities that fall completely within the span. Returns a tuple of `Span` objects. @@ -276,7 +276,7 @@ The named entities that fall completely within the span. Returns a tuple of | ----------- | ----------------------------------------------------------------- | | **RETURNS** | Entities in the span, one `Span` per entity. ~~Tuple[Span, ...]~~ | -## Span.noun_chunks {#noun_chunks tag="property" model="parser"} +## Span.noun_chunks {id="noun_chunks",tag="property",model="parser"} Iterate over the base noun phrases in the span. Yields base noun-phrase `Span` objects, if the document has been syntactically parsed. A base noun phrase, or @@ -302,7 +302,7 @@ raised. | ---------- | --------------------------------- | | **YIELDS** | Noun chunks in the span. ~~Span~~ | -## Span.as_doc {#as_doc tag="method"} +## Span.as_doc {id="as_doc",tag="method"} Create a new `Doc` object corresponding to the `Span`, with a copy of the data. @@ -326,7 +326,7 @@ time. | `array` | Precomputed array version of the original doc as generated by [`Doc.to_array`](/api/doc#to_array). ~~numpy.ndarray~~ | | **RETURNS** | A `Doc` object of the `Span`'s content. ~~Doc~~ | -## Span.root {#root tag="property" model="parser"} +## Span.root {id="root",tag="property",model="parser"} The token with the shortest path to the root of the sentence (or the root itself). If multiple tokens are equally high in the tree, the first token is @@ -347,7 +347,7 @@ taken. | ----------- | ------------------------- | | **RETURNS** | The root token. ~~Token~~ | -## Span.conjuncts {#conjuncts tag="property" model="parser"} +## Span.conjuncts {id="conjuncts",tag="property",model="parser"} A tuple of tokens coordinated to `span.root`. @@ -363,7 +363,7 @@ A tuple of tokens coordinated to `span.root`. | ----------- | --------------------------------------------- | | **RETURNS** | The coordinated tokens. ~~Tuple[Token, ...]~~ | -## Span.lefts {#lefts tag="property" model="parser"} +## Span.lefts {id="lefts",tag="property",model="parser"} Tokens that are to the left of the span, whose heads are within the span. @@ -379,7 +379,7 @@ Tokens that are to the left of the span, whose heads are within the span. | ---------- | ---------------------------------------------- | | **YIELDS** | A left-child of a token of the span. ~~Token~~ | -## Span.rights {#rights tag="property" model="parser"} +## Span.rights {id="rights",tag="property",model="parser"} Tokens that are to the right of the span, whose heads are within the span. @@ -395,7 +395,7 @@ Tokens that are to the right of the span, whose heads are within the span. | ---------- | ----------------------------------------------- | | **YIELDS** | A right-child of a token of the span. ~~Token~~ | -## Span.n_lefts {#n_lefts tag="property" model="parser"} +## Span.n_lefts {id="n_lefts",tag="property",model="parser"} The number of tokens that are to the left of the span, whose heads are within the span. @@ -411,7 +411,7 @@ the span. | ----------- | ---------------------------------------- | | **RETURNS** | The number of left-child tokens. ~~int~~ | -## Span.n_rights {#n_rights tag="property" model="parser"} +## Span.n_rights {id="n_rights",tag="property",model="parser"} The number of tokens that are to the right of the span, whose heads are within the span. @@ -427,7 +427,7 @@ the span. | ----------- | ----------------------------------------- | | **RETURNS** | The number of right-child tokens. ~~int~~ | -## Span.subtree {#subtree tag="property" model="parser"} +## Span.subtree {id="subtree",tag="property",model="parser"} Tokens within the span and tokens which descend from them. @@ -443,7 +443,7 @@ Tokens within the span and tokens which descend from them. | ---------- | ----------------------------------------------------------- | | **YIELDS** | A token within the span, or a descendant from it. ~~Token~~ | -## Span.has_vector {#has_vector tag="property" model="vectors"} +## Span.has_vector {id="has_vector",tag="property",model="vectors"} A boolean value indicating whether a word vector is associated with the object. @@ -458,7 +458,7 @@ A boolean value indicating whether a word vector is associated with the object. | ----------- | ----------------------------------------------------- | | **RETURNS** | Whether the span has a vector data attached. ~~bool~~ | -## Span.vector {#vector tag="property" model="vectors"} +## Span.vector {id="vector",tag="property",model="vectors"} A real-valued meaning representation. Defaults to an average of the token vectors. @@ -475,7 +475,7 @@ vectors. | ----------- | ----------------------------------------------------------------------------------------------- | | **RETURNS** | A 1-dimensional array representing the span's vector. ~~`numpy.ndarray[ndim=1, dtype=float32]~~ | -## Span.vector_norm {#vector_norm tag="property" model="vectors"} +## Span.vector_norm {id="vector_norm",tag="property",model="vectors"} The L2 norm of the span's vector representation. @@ -492,7 +492,7 @@ The L2 norm of the span's vector representation. | ----------- | --------------------------------------------------- | | **RETURNS** | The L2 norm of the vector representation. ~~float~~ | -## Span.sent {#sent tag="property" model="sentences"} +## Span.sent {id="sent",tag="property",model="sentences"} The sentence span that this span is a part of. This property is only available when [sentence boundaries](/usage/linguistic-features#sbd) have been set on the @@ -520,7 +520,7 @@ sent = doc[sent.start : max(sent.end, span.end)] | ----------- | ------------------------------------------------------- | | **RETURNS** | The sentence span that this span is a part of. ~~Span~~ | -## Span.sents {#sents tag="property" model="sentences" new="3.2.1"} +## Span.sents {id="sents",tag="property",model="sentences",version="3.2.1"} Returns a generator over the sentences the span belongs to. This property is only available when [sentence boundaries](/usage/linguistic-features#sbd) have @@ -542,7 +542,7 @@ overlaps with will be returned. | ----------- | -------------------------------------------------------------------------- | | **RETURNS** | A generator yielding sentences this `Span` is a part of ~~Iterable[Span]~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | -------------- | ----------------------------------------------------------------------------------------------------------------------------- | diff --git a/website/docs/api/spancategorizer.md b/website/docs/api/spancategorizer.mdx similarity index 94% rename from website/docs/api/spancategorizer.md rename to website/docs/api/spancategorizer.mdx index 58a06bcf5..f39c0aff9 100644 --- a/website/docs/api/spancategorizer.md +++ b/website/docs/api/spancategorizer.mdx @@ -2,7 +2,7 @@ title: SpanCategorizer tag: class,experimental source: spacy/pipeline/spancat.py -new: 3.1 +version: 3.1 teaser: 'Pipeline component for labeling potentially overlapping spans of text' api_base_class: /api/pipe api_string_name: spancat @@ -16,7 +16,7 @@ that predicts zero or more labels for each candidate. Predicted spans will be saved in a [`SpanGroup`](/api/spangroup) on the doc. Individual span scores can be found in `spangroup.attrs["scores"]`. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions will be saved to `Doc.spans[spans_key]` as a [`SpanGroup`](/api/spangroup). The scores for the spans in the `SpanGroup` will @@ -29,7 +29,7 @@ be saved in `SpanGroup.attrs["scores"]`. | `Doc.spans[spans_key]` | The annotated spans. ~~SpanGroup~~ | | `Doc.spans[spans_key].attrs["scores"]` | The score for each span in the `SpanGroup`. ~~Floats1d~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -65,7 +65,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/spancat.py ``` -## SpanCategorizer.\_\_init\_\_ {#init tag="method"} +## SpanCategorizer.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -97,7 +97,7 @@ shortcut for this and instantiate the component using its string name and | `threshold` | Minimum probability to consider a prediction positive. Spans with a positive prediction will be saved on the Doc. Defaults to `0.5`. ~~float~~ | | `max_positive` | Maximum number of labels to consider positive per span. Defaults to `None`, indicating no limit. ~~Optional[int]~~ | -## SpanCategorizer.\_\_call\_\_ {#call tag="method"} +## SpanCategorizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -120,7 +120,7 @@ delegate to the [`predict`](/api/spancategorizer#predict) and | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## SpanCategorizer.pipe {#pipe tag="method"} +## SpanCategorizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -144,7 +144,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/spancategorizer#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## SpanCategorizer.initialize {#initialize tag="method"} +## SpanCategorizer.initialize {id="initialize",tag="method"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -181,7 +181,7 @@ config. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | -## SpanCategorizer.predict {#predict tag="method"} +## SpanCategorizer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects without modifying them. @@ -198,7 +198,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## SpanCategorizer.set_annotations {#set_annotations tag="method"} +## SpanCategorizer.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores. @@ -215,7 +215,7 @@ Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `SpanCategorizer.predict`. | -## SpanCategorizer.update {#update tag="method"} +## SpanCategorizer.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -239,7 +239,7 @@ Delegates to [`predict`](/api/spancategorizer#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## SpanCategorizer.set_candidates {#set_candidates tag="method", new="3.3"} +## SpanCategorizer.set_candidates {id="set_candidates",tag="method", version="3.3"} Use the suggester to add a list of [`Span`](/api/span) candidates to a list of [`Doc`](/api/doc) objects. This method is intended to be used for debugging @@ -257,7 +257,7 @@ purposes. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `candidates_key` | Key of the Doc.spans dict to save the candidate spans under. ~~str~~ | -## SpanCategorizer.get_loss {#get_loss tag="method"} +## SpanCategorizer.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -276,7 +276,7 @@ predicted scores. | `spans_scores` | Scores representing the model's predictions. ~~Tuple[Ragged, Floats2d]~~ | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## SpanCategorizer.create_optimizer {#create_optimizer tag="method"} +## SpanCategorizer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -291,7 +291,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## SpanCategorizer.use_params {#use_params tag="method, contextmanager"} +## SpanCategorizer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model to use the given parameter values. @@ -307,7 +307,7 @@ Modify the pipe's model to use the given parameter values. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## SpanCategorizer.add_label {#add_label tag="method"} +## SpanCategorizer.add_label {id="add_label",tag="method"} Add a new label to the pipe. Raises an error if the output dimension is already set, or if the model has already been fully [initialized](#initialize). Note @@ -329,7 +329,7 @@ automatically. | `label` | The label to add. ~~str~~ | | **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | -## SpanCategorizer.to_disk {#to_disk tag="method"} +## SpanCategorizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -346,7 +346,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## SpanCategorizer.from_disk {#from_disk tag="method"} +## SpanCategorizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -364,7 +364,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `SpanCategorizer` object. ~~SpanCategorizer~~ | -## SpanCategorizer.to_bytes {#to_bytes tag="method"} +## SpanCategorizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -381,7 +381,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `SpanCategorizer` object. ~~bytes~~ | -## SpanCategorizer.from_bytes {#from_bytes tag="method"} +## SpanCategorizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -400,7 +400,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `SpanCategorizer` object. ~~SpanCategorizer~~ | -## SpanCategorizer.labels {#labels tag="property"} +## SpanCategorizer.labels {id="labels",tag="property"} The labels currently added to the component. @@ -415,7 +415,7 @@ The labels currently added to the component. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## SpanCategorizer.label_data {#label_data tag="property"} +## SpanCategorizer.label_data {id="label_data",tag="property"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -433,7 +433,7 @@ the model with a pre-defined label set. | ----------- | ---------------------------------------------------------- | | **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from @@ -451,9 +451,9 @@ serialization by passing in the string names via the `exclude` argument. | `cfg` | The config file. You usually don't want to exclude this. | | `model` | The binary model data. You usually don't want to exclude this. | -## Suggesters {#suggesters tag="registered functions" source="spacy/pipeline/spancat.py"} +## Suggesters {id="suggesters",tag="registered functions",source="spacy/pipeline/spancat.py"} -### spacy.ngram_suggester.v1 {#ngram_suggester} +### spacy.ngram_suggester.v1 {id="ngram_suggester"} > #### Example Config > @@ -471,7 +471,7 @@ integers. The array has two columns, indicating the start and end position. | `sizes` | The phrase lengths to suggest. For example, `[1, 2]` will suggest phrases consisting of 1 or 2 tokens. ~~List[int]~~ | | **CREATES** | The suggester function. ~~Callable[[Iterable[Doc], Optional[Ops]], Ragged]~~ | -### spacy.ngram_range_suggester.v1 {#ngram_range_suggester} +### spacy.ngram_range_suggester.v1 {id="ngram_range_suggester"} > #### Example Config > diff --git a/website/docs/api/spangroup.md b/website/docs/api/spangroup.mdx similarity index 92% rename from website/docs/api/spangroup.md rename to website/docs/api/spangroup.mdx index bd9659acb..cd0accb6a 100644 --- a/website/docs/api/spangroup.md +++ b/website/docs/api/spangroup.mdx @@ -2,7 +2,7 @@ title: SpanGroup tag: class source: spacy/tokens/span_group.pyx -new: 3 +version: 3 --- A group of arbitrary, potentially overlapping [`Span`](/api/span) objects that @@ -13,7 +13,7 @@ into a `SpanGroup` object for you automatically on assignment. `SpanGroup` objects behave similar to `list`s, so you can append `Span` objects to them or access a member at a given index. -## SpanGroup.\_\_init\_\_ {#init tag="method"} +## SpanGroup.\_\_init\_\_ {id="init",tag="method"} Create a `SpanGroup`. @@ -42,7 +42,7 @@ Create a `SpanGroup`. | `attrs` | Optional JSON-serializable attributes to attach to the span group. ~~Dict[str, Any]~~ | | `spans` | The spans to add to the span group. ~~Iterable[Span]~~ | -## SpanGroup.doc {#doc tag="property"} +## SpanGroup.doc {id="doc",tag="property"} The [`Doc`](/api/doc) object the span group is referring to. @@ -68,7 +68,7 @@ the scope of your function. | ----------- | ------------------------------- | | **RETURNS** | The reference document. ~~Doc~~ | -## SpanGroup.has_overlap {#has_overlap tag="property"} +## SpanGroup.has_overlap {id="has_overlap",tag="property"} Check whether the span group contains overlapping spans. @@ -86,7 +86,7 @@ Check whether the span group contains overlapping spans. | ----------- | -------------------------------------------------- | | **RETURNS** | Whether the span group contains overlaps. ~~bool~~ | -## SpanGroup.\_\_len\_\_ {#len tag="method"} +## SpanGroup.\_\_len\_\_ {id="len",tag="method"} Get the number of spans in the group. @@ -102,7 +102,7 @@ Get the number of spans in the group. | ----------- | ----------------------------------------- | | **RETURNS** | The number of spans in the group. ~~int~~ | -## SpanGroup.\_\_getitem\_\_ {#getitem tag="method"} +## SpanGroup.\_\_getitem\_\_ {id="getitem",tag="method"} Get a span from the group. Note that a copy of the span is returned, so if any changes are made to this span, they are not reflected in the corresponding @@ -125,7 +125,7 @@ changes to be reflected in the span group. | `i` | The item index. ~~int~~ | | **RETURNS** | The span at the given index. ~~Span~~ | -## SpanGroup.\_\_setitem\_\_ {#setitem tag="method", new="3.3"} +## SpanGroup.\_\_setitem\_\_ {id="setitem",tag="method", version="3.3"} Set a span in the span group. @@ -144,7 +144,7 @@ Set a span in the span group. | `i` | The item index. ~~int~~ | | `span` | The new value. ~~Span~~ | -## SpanGroup.\_\_delitem\_\_ {#delitem tag="method", new="3.3"} +## SpanGroup.\_\_delitem\_\_ {id="delitem",tag="method", version="3.3"} Delete a span from the span group. @@ -161,7 +161,7 @@ Delete a span from the span group. | ---- | ----------------------- | | `i` | The item index. ~~int~~ | -## SpanGroup.\_\_add\_\_ {#add tag="method", new="3.3"} +## SpanGroup.\_\_add\_\_ {id="add",tag="method", version="3.3"} Concatenate the current span group with another span group and return the result in a new span group. Any `attrs` from the first span group will have precedence @@ -182,7 +182,7 @@ over `attrs` in the second. | `other` | The span group or spans to concatenate. ~~Union[SpanGroup, Iterable[Span]]~~ | | **RETURNS** | The new span group. ~~SpanGroup~~ | -## SpanGroup.\_\_iadd\_\_ {#iadd tag="method", new="3.3"} +## SpanGroup.\_\_iadd\_\_ {id="iadd",tag="method", version="3.3"} Append an iterable of spans or the content of a span group to the current span group. Any `attrs` in the other span group will be added for keys that are not @@ -202,7 +202,7 @@ already present in the current span group. | `other` | The span group or spans to append. ~~Union[SpanGroup, Iterable[Span]]~~ | | **RETURNS** | The span group. ~~SpanGroup~~ | -## SpanGroup.\_\_iter\_\_ {#iter tag="method" new="3.5"} +## SpanGroup.\_\_iter\_\_ {id="iter",tag="method",version="3.5"} Iterate over the spans in this span group. @@ -219,7 +219,8 @@ Iterate over the spans in this span group. | ---------- | ----------------------------------- | | **YIELDS** | A span in this span group. ~~Span~~ | -## SpanGroup.append {#append tag="method"} + +## SpanGroup.append {id="append",tag="method"} Add a [`Span`](/api/span) object to the group. The span must refer to the same [`Doc`](/api/doc) object as the span group. @@ -237,7 +238,7 @@ Add a [`Span`](/api/span) object to the group. The span must refer to the same | ------ | ---------------------------- | | `span` | The span to append. ~~Span~~ | -## SpanGroup.extend {#extend tag="method"} +## SpanGroup.extend {id="extend",tag="method"} Add multiple [`Span`](/api/span) objects or contents of another `SpanGroup` to the group. All spans must refer to the same [`Doc`](/api/doc) object as the span @@ -258,7 +259,7 @@ group. | ------- | -------------------------------------------------------- | | `spans` | The spans to add. ~~Union[SpanGroup, Iterable["Span"]]~~ | -## SpanGroup.copy {#copy tag="method", new="3.3"} +## SpanGroup.copy {id="copy",tag="method", version="3.3"} Return a copy of the span group. @@ -277,7 +278,7 @@ Return a copy of the span group. | `doc` | The document to which the copy is bound. Defaults to `None` for the current doc. ~~Optional[Doc]~~ | | **RETURNS** | A copy of the `SpanGroup` object. ~~SpanGroup~~ | -## SpanGroup.to_bytes {#to_bytes tag="method"} +## SpanGroup.to_bytes {id="to_bytes",tag="method"} Serialize the span group to a bytestring. @@ -293,7 +294,7 @@ Serialize the span group to a bytestring. | ----------- | ------------------------------------- | | **RETURNS** | The serialized `SpanGroup`. ~~bytes~~ | -## SpanGroup.from_bytes {#from_bytes tag="method"} +## SpanGroup.from_bytes {id="from_bytes",tag="method"} Load the span group from a bytestring. Modifies the object in place and returns it. diff --git a/website/docs/api/spanruler.md b/website/docs/api/spanruler.mdx similarity index 94% rename from website/docs/api/spanruler.md rename to website/docs/api/spanruler.mdx index 31f04ccf9..d2d41f620 100644 --- a/website/docs/api/spanruler.md +++ b/website/docs/api/spanruler.mdx @@ -2,7 +2,7 @@ title: SpanRuler tag: class source: spacy/pipeline/span_ruler.py -new: 3.3 +version: 3.3 teaser: 'Pipeline component for rule-based span and named entity recognition' api_string_name: span_ruler api_trainable: false @@ -13,7 +13,7 @@ The span ruler lets you add spans to [`Doc.spans`](/api/doc#spans) and/or usage examples, see the docs on [rule-based span matching](/usage/rule-based-matching#spanruler). -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Matches will be saved to `Doc.spans[spans_key]` as a [`SpanGroup`](/api/spangroup) and/or to `Doc.ents`, where the annotation is @@ -28,7 +28,7 @@ saved in the `Token.ent_type` and `Token.ent_iob` fields. | `Token.ent_type` | The label part of the named entity tag (hash). ~~int~~ | | `Token.ent_type_` | The label part of the named entity tag. ~~str~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -62,7 +62,7 @@ how the component should be configured. You can override its settings via the %%GITHUB_SPACY/spacy/pipeline/span_ruler.py ``` -## SpanRuler.\_\_init\_\_ {#init tag="method"} +## SpanRuler.\_\_init\_\_ {id="init",tag="method"} Initialize the span ruler. If patterns are supplied here, they need to be a list of dictionaries with a `"label"` and `"pattern"` key. A pattern can either be a @@ -95,7 +95,7 @@ token pattern (list) or a phrase pattern (string). For example: | `overwrite` | Whether to remove any existing spans under `Doc.spans[spans key]` if `spans_key` is set, or to remove any ents under `Doc.ents` if `annotate_ents` is set. Defaults to `True`. ~~bool~~ | | `scorer` | The scoring method. Defaults to [`Scorer.score_spans`](/api/scorer#score_spans) for `Doc.spans[spans_key]` with overlapping spans allowed. ~~Optional[Callable]~~ | -## SpanRuler.initialize {#initialize tag="method"} +## SpanRuler.initialize {id="initialize",tag="method"} Initialize the component with data and used before training to load in rules from a [pattern file](/usage/rule-based-matching/#spanruler-files). This method @@ -127,7 +127,7 @@ config. Any existing patterns are removed on initialization. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `patterns` | The list of patterns. Defaults to `None`. ~~Optional[Sequence[Dict[str, Union[str, List[Dict[str, Any]]]]]]~~ | -## SpanRuler.\_\len\_\_ {#len tag="method"} +## SpanRuler.\_\_len\_\_ {id="len",tag="method"} The number of all patterns added to the span ruler. @@ -144,7 +144,7 @@ The number of all patterns added to the span ruler. | ----------- | ------------------------------- | | **RETURNS** | The number of patterns. ~~int~~ | -## SpanRuler.\_\_contains\_\_ {#contains tag="method"} +## SpanRuler.\_\_contains\_\_ {id="contains",tag="method"} Whether a label is present in the patterns. @@ -162,7 +162,7 @@ Whether a label is present in the patterns. | `label` | The label to check. ~~str~~ | | **RETURNS** | Whether the span ruler contains the label. ~~bool~~ | -## SpanRuler.\_\_call\_\_ {#call tag="method"} +## SpanRuler.\_\_call\_\_ {id="call",tag="method"} Find matches in the `Doc` and add them to `doc.spans[span_key]` and/or `doc.ents`. Typically, this happens automatically after the component has been @@ -186,7 +186,7 @@ will be removed. | `doc` | The `Doc` object to process, e.g. the `Doc` in the pipeline. ~~Doc~~ | | **RETURNS** | The modified `Doc` with added spans/entities. ~~Doc~~ | -## SpanRuler.add_patterns {#add_patterns tag="method"} +## SpanRuler.add_patterns {id="add_patterns",tag="method"} Add patterns to the span ruler. A pattern can either be a token pattern (list of dicts) or a phrase pattern (string). For more details, see the usage guide on @@ -207,7 +207,7 @@ dicts) or a phrase pattern (string). For more details, see the usage guide on | ---------- | ---------------------------------------------------------------- | | `patterns` | The patterns to add. ~~List[Dict[str, Union[str, List[dict]]]]~~ | -## SpanRuler.remove {#remove tag="method"} +## SpanRuler.remove {id="remove",tag="method"} Remove patterns by label from the span ruler. A `ValueError` is raised if the label does not exist in any patterns. @@ -225,7 +225,7 @@ label does not exist in any patterns. | ------- | -------------------------------------- | | `label` | The label of the pattern rule. ~~str~~ | -## SpanRuler.remove_by_id {#remove_by_id tag="method"} +## SpanRuler.remove_by_id {id="remove_by_id",tag="method"} Remove patterns by ID from the span ruler. A `ValueError` is raised if the ID does not exist in any patterns. @@ -243,7 +243,7 @@ does not exist in any patterns. | ------------ | ----------------------------------- | | `pattern_id` | The ID of the pattern rule. ~~str~~ | -## SpanRuler.clear {#clear tag="method"} +## SpanRuler.clear {id="clear",tag="method"} Remove all patterns the span ruler. @@ -256,7 +256,7 @@ Remove all patterns the span ruler. > ruler.clear() > ``` -## SpanRuler.to_disk {#to_disk tag="method"} +## SpanRuler.to_disk {id="to_disk",tag="method"} Save the span ruler patterns to a directory. The patterns will be saved as newline-delimited JSON (JSONL). @@ -272,7 +272,7 @@ newline-delimited JSON (JSONL). | ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | | `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | -## SpanRuler.from_disk {#from_disk tag="method"} +## SpanRuler.from_disk {id="from_disk",tag="method"} Load the span ruler from a path. @@ -288,7 +288,7 @@ Load the span ruler from a path. | `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ | -## SpanRuler.to_bytes {#to_bytes tag="method"} +## SpanRuler.to_bytes {id="to_bytes",tag="method"} Serialize the span ruler to a bytestring. @@ -303,7 +303,7 @@ Serialize the span ruler to a bytestring. | ----------- | ---------------------------------- | | **RETURNS** | The serialized patterns. ~~bytes~~ | -## SpanRuler.from_bytes {#from_bytes tag="method"} +## SpanRuler.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -320,7 +320,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `bytes_data` | The bytestring to load. ~~bytes~~ | | **RETURNS** | The modified `SpanRuler` object. ~~SpanRuler~~ | -## SpanRuler.labels {#labels tag="property"} +## SpanRuler.labels {id="labels",tag="property"} All labels present in the match patterns. @@ -328,7 +328,7 @@ All labels present in the match patterns. | ----------- | -------------------------------------- | | **RETURNS** | The string labels. ~~Tuple[str, ...]~~ | -## SpanRuler.ids {#ids tag="property"} +## SpanRuler.ids {id="ids",tag="property"} All IDs present in the `id` property of the match patterns. @@ -336,7 +336,7 @@ All IDs present in the `id` property of the match patterns. | ----------- | ----------------------------------- | | **RETURNS** | The string IDs. ~~Tuple[str, ...]~~ | -## SpanRuler.patterns {#patterns tag="property"} +## SpanRuler.patterns {id="patterns",tag="property"} All patterns that were added to the span ruler. @@ -344,7 +344,7 @@ All patterns that were added to the span ruler. | ----------- | ---------------------------------------------------------------------------------------- | | **RETURNS** | The original patterns, one dictionary per pattern. ~~List[Dict[str, Union[str, dict]]]~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | ---------------- | -------------------------------------------------------------------------------- | diff --git a/website/docs/api/stringstore.md b/website/docs/api/stringstore.mdx similarity index 89% rename from website/docs/api/stringstore.md rename to website/docs/api/stringstore.mdx index cd414b1f0..47d3715c1 100644 --- a/website/docs/api/stringstore.md +++ b/website/docs/api/stringstore.mdx @@ -8,7 +8,7 @@ Look up strings by 64-bit hashes. As of v2.0, spaCy uses hash values instead of integer IDs. This ensures that strings always map to the same ID, even from different `StringStores`. -## StringStore.\_\_init\_\_ {#init tag="method"} +## StringStore.\_\_init\_\_ {id="init",tag="method"} Create the `StringStore`. @@ -23,7 +23,7 @@ Create the `StringStore`. | --------- | ---------------------------------------------------------------------- | | `strings` | A sequence of strings to add to the store. ~~Optional[Iterable[str]]~~ | -## StringStore.\_\_len\_\_ {#len tag="method"} +## StringStore.\_\_len\_\_ {id="len",tag="method"} Get the number of strings in the store. @@ -38,7 +38,7 @@ Get the number of strings in the store. | ----------- | ------------------------------------------- | | **RETURNS** | The number of strings in the store. ~~int~~ | -## StringStore.\_\_getitem\_\_ {#getitem tag="method"} +## StringStore.\_\_getitem\_\_ {id="getitem",tag="method"} Retrieve a string from a given hash, or vice versa. @@ -56,7 +56,7 @@ Retrieve a string from a given hash, or vice versa. | `string_or_id` | The value to encode. ~~Union[bytes, str, int]~~ | | **RETURNS** | The value to be retrieved. ~~Union[str, int]~~ | -## StringStore.\_\_contains\_\_ {#contains tag="method"} +## StringStore.\_\_contains\_\_ {id="contains",tag="method"} Check whether a string is in the store. @@ -73,7 +73,7 @@ Check whether a string is in the store. | `string` | The string to check. ~~str~~ | | **RETURNS** | Whether the store contains the string. ~~bool~~ | -## StringStore.\_\_iter\_\_ {#iter tag="method"} +## StringStore.\_\_iter\_\_ {id="iter",tag="method"} Iterate over the strings in the store, in order. Note that a newly initialized store will always include an empty string `""` at position `0`. @@ -90,7 +90,7 @@ store will always include an empty string `""` at position `0`. | ---------- | ------------------------------ | | **YIELDS** | A string in the store. ~~str~~ | -## StringStore.add {#add tag="method" new="2"} +## StringStore.add {id="add",tag="method",version="2"} Add a string to the `StringStore`. @@ -110,7 +110,7 @@ Add a string to the `StringStore`. | `string` | The string to add. ~~str~~ | | **RETURNS** | The string's hash value. ~~int~~ | -## StringStore.to_disk {#to_disk tag="method" new="2"} +## StringStore.to_disk {id="to_disk",tag="method",version="2"} Save the current state to a directory. @@ -124,7 +124,7 @@ Save the current state to a directory. | ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | | `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | -## StringStore.from_disk {#from_disk tag="method" new="2"} +## StringStore.from_disk {id="from_disk",tag="method",version="2"} Loads state from a directory. Modifies the object in place and returns it. @@ -140,7 +140,7 @@ Loads state from a directory. Modifies the object in place and returns it. | `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | **RETURNS** | The modified `StringStore` object. ~~StringStore~~ | -## StringStore.to_bytes {#to_bytes tag="method"} +## StringStore.to_bytes {id="to_bytes",tag="method"} Serialize the current state to a binary string. @@ -154,7 +154,7 @@ Serialize the current state to a binary string. | ----------- | ---------------------------------------------------------- | | **RETURNS** | The serialized form of the `StringStore` object. ~~bytes~~ | -## StringStore.from_bytes {#from_bytes tag="method"} +## StringStore.from_bytes {id="from_bytes",tag="method"} Load state from a binary string. @@ -171,9 +171,9 @@ Load state from a binary string. | `bytes_data` | The data to load from. ~~bytes~~ | | **RETURNS** | The `StringStore` object. ~~StringStore~~ | -## Utilities {#util} +## Utilities {id="util"} -### strings.hash_string {#hash_string tag="function"} +### strings.hash_string {id="hash_string",tag="function"} Get a 64-bit hash for a given string. diff --git a/website/docs/api/tagger.md b/website/docs/api/tagger.mdx similarity index 95% rename from website/docs/api/tagger.md rename to website/docs/api/tagger.mdx index 90a49b197..ee38de81c 100644 --- a/website/docs/api/tagger.md +++ b/website/docs/api/tagger.mdx @@ -14,7 +14,7 @@ part-of-speech tag set. In the pre-trained pipelines, the tag schemas vary by language; see the [individual model pages](/models) for details. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions are assigned to `Token.tag`. @@ -23,7 +23,7 @@ Predictions are assigned to `Token.tag`. | `Token.tag` | The part of speech (hash). ~~int~~ | | `Token.tag_` | The part of speech. ~~str~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -51,7 +51,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/tagger.pyx ``` -## Tagger.\_\_init\_\_ {#init tag="method"} +## Tagger.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -81,7 +81,7 @@ shortcut for this and instantiate the component using its string name and | `overwrite` 3.2 | Whether existing annotation is overwritten. Defaults to `False`. ~~bool~~ | | `scorer` 3.2 | The scoring method. Defaults to [`Scorer.score_token_attr`](/api/scorer#score_token_attr) for the attribute `"tag"`. ~~Optional[Callable]~~ | -## Tagger.\_\_call\_\_ {#call tag="method"} +## Tagger.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -104,7 +104,7 @@ and all pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## Tagger.pipe {#pipe tag="method"} +## Tagger.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -127,7 +127,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/tagger#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## Tagger.initialize {#initialize tag="method" new="3"} +## Tagger.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -170,7 +170,7 @@ This method was previously called `begin_training`. | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | -## Tagger.predict {#predict tag="method"} +## Tagger.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects, without modifying them. @@ -187,7 +187,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## Tagger.set_annotations {#set_annotations tag="method"} +## Tagger.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -204,7 +204,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `Tagger.predict`. | -## Tagger.update {#update tag="method"} +## Tagger.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -228,7 +228,7 @@ Delegates to [`predict`](/api/tagger#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Tagger.rehearse {#rehearse tag="method,experimental" new="3"} +## Tagger.rehearse {id="rehearse",tag="method,experimental",version="3"} Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the current model to make predictions similar to an initial model, to try to address @@ -251,7 +251,7 @@ the "catastrophic forgetting" problem. This feature is experimental. | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Tagger.get_loss {#get_loss tag="method"} +## Tagger.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -270,7 +270,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## Tagger.create_optimizer {#create_optimizer tag="method"} +## Tagger.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -285,7 +285,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## Tagger.use_params {#use_params tag="method, contextmanager"} +## Tagger.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model, to use the given parameter values. At the end of the context, the original parameters are restored. @@ -302,7 +302,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## Tagger.add_label {#add_label tag="method"} +## Tagger.add_label {id="add_label",tag="method"} Add a new label to the pipe. Raises an error if the output dimension is already set, or if the model has already been fully [initialized](#initialize). Note @@ -324,7 +324,7 @@ automatically. | `label` | The label to add. ~~str~~ | | **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | -## Tagger.to_disk {#to_disk tag="method"} +## Tagger.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -341,7 +341,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Tagger.from_disk {#from_disk tag="method"} +## Tagger.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -359,7 +359,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Tagger` object. ~~Tagger~~ | -## Tagger.to_bytes {#to_bytes tag="method"} +## Tagger.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -376,7 +376,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Tagger` object. ~~bytes~~ | -## Tagger.from_bytes {#from_bytes tag="method"} +## Tagger.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -395,7 +395,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Tagger` object. ~~Tagger~~ | -## Tagger.labels {#labels tag="property"} +## Tagger.labels {id="labels",tag="property"} The labels currently added to the component. @@ -410,7 +410,7 @@ The labels currently added to the component. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## Tagger.label_data {#label_data tag="property" new="3"} +## Tagger.label_data {id="label_data",tag="property",version="3"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -428,7 +428,7 @@ pre-defined label set. | ----------- | ---------------------------------------------------------- | | **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/textcategorizer.md b/website/docs/api/textcategorizer.mdx similarity index 94% rename from website/docs/api/textcategorizer.md rename to website/docs/api/textcategorizer.mdx index f5f8706ec..a259b7b3c 100644 --- a/website/docs/api/textcategorizer.md +++ b/website/docs/api/textcategorizer.mdx @@ -2,7 +2,7 @@ title: TextCategorizer tag: class source: spacy/pipeline/textcat.py -new: 2 +version: 2 teaser: 'Pipeline component for text classification' api_base_class: /api/pipe api_string_name: textcat @@ -29,7 +29,7 @@ only. -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} Predictions will be saved to `doc.cats` as a dictionary, where the key is the name of the category and the value is a score between 0 and 1 (inclusive). For @@ -49,7 +49,7 @@ supported. | ---------- | ------------------------------------- | | `Doc.cats` | Category scores. ~~Dict[str, float]~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -93,7 +93,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/textcat_multilabel.py ``` -## TextCategorizer.\_\_init\_\_ {#init tag="method"} +## TextCategorizer.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -125,7 +125,7 @@ shortcut for this and instantiate the component using its string name and | `threshold` | Cutoff to consider a prediction "positive", relevant for `textcat_multilabel` when calculating accuracy scores. ~~float~~ | | `scorer` | The scoring method. Defaults to [`Scorer.score_cats`](/api/scorer#score_cats) for the attribute `"cats"`. ~~Optional[Callable]~~ | -## TextCategorizer.\_\_call\_\_ {#call tag="method"} +## TextCategorizer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -148,7 +148,7 @@ delegate to the [`predict`](/api/textcategorizer#predict) and | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## TextCategorizer.pipe {#pipe tag="method"} +## TextCategorizer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -172,7 +172,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/textcategorizer#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## TextCategorizer.initialize {#initialize tag="method" new="3"} +## TextCategorizer.initialize {id="initialize",tag="method",version="3"} Initialize the component for training. `get_examples` should be a function that returns an iterable of [`Example`](/api/example) objects. **At least one example @@ -217,7 +217,7 @@ This method was previously called `begin_training`. | `labels` | The label information to add to the component, as provided by the [`label_data`](#label_data) property after initialization. To generate a reusable JSON file from your data, you should run the [`init labels`](/api/cli#init-labels) command. If no labels are provided, the `get_examples` callback is used to extract the labels from the data, which may be a lot slower. ~~Optional[Iterable[str]]~~ | | `positive_label` | The positive label for a binary task with exclusive classes, `None` otherwise and by default. This parameter is only used during scoring. It is not available when using the `textcat_multilabel` component. ~~Optional[str]~~ | -## TextCategorizer.predict {#predict tag="method"} +## TextCategorizer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects without modifying them. @@ -234,7 +234,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## TextCategorizer.set_annotations {#set_annotations tag="method"} +## TextCategorizer.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores. @@ -251,7 +251,7 @@ Modify a batch of [`Doc`](/api/doc) objects using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `TextCategorizer.predict`. | -## TextCategorizer.update {#update tag="method"} +## TextCategorizer.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -275,7 +275,7 @@ Delegates to [`predict`](/api/textcategorizer#predict) and | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## TextCategorizer.rehearse {#rehearse tag="method,experimental" new="3"} +## TextCategorizer.rehearse {id="rehearse",tag="method,experimental",version="3"} Perform a "rehearsal" update from a batch of data. Rehearsal updates teach the current model to make predictions similar to an initial model to try to address @@ -298,7 +298,7 @@ the "catastrophic forgetting" problem. This feature is experimental. | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## TextCategorizer.get_loss {#get_loss tag="method"} +## TextCategorizer.get_loss {id="get_loss",tag="method"} Find the loss and gradient of loss for the batch of documents and their predicted scores. @@ -317,7 +317,7 @@ predicted scores. | `scores` | Scores representing the model's predictions. | | **RETURNS** | The loss and the gradient, i.e. `(loss, gradient)`. ~~Tuple[float, float]~~ | -## TextCategorizer.score {#score tag="method" new="3"} +## TextCategorizer.score {id="score",tag="method",version="3"} Score a batch of examples. @@ -333,7 +333,7 @@ Score a batch of examples. | _keyword-only_ | | | **RETURNS** | The scores, produced by [`Scorer.score_cats`](/api/scorer#score_cats). ~~Dict[str, Union[float, Dict[str, float]]]~~ | -## TextCategorizer.create_optimizer {#create_optimizer tag="method"} +## TextCategorizer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -348,7 +348,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## TextCategorizer.use_params {#use_params tag="method, contextmanager"} +## TextCategorizer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model to use the given parameter values. @@ -364,7 +364,7 @@ Modify the pipe's model to use the given parameter values. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## TextCategorizer.add_label {#add_label tag="method"} +## TextCategorizer.add_label {id="add_label",tag="method"} Add a new label to the pipe. Raises an error if the output dimension is already set, or if the model has already been fully [initialized](#initialize). Note @@ -386,7 +386,7 @@ automatically. | `label` | The label to add. ~~str~~ | | **RETURNS** | `0` if the label is already present, otherwise `1`. ~~int~~ | -## TextCategorizer.to_disk {#to_disk tag="method"} +## TextCategorizer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -403,7 +403,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## TextCategorizer.from_disk {#from_disk tag="method"} +## TextCategorizer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -421,7 +421,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `TextCategorizer` object. ~~TextCategorizer~~ | -## TextCategorizer.to_bytes {#to_bytes tag="method"} +## TextCategorizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -438,7 +438,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `TextCategorizer` object. ~~bytes~~ | -## TextCategorizer.from_bytes {#from_bytes tag="method"} +## TextCategorizer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -457,7 +457,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `TextCategorizer` object. ~~TextCategorizer~~ | -## TextCategorizer.labels {#labels tag="property"} +## TextCategorizer.labels {id="labels",tag="property"} The labels currently added to the component. @@ -472,7 +472,7 @@ The labels currently added to the component. | ----------- | ------------------------------------------------------ | | **RETURNS** | The labels added to the component. ~~Tuple[str, ...]~~ | -## TextCategorizer.label_data {#label_data tag="property" new="3"} +## TextCategorizer.label_data {id="label_data",tag="property",version="3"} The labels currently added to the component and their internal meta information. This is the data generated by [`init labels`](/api/cli#init-labels) and used by @@ -490,7 +490,7 @@ the model with a pre-defined label set. | ----------- | ---------------------------------------------------------- | | **RETURNS** | The label data added to the component. ~~Tuple[str, ...]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/tok2vec.md b/website/docs/api/tok2vec.mdx similarity index 94% rename from website/docs/api/tok2vec.md rename to website/docs/api/tok2vec.mdx index 2dcb1a013..a1bb1265e 100644 --- a/website/docs/api/tok2vec.md +++ b/website/docs/api/tok2vec.mdx @@ -1,7 +1,7 @@ --- title: Tok2Vec source: spacy/pipeline/tok2vec.py -new: 3 +version: 3 teaser: null api_base_class: /api/pipe api_string_name: tok2vec @@ -23,7 +23,7 @@ components can backpropagate to the shared weights. This implementation is used because it allows us to avoid relying on object identity within the models to achieve the parameter sharing. -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -48,7 +48,7 @@ architectures and their arguments and hyperparameters. %%GITHUB_SPACY/spacy/pipeline/tok2vec.py ``` -## Tok2Vec.\_\_init\_\_ {#init tag="method"} +## Tok2Vec.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -75,7 +75,7 @@ shortcut for this and instantiate the component using its string name and | `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) powering the pipeline component. ~~Model[List[Doc], List[Floats2d]~~ | | `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | -## Tok2Vec.\_\_call\_\_ {#call tag="method"} +## Tok2Vec.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document and add context-sensitive embeddings to the `Doc.tensor` attribute, allowing them to be used as features by downstream @@ -100,7 +100,7 @@ pipeline components are applied to the `Doc` in order. Both | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## Tok2Vec.pipe {#pipe tag="method"} +## Tok2Vec.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -123,7 +123,7 @@ and [`set_annotations`](/api/tok2vec#set_annotations) methods. | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## Tok2Vec.initialize {#initialize tag="method"} +## Tok2Vec.initialize {id="initialize",tag="method"} Initialize the component for training and return an [`Optimizer`](https://thinc.ai/docs/api-optimizers). `get_examples` should be a @@ -148,7 +148,7 @@ by [`Language.initialize`](/api/language#initialize). | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | -## Tok2Vec.predict {#predict tag="method"} +## Tok2Vec.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects without modifying them. @@ -165,7 +165,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## Tok2Vec.set_annotations {#set_annotations tag="method"} +## Tok2Vec.set_annotations {id="set_annotations",tag="method"} Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. @@ -182,7 +182,7 @@ Modify a batch of [`Doc`](/api/doc) objects, using pre-computed scores. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `Tok2Vec.predict`. | -## Tok2Vec.update {#update tag="method"} +## Tok2Vec.update {id="update",tag="method"} Learn from a batch of [`Example`](/api/example) objects containing the predictions and gold-standard annotations, and update the component's model. @@ -205,7 +205,7 @@ Delegates to [`predict`](/api/tok2vec#predict). | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Tok2Vec.create_optimizer {#create_optimizer tag="method"} +## Tok2Vec.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -220,7 +220,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## Tok2Vec.use_params {#use_params tag="method, contextmanager"} +## Tok2Vec.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model to use the given parameter values. At the end of the context, the original parameters are restored. @@ -237,7 +237,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## Tok2Vec.to_disk {#to_disk tag="method"} +## Tok2Vec.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -254,7 +254,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Tok2Vec.from_disk {#from_disk tag="method"} +## Tok2Vec.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -272,7 +272,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Tok2Vec` object. ~~Tok2Vec~~ | -## Tok2Vec.to_bytes {#to_bytes tag="method"} +## Tok2Vec.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -289,7 +289,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Tok2Vec` object. ~~bytes~~ | -## Tok2Vec.from_bytes {#from_bytes tag="method"} +## Tok2Vec.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -308,7 +308,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Tok2Vec` object. ~~Tok2Vec~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/token.md b/website/docs/api/token.mdx similarity index 96% rename from website/docs/api/token.md rename to website/docs/api/token.mdx index 89bd77447..63ee1080b 100644 --- a/website/docs/api/token.md +++ b/website/docs/api/token.mdx @@ -5,7 +5,7 @@ tag: class source: spacy/tokens/token.pyx --- -## Token.\_\_init\_\_ {#init tag="method"} +## Token.\_\_init\_\_ {id="init",tag="method"} Construct a `Token` object. @@ -23,7 +23,7 @@ Construct a `Token` object. | `doc` | The parent document. ~~Doc~~ | | `offset` | The index of the token within the document. ~~int~~ | -## Token.\_\_len\_\_ {#len tag="method"} +## Token.\_\_len\_\_ {id="len",tag="method"} The number of unicode characters in the token, i.e. `token.text`. @@ -39,7 +39,7 @@ The number of unicode characters in the token, i.e. `token.text`. | ----------- | ------------------------------------------------------ | | **RETURNS** | The number of unicode characters in the token. ~~int~~ | -## Token.set_extension {#set_extension tag="classmethod" new="2"} +## Token.set_extension {id="set_extension",tag="classmethod",version="2"} Define a custom attribute on the `Token` which becomes available via `Token._`. For details, see the documentation on @@ -64,7 +64,7 @@ For details, see the documentation on | `setter` | Setter function that takes the `Token` and a value, and modifies the object. Is called when the user writes to the `Token._` attribute. ~~Optional[Callable[[Token, Any], None]]~~ | | `force` | Force overwriting existing attribute. ~~bool~~ | -## Token.get_extension {#get_extension tag="classmethod" new="2"} +## Token.get_extension {id="get_extension",tag="classmethod",version="2"} Look up a previously registered extension by name. Returns a 4-tuple `(default, method, getter, setter)` if the extension is registered. Raises a @@ -84,7 +84,7 @@ Look up a previously registered extension by name. Returns a 4-tuple | `name` | Name of the extension. ~~str~~ | | **RETURNS** | A `(default, method, getter, setter)` tuple of the extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | -## Token.has_extension {#has_extension tag="classmethod" new="2"} +## Token.has_extension {id="has_extension",tag="classmethod",version="2"} Check whether an extension has been registered on the `Token` class. @@ -101,7 +101,7 @@ Check whether an extension has been registered on the `Token` class. | `name` | Name of the extension to check. ~~str~~ | | **RETURNS** | Whether the extension has been registered. ~~bool~~ | -## Token.remove_extension {#remove_extension tag="classmethod" new=""2.0.11""} +## Token.remove_extension {id="remove_extension",tag="classmethod",version="2.0.11"} Remove a previously registered extension. @@ -119,7 +119,7 @@ Remove a previously registered extension. | `name` | Name of the extension. ~~str~~ | | **RETURNS** | A `(default, method, getter, setter)` tuple of the removed extension. ~~Tuple[Optional[Any], Optional[Callable], Optional[Callable], Optional[Callable]]~~ | -## Token.check_flag {#check_flag tag="method"} +## Token.check_flag {id="check_flag",tag="method"} Check the value of a boolean flag. @@ -137,7 +137,7 @@ Check the value of a boolean flag. | `flag_id` | The attribute ID of the flag to check. ~~int~~ | | **RETURNS** | Whether the flag is set. ~~bool~~ | -## Token.similarity {#similarity tag="method" model="vectors"} +## Token.similarity {id="similarity",tag="method",model="vectors"} Compute a semantic similarity estimate. Defaults to cosine over vectors. @@ -155,7 +155,7 @@ Compute a semantic similarity estimate. Defaults to cosine over vectors. | other | The object to compare with. By default, accepts `Doc`, `Span`, `Token` and `Lexeme` objects. ~~Union[Doc, Span, Token, Lexeme]~~ | | **RETURNS** | A scalar similarity score. Higher is more similar. ~~float~~ | -## Token.nbor {#nbor tag="method"} +## Token.nbor {id="nbor",tag="method"} Get a neighboring token. @@ -172,7 +172,7 @@ Get a neighboring token. | `i` | The relative position of the token to get. Defaults to `1`. ~~int~~ | | **RETURNS** | The token at position `self.doc[self.i+i]`. ~~Token~~ | -## Token.set_morph {#set_morph tag="method"} +## Token.set_morph {id="set_morph",tag="method"} Set the morphological analysis from a UD FEATS string, hash value of a UD FEATS string, features dict or `MorphAnalysis`. The value `None` can be used to reset @@ -191,7 +191,7 @@ the morph to an unset state. | -------- | --------------------------------------------------------------------------------- | | features | The morphological features to set. ~~Union[int, dict, str, MorphAnalysis, None]~~ | -## Token.has_morph {#has_morph tag="method"} +## Token.has_morph {id="has_morph",tag="method"} Check whether the token has annotated morph information. Return `False` when the morph annotation is unset/missing. @@ -200,7 +200,7 @@ morph annotation is unset/missing. | ----------- | --------------------------------------------- | | **RETURNS** | Whether the morph annotation is set. ~~bool~~ | -## Token.is_ancestor {#is_ancestor tag="method" model="parser"} +## Token.is_ancestor {id="is_ancestor",tag="method",model="parser"} Check whether this token is a parent, grandparent, etc. of another in the dependency tree. @@ -219,7 +219,7 @@ dependency tree. | descendant | Another token. ~~Token~~ | | **RETURNS** | Whether this token is the ancestor of the descendant. ~~bool~~ | -## Token.ancestors {#ancestors tag="property" model="parser"} +## Token.ancestors {id="ancestors",tag="property",model="parser"} A sequence of the token's syntactic ancestors (parents, grandparents, etc). @@ -237,7 +237,7 @@ A sequence of the token's syntactic ancestors (parents, grandparents, etc). | ---------- | ------------------------------------------------------------------------------- | | **YIELDS** | A sequence of ancestor tokens such that `ancestor.is_ancestor(self)`. ~~Token~~ | -## Token.conjuncts {#conjuncts tag="property" model="parser"} +## Token.conjuncts {id="conjuncts",tag="property",model="parser"} A tuple of coordinated tokens, not including the token itself. @@ -253,7 +253,7 @@ A tuple of coordinated tokens, not including the token itself. | ----------- | --------------------------------------------- | | **RETURNS** | The coordinated tokens. ~~Tuple[Token, ...]~~ | -## Token.children {#children tag="property" model="parser"} +## Token.children {id="children",tag="property",model="parser"} A sequence of the token's immediate syntactic children. @@ -269,7 +269,7 @@ A sequence of the token's immediate syntactic children. | ---------- | ------------------------------------------------------- | | **YIELDS** | A child token such that `child.head == self`. ~~Token~~ | -## Token.lefts {#lefts tag="property" model="parser"} +## Token.lefts {id="lefts",tag="property",model="parser"} The leftward immediate children of the word in the syntactic dependency parse. @@ -285,7 +285,7 @@ The leftward immediate children of the word in the syntactic dependency parse. | ---------- | ------------------------------------ | | **YIELDS** | A left-child of the token. ~~Token~~ | -## Token.rights {#rights tag="property" model="parser"} +## Token.rights {id="rights",tag="property",model="parser"} The rightward immediate children of the word in the syntactic dependency parse. @@ -301,7 +301,7 @@ The rightward immediate children of the word in the syntactic dependency parse. | ---------- | ------------------------------------- | | **YIELDS** | A right-child of the token. ~~Token~~ | -## Token.n_lefts {#n_lefts tag="property" model="parser"} +## Token.n_lefts {id="n_lefts",tag="property",model="parser"} The number of leftward immediate children of the word in the syntactic dependency parse. @@ -317,7 +317,7 @@ dependency parse. | ----------- | ---------------------------------------- | | **RETURNS** | The number of left-child tokens. ~~int~~ | -## Token.n_rights {#n_rights tag="property" model="parser"} +## Token.n_rights {id="n_rights",tag="property",model="parser"} The number of rightward immediate children of the word in the syntactic dependency parse. @@ -333,7 +333,7 @@ dependency parse. | ----------- | ----------------------------------------- | | **RETURNS** | The number of right-child tokens. ~~int~~ | -## Token.subtree {#subtree tag="property" model="parser"} +## Token.subtree {id="subtree",tag="property",model="parser"} A sequence containing the token and all the token's syntactic descendants. @@ -349,7 +349,7 @@ A sequence containing the token and all the token's syntactic descendants. | ---------- | ------------------------------------------------------------------------------------ | | **YIELDS** | A descendant token such that `self.is_ancestor(token)` or `token == self`. ~~Token~~ | -## Token.has_vector {#has_vector tag="property" model="vectors"} +## Token.has_vector {id="has_vector",tag="property",model="vectors"} A boolean value indicating whether a word vector is associated with the token. @@ -365,7 +365,7 @@ A boolean value indicating whether a word vector is associated with the token. | ----------- | ------------------------------------------------------ | | **RETURNS** | Whether the token has a vector data attached. ~~bool~~ | -## Token.vector {#vector tag="property" model="vectors"} +## Token.vector {id="vector",tag="property",model="vectors"} A real-valued meaning representation. @@ -382,7 +382,7 @@ A real-valued meaning representation. | ----------- | ----------------------------------------------------------------------------------------------- | | **RETURNS** | A 1-dimensional array representing the token's vector. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Token.vector_norm {#vector_norm tag="property" model="vectors"} +## Token.vector_norm {id="vector_norm",tag="property",model="vectors"} The L2 norm of the token's vector representation. @@ -401,7 +401,7 @@ The L2 norm of the token's vector representation. | ----------- | --------------------------------------------------- | | **RETURNS** | The L2 norm of the vector representation. ~~float~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/website/docs/api/tokenizer.md b/website/docs/api/tokenizer.mdx similarity index 95% rename from website/docs/api/tokenizer.md rename to website/docs/api/tokenizer.mdx index 6eb7e8024..0a579ab4c 100644 --- a/website/docs/api/tokenizer.md +++ b/website/docs/api/tokenizer.mdx @@ -20,7 +20,7 @@ The tokenizer is typically created automatically when a like punctuation and special case rules from the [`Language.Defaults`](/api/language#defaults) provided by the language subclass. -## Tokenizer.\_\_init\_\_ {#init tag="method"} +## Tokenizer.\_\_init\_\_ {id="init",tag="method"} Create a `Tokenizer` to create `Doc` objects given unicode text. For examples of how to construct a custom tokenizer with different tokenization rules, see the @@ -55,7 +55,7 @@ how to construct a custom tokenizer with different tokenization rules, see the | `url_match` | A function matching the signature of `re.compile(string).match` to find token matches after considering prefixes and suffixes. ~~Optional[Callable[[str], Optional[Match]]]~~ | | `faster_heuristics` 3.3.0 | Whether to restrict the final `Matcher`-based pass for rules to those containing affixes or space. Defaults to `True`. ~~bool~~ | -## Tokenizer.\_\_call\_\_ {#call tag="method"} +## Tokenizer.\_\_call\_\_ {id="call",tag="method"} Tokenize a string. @@ -71,7 +71,7 @@ Tokenize a string. | `string` | The string to tokenize. ~~str~~ | | **RETURNS** | A container for linguistic annotations. ~~Doc~~ | -## Tokenizer.pipe {#pipe tag="method"} +## Tokenizer.pipe {id="pipe",tag="method"} Tokenize a stream of texts. @@ -89,7 +89,7 @@ Tokenize a stream of texts. | `batch_size` | The number of texts to accumulate in an internal buffer. Defaults to `1000`. ~~int~~ | | **YIELDS** | The tokenized `Doc` objects, in order. ~~Doc~~ | -## Tokenizer.find_infix {#find_infix tag="method"} +## Tokenizer.find_infix {id="find_infix",tag="method"} Find internal split points of the string. @@ -98,7 +98,7 @@ Find internal split points of the string. | `string` | The string to split. ~~str~~ | | **RETURNS** | A list of `re.MatchObject` objects that have `.start()` and `.end()` methods, denoting the placement of internal segment separators, e.g. hyphens. ~~List[Match]~~ | -## Tokenizer.find_prefix {#find_prefix tag="method"} +## Tokenizer.find_prefix {id="find_prefix",tag="method"} Find the length of a prefix that should be segmented from the string, or `None` if no prefix rules match. @@ -108,7 +108,7 @@ if no prefix rules match. | `string` | The string to segment. ~~str~~ | | **RETURNS** | The length of the prefix if present, otherwise `None`. ~~Optional[int]~~ | -## Tokenizer.find_suffix {#find_suffix tag="method"} +## Tokenizer.find_suffix {id="find_suffix",tag="method"} Find the length of a suffix that should be segmented from the string, or `None` if no suffix rules match. @@ -118,7 +118,7 @@ if no suffix rules match. | `string` | The string to segment. ~~str~~ | | **RETURNS** | The length of the suffix if present, otherwise `None`. ~~Optional[int]~~ | -## Tokenizer.add_special_case {#add_special_case tag="method"} +## Tokenizer.add_special_case {id="add_special_case",tag="method"} Add a special-case tokenization rule. This mechanism is also used to add custom tokenizer exceptions to the language data. See the usage guide on the @@ -139,7 +139,7 @@ details and examples. | `string` | The string to specially tokenize. ~~str~~ | | `token_attrs` | A sequence of dicts, where each dict describes a token and its attributes. The `ORTH` fields of the attributes must exactly match the string when they are concatenated. ~~Iterable[Dict[int, str]]~~ | -## Tokenizer.explain {#explain tag="method"} +## Tokenizer.explain {id="explain",tag="method"} Tokenize a string with a slow debugging tokenizer that provides information about which tokenizer rule or pattern was matched for each token. The tokens @@ -158,7 +158,7 @@ produced are identical to `Tokenizer.__call__` except for whitespace tokens. | `string` | The string to tokenize with the debugging tokenizer. ~~str~~ | | **RETURNS** | A list of `(pattern_string, token_string)` tuples. ~~List[Tuple[str, str]]~~ | -## Tokenizer.to_disk {#to_disk tag="method"} +## Tokenizer.to_disk {id="to_disk",tag="method"} Serialize the tokenizer to disk. @@ -175,7 +175,7 @@ Serialize the tokenizer to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Tokenizer.from_disk {#from_disk tag="method"} +## Tokenizer.from_disk {id="from_disk",tag="method"} Load the tokenizer from disk. Modifies the object in place and returns it. @@ -193,7 +193,7 @@ Load the tokenizer from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Tokenizer` object. ~~Tokenizer~~ | -## Tokenizer.to_bytes {#to_bytes tag="method"} +## Tokenizer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -210,7 +210,7 @@ Serialize the tokenizer to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Tokenizer` object. ~~bytes~~ | -## Tokenizer.from_bytes {#from_bytes tag="method"} +## Tokenizer.from_bytes {id="from_bytes",tag="method"} Load the tokenizer from a bytestring. Modifies the object in place and returns it. @@ -230,7 +230,7 @@ it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Tokenizer` object. ~~Tokenizer~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | ---------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -241,7 +241,7 @@ it. | `token_match` | A function matching the signature of `re.compile(string).match` to find token matches. Returns an `re.MatchObject` or `None`. ~~Optional[Callable[[str], Optional[Match]]]~~ | | `rules` | A dictionary of tokenizer exceptions and special cases. ~~Optional[Dict[str, List[Dict[int, str]]]]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/api/top-level.md b/website/docs/api/top-level.mdx similarity index 93% rename from website/docs/api/top-level.md rename to website/docs/api/top-level.mdx index 9d3e463d8..a222cfa8f 100644 --- a/website/docs/api/top-level.md +++ b/website/docs/api/top-level.mdx @@ -13,9 +13,9 @@ menu: - ['Utility Functions', 'util'] --- -## spaCy {#spacy hidden="true"} +## spaCy {id="spacy",hidden="true"} -### spacy.load {#spacy.load tag="function"} +### spacy.load {id="spacy.load",tag="function"} Load a pipeline using the name of an installed [package](/usage/saving-loading#models), a string path or a `Path`-like object. @@ -61,8 +61,7 @@ Essentially, `spacy.load()` is a convenience wrapper that reads the pipeline's information to construct a `Language` object, loads in the model data and weights, and returns it. -```python -### Abstract example +```python {title="Abstract example"} cls = spacy.util.get_lang_class(lang) # 1. Get Language class, e.g. English nlp = cls() # 2. Initialize it for name in pipeline: @@ -70,7 +69,7 @@ for name in pipeline: nlp.from_disk(data_path) # 4. Load in the binary data ``` -### spacy.blank {#spacy.blank tag="function" new="2"} +### spacy.blank {id="spacy.blank",tag="function",version="2"} Create a blank pipeline of a given language class. This function is the twin of `spacy.load()`. @@ -91,7 +90,7 @@ Create a blank pipeline of a given language class. This function is the twin of | `meta` | Optional meta overrides for [`nlp.meta`](/api/language#meta). ~~Dict[str, Any]~~ | | **RETURNS** | An empty `Language` object of the appropriate subclass. ~~Language~~ | -### spacy.info {#spacy.info tag="function"} +### spacy.info {id="spacy.info",tag="function"} The same as the [`info` command](/api/cli#info). Pretty-print information about your installation, installed pipelines and local setup from within spaCy. @@ -111,7 +110,7 @@ your installation, installed pipelines and local setup from within spaCy. | `markdown` | Print information as Markdown. ~~bool~~ | | `silent` | Don't print anything, just return. ~~bool~~ | -### spacy.explain {#spacy.explain tag="function"} +### spacy.explain {id="spacy.explain",tag="function"} Get a description for a given POS tag, dependency label or entity type. For a list of available terms, see [`glossary.py`](%%GITHUB_SPACY/spacy/glossary.py). @@ -134,7 +133,7 @@ list of available terms, see [`glossary.py`](%%GITHUB_SPACY/spacy/glossary.py). | `term` | Term to explain. ~~str~~ | | **RETURNS** | The explanation, or `None` if not found in the glossary. ~~Optional[str]~~ | -### spacy.prefer_gpu {#spacy.prefer_gpu tag="function" new="2.0.14"} +### spacy.prefer_gpu {id="spacy.prefer_gpu",tag="function",version="2.0.14"} Allocate data and perform operations on [GPU](/usage/#gpu), if available. If data has already been allocated on CPU, it will not be moved. Ideally, this @@ -162,7 +161,7 @@ ensure that the model is loaded on the correct device. See | `gpu_id` | Device index to select. Defaults to `0`. ~~int~~ | | **RETURNS** | Whether the GPU was activated. ~~bool~~ | -### spacy.require_gpu {#spacy.require_gpu tag="function" new="2.0.14"} +### spacy.require_gpu {id="spacy.require_gpu",tag="function",version="2.0.14"} Allocate data and perform operations on [GPU](/usage/#gpu). Will raise an error if no GPU is available. If data has already been allocated on CPU, it will not @@ -190,7 +189,7 @@ ensure that the model is loaded on the correct device. See | `gpu_id` | Device index to select. Defaults to `0`. ~~int~~ | | **RETURNS** | `True` ~~bool~~ | -### spacy.require_cpu {#spacy.require_cpu tag="function" new="3.0.0"} +### spacy.require_cpu {id="spacy.require_cpu",tag="function",version="3.0.0"} Allocate data and perform operations on CPU. If data has already been allocated on GPU, it will not be moved. Ideally, this function should be called right @@ -216,12 +215,12 @@ ensure that the model is loaded on the correct device. See | ----------- | --------------- | | **RETURNS** | `True` ~~bool~~ | -## displaCy {#displacy source="spacy/displacy"} +## displaCy {id="displacy",source="spacy/displacy"} As of v2.0, spaCy comes with a built-in visualization suite. For more info and examples, see the usage guide on [visualizing spaCy](/usage/visualizers). -### displacy.serve {#displacy.serve tag="method" new="2"} +### displacy.serve {id="displacy.serve",tag="method",version="2"} Serve a dependency parse tree or named entity visualization to view it in your browser. Will run a simple web server. @@ -249,7 +248,7 @@ browser. Will run a simple web server. | `host` | Host to serve visualization. Defaults to `"0.0.0.0"`. ~~str~~ | | `auto_select_port` | If `True`, automatically switch to a different port if the specified port is already in use. Defaults to `False`. ~~bool~~ | -### displacy.render {#displacy.render tag="method" new="2"} +### displacy.render {id="displacy.render",tag="method",version="2"} Render a dependency parse tree or named entity visualization. @@ -274,7 +273,7 @@ Render a dependency parse tree or named entity visualization. | `jupyter` | Explicitly enable or disable "[Jupyter](http://jupyter.org/) mode" to return markup ready to be rendered in a notebook. Detected automatically if `None` (default). ~~Optional[bool]~~ | | **RETURNS** | The rendered HTML markup. ~~str~~ | -### displacy.parse_deps {#displacy.parse_deps tag="method" new="2"} +### displacy.parse_deps {id="displacy.parse_deps",tag="method",version="2"} Generate dependency parse in `{'words': [], 'arcs': []}` format. For use with the `manual=True` argument in `displacy.render`. @@ -296,7 +295,7 @@ the `manual=True` argument in `displacy.render`. | `options` | Dependency parse specific visualisation options. ~~Dict[str, Any]~~ | | **RETURNS** | Generated dependency parse keyed by words and arcs. ~~dict~~ | -### displacy.parse_ents {#displacy.parse_ents tag="method" new="2"} +### displacy.parse_ents {id="displacy.parse_ents",tag="method",version="2"} Generate named entities in `[{start: i, end: i, label: 'label'}]` format. For use with the `manual=True` argument in `displacy.render`. @@ -318,7 +317,7 @@ use with the `manual=True` argument in `displacy.render`. | `options` | NER-specific visualisation options. ~~Dict[str, Any]~~ | | **RETURNS** | Generated entities keyed by text (original text) and ents. ~~dict~~ | -### displacy.parse_spans {#displacy.parse_spans tag="method" new="2"} +### displacy.parse_spans {id="displacy.parse_spans",tag="method",version="2"} Generate spans in `[{start_token: i, end_token: i, label: 'label'}]` format. For use with the `manual=True` argument in `displacy.render`. @@ -341,12 +340,12 @@ use with the `manual=True` argument in `displacy.render`. | `options` | Span-specific visualisation options. ~~Dict[str, Any]~~ | | **RETURNS** | Generated entities keyed by text (original text) and ents. ~~dict~~ | -### Visualizer options {#displacy_options} +### Visualizer options {id="displacy_options"} The `options` argument lets you specify additional settings for each visualizer. If a setting is not present in the options, the default value will be used. -#### Dependency Visualizer options {#options-dep} +#### Dependency Visualizer options {id="options-dep"} > #### Example > @@ -372,7 +371,7 @@ If a setting is not present in the options, the default value will be used. | `word_spacing` | Vertical spacing between words and arcs in px. Defaults to `45`. ~~int~~ | | `distance` | Distance between words in px. Defaults to `175` in regular mode and `150` in compact mode. ~~int~~ | -#### Named Entity Visualizer options {#displacy_options-ent} +#### Named Entity Visualizer options {id="displacy_options-ent"} > #### Example > @@ -389,7 +388,7 @@ If a setting is not present in the options, the default value will be used. | `template` | Optional template to overwrite the HTML used to render entity spans. Should be a format string and can use `{bg}`, `{text}` and `{label}`. See [`templates.py`](%%GITHUB_SPACY/spacy/displacy/templates.py) for examples. ~~Optional[str]~~ | | `kb_url_template` 3.2.1 | Optional template to construct the KB url for the entity to link to. Expects a python f-string format with single field to fill in. ~~Optional[str]~~ | -#### Span Visualizer options {#displacy_options-span} +#### Span Visualizer options {id="displacy_options-span"} > #### Example > @@ -420,7 +419,7 @@ span. If you wish to link an entity to their URL then consider using the should redirect you to their Wikidata page, in this case `https://www.wikidata.org/wiki/Q95`. -## registry {#registry source="spacy/util.py" new="3"} +## registry {id="registry",source="spacy/util.py",version="3"} spaCy's function registry extends [Thinc's `registry`](https://thinc.ai/docs/api-config#registry) and allows you @@ -470,7 +469,7 @@ factories. | `scorers` | Registry for functions that create scoring methods for user with the [`Scorer`](/api/scorer). Scoring methods are called with `Iterable[Example]` and arbitrary `\*\*kwargs` and return scores as `Dict[str, Any]`. | | `tokenizers` | Registry for tokenizer factories. Registered functions should return a callback that receives the `nlp` object and returns a [`Tokenizer`](/api/tokenizer) or a custom callable. | -### spacy-transformers registry {#registry-transformers} +### spacy-transformers registry {id="registry-transformers"} The following registries are added by the [`spacy-transformers`](https://github.com/explosion/spacy-transformers) package. @@ -495,7 +494,7 @@ See the [`Transformer`](/api/transformer) API reference and | [`span_getters`](/api/transformer#span_getters) | Registry for functions that take a batch of `Doc` objects and return a list of `Span` objects to process by the transformer, e.g. sentences. | | [`annotation_setters`](/api/transformer#annotation_setters) | Registry for functions that create annotation setters. Annotation setters are functions that take a batch of `Doc` objects and a [`FullTransformerBatch`](/api/transformer#fulltransformerbatch) and can set additional annotations on the `Doc`. | -## Loggers {#loggers source="spacy/training/loggers.py" new="3"} +## Loggers {id="loggers",source="spacy/training/loggers.py",version="3"} A logger records the training results. When a logger is created, two functions are returned: one for logging the information for each training step, and a @@ -531,7 +530,7 @@ saves them to a `jsonl` file. -```cli +```bash $ python -m spacy train config.cfg ``` @@ -571,7 +570,7 @@ start decreasing across epochs. | `console_output` | Whether the logger should print the logs in the console (default: `True`). ~~bool~~ | | `output_file` | The file to save the training logs to (default: `None`). ~~Optional[Union[str, Path]]~~ | -#### spacy.ConsoleLogger.v3 {#ConsoleLogger tag="registered function"} +#### spacy.ConsoleLogger.v3 {id="ConsoleLogger",tag="registered function"} > #### Example config > @@ -593,9 +592,9 @@ optionally saves them to a `jsonl` file. | `console_output` | Whether the logger should print the logs in the console (default: `True`). ~~bool~~ | | `output_file` | The file to save the training logs to (default: `None`). ~~Optional[Union[str, Path]]~~ | -## Readers {#readers} +## Readers {id="readers"} -### File readers {#file-readers source="github.com/explosion/srsly" new="3"} +### File readers {id="file-readers",source="github.com/explosion/srsly",version="3"} The following file readers are provided by our serialization library [`srsly`](https://github.com/explosion/srsly). All registered functions take one @@ -625,7 +624,7 @@ blocks that are **not executed at runtime** – for example, in `[training]` and -#### spacy.read_labels.v1 {#read_labels tag="registered function"} +#### spacy.read_labels.v1 {id="read_labels",tag="registered function"} Read a JSON-formatted labels file generated with [`init labels`](/api/cli#init-labels). Typically used in the @@ -651,7 +650,7 @@ label sets. | `require` | Whether to require the file to exist. If set to `False` and the labels file doesn't exist, the loader will return `None` and the `initialize` method will extract the labels from the data. Defaults to `False`. ~~bool~~ | | **CREATES** | The list of labels. ~~List[str]~~ | -### Corpus readers {#corpus-readers source="spacy/training/corpus.py" new="3"} +### Corpus readers {id="corpus-readers",source="spacy/training/corpus.py",version="3"} Corpus readers are registered functions that load data and return a function that takes the current `nlp` object and yields [`Example`](/api/example) objects @@ -661,7 +660,7 @@ with your own registered function in the [`@readers` registry](/api/top-level#registry) to customize the data loading and streaming. -#### spacy.Corpus.v1 {#corpus tag="registered function"} +#### spacy.Corpus.v1 {id="corpus",tag="registered function"} The `Corpus` reader manages annotated corpora and can be used for training and development datasets in the [DocBin](/api/docbin) (`.spacy`) format. Also see @@ -690,7 +689,7 @@ the [`Corpus`](/api/corpus) class. | `augmenter` | Apply some simply data augmentation, where we replace tokens with variations. This is especially useful for punctuation and case replacement, to help generalize beyond corpora that don't have smart-quotes, or only have smart quotes, etc. Defaults to `None`. ~~Optional[Callable]~~ | | **CREATES** | The corpus reader. ~~Corpus~~ | -#### spacy.JsonlCorpus.v1 {#jsonlcorpus tag="registered function"} +#### spacy.JsonlCorpus.v1 {id="jsonlcorpus",tag="registered function"} Create [`Example`](/api/example) objects from a JSONL (newline-delimited JSON) file of texts keyed by `"text"`. Can be used to read the raw text corpus for @@ -719,7 +718,7 @@ JSONL file. Also see the [`JsonlCorpus`](/api/corpus#jsonlcorpus) class. | `limit` | Limit corpus to a subset of examples, e.g. for debugging. Defaults to `0` for no limit. ~~int~~ | | **CREATES** | The corpus reader. ~~JsonlCorpus~~ | -## Batchers {#batchers source="spacy/training/batchers.py" new="3"} +## Batchers {id="batchers",source="spacy/training/batchers.py",version="3"} A data batcher implements a batching strategy that essentially turns a stream of items into a stream of batches, with each batch consisting of one item or a list @@ -733,7 +732,7 @@ Instead of using one of the built-in batchers listed here, you can also [implement your own](/usage/training#custom-code-readers-batchers), which may or may not use a custom schedule. -### spacy.batch_by_words.v1 {#batch_by_words tag="registered function"} +### spacy.batch_by_words.v1 {id="batch_by_words",tag="registered function"} Create minibatches of roughly a given number of words. If any examples are longer than the specified batch length, they will appear in a batch by @@ -761,7 +760,7 @@ themselves, or be discarded if `discard_oversize` is set to `True`. The argument | `get_length` | Optional function that receives a sequence item and returns its length. Defaults to the built-in `len()` if not set. ~~Optional[Callable[[Any], int]]~~ | | **CREATES** | The batcher that takes an iterable of items and returns batches. ~~Callable[[Iterable[Any]], Iterable[List[Any]]]~~ | -### spacy.batch_by_sequence.v1 {#batch_by_sequence tag="registered function"} +### spacy.batch_by_sequence.v1 {id="batch_by_sequence",tag="registered function"} > #### Example config > @@ -780,7 +779,7 @@ Create a batcher that creates batches of the specified size. | `get_length` | Optional function that receives a sequence item and returns its length. Defaults to the built-in `len()` if not set. ~~Optional[Callable[[Any], int]]~~ | | **CREATES** | The batcher that takes an iterable of items and returns batches. ~~Callable[[Iterable[Any]], Iterable[List[Any]]]~~ | -### spacy.batch_by_padded.v1 {#batch_by_padded tag="registered function"} +### spacy.batch_by_padded.v1 {id="batch_by_padded",tag="registered function"} > #### Example config > @@ -806,7 +805,7 @@ sequences in the batch. | `get_length` | Optional function that receives a sequence item and returns its length. Defaults to the built-in `len()` if not set. ~~Optional[Callable[[Any], int]]~~ | | **CREATES** | The batcher that takes an iterable of items and returns batches. ~~Callable[[Iterable[Any]], Iterable[List[Any]]]~~ | -## Augmenters {#augmenters source="spacy/training/augment.py" new="3"} +## Augmenters {id="augmenters",source="spacy/training/augment.py",version="3"} Data augmentation is the process of applying small modifications to the training data. It can be especially useful for punctuation and case replacement – for @@ -815,7 +814,7 @@ variations using regular quotes, or to make the model less sensitive to capitalization by including a mix of capitalized and lowercase examples. See the [usage guide](/usage/training#data-augmentation) for details and examples. -### spacy.orth_variants.v1 {#orth_variants tag="registered function"} +### spacy.orth_variants.v1 {id="orth_variants",tag="registered function"} > #### Example config > @@ -842,7 +841,7 @@ beyond corpora that don't have smart quotes, or only have smart quotes etc. | `orth_variants` | A dictionary containing the single and paired orth variants. Typically loaded from a JSON file. See [`en_orth_variants.json`](https://github.com/explosion/spacy-lookups-data/blob/master/spacy_lookups_data/data/en_orth_variants.json) for an example. ~~Dict[str, Dict[List[Union[str, List[str]]]]]~~ | | **CREATES** | A function that takes the current `nlp` object and an [`Example`](/api/example) and yields augmented `Example` objects. ~~Callable[[Language, Example], Iterator[Example]]~~ | -### spacy.lower_case.v1 {#lower_case tag="registered function"} +### spacy.lower_case.v1 {id="lower_case",tag="registered function"} > #### Example config > @@ -861,12 +860,12 @@ useful for making the model less sensitive to capitalization. | `level` | The percentage of texts that will be augmented. ~~float~~ | | **CREATES** | A function that takes the current `nlp` object and an [`Example`](/api/example) and yields augmented `Example` objects. ~~Callable[[Language, Example], Iterator[Example]]~~ | -## Callbacks {#callbacks source="spacy/training/callbacks.py" new="3"} +## Callbacks {id="callbacks",source="spacy/training/callbacks.py",version="3"} The config supports [callbacks](/usage/training#custom-code-nlp-callbacks) at several points in the lifecycle that can be used modify the `nlp` object. -### spacy.copy_from_base_model.v1 {#copy_from_base_model tag="registered function"} +### spacy.copy_from_base_model.v1 {id="copy_from_base_model",tag="registered function"} > #### Example config > @@ -890,7 +889,7 @@ from the specified model. Intended for use in `[initialize.before_init]`. | `vocab` | The pipeline to copy the vocab from. The vocab includes the lookups and vectors. Defaults to `None`. ~~Optional[str]~~ | | **CREATES** | A function that takes the current `nlp` object and modifies its `tokenizer` and `vocab`. ~~Callable[[Language], None]~~ | -### spacy.models_with_nvtx_range.v1 {#models_with_nvtx_range tag="registered function"} +### spacy.models_with_nvtx_range.v1 {id="models_with_nvtx_range",tag="registered function"} > #### Example config > @@ -910,7 +909,7 @@ backprop passes. | `backprop_color` | Color identifier for backpropagation passes. Defaults to `-1`. ~~int~~ | | **CREATES** | A function that takes the current `nlp` and wraps forward/backprop passes in NVTX ranges. ~~Callable[[Language], Language]~~ | -### spacy.models_and_pipes_with_nvtx_range.v1 {#models_and_pipes_with_nvtx_range tag="registered function" new="3.4"} +### spacy.models_and_pipes_with_nvtx_range.v1 {id="models_and_pipes_with_nvtx_range",tag="registered function",version="3.4"} > #### Example config > @@ -931,9 +930,9 @@ methods are wrapped: `pipe`, `predict`, `set_annotations`, `update`, `rehearse`, | `additional_pipe_functions` | Additional pipeline methods to wrap. Keys are pipeline names and values are lists of method identifiers. Defaults to `None`. ~~Optional[Dict[str, List[str]]]~~ | | **CREATES** | A function that takes the current `nlp` and wraps pipe models and methods in NVTX ranges. ~~Callable[[Language], Language]~~ | -## Training data and alignment {#gold source="spacy/training"} +## Training data and alignment {id="gold",source="spacy/training"} -### training.offsets_to_biluo_tags {#offsets_to_biluo_tags tag="function"} +### training.offsets_to_biluo_tags {id="offsets_to_biluo_tags",tag="function"} Encode labelled spans into per-token tags, using the [BILUO scheme](/usage/linguistic-features#accessing-ner) (Begin, In, Last, Unit, @@ -970,7 +969,7 @@ This method was previously available as `spacy.gold.biluo_tags_from_offsets`. | `missing` | The label used for missing values, e.g. if tokenization doesn't align with the entity offsets. Defaults to `"O"`. ~~str~~ | | **RETURNS** | A list of strings, describing the [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | -### training.biluo_tags_to_offsets {#biluo_tags_to_offsets tag="function"} +### training.biluo_tags_to_offsets {id="biluo_tags_to_offsets",tag="function"} Encode per-token tags following the [BILUO scheme](/usage/linguistic-features#accessing-ner) into entity offsets. @@ -998,7 +997,7 @@ This method was previously available as `spacy.gold.offsets_from_biluo_tags`. | `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags with each tag describing one token. Each tag string will be of the form of either `""`, `"O"` or `"{action}-{label}"`, where action is one of `"B"`, `"I"`, `"L"`, `"U"`. ~~List[str]~~ | | **RETURNS** | A sequence of `(start, end, label)` triples. `start` and `end` will be character-offset integers denoting the slice into the original string. ~~List[Tuple[int, int, str]]~~ | -### training.biluo_tags_to_spans {#biluo_tags_to_spans tag="function" new="2.1"} +### training.biluo_tags_to_spans {id="biluo_tags_to_spans",tag="function",version="2.1"} Encode per-token tags following the [BILUO scheme](/usage/linguistic-features#accessing-ner) into @@ -1027,7 +1026,7 @@ This method was previously available as `spacy.gold.spans_from_biluo_tags`. | `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags with each tag describing one token. Each tag string will be of the form of either `""`, `"O"` or `"{action}-{label}"`, where action is one of `"B"`, `"I"`, `"L"`, `"U"`. ~~List[str]~~ | | **RETURNS** | A sequence of `Span` objects with added entity labels. ~~List[Span]~~ | -### training.biluo_to_iob {#biluo_to_iob tag="function"} +### training.biluo_to_iob {id="biluo_to_iob",tag="function"} Convert a sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags to [IOB](/usage/linguistic-features#accessing-ner) tags. This is useful if you want @@ -1048,7 +1047,7 @@ use the BILUO tags with a model that only supports IOB tags. | `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | | **RETURNS** | A list of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | -### training.iob_to_biluo {#iob_to_biluo tag="function"} +### training.iob_to_biluo {id="iob_to_biluo",tag="function"} Convert a sequence of [IOB](/usage/linguistic-features#accessing-ner) tags to [BILUO](/usage/linguistic-features#accessing-ner) tags. This is useful if you @@ -1075,7 +1074,55 @@ This method was previously available as `spacy.gold.iob_to_biluo`. | `tags` | A sequence of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | | **RETURNS** | A list of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | -## Utility functions {#util source="spacy/util.py"} +### training.biluo_to_iob {id="biluo_to_iob",tag="function"} + +Convert a sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags to +[IOB](/usage/linguistic-features#accessing-ner) tags. This is useful if you want +use the BILUO tags with a model that only supports IOB tags. + +> #### Example +> +> ```python +> from spacy.training import biluo_to_iob +> +> tags = ["O", "O", "B-LOC", "I-LOC", "L-LOC", "O"] +> iob_tags = biluo_to_iob(tags) +> assert iob_tags == ["O", "O", "B-LOC", "I-LOC", "I-LOC", "O"] +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------------------------------------- | +| `tags` | A sequence of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | +| **RETURNS** | A list of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | + +### training.iob_to_biluo {id="iob_to_biluo",tag="function"} + +Convert a sequence of [IOB](/usage/linguistic-features#accessing-ner) tags to +[BILUO](/usage/linguistic-features#accessing-ner) tags. This is useful if you +want use the IOB tags with a model that only supports BILUO tags. + + + +This method was previously available as `spacy.gold.iob_to_biluo`. + + + +> #### Example +> +> ```python +> from spacy.training import iob_to_biluo +> +> tags = ["O", "O", "B-LOC", "I-LOC", "O"] +> biluo_tags = iob_to_biluo(tags) +> assert biluo_tags == ["O", "O", "B-LOC", "L-LOC", "O"] +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------------------- | +| `tags` | A sequence of [IOB](/usage/linguistic-features#accessing-ner) tags. ~~Iterable[str]~~ | +| **RETURNS** | A list of [BILUO](/usage/linguistic-features#accessing-ner) tags. ~~List[str]~~ | + +## Utility functions {id="util",source="spacy/util.py"} spaCy comes with a small collection of utility functions located in [`spacy/util.py`](%%GITHUB_SPACY/spacy/util.py). Because utility functions are @@ -1085,7 +1132,7 @@ use and we'll try to ensure backwards compatibility. However, we recommend having additional tests in place if your application depends on any of spaCy's utilities. -### util.get_lang_class {#util.get_lang_class tag="function"} +### util.get_lang_class {id="util.get_lang_class",tag="function"} Import and load a `Language` class. Allows lazy-loading [language data](/usage/linguistic-features#language-data) and importing @@ -1106,7 +1153,7 @@ custom language class, you can register it using the | `lang` | Two-letter language code, e.g. `"en"`. ~~str~~ | | **RETURNS** | The respective subclass. ~~Language~~ | -### util.lang_class_is_loaded {#util.lang_class_is_loaded tag="function" new="2.1"} +### util.lang_class_is_loaded {id="util.lang_class_is_loaded",tag="function",version="2.1"} Check whether a `Language` subclass is already loaded. `Language` subclasses are loaded lazily to avoid expensive setup code associated with the language data. @@ -1124,7 +1171,7 @@ loaded lazily to avoid expensive setup code associated with the language data. | `name` | Two-letter language code, e.g. `"en"`. ~~str~~ | | **RETURNS** | Whether the class has been loaded. ~~bool~~ | -### util.load_model {#util.load_model tag="function" new="2"} +### util.load_model {id="util.load_model",tag="function",version="2"} Load a pipeline from a package or data path. If called with a string name, spaCy will assume the pipeline is a Python package and import and call its `load()` @@ -1152,7 +1199,7 @@ and create a `Language` object. The model data will then be loaded in via | `config` 3 | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ | | **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ | -### util.load_model_from_init_py {#util.load_model_from_init_py tag="function" new="2"} +### util.load_model_from_init_py {id="util.load_model_from_init_py",tag="function",version="2"} A helper function to use in the `load()` method of a pipeline package's [`__init__.py`](https://github.com/explosion/spacy-models/tree/master/template/model/xx_model_name/__init__.py). @@ -1177,7 +1224,7 @@ A helper function to use in the `load()` method of a pipeline package's | `config` 3 | Config overrides as nested dict or flat dict keyed by section values in dot notation, e.g. `"nlp.pipeline"`. ~~Union[Dict[str, Any], Config]~~ | | **RETURNS** | `Language` class with the loaded pipeline. ~~Language~~ | -### util.load_config {#util.load_config tag="function" new="3"} +### util.load_config {id="util.load_config",tag="function",version="3"} Load a pipeline's [`config.cfg`](/api/data-formats#config) from a file path. The config typically includes details about the components and how they're created, @@ -1197,7 +1244,7 @@ as well as all training settings and hyperparameters. | `interpolate` | Whether to interpolate the config and replace variables like `${paths.train}` with their values. Defaults to `False`. ~~bool~~ | | **RETURNS** | The pipeline's config. ~~Config~~ | -### util.load_meta {#util.load_meta tag="function" new="3"} +### util.load_meta {id="util.load_meta",tag="function",version="3"} Get a pipeline's [`meta.json`](/api/data-formats#meta) from a file path and validate its contents. The meta typically includes details about author, @@ -1214,7 +1261,7 @@ licensing, data sources and version. | `path` | Path to the pipeline's `meta.json`. ~~Union[str, Path]~~ | | **RETURNS** | The pipeline's meta data. ~~Dict[str, Any]~~ | -### util.get_installed_models {#util.get_installed_models tag="function" new="3"} +### util.get_installed_models {id="util.get_installed_models",tag="function",version="3"} List all pipeline packages installed in the current environment. This will include any spaCy pipeline that was packaged with @@ -1232,7 +1279,7 @@ object. | ----------- | ------------------------------------------------------------------------------------- | | **RETURNS** | The string names of the pipelines installed in the current environment. ~~List[str]~~ | -### util.is_package {#util.is_package tag="function"} +### util.is_package {id="util.is_package",tag="function"} Check if string maps to a package installed via pip. Mainly used to validate [pipeline packages](/usage/models). @@ -1249,7 +1296,7 @@ Check if string maps to a package installed via pip. Mainly used to validate | `name` | Name of package. ~~str~~ | | **RETURNS** | `True` if installed package, `False` if not. ~~bool~~ | -### util.get_package_path {#util.get_package_path tag="function" new="2"} +### util.get_package_path {id="util.get_package_path",tag="function",version="2"} Get path to an installed package. Mainly used to resolve the location of [pipeline packages](/usage/models). Currently imports the package to find its @@ -1267,7 +1314,7 @@ path. | `package_name` | Name of installed package. ~~str~~ | | **RETURNS** | Path to pipeline package directory. ~~Path~~ | -### util.is_in_jupyter {#util.is_in_jupyter tag="function" new="2"} +### util.is_in_jupyter {id="util.is_in_jupyter",tag="function",version="2"} Check if user is running spaCy from a [Jupyter](https://jupyter.org) notebook by detecting the IPython kernel. Mainly used for the @@ -1286,7 +1333,7 @@ detecting the IPython kernel. Mainly used for the | ----------- | ---------------------------------------------- | | **RETURNS** | `True` if in Jupyter, `False` if not. ~~bool~~ | -### util.compile_prefix_regex {#util.compile_prefix_regex tag="function"} +### util.compile_prefix_regex {id="util.compile_prefix_regex",tag="function"} Compile a sequence of prefix rules into a regex object. @@ -1303,7 +1350,7 @@ Compile a sequence of prefix rules into a regex object. | `entries` | The prefix rules, e.g. [`lang.punctuation.TOKENIZER_PREFIXES`](%%GITHUB_SPACY/spacy/lang/punctuation.py). ~~Iterable[Union[str, Pattern]]~~ | | **RETURNS** | The regex object to be used for [`Tokenizer.prefix_search`](/api/tokenizer#attributes). ~~Pattern~~ | -### util.compile_suffix_regex {#util.compile_suffix_regex tag="function"} +### util.compile_suffix_regex {id="util.compile_suffix_regex",tag="function"} Compile a sequence of suffix rules into a regex object. @@ -1320,7 +1367,7 @@ Compile a sequence of suffix rules into a regex object. | `entries` | The suffix rules, e.g. [`lang.punctuation.TOKENIZER_SUFFIXES`](%%GITHUB_SPACY/spacy/lang/punctuation.py). ~~Iterable[Union[str, Pattern]]~~ | | **RETURNS** | The regex object to be used for [`Tokenizer.suffix_search`](/api/tokenizer#attributes). ~~Pattern~~ | -### util.compile_infix_regex {#util.compile_infix_regex tag="function"} +### util.compile_infix_regex {id="util.compile_infix_regex",tag="function"} Compile a sequence of infix rules into a regex object. @@ -1337,7 +1384,7 @@ Compile a sequence of infix rules into a regex object. | `entries` | The infix rules, e.g. [`lang.punctuation.TOKENIZER_INFIXES`](%%GITHUB_SPACY/spacy/lang/punctuation.py). ~~Iterable[Union[str, Pattern]]~~ | | **RETURNS** | The regex object to be used for [`Tokenizer.infix_finditer`](/api/tokenizer#attributes). ~~Pattern~~ | -### util.minibatch {#util.minibatch tag="function" new="2"} +### util.minibatch {id="util.minibatch",tag="function",version="2"} Iterate over batches of items. `size` may be an iterator, so that batch-size can vary on each step. @@ -1356,7 +1403,7 @@ vary on each step. | `size` | The batch size(s). ~~Union[int, Sequence[int]]~~ | | **YIELDS** | The batches. | -### util.filter_spans {#util.filter_spans tag="function" new="2.1.4"} +### util.filter_spans {id="util.filter_spans",tag="function",version="2.1.4"} Filter a sequence of [`Span`](/api/span) objects and remove duplicates or overlaps. Useful for creating named entities (where one token can only be part @@ -1377,7 +1424,7 @@ of one entity) or when merging spans with | `spans` | The spans to filter. ~~Iterable[Span]~~ | | **RETURNS** | The filtered spans. ~~List[Span]~~ | -### util.get_words_and_spaces {#get_words_and_spaces tag="function" new="3"} +### util.get_words_and_spaces {id="get_words_and_spaces",tag="function",version="3"} Given a list of words and a text, reconstruct the original tokens and return a list of words and spaces that can be used to create a [`Doc`](/api/doc#init). diff --git a/website/docs/api/transformer.md b/website/docs/api/transformer.mdx similarity index 95% rename from website/docs/api/transformer.md rename to website/docs/api/transformer.mdx index e747ad383..ad8ecce54 100644 --- a/website/docs/api/transformer.md +++ b/website/docs/api/transformer.mdx @@ -3,7 +3,7 @@ title: Transformer teaser: Pipeline component for multi-task learning with transformer models tag: class source: github.com/explosion/spacy-transformers/blob/master/spacy_transformers/pipeline_component.py -new: 3 +version: 3 api_base_class: /api/pipe api_string_name: transformer --- @@ -44,7 +44,7 @@ package also adds the function registries [`@span_getters`](#span_getters) and functions. For more details, see the [usage documentation](/usage/embeddings-transformers). -## Assigned Attributes {#assigned-attributes} +## Assigned Attributes {id="assigned-attributes"} The component sets the following [custom extension attribute](/usage/processing-pipeline#custom-components-attributes): @@ -53,7 +53,7 @@ The component sets the following | ---------------- | ------------------------------------------------------------------------ | | `Doc._.trf_data` | Transformer tokens and outputs for the `Doc` object. ~~TransformerData~~ | -## Config and implementation {#config} +## Config and implementation {id="config"} The default config is defined by the pipeline component factory and describes how the component should be configured. You can override its settings via the @@ -81,7 +81,7 @@ on the transformer architectures and their arguments and hyperparameters. https://github.com/explosion/spacy-transformers/blob/master/spacy_transformers/pipeline_component.py ``` -## Transformer.\_\_init\_\_ {#init tag="method"} +## Transformer.\_\_init\_\_ {id="init",tag="method"} > #### Example > @@ -124,7 +124,7 @@ component using its string name and [`nlp.add_pipe`](/api/language#create_pipe). | `name` | String name of the component instance. Used to add entries to the `losses` during training. ~~str~~ | | `max_batch_items` | Maximum size of a padded batch. Defaults to `128*32`. ~~int~~ | -## Transformer.\_\_call\_\_ {#call tag="method"} +## Transformer.\_\_call\_\_ {id="call",tag="method"} Apply the pipe to one document. The document is modified in place, and returned. This usually happens under the hood when the `nlp` object is called on a text @@ -147,7 +147,7 @@ to the [`predict`](/api/transformer#predict) and | `doc` | The document to process. ~~Doc~~ | | **RETURNS** | The processed document. ~~Doc~~ | -## Transformer.pipe {#pipe tag="method"} +## Transformer.pipe {id="pipe",tag="method"} Apply the pipe to a stream of documents. This usually happens under the hood when the `nlp` object is called on a text and all pipeline components are @@ -171,7 +171,7 @@ applied to the `Doc` in order. Both [`__call__`](/api/transformer#call) and | `batch_size` | The number of documents to buffer. Defaults to `128`. ~~int~~ | | **YIELDS** | The processed documents in order. ~~Doc~~ | -## Transformer.initialize {#initialize tag="method"} +## Transformer.initialize {id="initialize",tag="method"} Initialize the component for training and return an [`Optimizer`](https://thinc.ai/docs/api-optimizers). `get_examples` should be a @@ -196,7 +196,7 @@ by [`Language.initialize`](/api/language#initialize). | _keyword-only_ | | | `nlp` | The current `nlp` object. Defaults to `None`. ~~Optional[Language]~~ | -## Transformer.predict {#predict tag="method"} +## Transformer.predict {id="predict",tag="method"} Apply the component's model to a batch of [`Doc`](/api/doc) objects without modifying them. @@ -213,7 +213,7 @@ modifying them. | `docs` | The documents to predict. ~~Iterable[Doc]~~ | | **RETURNS** | The model's prediction for each document. | -## Transformer.set_annotations {#set_annotations tag="method"} +## Transformer.set_annotations {id="set_annotations",tag="method"} Assign the extracted features to the `Doc` objects. By default, the [`TransformerData`](/api/transformer#transformerdata) object is written to the @@ -233,7 +233,7 @@ callback is then called, if provided. | `docs` | The documents to modify. ~~Iterable[Doc]~~ | | `scores` | The scores to set, produced by `Transformer.predict`. | -## Transformer.update {#update tag="method"} +## Transformer.update {id="update",tag="method"} Prepare for an update to the transformer. Like the [`Tok2Vec`](/api/tok2vec) component, the `Transformer` component is unusual in that it does not receive @@ -266,7 +266,7 @@ and call the optimizer, while the others simply increment the gradients. | `losses` | Optional record of the loss during training. Updated using the component name as the key. ~~Optional[Dict[str, float]]~~ | | **RETURNS** | The updated `losses` dictionary. ~~Dict[str, float]~~ | -## Transformer.create_optimizer {#create_optimizer tag="method"} +## Transformer.create_optimizer {id="create_optimizer",tag="method"} Create an optimizer for the pipeline component. @@ -281,7 +281,7 @@ Create an optimizer for the pipeline component. | ----------- | ---------------------------- | | **RETURNS** | The optimizer. ~~Optimizer~~ | -## Transformer.use_params {#use_params tag="method, contextmanager"} +## Transformer.use_params {id="use_params",tag="method, contextmanager"} Modify the pipe's model to use the given parameter values. At the end of the context, the original parameters are restored. @@ -298,7 +298,7 @@ context, the original parameters are restored. | -------- | -------------------------------------------------- | | `params` | The parameter values to use in the model. ~~dict~~ | -## Transformer.to_disk {#to_disk tag="method"} +## Transformer.to_disk {id="to_disk",tag="method"} Serialize the pipe to disk. @@ -315,7 +315,7 @@ Serialize the pipe to disk. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Transformer.from_disk {#from_disk tag="method"} +## Transformer.from_disk {id="from_disk",tag="method"} Load the pipe from disk. Modifies the object in place and returns it. @@ -333,7 +333,7 @@ Load the pipe from disk. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Transformer` object. ~~Transformer~~ | -## Transformer.to_bytes {#to_bytes tag="method"} +## Transformer.to_bytes {id="to_bytes",tag="method"} > #### Example > @@ -350,7 +350,7 @@ Serialize the pipe to a bytestring. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Transformer` object. ~~bytes~~ | -## Transformer.from_bytes {#from_bytes tag="method"} +## Transformer.from_bytes {id="from_bytes",tag="method"} Load the pipe from a bytestring. Modifies the object in place and returns it. @@ -369,7 +369,7 @@ Load the pipe from a bytestring. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Transformer` object. ~~Transformer~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from @@ -387,7 +387,7 @@ serialization by passing in the string names via the `exclude` argument. | `cfg` | The config file. You usually don't want to exclude this. | | `model` | The binary model data. You usually don't want to exclude this. | -## TransformerData {#transformerdata tag="dataclass"} +## TransformerData {id="transformerdata",tag="dataclass"} Transformer tokens and outputs for one `Doc` object. The transformer models return tensors that refer to a whole padded batch of documents. These tensors @@ -405,7 +405,7 @@ by this class. Instances of this class are typically assigned to the | `align` | Alignment from the `Doc`'s tokenization to the wordpieces. This is a ragged array, where `align.lengths[i]` indicates the number of wordpiece tokens that token `i` aligns against. The actual indices are provided at `align[i].dataXd`. ~~Ragged~~ | | `width` | The width of the last hidden layer. ~~int~~ | -### TransformerData.empty {#transformerdata-emoty tag="classmethod"} +### TransformerData.empty {id="transformerdata-emoty",tag="classmethod"} Create an empty `TransformerData` container. @@ -425,7 +425,7 @@ model. -## FullTransformerBatch {#fulltransformerbatch tag="dataclass"} +## FullTransformerBatch {id="fulltransformerbatch",tag="dataclass"} Holds a batch of input and output objects for a transformer model. The data can then be split to a list of [`TransformerData`](/api/transformer#transformerdata) @@ -440,7 +440,7 @@ objects to associate the outputs to each [`Doc`](/api/doc) in the batch. | `align` | Alignment from the spaCy tokenization to the wordpieces. This is a ragged array, where `align.lengths[i]` indicates the number of wordpiece tokens that token `i` aligns against. The actual indices are provided at `align[i].dataXd`. ~~Ragged~~ | | `doc_data` | The outputs, split per `Doc` object. ~~List[TransformerData]~~ | -### FullTransformerBatch.unsplit_by_doc {#fulltransformerbatch-unsplit_by_doc tag="method"} +### FullTransformerBatch.unsplit_by_doc {id="fulltransformerbatch-unsplit_by_doc",tag="method"} Return a new `FullTransformerBatch` from a split batch of activations, using the current object's spans, tokens and alignment. This is used during the backward @@ -452,7 +452,7 @@ model. | `arrays` | The split batch of activations. ~~List[List[Floats3d]]~~ | | **RETURNS** | The transformer batch. ~~FullTransformerBatch~~ | -### FullTransformerBatch.split_by_doc {#fulltransformerbatch-split_by_doc tag="method"} +### FullTransformerBatch.split_by_doc {id="fulltransformerbatch-split_by_doc",tag="method"} Split a `TransformerData` object that represents a batch into a list with one `TransformerData` per `Doc`. @@ -468,7 +468,7 @@ In `spacy-transformers` v1.0, the model output is stored in -## Span getters {#span_getters source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/span_getters.py"} +## Span getters {id="span_getters",source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/span_getters.py"} Span getters are functions that take a batch of [`Doc`](/api/doc) objects and return a lists of [`Span`](/api/span) objects for each doc to be processed by @@ -498,7 +498,7 @@ using the `@spacy.registry.span_getters` decorator. | `docs` | A batch of `Doc` objects. ~~Iterable[Doc]~~ | | **RETURNS** | The spans to process by the transformer. ~~List[List[Span]]~~ | -### doc_spans.v1 {#doc_spans tag="registered function"} +### doc_spans.v1 {id="doc_spans",tag="registered function"} > #### Example config > @@ -511,7 +511,7 @@ Create a span getter that uses the whole document as its spans. This is the best approach if your [`Doc`](/api/doc) objects already refer to relatively short texts. -### sent_spans.v1 {#sent_spans tag="registered function"} +### sent_spans.v1 {id="sent_spans",tag="registered function"} > #### Example config > @@ -531,7 +531,7 @@ To set sentence boundaries with the `sentencizer` during training, add a [`[training.annotating_components]`](/usage/training#annotating-components) to have it set the sentence boundaries before the `transformer` component runs. -### strided_spans.v1 {#strided_spans tag="registered function"} +### strided_spans.v1 {id="strided_spans",tag="registered function"} > #### Example config > @@ -553,7 +553,7 @@ right context. | `window` | The window size. ~~int~~ | | `stride` | The stride size. ~~int~~ | -## Annotation setters {#annotation_setters tag="registered functions" source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/annotation_setters.py"} +## Annotation setters {id="annotation_setters",tag="registered functions",source="github.com/explosion/spacy-transformers/blob/master/spacy_transformers/annotation_setters.py"} Annotation setters are functions that take a batch of `Doc` objects and a [`FullTransformerBatch`](/api/transformer#fulltransformerbatch) and can set diff --git a/website/docs/api/vectors.md b/website/docs/api/vectors.mdx similarity index 94% rename from website/docs/api/vectors.md rename to website/docs/api/vectors.mdx index d4702b592..d6033c096 100644 --- a/website/docs/api/vectors.md +++ b/website/docs/api/vectors.mdx @@ -3,7 +3,7 @@ title: Vectors teaser: Store, save and load word vectors tag: class source: spacy/vectors.pyx -new: 2 +version: 2 --- Vectors data is kept in the `Vectors.data` attribute, which should be an @@ -25,7 +25,7 @@ As of spaCy v3.2, `Vectors` supports two types of vector tables: the sum of one or more rows as determined by the settings related to character ngrams and the hash table. -## Vectors.\_\_init\_\_ {#init tag="method"} +## Vectors.\_\_init\_\_ {id="init",tag="method"} Create a new vector store. With the default mode, you can set the vector values and keys directly on initialization, or supply a `shape` keyword argument to @@ -61,7 +61,7 @@ modified later. | `bow` 3.2 | The floret BOW string (default: `"<"`). ~~str~~ | | `eow` 3.2 | The floret EOW string (default: `">"`). ~~str~~ | -## Vectors.\_\_getitem\_\_ {#getitem tag="method"} +## Vectors.\_\_getitem\_\_ {id="getitem",tag="method"} Get a vector by key. If the key is not found in the table, a `KeyError` is raised. @@ -79,7 +79,7 @@ raised. | `key` | The key to get the vector for. ~~Union[int, str]~~ | | **RETURNS** | The vector for the key. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Vectors.\_\_setitem\_\_ {#setitem tag="method"} +## Vectors.\_\_setitem\_\_ {id="setitem",tag="method"} Set a vector for the given key. Not supported for `floret` mode. @@ -96,7 +96,7 @@ Set a vector for the given key. Not supported for `floret` mode. | `key` | The key to set the vector for. ~~int~~ | | `vector` | The vector to set. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Vectors.\_\_iter\_\_ {#iter tag="method"} +## Vectors.\_\_iter\_\_ {id="iter",tag="method"} Iterate over the keys in the table. In `floret` mode, the keys table is not used. @@ -112,7 +112,7 @@ used. | ---------- | --------------------------- | | **YIELDS** | A key in the table. ~~int~~ | -## Vectors.\_\_len\_\_ {#len tag="method"} +## Vectors.\_\_len\_\_ {id="len",tag="method"} Return the number of vectors in the table. @@ -127,7 +127,7 @@ Return the number of vectors in the table. | ----------- | ------------------------------------------- | | **RETURNS** | The number of vectors in the table. ~~int~~ | -## Vectors.\_\_contains\_\_ {#contains tag="method"} +## Vectors.\_\_contains\_\_ {id="contains",tag="method"} Check whether a key has been mapped to a vector entry in the table. In `floret` mode, returns `True` for all keys. @@ -145,7 +145,7 @@ mode, returns `True` for all keys. | `key` | The key to check. ~~int~~ | | **RETURNS** | Whether the key has a vector entry. ~~bool~~ | -## Vectors.add {#add tag="method"} +## Vectors.add {id="add",tag="method"} Add a key to the table, optionally setting a vector value as well. Keys can be mapped to an existing vector by setting `row`, or a new vector can be added. Not @@ -168,7 +168,7 @@ supported for `floret` mode. | `row` | An optional row number of a vector to map the key to. ~~int~~ | | **RETURNS** | The row the vector was added to. ~~int~~ | -## Vectors.resize {#resize tag="method"} +## Vectors.resize {id="resize",tag="method"} Resize the underlying vectors array. If `inplace=True`, the memory is reallocated. This may cause other references to the data to become invalid, so @@ -189,7 +189,7 @@ for `floret` mode. | `inplace` | Reallocate the memory. ~~bool~~ | | **RETURNS** | The removed items as a list of `(key, row)` tuples. ~~List[Tuple[int, int]]~~ | -## Vectors.keys {#keys tag="method"} +## Vectors.keys {id="keys",tag="method"} A sequence of the keys in the table. In `floret` mode, the keys table is not used. @@ -205,7 +205,7 @@ used. | ----------- | --------------------------- | | **RETURNS** | The keys. ~~Iterable[int]~~ | -## Vectors.values {#values tag="method"} +## Vectors.values {id="values",tag="method"} Iterate over vectors that have been assigned to at least one key. Note that some vectors may be unassigned, so the number of vectors returned may be less than @@ -222,7 +222,7 @@ the length of the vectors table. In `floret` mode, the keys table is not used. | ---------- | --------------------------------------------------------------- | | **YIELDS** | A vector in the table. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Vectors.items {#items tag="method"} +## Vectors.items {id="items",tag="method"} Iterate over `(key, vector)` pairs, in order. In `floret` mode, the keys table is empty. @@ -238,7 +238,7 @@ is empty. | ---------- | ------------------------------------------------------------------------------------- | | **YIELDS** | `(key, vector)` pairs, in order. ~~Tuple[int, numpy.ndarray[ndim=1, dtype=float32]]~~ | -## Vectors.find {#find tag="method"} +## Vectors.find {id="find",tag="method"} Look up one or more keys by row, or vice versa. Not supported for `floret` mode. @@ -260,7 +260,7 @@ Look up one or more keys by row, or vice versa. Not supported for `floret` mode. | `rows` | Find the keys that point to the rows. Returns `numpy.ndarray`. ~~Iterable[int]~~ | | **RETURNS** | The requested key, keys, row or rows. ~~Union[int, numpy.ndarray[ndim=1, dtype=float32]]~~ | -## Vectors.shape {#shape tag="property"} +## Vectors.shape {id="shape",tag="property"} Get `(rows, dims)` tuples of number of rows and number of dimensions in the vector table. @@ -279,7 +279,7 @@ vector table. | ----------- | ------------------------------------------ | | **RETURNS** | A `(rows, dims)` pair. ~~Tuple[int, int]~~ | -## Vectors.size {#size tag="property"} +## Vectors.size {id="size",tag="property"} The vector size, i.e. `rows * dims`. @@ -294,7 +294,7 @@ The vector size, i.e. `rows * dims`. | ----------- | ------------------------ | | **RETURNS** | The vector size. ~~int~~ | -## Vectors.is_full {#is_full tag="property"} +## Vectors.is_full {id="is_full",tag="property"} Whether the vectors table is full and has no slots are available for new keys. If a table is full, it can be resized using @@ -313,7 +313,7 @@ full and cannot be resized. | ----------- | ------------------------------------------- | | **RETURNS** | Whether the vectors table is full. ~~bool~~ | -## Vectors.n_keys {#n_keys tag="property"} +## Vectors.n_keys {id="n_keys",tag="property"} Get the number of keys in the table. Note that this is the number of _all_ keys, not just unique vectors. If several keys are mapped to the same vectors, they @@ -331,7 +331,7 @@ will be counted individually. In `floret` mode, the keys table is not used. | ----------- | ----------------------------------------------------------------------------- | | **RETURNS** | The number of all keys in the table. Returns `-1` for floret vectors. ~~int~~ | -## Vectors.most_similar {#most_similar tag="method"} +## Vectors.most_similar {id="most_similar",tag="method"} For each of the given vectors, find the `n` most similar entries to it by cosine. Queries are by vector. Results are returned as a @@ -356,7 +356,7 @@ supported for `floret` mode. | `sort` | Whether to sort the entries returned by score. Defaults to `True`. ~~bool~~ | | **RETURNS** | The most similar entries as a `(keys, best_rows, scores)` tuple. ~~Tuple[numpy.ndarray, numpy.ndarray, numpy.ndarray]~~ | -## Vectors.get_batch {#get_batch tag="method" new="3.2"} +## Vectors.get_batch {id="get_batch",tag="method",version="3.2"} Get the vectors for the provided keys efficiently as a batch. @@ -371,7 +371,7 @@ Get the vectors for the provided keys efficiently as a batch. | ------ | --------------------------------------- | | `keys` | The keys. ~~Iterable[Union[int, str]]~~ | -## Vectors.to_ops {#to_ops tag="method"} +## Vectors.to_ops {id="to_ops",tag="method"} Change the embedding matrix to use different Thinc ops. @@ -388,7 +388,7 @@ Change the embedding matrix to use different Thinc ops. | ----- | -------------------------------------------------------- | | `ops` | The Thinc ops to switch the embedding matrix to. ~~Ops~~ | -## Vectors.to_disk {#to_disk tag="method"} +## Vectors.to_disk {id="to_disk",tag="method"} Save the current state to a directory. @@ -403,7 +403,7 @@ Save the current state to a directory. | ------ | ------------------------------------------------------------------------------------------------------------------------------------------ | | `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | -## Vectors.from_disk {#from_disk tag="method"} +## Vectors.from_disk {id="from_disk",tag="method"} Loads state from a directory. Modifies the object in place and returns it. @@ -419,7 +419,7 @@ Loads state from a directory. Modifies the object in place and returns it. | `path` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | | **RETURNS** | The modified `Vectors` object. ~~Vectors~~ | -## Vectors.to_bytes {#to_bytes tag="method"} +## Vectors.to_bytes {id="to_bytes",tag="method"} Serialize the current state to a binary string. @@ -433,7 +433,7 @@ Serialize the current state to a binary string. | ----------- | ------------------------------------------------------ | | **RETURNS** | The serialized form of the `Vectors` object. ~~bytes~~ | -## Vectors.from_bytes {#from_bytes tag="method"} +## Vectors.from_bytes {id="from_bytes",tag="method"} Load state from a binary string. @@ -451,7 +451,7 @@ Load state from a binary string. | `data` | The data to load from. ~~bytes~~ | | **RETURNS** | The `Vectors` object. ~~Vectors~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} | Name | Description | | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/website/docs/api/vocab.md b/website/docs/api/vocab.mdx similarity index 94% rename from website/docs/api/vocab.md rename to website/docs/api/vocab.mdx index 5e4de219a..131e4ce0a 100644 --- a/website/docs/api/vocab.md +++ b/website/docs/api/vocab.mdx @@ -10,7 +10,7 @@ The `Vocab` object provides a lookup table that allows you to access [`StringStore`](/api/stringstore). It also owns underlying C-data that is shared between `Doc` objects. -## Vocab.\_\_init\_\_ {#init tag="method"} +## Vocab.\_\_init\_\_ {id="init",tag="method"} Create the vocabulary. @@ -31,7 +31,7 @@ Create the vocabulary. | `writing_system` | A dictionary describing the language's writing system. Typically provided by [`Language.Defaults`](/api/language#defaults). ~~Dict[str, Any]~~ | | `get_noun_chunks` | A function that yields base noun phrases used for [`Doc.noun_chunks`](/api/doc#noun_chunks). ~~Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]~~ | -## Vocab.\_\_len\_\_ {#len tag="method"} +## Vocab.\_\_len\_\_ {id="len",tag="method"} Get the current number of lexemes in the vocabulary. @@ -46,7 +46,7 @@ Get the current number of lexemes in the vocabulary. | ----------- | ------------------------------------------------ | | **RETURNS** | The number of lexemes in the vocabulary. ~~int~~ | -## Vocab.\_\_getitem\_\_ {#getitem tag="method"} +## Vocab.\_\_getitem\_\_ {id="getitem",tag="method"} Retrieve a lexeme, given an int ID or a string. If a previously unseen string is given, a new lexeme is created and stored. @@ -63,7 +63,7 @@ given, a new lexeme is created and stored. | `id_or_string` | The hash value of a word, or its string. ~~Union[int, str]~~ | | **RETURNS** | The lexeme indicated by the given ID. ~~Lexeme~~ | -## Vocab.\_\_iter\_\_ {#iter tag="method"} +## Vocab.\_\_iter\_\_ {id="iter",tag="method"} Iterate over the lexemes in the vocabulary. @@ -77,7 +77,7 @@ Iterate over the lexemes in the vocabulary. | ---------- | -------------------------------------- | | **YIELDS** | An entry in the vocabulary. ~~Lexeme~~ | -## Vocab.\_\_contains\_\_ {#contains tag="method"} +## Vocab.\_\_contains\_\_ {id="contains",tag="method"} Check whether the string has an entry in the vocabulary. To get the ID for a given string, you need to look it up in @@ -97,7 +97,7 @@ given string, you need to look it up in | `string` | The ID string. ~~str~~ | | **RETURNS** | Whether the string has an entry in the vocabulary. ~~bool~~ | -## Vocab.add_flag {#add_flag tag="method"} +## Vocab.add_flag {id="add_flag",tag="method"} Set a new boolean flag to words in the vocabulary. The `flag_getter` function will be called over the words currently in the vocab, and then applied to new @@ -122,7 +122,7 @@ using `token.check_flag(flag_id)`. | `flag_id` | An integer between `1` and `63` (inclusive), specifying the bit at which the flag will be stored. If `-1`, the lowest available bit will be chosen. ~~int~~ | | **RETURNS** | The integer ID by which the flag value can be checked. ~~int~~ | -## Vocab.reset_vectors {#reset_vectors tag="method" new="2"} +## Vocab.reset_vectors {id="reset_vectors",tag="method",version="2"} Drop the current vector table. Because all vectors must be the same width, you have to call this to change the size of the vectors. Only one of the `width` and @@ -140,7 +140,7 @@ have to call this to change the size of the vectors. Only one of the `width` and | `width` | The new width. ~~int~~ | | `shape` | The new shape. ~~int~~ | -## Vocab.prune_vectors {#prune_vectors tag="method" new="2"} +## Vocab.prune_vectors {id="prune_vectors",tag="method",version="2"} Reduce the current vector table to `nr_row` unique entries. Words mapped to the discarded vectors will be remapped to the closest vector among those remaining. @@ -165,7 +165,7 @@ cosines are calculated in minibatches to reduce memory usage. | `batch_size` | Batch of vectors for calculating the similarities. Larger batch sizes might be faster, while temporarily requiring more memory. ~~int~~ | | **RETURNS** | A dictionary keyed by removed words mapped to `(string, score)` tuples, where `string` is the entry the removed word was mapped to, and `score` the similarity score between the two words. ~~Dict[str, Tuple[str, float]]~~ | -## Vocab.deduplicate_vectors {#deduplicate_vectors tag="method" new="3.3"} +## Vocab.deduplicate_vectors {id="deduplicate_vectors",tag="method",version="3.3"} > #### Example > @@ -176,7 +176,7 @@ cosines are calculated in minibatches to reduce memory usage. Remove any duplicate rows from the current vector table, maintaining the mappings for all words in the vectors. -## Vocab.get_vector {#get_vector tag="method" new="2"} +## Vocab.get_vector {id="get_vector",tag="method",version="2"} Retrieve a vector for a word in the vocabulary. Words can be looked up by string or hash value. If the current vectors do not contain an entry for the word, a @@ -194,7 +194,7 @@ or hash value. If the current vectors do not contain an entry for the word, a | `orth` | The hash value of a word, or its unicode string. ~~Union[int, str]~~ | | **RETURNS** | A word vector. Size and shape are determined by the `Vocab.vectors` instance. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Vocab.set_vector {#set_vector tag="method" new="2"} +## Vocab.set_vector {id="set_vector",tag="method",version="2"} Set a vector for a word in the vocabulary. Words can be referenced by string or hash value. @@ -210,7 +210,7 @@ hash value. | `orth` | The hash value of a word, or its unicode string. ~~Union[int, str]~~ | | `vector` | The vector to set. ~~numpy.ndarray[ndim=1, dtype=float32]~~ | -## Vocab.has_vector {#has_vector tag="method" new="2"} +## Vocab.has_vector {id="has_vector",tag="method",version="2"} Check whether a word has a vector. Returns `False` if no vectors are loaded. Words can be looked up by string or hash value. @@ -227,7 +227,7 @@ Words can be looked up by string or hash value. | `orth` | The hash value of a word, or its unicode string. ~~Union[int, str]~~ | | **RETURNS** | Whether the word has a vector. ~~bool~~ | -## Vocab.to_disk {#to_disk tag="method" new="2"} +## Vocab.to_disk {id="to_disk",tag="method",version="2"} Save the current state to a directory. @@ -243,7 +243,7 @@ Save the current state to a directory. | _keyword-only_ | | | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | -## Vocab.from_disk {#from_disk tag="method" new="2"} +## Vocab.from_disk {id="from_disk",tag="method",version="2"} Loads state from a directory. Modifies the object in place and returns it. @@ -261,7 +261,7 @@ Loads state from a directory. Modifies the object in place and returns it. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `Vocab` object. ~~Vocab~~ | -## Vocab.to_bytes {#to_bytes tag="method"} +## Vocab.to_bytes {id="to_bytes",tag="method"} Serialize the current state to a binary string. @@ -277,7 +277,7 @@ Serialize the current state to a binary string. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The serialized form of the `Vocab` object. ~~Vocab~~ | -## Vocab.from_bytes {#from_bytes tag="method"} +## Vocab.from_bytes {id="from_bytes",tag="method"} Load state from a binary string. @@ -297,7 +297,7 @@ Load state from a binary string. | `exclude` | String names of [serialization fields](#serialization-fields) to exclude. ~~Iterable[str]~~ | | **RETURNS** | The `Vocab` object. ~~Vocab~~ | -## Attributes {#attributes} +## Attributes {id="attributes"} > #### Example > @@ -317,7 +317,7 @@ Load state from a binary string. | `writing_system` | A dict with information about the language's writing system. ~~Dict[str, Any]~~ | | `get_noun_chunks` 3.0 | A function that yields base noun phrases used for [`Doc.noun_chunks`](/api/doc#noun_chunks). ~~Optional[Callable[[Union[Doc, Span], Iterator[Tuple[int, int, int]]]]]~~ | -## Serialization fields {#serialization-fields} +## Serialization fields {id="serialization-fields"} During serialization, spaCy will export several data fields used to restore different aspects of the object. If needed, you can exclude them from diff --git a/website/docs/images/displacy-dep-founded.html b/website/docs/images/displacy-dep-founded.html deleted file mode 100644 index e22984ee1..000000000 --- a/website/docs/images/displacy-dep-founded.html +++ /dev/null @@ -1,58 +0,0 @@ - - - Smith - - - - - founded - - - - - a - - - - - healthcare - - - - - company - - - - - - - nsubj - - - - - - - - det - - - - - - - - compound - - - - - - - - dobj - - - - diff --git a/website/docs/images/displacy-ent-custom.html b/website/docs/images/displacy-ent-custom.html deleted file mode 100644 index 709c6f631..000000000 --- a/website/docs/images/displacy-ent-custom.html +++ /dev/null @@ -1,33 +0,0 @@ -

But - Google - ORGis starting from behind. The company made a late push into hardware, and - Apple - ORG’s Siri, available on iPhones, and - Amazon - ORG’s Alexa software, which runs on its Echo and Dot devices, have clear leads in consumer - adoption.
diff --git a/website/docs/images/displacy-ent-snek.html b/website/docs/images/displacy-ent-snek.html deleted file mode 100644 index c8b416d8d..000000000 --- a/website/docs/images/displacy-ent-snek.html +++ /dev/null @@ -1,26 +0,0 @@ -
- 🌱🌿 - 🐍 - SNEK - ____ 🌳🌲 ____ - 👨‍🌾 - HUMAN - 🏘️ -
diff --git a/website/docs/images/displacy-ent1.html b/website/docs/images/displacy-ent1.html deleted file mode 100644 index 708df8093..000000000 --- a/website/docs/images/displacy-ent1.html +++ /dev/null @@ -1,37 +0,0 @@ -
- - Apple - ORG - - is looking at buying - - U.K. - GPE - - startup for - - $1 billion - MONEY - -
diff --git a/website/docs/images/displacy-ent2.html b/website/docs/images/displacy-ent2.html deleted file mode 100644 index 5e1833ca0..000000000 --- a/website/docs/images/displacy-ent2.html +++ /dev/null @@ -1,39 +0,0 @@ -
- When - - Sebastian Thrun - PERSON - - started working on self-driving cars at - - Google - ORG - - in - - 2007 - DATE - - , few people outside of the company took him seriously. -
diff --git a/website/docs/images/displacy-long2.html b/website/docs/images/displacy-long2.html deleted file mode 100644 index abe18c42a..000000000 --- a/website/docs/images/displacy-long2.html +++ /dev/null @@ -1,84 +0,0 @@ - - - Autonomous - ADJ - - - - cars - NOUN - - - - shift - VERB - - - - insurance - NOUN - - - - liability - NOUN - - - - toward - ADP - - - - manufacturers - NOUN - - - - - - amod - - - - - - - - nsubj - - - - - - - - compound - - - - - - - - dobj - - - - - - - - prep - - - - - - - - pobj - - - - diff --git a/website/docs/images/displacy-span-custom.html b/website/docs/images/displacy-span-custom.html deleted file mode 100644 index 97dd3b140..000000000 --- a/website/docs/images/displacy-span-custom.html +++ /dev/null @@ -1,31 +0,0 @@ -
- Welcome to the - - Bank - - - - - BANK - - - - - of - - - - - China - - - - - . -
\ No newline at end of file diff --git a/website/docs/images/displacy-span.html b/website/docs/images/displacy-span.html deleted file mode 100644 index 9bbc6403c..000000000 --- a/website/docs/images/displacy-span.html +++ /dev/null @@ -1,41 +0,0 @@ -
- Welcome to the - - Bank - - - - - ORG - - - - - of - - - - - - China - - - - - - - GPE - - - - . -
\ No newline at end of file diff --git a/website/docs/index.md b/website/docs/index.md deleted file mode 100644 index 48e487d08..000000000 --- a/website/docs/index.md +++ /dev/null @@ -1,6 +0,0 @@ ---- ---- - -import Landing from 'widgets/landing.js' - - diff --git a/website/docs/models/index.md b/website/docs/models/index.mdx similarity index 95% rename from website/docs/models/index.md rename to website/docs/models/index.mdx index 203555651..371e4460f 100644 --- a/website/docs/models/index.md +++ b/website/docs/models/index.mdx @@ -7,7 +7,7 @@ menu: - ['Pipeline Design', 'design'] --- - +{/* TODO: include interactive demo */} ### Quickstart {hidden="true"} @@ -16,11 +16,9 @@ menu: > For more details on how to use trained pipelines with spaCy, see the > [usage guide](/usage/models). -import QuickstartModels from 'widgets/quickstart-models.js' - -## Package naming conventions {#conventions} +## Package naming conventions {id="conventions"} In general, spaCy expects all pipeline packages to follow the naming convention of `[lang]\_[name]`. For spaCy's pipelines, we also chose to divide the name @@ -45,7 +43,7 @@ For example, [`en_core_web_sm`](/models/en#en_core_web_sm) is a small English pipeline trained on written web text (blogs, news, comments), that includes vocabulary, syntax and entities. -### Package versioning {#model-versioning} +### Package versioning {id="model-versioning"} Additionally, the pipeline package versioning reflects both the compatibility with spaCy, as well as the model version. A package version `a.b.c` translates @@ -62,7 +60,7 @@ For a detailed compatibility overview, see the This is also the source of spaCy's internal compatibility check, performed when you run the [`download`](/api/cli#download) command. -## Trained pipeline design {#design} +## Trained pipeline design {id="design"} The spaCy v3 trained pipelines are designed to be efficient and configurable. For example, multiple components can share a common "token-to-vector" model and @@ -89,9 +87,9 @@ Main changes from spaCy v2 models: - The lemmatizer tables and processing move from the vocab and tagger to a separate `lemmatizer` component. -### CNN/CPU pipeline design {#design-cnn} +### CNN/CPU pipeline design {id="design-cnn"} -![Components and their dependencies in the CNN pipelines](../images/pipeline-design.svg) +![Components and their dependencies in the CNN pipelines](/images/pipeline-design.svg) In the `sm`/`md`/`lg` models: @@ -132,13 +130,13 @@ vector keys for default vectors. - [`Vectors.most_similar`](/api/vectors#most_similar) is not supported because there's no fixed list of vectors to compare your vectors to. -### Transformer pipeline design {#design-trf} +### Transformer pipeline design {id="design-trf"} In the transformer (`trf`) models, the `tagger`, `parser` and `ner` (if present) all listen to the `transformer` component. The `attribute_ruler` and `lemmatizer` have the same configuration as in the CNN models. -### Modifying the default pipeline {#design-modify} +### Modifying the default pipeline {id="design-modify"} For faster processing, you may only want to run a subset of the components in a trained pipeline. The `disable` and `exclude` arguments to @@ -189,8 +187,8 @@ than the rule-based `sentencizer`. #### Switch from trainable lemmatizer to default lemmatizer -Since v3.3, a number of pipelines use a trainable lemmatizer. You can check whether -the lemmatizer is trainable: +Since v3.3, a number of pipelines use a trainable lemmatizer. You can check +whether the lemmatizer is trainable: ```python nlp = spacy.load("de_core_web_sm") diff --git a/website/docs/styleguide.md b/website/docs/styleguide.mdx similarity index 86% rename from website/docs/styleguide.md rename to website/docs/styleguide.mdx index 47bca1ed4..276137aab 100644 --- a/website/docs/styleguide.md +++ b/website/docs/styleguide.mdx @@ -42,9 +42,7 @@ enough, JSX components can be used. > For more details on editing the site locally, see the installation > instructions and markdown reference below. -## Logo {#logo source="website/src/images/logo.svg"} - -import { Logos } from 'widgets/styleguide' +## Logo {id="logo",source="website/src/images/logo.svg"} If you would like to use the spaCy logo on your site, please get in touch and ask us first. However, if you want to show support and tell others that your @@ -53,9 +51,7 @@ project is using spaCy, you can grab one of our -## Colors {#colors} - -import { Colors, Patterns } from 'widgets/styleguide' +## Colors {id="colors"} @@ -63,17 +59,16 @@ import { Colors, Patterns } from 'widgets/styleguide' -## Typography {#typography} - -import { H1, H2, H3, H4, H5, Label, InlineList, Comment } from -'components/typography' +## Typography {id="typography"} > #### Markdown > -> ```markdown_ +> ```markdown > ## Headline 2 -> ## Headline 2 {#some_id} -> ## Headline 2 {#some_id tag="method"} +> +> ## Headline 2 {id="some_id"} +> +> ## Headline 2 {id="some_id" tag="method"} > ``` > > #### JSX @@ -101,12 +96,11 @@ in the sidebar menu.
-

Headline 1

-

Headline 2

-

Headline 3

-

Headline 4

-
Headline 5
- +

Headline 2

+

Headline 3

+

Headline 4

+
Headline 5
+
--- @@ -116,16 +110,16 @@ example, to add a tag for the documented type or mark features that have been introduced in a specific version or require statistical models to be loaded. Tags are also available as standalone `` components. -| Argument | Example | Result | -| -------- | -------------------------- | ----------------------------------------- | -| `tag` | `{tag="method"}` | method | -| `new` | `{new="3"}` | 3 | -| `model` | `{model="tagger, parser"}` | tagger, parser | -| `hidden` | `{hidden="true"}` | | +| Argument | Example | Result | +| --------- | -------------------------- | ----------------------------------------- | +| `tag` | `{tag="method"}` | method | +| `version` | `{version="3"}` | 3 | +| `model` | `{model="tagger, parser"}` | tagger, parser | +| `hidden` | `{hidden="true"}` | | -## Elements {#elements} +## Elements {id="elements"} -### Links {#links} +### Links {id="links"} > #### Markdown > @@ -147,9 +141,7 @@ Special link styles are used depending on the link URL. - [I am a link to a model](/models/en#en_core_web_sm) - [I am a link to GitHub](https://github.com/explosion/spaCy) -### Abbreviations {#abbr} - -import { Abbr } from 'components/typography' +### Abbreviations {id="abbr"} > #### JSX > @@ -161,13 +153,11 @@ Some text with an abbreviation. On small screens, I collapse and the explanation text is displayed next to the abbreviation. -### Tags {#tags} - -import Tag from 'components/tag' +### Tags {id="tags"} > ```jsx > method -> 4 +> 4 > tagger, parser > ``` @@ -180,16 +170,13 @@ new anymore. Setting `variant="model"` takes a description of model capabilities and can be used to mark features that require a respective model to be installed. - +

+ method + 4 + tagger, parser +

-method 4 tagger, -parser - - - -### Buttons {#buttons} - -import Button from 'components/button' +### Buttons {id="buttons"} > ```jsx > @@ -200,21 +187,29 @@ Link buttons come in two variants, `primary` and `secondary` and two sizes, with an optional `large` size modifier. Since they're mostly used as enhanced links, the buttons are implemented as styled links instead of native button elements. - - +

+ -
+{' '} - - + +

+ +

+ + +{' '} + + +

## Components -### Table {#table} +### Table {id="table"} > #### Markdown > -> ```markdown_ +> ```markdown > | Header 1 | Header 2 | > | -------- | -------- | > | Column 1 | Column 2 | @@ -248,7 +243,7 @@ be italicized: > #### Markdown > -> ```markdown_ +> ```markdown > | Header 1 | Header 2 | Header 3 | > | -------- | -------- | -------- | > | Column 1 | Column 2 | Column 3 | @@ -262,11 +257,11 @@ be italicized: | _Hello_ | | | | Column 1 | Column 2 | Column 3 | -### Type Annotations {#type-annotations} +### Type Annotations {id="type-annotations"} > #### Markdown > -> ```markdown_ +> ```markdown > ~~Model[List[Doc], Floats2d]~~ > ``` > @@ -295,9 +290,9 @@ always be the **last element** in the row. > #### Markdown > -> ```markdown_ -> | Header 1 | Header 2 | -> | -------- | ----------------------- | +> ```markdown +> | Header 1 | Header 2 | +> | -------- | ---------------------- | > | Column 1 | Column 2 ~~List[Doc]~~ | > ``` @@ -307,11 +302,11 @@ always be the **last element** in the row. | `model` | The Thinc [`Model`](https://thinc.ai/docs/api-model) wrapping the transformer. ~~Model[List[Doc], FullTransformerBatch]~~ | | `set_extra_annotations` | Function that takes a batch of `Doc` objects and transformer outputs and can set additional annotations on the `Doc`. ~~Callable[[List[Doc], FullTransformerBatch], None]~~ | -### List {#list} +### List {id="list"} > #### Markdown > -> ```markdown_ +> ```markdown > 1. One > 2. Two > ``` @@ -338,12 +333,13 @@ automatically. 3. Lorem ipsum dolor 4. consectetur adipiscing elit -### Aside {#aside} +### Aside {id="aside"} > #### Markdown > -> ```markdown_ +> ```markdown > > #### Aside title +> > > > This is aside text. > ``` > @@ -363,11 +359,11 @@ To make them easier to use in Markdown, paragraphs formatted as blockquotes will turn into asides by default. Level 4 headlines (with a leading `####`) will become aside titles. -### Code Block {#code-block} +### Code Block {id="code-block"} > #### Markdown > -> ````markdown_ +> ````markdown > ```python > ### This is a title > import spacy @@ -388,8 +384,7 @@ to raw text with no highlighting. An optional label can be added as the first line with the prefix `####` (Python-like) and `///` (JavaScript-like). the indented block as plain text and preserve whitespace. -```python -### Using spaCy +```python {title="Using spaCy"} import spacy nlp = spacy.load("en_core_web_sm") doc = nlp("This is a sentence.") @@ -403,7 +398,7 @@ adding `{highlight="..."}` to the headline. Acceptable ranges are spans like > #### Markdown > -> ````markdown_ +> ````markdown > ```python > ### This is a title {highlight="1-2"} > import spacy @@ -411,8 +406,7 @@ adding `{highlight="..."}` to the headline. Acceptable ranges are spans like > ``` > ```` -```python -### Using the matcher {highlight="5-7"} +```python {title="Using the matcher",highlight="5-7"} import spacy from spacy.matcher import Matcher @@ -431,7 +425,7 @@ interactive widget defaults to a regular code block. > #### Markdown > -> ````markdown_ +> ````markdown > ```python > ### {executable="true"} > import spacy @@ -439,8 +433,7 @@ interactive widget defaults to a regular code block. > ``` > ```` -```python -### {executable="true"} +```python {executable="true"} import spacy nlp = spacy.load("en_core_web_sm") doc = nlp("This is a sentence.") @@ -454,7 +447,7 @@ original file is shown at the top of the widget. > #### Markdown > -> ````markdown_ +> ````markdown > ```python > https://github.com/... > ``` @@ -470,9 +463,7 @@ original file is shown at the top of the widget. https://github.com/explosion/spaCy/tree/master/spacy/language.py ``` -### Infobox {#infobox} - -import Infobox from 'components/infobox' +### Infobox {id="infobox"} > #### JSX > @@ -508,9 +499,7 @@ blocks. -### Accordion {#accordion} - -import Accordion from 'components/accordion' +### Accordion {id="accordion"} > #### JSX > @@ -537,9 +526,9 @@ sit amet dignissim justo congue. -## Markdown reference {#markdown} +## Markdown reference {id="markdown"} -All page content and page meta lives in the `.md` files in the `/docs` +All page content and page meta lives in the `.mdx` files in the `/docs` directory. The frontmatter block at the top of each file defines the page title and other settings like the sidebar menu. @@ -548,7 +537,7 @@ and other settings like the sidebar menu. title: Page title --- -## Headline starting a section {#some_id} +## Headline starting a section {id="some_id"} This is a regular paragraph with a [link](https://spacy.io) and **bold text**. @@ -562,8 +551,7 @@ This is a regular paragraph with a [link](https://spacy.io) and **bold text**. | -------- | -------- | | Column 1 | Column 2 | -```python -### Code block title {highlight="2-3"} +```python {title="Code block title",highlight="2-3"} import spacy nlp = spacy.load("en_core_web_sm") doc = nlp("Hello world") @@ -585,7 +573,7 @@ In addition to the native markdown elements, you can use the components [abbr]: https://spacy.io/styleguide#abbr [tag]: https://spacy.io/styleguide#tag -## Editorial {#editorial} +## Editorial {id="editorial"} - "spaCy" should always be spelled with a lowercase "s" and a capital "C", unless it specifically refers to the Python package or Python import `spacy` @@ -609,21 +597,16 @@ In addition to the native markdown elements, you can use the components - ❌ The [`Span`](/api/span) and [`Token`](/api/token) objects are views of a [`Doc`](/api/doc). [`Span.as_doc`](/api/span#as_doc) creates a [`Doc`](/api/doc) object from a [`Span`](/api/span). - -* Other things we format as code are: references to trained pipeline packages +- Other things we format as code are: references to trained pipeline packages like `en_core_web_sm` or file names like `code.py` or `meta.json`. - - ✅ After training, the `config.cfg` is saved to disk. - -* [Type annotations](#type-annotations) are a special type of code formatting, +- [Type annotations](#type-annotations) are a special type of code formatting, expressed by wrapping the text in `~~` instead of backticks. The result looks like this: ~~List[Doc]~~. All references to known types will be linked automatically. - - ✅ The model has the input type ~~List[Doc]~~ and it outputs a ~~List[Array2d]~~. - -* We try to keep links meaningful but short. +- We try to keep links meaningful but short. - ✅ For details, see the usage guide on [training with custom code](/usage/training#custom-code). - ❌ For details, see diff --git a/website/docs/usage/101/_architecture.md b/website/docs/usage/101/_architecture.mdx similarity index 96% rename from website/docs/usage/101/_architecture.md rename to website/docs/usage/101/_architecture.mdx index 4ebca2756..5727c6921 100644 --- a/website/docs/usage/101/_architecture.md +++ b/website/docs/usage/101/_architecture.mdx @@ -14,9 +14,9 @@ of the pipeline. The `Language` object coordinates these components. It takes raw text and sends it through the pipeline, returning an **annotated document**. It also orchestrates training and serialization. -![Library architecture](../../images/architecture.svg) +![Library architecture {{w:1080, h:1254}}](/images/architecture.svg) -### Container objects {#architecture-containers} +### Container objects {id="architecture-containers"} | Name | Description | | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -29,7 +29,7 @@ It also orchestrates training and serialization. | [`SpanGroup`](/api/spangroup) | A named collection of spans belonging to a `Doc`. | | [`Token`](/api/token) | An individual token — i.e. a word, punctuation symbol, whitespace, etc. | -### Processing pipeline {#architecture-pipeline} +### Processing pipeline {id="architecture-pipeline"} The processing pipeline consists of one or more **pipeline components** that are called on the `Doc` in order. The tokenizer runs before the components. Pipeline @@ -39,7 +39,7 @@ rule-based modifications to the `Doc`. spaCy provides a range of built-in components for different language processing tasks and also allows adding [custom components](/usage/processing-pipelines#custom-components). -![The processing pipeline](../../images/pipeline.svg) +![The processing pipeline](/images/pipeline.svg) | Name | Description | | ----------------------------------------------- | ------------------------------------------------------------------------------------------- | @@ -61,7 +61,7 @@ components for different language processing tasks and also allows adding | [`Transformer`](/api/transformer) | Use a transformer model and set its outputs. | | [Other functions](/api/pipeline-functions) | Automatically apply something to the `Doc`, e.g. to merge spans of tokens. | -### Matchers {#architecture-matchers} +### Matchers {id="architecture-matchers"} Matchers help you find and extract information from [`Doc`](/api/doc) objects based on match patterns describing the sequences you're looking for. A matcher @@ -73,7 +73,7 @@ operates on a `Doc` and gives you access to the matched tokens **in context**. | [`Matcher`](/api/matcher) | Match sequences of tokens, based on pattern rules, similar to regular expressions. | | [`PhraseMatcher`](/api/phrasematcher) | Match sequences of tokens based on phrases. | -### Other classes {#architecture-other} +### Other classes {id="architecture-other"} | Name | Description | | ------------------------------------------------ | -------------------------------------------------------------------------------------------------- | diff --git a/website/docs/usage/101/_language-data.md b/website/docs/usage/101/_language-data.mdx similarity index 100% rename from website/docs/usage/101/_language-data.md rename to website/docs/usage/101/_language-data.mdx diff --git a/website/docs/usage/101/_named-entities.md b/website/docs/usage/101/_named-entities.mdx similarity index 75% rename from website/docs/usage/101/_named-entities.md rename to website/docs/usage/101/_named-entities.mdx index 2abc45cbd..9ae4134d8 100644 --- a/website/docs/usage/101/_named-entities.md +++ b/website/docs/usage/101/_named-entities.mdx @@ -1,14 +1,13 @@ A named entity is a "real-world object" that's assigned a name – for example, a person, a country, a product or a book title. spaCy can **recognize various -types of named entities in a document, by asking the model for a -prediction**. Because models are statistical and strongly depend on the -examples they were trained on, this doesn't always work _perfectly_ and might -need some tuning later, depending on your use case. +types of named entities in a document, by asking the model for a prediction**. +Because models are statistical and strongly depend on the examples they were +trained on, this doesn't always work _perfectly_ and might need some tuning +later, depending on your use case. Named entities are available as the `ents` property of a `Doc`: -```python -### {executable="true"} +```python {executable="true"} import spacy nlp = spacy.load("en_core_web_sm") @@ -32,7 +31,8 @@ for ent in doc.ents: Using spaCy's built-in [displaCy visualizer](/usage/visualizers), here's what our example sentence and its named entities look like: -import DisplaCyEntHtml from 'images/displacy-ent1.html'; import { Iframe } from -'components/embed' - -