From b64243ed557a1cc591a2f436449072b27f432de7 Mon Sep 17 00:00:00 2001 From: Luka Dragar Date: Fri, 5 Aug 2022 10:10:18 +0200 Subject: [PATCH 01/24] Updates to Slovenian language (#11162) * Added examples for Slovene * Update spacy/lang/sl/examples.py Co-authored-by: Adriane Boyd * Corrected a typo in one of the sentences * Updated support for Slovenian * Some minor changes to corrections * Added forint currency * Corrected HYPHENS_PERMITTED regex and some formatting * Minor changes * Un-xfail tokenizer test * Format Co-authored-by: Luka Dragar Co-authored-by: Adriane Boyd --- spacy/lang/sl/__init__.py | 8 + spacy/lang/sl/lex_attrs.py | 145 ++++++++++ spacy/lang/sl/punctuation.py | 84 ++++++ spacy/lang/sl/stop_words.py | 394 +++++--------------------- spacy/lang/sl/tokenizer_exceptions.py | 272 ++++++++++++++++++ spacy/tests/lang/sl/test_text.py | 1 - 6 files changed, 585 insertions(+), 319 deletions(-) create mode 100644 spacy/lang/sl/lex_attrs.py create mode 100644 spacy/lang/sl/punctuation.py create mode 100644 spacy/lang/sl/tokenizer_exceptions.py diff --git a/spacy/lang/sl/__init__.py b/spacy/lang/sl/__init__.py index 9ddd676bf..0070e9fa1 100644 --- a/spacy/lang/sl/__init__.py +++ b/spacy/lang/sl/__init__.py @@ -1,9 +1,17 @@ +from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_INFIXES, TOKENIZER_SUFFIXES, TOKENIZER_PREFIXES from .stop_words import STOP_WORDS +from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from ...language import Language, BaseDefaults class SlovenianDefaults(BaseDefaults): stop_words = STOP_WORDS + tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + infixes = TOKENIZER_INFIXES + suffixes = TOKENIZER_SUFFIXES + lex_attr_getters = LEX_ATTRS class Slovenian(Language): diff --git a/spacy/lang/sl/lex_attrs.py b/spacy/lang/sl/lex_attrs.py new file mode 100644 index 000000000..958152e37 --- /dev/null +++ b/spacy/lang/sl/lex_attrs.py @@ -0,0 +1,145 @@ +from ...attrs import LIKE_NUM +from ...attrs import IS_CURRENCY +import unicodedata + + +_num_words = set( + """ + nula ničla nič ena dva tri štiri pet šest sedem osem + devet deset enajst dvanajst trinajst štirinajst petnajst + šestnajst sedemnajst osemnajst devetnajst dvajset trideset štirideset + petdeset šestdest sedemdeset osemdeset devedeset sto tisoč + milijon bilijon trilijon kvadrilijon nešteto + + en eden enega enemu ennem enim enih enima enimi ene eni eno + dveh dvema dvem dvoje trije treh trem tremi troje štirje štirih štirim štirimi + petih petim petimi šestih šestim šestimi sedmih sedmim sedmimi osmih osmim osmimi + devetih devetim devetimi desetih desetim desetimi enajstih enajstim enajstimi + dvanajstih dvanajstim dvanajstimi trinajstih trinajstim trinajstimi + šestnajstih šestnajstim šestnajstimi petnajstih petnajstim petnajstimi + sedemnajstih sedemnajstim sedemnajstimi osemnajstih osemnajstim osemnajstimi + devetnajstih devetnajstim devetnajstimi dvajsetih dvajsetim dvajsetimi + """.split() +) + +_ordinal_words = set( + """ + prvi drugi tretji četrti peti šesti sedmi osmi + deveti deseti enajsti dvanajsti trinajsti štirinajsti + petnajsti šestnajsti sedemnajsti osemnajsti devetnajsti + dvajseti trideseti štirideseti petdeseti šestdeseti sedemdeseti + osemdeseti devetdeseti stoti tisoči milijonti bilijonti + trilijonti kvadrilijonti nešteti + + prva druga tretja četrta peta šesta sedma osma + deveta deseta enajsta dvanajsta trinajsta štirnajsta + petnajsta šestnajsta sedemnajsta osemnajsta devetnajsta + dvajseta trideseta štirideseta petdeseta šestdeseta sedemdeseta + osemdeseta devetdeseta stota tisoča milijonta bilijonta + trilijonta kvadrilijonta nešteta + + prvo drugo tretje četrto peto šestro sedmo osmo + deveto deseto enajsto dvanajsto trinajsto štirnajsto + petnajsto šestnajsto sedemnajsto osemnajsto devetnajsto + dvajseto trideseto štirideseto petdeseto šestdeseto sedemdeseto + osemdeseto devetdeseto stoto tisočo milijonto bilijonto + trilijonto kvadrilijonto nešteto + + prvega drugega tretjega četrtega petega šestega sedmega osmega + devega desetega enajstega dvanajstega trinajstega štirnajstega + petnajstega šestnajstega sedemnajstega osemnajstega devetnajstega + dvajsetega tridesetega štiridesetega petdesetega šestdesetega sedemdesetega + osemdesetega devetdesetega stotega tisočega milijontega bilijontega + trilijontega kvadrilijontega neštetega + + prvemu drugemu tretjemu četrtemu petemu šestemu sedmemu osmemu devetemu desetemu + enajstemu dvanajstemu trinajstemu štirnajstemu petnajstemu šestnajstemu sedemnajstemu + osemnajstemu devetnajstemu dvajsetemu tridesetemu štiridesetemu petdesetemu šestdesetemu + sedemdesetemu osemdesetemu devetdesetemu stotemu tisočemu milijontemu bilijontemu + trilijontemu kvadrilijontemu neštetemu + + prvem drugem tretjem četrtem petem šestem sedmem osmem devetem desetem + enajstem dvanajstem trinajstem štirnajstem petnajstem šestnajstem sedemnajstem + osemnajstem devetnajstem dvajsetem tridesetem štiridesetem petdesetem šestdesetem + sedemdesetem osemdesetem devetdesetem stotem tisočem milijontem bilijontem + trilijontem kvadrilijontem neštetem + + prvim drugim tretjim četrtim petim šestim sedtim osmim devetim desetim + enajstim dvanajstim trinajstim štirnajstim petnajstim šestnajstim sedemnajstim + osemnajstim devetnajstim dvajsetim tridesetim štiridesetim petdesetim šestdesetim + sedemdesetim osemdesetim devetdesetim stotim tisočim milijontim bilijontim + trilijontim kvadrilijontim neštetim + + prvih drugih tretjih četrthih petih šestih sedmih osmih deveth desetih + enajstih dvanajstih trinajstih štirnajstih petnajstih šestnajstih sedemnajstih + osemnajstih devetnajstih dvajsetih tridesetih štiridesetih petdesetih šestdesetih + sedemdesetih osemdesetih devetdesetih stotih tisočih milijontih bilijontih + trilijontih kvadrilijontih nešteth + + prvima drugima tretjima četrtima petima šestima sedmima osmima devetima desetima + enajstima dvanajstima trinajstima štirnajstima petnajstima šestnajstima sedemnajstima + osemnajstima devetnajstima dvajsetima tridesetima štiridesetima petdesetima šestdesetima + sedemdesetima osemdesetima devetdesetima stotima tisočima milijontima bilijontima + trilijontima kvadrilijontima neštetima + + prve druge četrte pete šeste sedme osme devete desete + enajste dvanajste trinajste štirnajste petnajste šestnajste sedemnajste + osemnajste devetnajste dvajsete tridesete štiridesete petdesete šestdesete + sedemdesete osemdesete devetdesete stote tisoče milijonte bilijonte + trilijonte kvadrilijonte neštete + + prvimi drugimi tretjimi četrtimi petimi šestimi sedtimi osmimi devetimi desetimi + enajstimi dvanajstimi trinajstimi štirnajstimi petnajstimi šestnajstimi sedemnajstimi + osemnajstimi devetnajstimi dvajsetimi tridesetimi štiridesetimi petdesetimi šestdesetimi + sedemdesetimi osemdesetimi devetdesetimi stotimi tisočimi milijontimi bilijontimi + trilijontimi kvadrilijontimi neštetimi + """.split() +) + +_currency_words = set( + """ + evro evra evru evrom evrov evroma evrih evrom evre evri evr eur + cent centa centu cenom centov centoma centih centom cente centi + dolar dolarja dolarji dolarju dolarjem dolarjev dolarjema dolarjih dolarje usd + tolar tolarja tolarji tolarju tolarjem tolarjev tolarjema tolarjih tolarje tol + dinar dinarja dinarji dinarju dinarjem dinarjev dinarjema dinarjih dinarje din + funt funta funti funtu funtom funtov funtoma funtih funte gpb + forint forinta forinti forintu forintom forintov forintoma forintih forinte + zlot zlota zloti zlotu zlotom zlotov zlotoma zlotih zlote + rupij rupija rupiji rupiju rupijem rupijev rupijema rupijih rupije + jen jena jeni jenu jenom jenov jenoma jenih jene + kuna kuni kune kuno kun kunama kunah kunam kunami + marka marki marke markama markah markami + """.split() +) + + +def like_num(text): + if text.startswith(("+", "-", "±", "~")): + text = text[1:] + text = text.replace(",", "").replace(".", "") + if text.isdigit(): + return True + if text.count("/") == 1: + num, denom = text.split("/") + if num.isdigit() and denom.isdigit(): + return True + text_lower = text.lower() + if text_lower in _num_words: + return True + if text_lower in _ordinal_words: + return True + return False + + +def is_currency(text): + text_lower = text.lower() + if text in _currency_words: + return True + for char in text: + if unicodedata.category(char) != "Sc": + return False + return True + + +LEX_ATTRS = {LIKE_NUM: like_num, IS_CURRENCY: is_currency} diff --git a/spacy/lang/sl/punctuation.py b/spacy/lang/sl/punctuation.py new file mode 100644 index 000000000..b6ca1830e --- /dev/null +++ b/spacy/lang/sl/punctuation.py @@ -0,0 +1,84 @@ +from ..char_classes import ( + LIST_ELLIPSES, + LIST_ICONS, + HYPHENS, + LIST_PUNCT, + LIST_QUOTES, + CURRENCY, + UNITS, + PUNCT, + LIST_CURRENCY, + CONCAT_QUOTES, +) +from ..char_classes import CONCAT_QUOTES, ALPHA_LOWER, ALPHA_UPPER, ALPHA +from ..char_classes import merge_chars +from ..punctuation import TOKENIZER_PREFIXES as BASE_TOKENIZER_PREFIXES + + +INCLUDE_SPECIAL = ["\\+", "\\/", "\\•", "\\¯", "\\=", "\\×"] + HYPHENS.split("|") + +_prefixes = INCLUDE_SPECIAL + BASE_TOKENIZER_PREFIXES + +_suffixes = ( + INCLUDE_SPECIAL + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + [ + r"(?<=°[FfCcKk])\.", + r"(?<=[0-9])(?:{c})".format(c=CURRENCY), + r"(?<=[0-9])(?:{u})".format(u=UNITS), + r"(?<=[{al}{e}{p}(?:{q})])\.".format( + al=ALPHA_LOWER, e=r"%²\-\+", q=CONCAT_QUOTES, p=PUNCT + ), + r"(?<=[{au}][{au}])\.".format(au=ALPHA_UPPER), + # split initials like J.K. Rowling + r"(?<=[A-Z]\.)(?:[A-Z].)", + ] +) + +# a list of all suffixes following a hyphen that are shouldn't split (eg. BTC-jev) +# source: Obeliks tokenizer - https://github.com/clarinsi/obeliks/blob/master/obeliks/res/TokRulesPart1.txt +CONCAT_QUOTES = CONCAT_QUOTES.replace("'", "") +HYPHENS_PERMITTED = ( + "((a)|(evemu)|(evskega)|(i)|(jevega)|(jevska)|(jevskimi)|(jinemu)|(oma)|(ovim)|" + "(ovski)|(e)|(evi)|(evskem)|(ih)|(jevem)|(jevske)|(jevsko)|(jini)|(ov)|(ovima)|" + "(ovskih)|(em)|(evih)|(evskemu)|(ja)|(jevemu)|(jevskega)|(ji)|(jinih)|(ova)|" + "(ovimi)|(ovskim)|(ema)|(evim)|(evski)|(je)|(jevi)|(jevskem)|(jih)|(jinim)|" + "(ove)|(ovo)|(ovskima)|(ev)|(evima)|(evskih)|(jem)|(jevih)|(jevskemu)|(jin)|" + "(jinima)|(ovega)|(ovska)|(ovskimi)|(eva)|(evimi)|(evskim)|(jema)|(jevim)|" + "(jevski)|(jina)|(jinimi)|(ovem)|(ovske)|(ovsko)|(eve)|(evo)|(evskima)|(jev)|" + "(jevima)|(jevskih)|(jine)|(jino)|(ovemu)|(ovskega)|(u)|(evega)|(evska)|" + "(evskimi)|(jeva)|(jevimi)|(jevskim)|(jinega)|(ju)|(ovi)|(ovskem)|(evem)|" + "(evske)|(evsko)|(jeve)|(jevo)|(jevskima)|(jinem)|(om)|(ovih)|(ovskemu)|" + "(ovec)|(ovca)|(ovcu)|(ovcem)|(ovcev)|(ovcema)|(ovcih)|(ovci)|(ovce)|(ovcimi)|" + "(evec)|(evca)|(evcu)|(evcem)|(evcev)|(evcema)|(evcih)|(evci)|(evce)|(evcimi)|" + "(jevec)|(jevca)|(jevcu)|(jevcem)|(jevcev)|(jevcema)|(jevcih)|(jevci)|(jevce)|" + "(jevcimi)|(ovka)|(ovke)|(ovki)|(ovko)|(ovk)|(ovkama)|(ovkah)|(ovkam)|(ovkami)|" + "(evka)|(evke)|(evki)|(evko)|(evk)|(evkama)|(evkah)|(evkam)|(evkami)|(jevka)|" + "(jevke)|(jevki)|(jevko)|(jevk)|(jevkama)|(jevkah)|(jevkam)|(jevkami)|(timi)|" + "(im)|(ima)|(a)|(imi)|(e)|(o)|(ega)|(ti)|(em)|(tih)|(emu)|(tim)|(i)|(tima)|" + "(ih)|(ta)|(te)|(to)|(tega)|(tem)|(temu))" +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])(?:{h})(?!{hp}$)(?=[{a}])".format( + a=ALPHA, h=HYPHENS, hp=HYPHENS_PERMITTED + ), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + ] +) + + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/lang/sl/stop_words.py b/spacy/lang/sl/stop_words.py index c9004ed5d..8491efcb5 100644 --- a/spacy/lang/sl/stop_words.py +++ b/spacy/lang/sl/stop_words.py @@ -1,326 +1,84 @@ # Source: https://github.com/stopwords-iso/stopwords-sl -# Removed various words that are not normally considered stop words, such as months. STOP_WORDS = set( """ -a -ali -b -bi -bil -bila -bile -bili -bilo -biti -blizu -bo -bodo -bolj -bom -bomo -boste -bova -boš -brez -c -cel -cela -celi -celo -d -da -daleč -dan -danes -do -dober -dobra -dobri -dobro -dokler -dol -dovolj -e -eden -en -ena -ene -eni -enkrat -eno -etc. +a ali + +b bi bil bila bile bili bilo biti blizu bo bodo bojo bolj bom bomo +boste bova boš brez + +c cel cela celi celo + +č če često četrta četrtek četrti četrto čez čigav + +d da daleč dan danes datum deset deseta deseti deseto devet +deveta deveti deveto do dober dobra dobri dobro dokler dol dolg +dolga dolgi dovolj drug druga drugi drugo dva dve + +e eden en ena ene eni enkrat eno etc. + f -g -g. -ga -ga. -gor -gospa -gospod -h -halo -i -idr. -ii -iii -in -iv -ix -iz -j -jaz -je -ji -jih -jim -jo -k -kadarkoli -kaj -kajti -kako -kakor -kamor -kamorkoli -kar -karkoli -katerikoli -kdaj -kdo -kdorkoli -ker -ki -kje -kjer -kjerkoli -ko -koderkoli -koga -komu -kot -l -le -lep -lepa -lepe -lepi -lepo -m -manj -me -med -medtem -mene -mi -midva -midve -mnogo -moj -moja -moje -mora -morajo -moram -moramo -morate -moraš -morem -mu -n -na -nad -naj -najina -najino -najmanj -naju -največ -nam -nas -nato -nazaj -naš -naša -naše -ne -nedavno -nek -neka -nekaj -nekatere -nekateri -nekatero -nekdo -neke -nekega -neki -nekje -neko -nekoga -nekoč -ni -nikamor -nikdar -nikjer -nikoli -nič -nje -njega -njegov -njegova -njegovo -njej -njemu -njen -njena -njeno -nji -njih -njihov -njihova -njihovo -njiju -njim -njo -njun -njuna -njuno -no -nocoj -npr. -o -ob -oba -obe -oboje -od -okoli -on -onadva -one -oni -onidve -oz. -p -pa -po -pod -pogosto -poleg -ponavadi -ponovno -potem -povsod -prbl. -precej -pred -prej -preko -pri -pribl. -približno -proti -r -redko -res -s -saj -sam -sama -same -sami -samo -se -sebe -sebi -sedaj -sem -seveda -si -sicer -skoraj -skozi -smo -so -spet -sta -ste -sva -t -ta -tak -taka -take -taki -tako -takoj -tam -te -tebe -tebi -tega -ti -tista -tiste -tisti -tisto -tj. -tja -to -toda -tu -tudi -tukaj -tvoj -tvoja -tvoje + +g g. ga ga. gor gospa gospod + +h halo + +i idr. ii iii in iv ix iz + +j jaz je ji jih jim jo jutri + +k kadarkoli kaj kajti kako kakor kamor kamorkoli kar karkoli +katerikoli kdaj kdo kdorkoli ker ki kje kjer kjerkoli +ko koder koderkoli koga komu kot kratek kratka kratke kratki + +l lahka lahke lahki lahko le lep lepa lepe lepi lepo leto + +m majhen majhna majhni malce malo manj me med medtem mene +mesec mi midva midve mnogo moj moja moje mora morajo moram +moramo morate moraš morem mu + +n na nad naj najina najino najmanj naju največ nam narobe +nas nato nazaj naš naša naše ne nedavno nedelja nek neka +nekaj nekatere nekateri nekatero nekdo neke nekega neki +nekje neko nekoga nekoč ni nikamor nikdar nikjer nikoli +nič nje njega njegov njegova njegovo njej njemu njen +njena njeno nji njih njihov njihova njihovo njiju njim +njo njun njuna njuno no nocoj npr. + +o ob oba obe oboje od odprt odprta odprti okoli on +onadva one oni onidve osem osma osmi osmo oz. + +p pa pet peta petek peti peto po pod pogosto poleg poln +polna polni polno ponavadi ponedeljek ponovno potem +povsod pozdravljen pozdravljeni prav prava prave pravi +pravo prazen prazna prazno prbl. precej pred prej preko +pri pribl. približno primer pripravljen pripravljena +pripravljeni proti prva prvi prvo + +r ravno redko res reč + +s saj sam sama same sami samo se sebe sebi sedaj sedem +sedma sedmi sedmo sem seveda si sicer skoraj skozi slab sm +so sobota spet sreda srednja srednji sta ste stran stvar sva + +š šest šesta šesti šesto štiri + +t ta tak taka take taki tako takoj tam te tebe tebi tega +težak težka težki težko ti tista tiste tisti tisto tj. +tja to toda torek tretja tretje tretji tri tu tudi tukaj +tvoj tvoja tvoje + u -v -vaju -vam -vas -vaš -vaša -vaše -ve -vedno -vendar -ves -več -vi -vidva -vii -viii -vsa -vsaj -vsak -vsaka -vsakdo -vsake -vsaki -vsakomur -vse -vsega -vsi -vso -včasih -x -z -za -zadaj -zadnji -zakaj -zdaj -zelo -zunaj -č -če -često -čez -čigav -š -ž -že + +v vaju vam vas vaš vaša vaše ve vedno velik velika veliki +veliko vendar ves več vi vidva vii viii visok visoka visoke +visoki vsa vsaj vsak vsaka vsakdo vsake vsaki vsakomur vse +vsega vsi vso včasih včeraj + +x + +z za zadaj zadnji zakaj zaprta zaprti zaprto zdaj zelo zunaj + +ž že """.split() ) diff --git a/spacy/lang/sl/tokenizer_exceptions.py b/spacy/lang/sl/tokenizer_exceptions.py new file mode 100644 index 000000000..3d4109228 --- /dev/null +++ b/spacy/lang/sl/tokenizer_exceptions.py @@ -0,0 +1,272 @@ +from typing import Dict, List +from ..tokenizer_exceptions import BASE_EXCEPTIONS +from ...symbols import ORTH, NORM +from ...util import update_exc + +_exc: Dict[str, List[Dict]] = {} + +_other_exc = { + "t.i.": [{ORTH: "t.", NORM: "tako"}, {ORTH: "i.", NORM: "imenovano"}], + "t.j.": [{ORTH: "t.", NORM: "to"}, {ORTH: "j.", NORM: "je"}], + "T.j.": [{ORTH: "T.", NORM: "to"}, {ORTH: "j.", NORM: "je"}], + "d.o.o.": [ + {ORTH: "d.", NORM: "družba"}, + {ORTH: "o.", NORM: "omejeno"}, + {ORTH: "o.", NORM: "odgovornostjo"}, + ], + "D.O.O.": [ + {ORTH: "D.", NORM: "družba"}, + {ORTH: "O.", NORM: "omejeno"}, + {ORTH: "O.", NORM: "odgovornostjo"}, + ], + "d.n.o.": [ + {ORTH: "d.", NORM: "družba"}, + {ORTH: "n.", NORM: "neomejeno"}, + {ORTH: "o.", NORM: "odgovornostjo"}, + ], + "D.N.O.": [ + {ORTH: "D.", NORM: "družba"}, + {ORTH: "N.", NORM: "neomejeno"}, + {ORTH: "O.", NORM: "odgovornostjo"}, + ], + "d.d.": [{ORTH: "d.", NORM: "delniška"}, {ORTH: "d.", NORM: "družba"}], + "D.D.": [{ORTH: "D.", NORM: "delniška"}, {ORTH: "D.", NORM: "družba"}], + "s.p.": [{ORTH: "s.", NORM: "samostojni"}, {ORTH: "p.", NORM: "podjetnik"}], + "S.P.": [{ORTH: "S.", NORM: "samostojni"}, {ORTH: "P.", NORM: "podjetnik"}], + "l.r.": [{ORTH: "l.", NORM: "lastno"}, {ORTH: "r.", NORM: "ročno"}], + "le-te": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "te"}], + "Le-te": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "te"}], + "le-ti": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "ti"}], + "Le-ti": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "ti"}], + "le-to": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "to"}], + "Le-to": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "to"}], + "le-ta": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "ta"}], + "Le-ta": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "ta"}], + "le-tega": [{ORTH: "le"}, {ORTH: "-"}, {ORTH: "tega"}], + "Le-tega": [{ORTH: "Le"}, {ORTH: "-"}, {ORTH: "tega"}], +} + +_exc.update(_other_exc) + + +for exc_data in [ + {ORTH: "adm.", NORM: "administracija"}, + {ORTH: "aer.", NORM: "aeronavtika"}, + {ORTH: "agr.", NORM: "agronomija"}, + {ORTH: "amer.", NORM: "ameriško"}, + {ORTH: "anat.", NORM: "anatomija"}, + {ORTH: "angl.", NORM: "angleški"}, + {ORTH: "ant.", NORM: "antonim"}, + {ORTH: "antr.", NORM: "antropologija"}, + {ORTH: "apr.", NORM: "april"}, + {ORTH: "arab.", NORM: "arabsko"}, + {ORTH: "arheol.", NORM: "arheologija"}, + {ORTH: "arhit.", NORM: "arhitektura"}, + {ORTH: "avg.", NORM: "avgust"}, + {ORTH: "avstr.", NORM: "avstrijsko"}, + {ORTH: "avt.", NORM: "avtomobilizem"}, + {ORTH: "bibl.", NORM: "biblijsko"}, + {ORTH: "biokem.", NORM: "biokemija"}, + {ORTH: "biol.", NORM: "biologija"}, + {ORTH: "bolg.", NORM: "bolgarski"}, + {ORTH: "bot.", NORM: "botanika"}, + {ORTH: "cit.", NORM: "citat"}, + {ORTH: "daj.", NORM: "dajalnik"}, + {ORTH: "del.", NORM: "deležnik"}, + {ORTH: "ed.", NORM: "ednina"}, + {ORTH: "etn.", NORM: "etnografija"}, + {ORTH: "farm.", NORM: "farmacija"}, + {ORTH: "filat.", NORM: "filatelija"}, + {ORTH: "filoz.", NORM: "filozofija"}, + {ORTH: "fin.", NORM: "finančništvo"}, + {ORTH: "fiz.", NORM: "fizika"}, + {ORTH: "fot.", NORM: "fotografija"}, + {ORTH: "fr.", NORM: "francoski"}, + {ORTH: "friz.", NORM: "frizerstvo"}, + {ORTH: "gastr.", NORM: "gastronomija"}, + {ORTH: "geogr.", NORM: "geografija"}, + {ORTH: "geol.", NORM: "geologija"}, + {ORTH: "geom.", NORM: "geometrija"}, + {ORTH: "germ.", NORM: "germanski"}, + {ORTH: "gl.", NORM: "glej"}, + {ORTH: "glag.", NORM: "glagolski"}, + {ORTH: "glasb.", NORM: "glasba"}, + {ORTH: "gled.", NORM: "gledališče"}, + {ORTH: "gost.", NORM: "gostinstvo"}, + {ORTH: "gozd.", NORM: "gozdarstvo"}, + {ORTH: "gr.", NORM: "grški"}, + {ORTH: "grad.", NORM: "gradbeništvo"}, + {ORTH: "hebr.", NORM: "hebrejsko"}, + {ORTH: "hrv.", NORM: "hrvaško"}, + {ORTH: "ide.", NORM: "indoevropsko"}, + {ORTH: "igr.", NORM: "igre"}, + {ORTH: "im.", NORM: "imenovalnik"}, + {ORTH: "iron.", NORM: "ironično"}, + {ORTH: "it.", NORM: "italijanski"}, + {ORTH: "itd.", NORM: "in tako dalje"}, + {ORTH: "itn.", NORM: "in tako naprej"}, + {ORTH: "ipd.", NORM: "in podobno"}, + {ORTH: "jap.", NORM: "japonsko"}, + {ORTH: "jul.", NORM: "julij"}, + {ORTH: "jun.", NORM: "junij"}, + {ORTH: "kit.", NORM: "kitajsko"}, + {ORTH: "knj.", NORM: "knjižno"}, + {ORTH: "knjiž.", NORM: "knjižno"}, + {ORTH: "kor.", NORM: "koreografija"}, + {ORTH: "lat.", NORM: "latinski"}, + {ORTH: "les.", NORM: "lesna stroka"}, + {ORTH: "lingv.", NORM: "lingvistika"}, + {ORTH: "lit.", NORM: "literarni"}, + {ORTH: "ljubk.", NORM: "ljubkovalno"}, + {ORTH: "lov.", NORM: "lovstvo"}, + {ORTH: "m.", NORM: "moški"}, + {ORTH: "mak.", NORM: "makedonski"}, + {ORTH: "mar.", NORM: "marec"}, + {ORTH: "mat.", NORM: "matematika"}, + {ORTH: "med.", NORM: "medicina"}, + {ORTH: "meh.", NORM: "mehiško"}, + {ORTH: "mest.", NORM: "mestnik"}, + {ORTH: "mdr.", NORM: "med drugim"}, + {ORTH: "min.", NORM: "mineralogija"}, + {ORTH: "mitol.", NORM: "mitologija"}, + {ORTH: "mn.", NORM: "množina"}, + {ORTH: "mont.", NORM: "montanistika"}, + {ORTH: "muz.", NORM: "muzikologija"}, + {ORTH: "nam.", NORM: "namenilnik"}, + {ORTH: "nar.", NORM: "narečno"}, + {ORTH: "nav.", NORM: "navadno"}, + {ORTH: "nedol.", NORM: "nedoločnik"}, + {ORTH: "nedov.", NORM: "nedovršni"}, + {ORTH: "neprav.", NORM: "nepravilno"}, + {ORTH: "nepreh.", NORM: "neprehodno"}, + {ORTH: "neskl.", NORM: "nesklonljiv(o)"}, + {ORTH: "nestrok.", NORM: "nestrokovno"}, + {ORTH: "num.", NORM: "numizmatika"}, + {ORTH: "npr.", NORM: "na primer"}, + {ORTH: "obrt.", NORM: "obrtništvo"}, + {ORTH: "okt.", NORM: "oktober"}, + {ORTH: "or.", NORM: "orodnik"}, + {ORTH: "os.", NORM: "oseba"}, + {ORTH: "otr.", NORM: "otroško"}, + {ORTH: "oz.", NORM: "oziroma"}, + {ORTH: "pal.", NORM: "paleontologija"}, + {ORTH: "papir.", NORM: "papirništvo"}, + {ORTH: "ped.", NORM: "pedagogika"}, + {ORTH: "pisar.", NORM: "pisarniško"}, + {ORTH: "pog.", NORM: "pogovorno"}, + {ORTH: "polit.", NORM: "politika"}, + {ORTH: "polj.", NORM: "poljsko"}, + {ORTH: "poljud.", NORM: "poljudno"}, + {ORTH: "preg.", NORM: "pregovor"}, + {ORTH: "preh.", NORM: "prehodno"}, + {ORTH: "pren.", NORM: "preneseno"}, + {ORTH: "prid.", NORM: "pridevnik"}, + {ORTH: "prim.", NORM: "primerjaj"}, + {ORTH: "prisl.", NORM: "prislov"}, + {ORTH: "psih.", NORM: "psihologija"}, + {ORTH: "psiht.", NORM: "psihiatrija"}, + {ORTH: "rad.", NORM: "radiotehnika"}, + {ORTH: "rač.", NORM: "računalništvo"}, + {ORTH: "rib.", NORM: "ribištvo"}, + {ORTH: "rod.", NORM: "rodilnik"}, + {ORTH: "rus.", NORM: "rusko"}, + {ORTH: "s.", NORM: "srednji"}, + {ORTH: "sam.", NORM: "samostalniški"}, + {ORTH: "sed.", NORM: "sedanjik"}, + {ORTH: "sep.", NORM: "september"}, + {ORTH: "slabš.", NORM: "slabšalno"}, + {ORTH: "slovan.", NORM: "slovansko"}, + {ORTH: "slovaš.", NORM: "slovaško"}, + {ORTH: "srb.", NORM: "srbsko"}, + {ORTH: "star.", NORM: "starinsko"}, + {ORTH: "stil.", NORM: "stilno"}, + {ORTH: "sv.", NORM: "svet(i)"}, + {ORTH: "teh.", NORM: "tehnika"}, + {ORTH: "tisk.", NORM: "tiskarstvo"}, + {ORTH: "tj.", NORM: "to je"}, + {ORTH: "tož.", NORM: "tožilnik"}, + {ORTH: "trg.", NORM: "trgovina"}, + {ORTH: "ukr.", NORM: "ukrajinski"}, + {ORTH: "um.", NORM: "umetnost"}, + {ORTH: "vel.", NORM: "velelnik"}, + {ORTH: "vet.", NORM: "veterina"}, + {ORTH: "vez.", NORM: "veznik"}, + {ORTH: "vn.", NORM: "visokonemško"}, + {ORTH: "voj.", NORM: "vojska"}, + {ORTH: "vrtn.", NORM: "vrtnarstvo"}, + {ORTH: "vulg.", NORM: "vulgarno"}, + {ORTH: "vznes.", NORM: "vzneseno"}, + {ORTH: "zal.", NORM: "založništvo"}, + {ORTH: "zastar.", NORM: "zastarelo"}, + {ORTH: "zgod.", NORM: "zgodovina"}, + {ORTH: "zool.", NORM: "zoologija"}, + {ORTH: "čeb.", NORM: "čebelarstvo"}, + {ORTH: "češ.", NORM: "češki"}, + {ORTH: "člov.", NORM: "človeškost"}, + {ORTH: "šah.", NORM: "šahovski"}, + {ORTH: "šalj.", NORM: "šaljivo"}, + {ORTH: "šp.", NORM: "španski"}, + {ORTH: "špan.", NORM: "špansko"}, + {ORTH: "šport.", NORM: "športni"}, + {ORTH: "štev.", NORM: "števnik"}, + {ORTH: "šved.", NORM: "švedsko"}, + {ORTH: "švic.", NORM: "švicarsko"}, + {ORTH: "ž.", NORM: "ženski"}, + {ORTH: "žarg.", NORM: "žargonsko"}, + {ORTH: "žel.", NORM: "železnica"}, + {ORTH: "živ.", NORM: "živost"}, +]: + _exc[exc_data[ORTH]] = [exc_data] + + +abbrv = """ +Co. Ch. DIPL. DR. Dr. Ev. Inc. Jr. Kr. Mag. M. MR. Mr. Mt. Murr. Npr. OZ. +Opr. Osn. Prim. Roj. ST. Sim. Sp. Sred. St. Sv. Škofl. Tel. UR. Zb. +a. aa. ab. abc. abit. abl. abs. abt. acc. accel. add. adj. adv. aet. afr. akad. al. alban. all. alleg. +alp. alt. alter. alžir. am. an. andr. ang. anh. anon. ans. antrop. apoc. app. approx. apt. ar. arc. arch. +arh. arr. as. asist. assist. assoc. asst. astr. attn. aug. avstral. az. b. bab. bal. bbl. bd. belg. bioinf. +biomed. bk. bl. bn. borg. bp. br. braz. brit. bros. broš. bt. bu. c. ca. cal. can. cand. cantab. cap. capt. +cat. cath. cc. cca. cd. cdr. cdre. cent. cerkv. cert. cf. cfr. ch. chap. chem. chr. chs. cic. circ. civ. cl. +cm. cmd. cnr. co. cod. col. coll. colo. com. comp. con. conc. cond. conn. cons. cont. coop. corr. cost. cp. +cpl. cr. crd. cres. cresc. ct. cu. d. dan. dat. davč. ddr. dec. ded. def. dem. dent. dept. dia. dip. dipl. +dir. disp. diss. div. do. doc. dok. dol. doo. dop. dott. dr. dram. druž. družb. drž. dt. duh. dur. dvr. dwt. e. +ea. ecc. eccl. eccles. econ. edn. egipt. egr. ekon. eksp. el. em. enc. eng. eo. ep. err. esp. esq. est. +et. etc. etnogr. etnol. ev. evfem. evr. ex. exc. excl. exp. expl. ext. exx. f. fa. facs. fak. faks. fas. +fasc. fco. fcp. feb. febr. fec. fed. fem. ff. fff. fid. fig. fil. film. fiziol. fiziot. flam. fm. fo. fol. folk. +frag. fran. franc. fsc. g. ga. gal. gdč. ge. gen. geod. geog. geotehnol. gg. gimn. glas. glav. gnr. go. gor. +gosp. gp. graf. gram. gren. grš. gs. h. hab. hf. hist. ho. hort. i. ia. ib. ibid. id. idr. idridr. ill. imen. +imp. impf. impr. in. inc. incl. ind. indus. inf. inform. ing. init. ins. int. inv. inšp. inštr. inž. is. islam. +ist. ital. iur. iz. izbr. izd. izg. izgr. izr. izv. j. jak. jam. jan. jav. je. jez. jr. jsl. jud. jug. +jugoslovan. jur. juž. jv. jz. k. kal. kan. kand. kat. kdo. kem. kip. kmet. kol. kom. komp. konf. kont. kost. kov. +kp. kpfw. kr. kraj. krat. kub. kult. kv. kval. l. la. lab. lb. ld. let. lib. lik. litt. lj. ljud. ll. loc. log. +loč. lt. ma. madž. mag. manag. manjš. masc. mass. mater. max. maxmax. mb. md. mech. medic. medij. medn. +mehč. mem. menedž. mes. mess. metal. meteor. meteorol. mex. mi. mikr. mil. minn. mio. misc. miss. mit. mk. +mkt. ml. mlad. mlle. mlr. mm. mme. množ. mo. moj. moš. možn. mr. mrd. mrs. ms. msc. msgr. mt. murr. mus. mut. +n. na. nad. nadalj. nadom. nagl. nakl. namer. nan. naniz. nasl. nat. navt. nač. ned. nem. nik. nizoz. nm. nn. +no. nom. norv. notr. nov. novogr. ns. o. ob. obd. obj. oblač. obl. oblik. obr. obraz. obs. obst. obt. obč. oc. +oct. od. odd. odg. odn. odst. odv. oec. off. ok. okla. okr. ont. oo. op. opis. opp. opr. orch. ord. ore. oreg. +org. orient. orig. ork. ort. oseb. osn. ot. ozir. ošk. p. pag. par. para. parc. parl. part. past. pat. pdk. +pen. perf. pert. perz. pesn. pet. pev. pf. pfc. ph. pharm. phil. pis. pl. po. pod. podr. podaljš. pogl. pogoj. pojm. +pok. pokr. pol. poljed. poljub. polu. pom. pomen. pon. ponov. pop. por. port. pos. posl. posn. pov. pp. ppl. pr. +praet. prav. pravopis. pravosl. preb. pred. predl. predm. predp. preds. pref. pregib. prel. prem. premen. prep. +pres. pret. prev. pribl. prih. pril. primerj. primor. prip. pripor. prir. prist. priv. proc. prof. prog. proiz. +prom. pron. prop. prot. protest. prov. ps. pss. pt. publ. pz. q. qld. qu. quad. que. r. racc. rastl. razgl. +razl. razv. rd. red. ref. reg. rel. relig. rep. repr. rer. resp. rest. ret. rev. revol. rež. rim. rist. rkp. rm. +roj. rom. romun. rp. rr. rt. rud. ruš. ry. sal. samogl. san. sc. scen. sci. scr. sdv. seg. sek. sen. sept. ser. +sev. sg. sgt. sh. sig. sigg. sign. sim. sin. sing. sinh. skand. skl. sklad. sklanj. sklep. skr. sl. slik. slov. +slovak. slovn. sn. so. sob. soc. sociol. sod. sopomen. sopr. sor. sov. sovj. sp. spec. spl. spr. spreg. sq. sr. +sre. sred. sredoz. srh. ss. ssp. st. sta. stan. stanstar. stcsl. ste. stim. stol. stom. str. stroj. strok. stsl. +stud. sup. supl. suppl. svet. sz. t. tab. tech. ted. tehn. tehnol. tek. teks. tekst. tel. temp. ten. teol. ter. +term. test. th. theol. tim. tip. tisočl. tit. tl. tol. tolmač. tom. tor. tov. tr. trad. traj. trans. tren. +trib. tril. trop. trp. trž. ts. tt. tu. tur. turiz. tvor. tvorb. tč. u. ul. umet. un. univ. up. upr. ur. urad. +us. ust. utr. v. va. val. var. varn. ven. ver. verb. vest. vezal. vic. vis. viv. viz. viš. vod. vok. vol. vpr. +vrst. vrstil. vs. vv. vzd. vzg. vzh. vzor. w. wed. wg. wk. x. y. z. zah. zaim. zak. zap. zasl. zavar. zač. zb. +združ. zg. zn. znan. znanstv. zoot. zun. zv. zvd. á. é. ć. č. čas. čet. čl. člen. čustv. đ. ľ. ł. ş. ŠT. š. šir. +škofl. škot. šol. št. števil. štud. ů. ű. žen. žival. +""".split() + +for orth in abbrv: + _exc[orth] = [{ORTH: orth}] + + +TOKENIZER_EXCEPTIONS = update_exc(BASE_EXCEPTIONS, _exc) diff --git a/spacy/tests/lang/sl/test_text.py b/spacy/tests/lang/sl/test_text.py index ddc5b6b5d..a2a932077 100644 --- a/spacy/tests/lang/sl/test_text.py +++ b/spacy/tests/lang/sl/test_text.py @@ -20,7 +20,6 @@ od katerih so te svoboščine odvisne, assert len(tokens) == 116 -@pytest.mark.xfail def test_ordinal_number(sl_tokenizer): text = "10. decembra 1948" tokens = sl_tokenizer(text) From 5fa8f4faca966fe58c5c8de861900724c7659f25 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Mon, 22 Aug 2022 11:27:14 +0200 Subject: [PATCH 02/24] Switch ru and uk lemmatizers to pymorphy3 (#11345) * Switch ru and uk lemmatizers to pymorphy3 * Switch to pymorphy3 in tests --- spacy/lang/ru/__init__.py | 2 +- spacy/lang/ru/lemmatizer.py | 15 ++++++++++++++- spacy/lang/uk/__init__.py | 2 +- spacy/lang/uk/lemmatizer.py | 13 ++++++++++++- spacy/tests/conftest.py | 10 +++++----- website/docs/api/lemmatizer.md | 6 +++--- website/meta/languages.json | 8 ++++---- 7 files changed, 40 insertions(+), 16 deletions(-) diff --git a/spacy/lang/ru/__init__.py b/spacy/lang/ru/__init__.py index c118c26ff..7d17628c4 100644 --- a/spacy/lang/ru/__init__.py +++ b/spacy/lang/ru/__init__.py @@ -28,7 +28,7 @@ class Russian(Language): assigns=["token.lemma"], default_config={ "model": None, - "mode": "pymorphy2", + "mode": "pymorphy3", "overwrite": False, "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, }, diff --git a/spacy/lang/ru/lemmatizer.py b/spacy/lang/ru/lemmatizer.py index 85180b1e4..720d3a8cb 100644 --- a/spacy/lang/ru/lemmatizer.py +++ b/spacy/lang/ru/lemmatizer.py @@ -19,7 +19,7 @@ class RussianLemmatizer(Lemmatizer): model: Optional[Model], name: str = "lemmatizer", *, - mode: str = "pymorphy2", + mode: str = "pymorphy3", overwrite: bool = False, scorer: Optional[Callable] = lemmatizer_score, ) -> None: @@ -33,6 +33,16 @@ class RussianLemmatizer(Lemmatizer): ) from None if getattr(self, "_morph", None) is None: self._morph = MorphAnalyzer() + elif mode == "pymorphy3": + try: + from pymorphy3 import MorphAnalyzer + except ImportError: + raise ImportError( + "The Russian lemmatizer mode 'pymorphy3' requires the " + "pymorphy3 library. Install it with: pip install pymorphy3" + ) from None + if getattr(self, "_morph", None) is None: + self._morph = MorphAnalyzer() super().__init__( vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer ) @@ -104,6 +114,9 @@ class RussianLemmatizer(Lemmatizer): return [analyses[0].normal_form] return [string] + def pymorphy3_lemmatize(self, token: Token) -> List[str]: + return self.pymorphy2_lemmatize(token) + def oc2ud(oc_tag: str) -> Tuple[str, Dict[str, str]]: gram_map = { diff --git a/spacy/lang/uk/__init__.py b/spacy/lang/uk/__init__.py index 737243b66..bfea9ff69 100644 --- a/spacy/lang/uk/__init__.py +++ b/spacy/lang/uk/__init__.py @@ -29,7 +29,7 @@ class Ukrainian(Language): assigns=["token.lemma"], default_config={ "model": None, - "mode": "pymorphy2", + "mode": "pymorphy3", "overwrite": False, "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"}, }, diff --git a/spacy/lang/uk/lemmatizer.py b/spacy/lang/uk/lemmatizer.py index a8bc56057..97ee80479 100644 --- a/spacy/lang/uk/lemmatizer.py +++ b/spacy/lang/uk/lemmatizer.py @@ -14,7 +14,7 @@ class UkrainianLemmatizer(RussianLemmatizer): model: Optional[Model], name: str = "lemmatizer", *, - mode: str = "pymorphy2", + mode: str = "pymorphy3", overwrite: bool = False, scorer: Optional[Callable] = lemmatizer_score, ) -> None: @@ -29,6 +29,17 @@ class UkrainianLemmatizer(RussianLemmatizer): ) from None if getattr(self, "_morph", None) is None: self._morph = MorphAnalyzer(lang="uk") + elif mode == "pymorphy3": + try: + from pymorphy3 import MorphAnalyzer + except ImportError: + raise ImportError( + "The Ukrainian lemmatizer mode 'pymorphy3' requires the " + "pymorphy3 library and dictionaries. Install them with: " + "pip install pymorphy3 pymorphy3-dicts-uk" + ) from None + if getattr(self, "_morph", None) is None: + self._morph = MorphAnalyzer(lang="uk") super().__init__( vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer ) diff --git a/spacy/tests/conftest.py b/spacy/tests/conftest.py index eb643ec2f..76de8f373 100644 --- a/spacy/tests/conftest.py +++ b/spacy/tests/conftest.py @@ -323,13 +323,13 @@ def ro_tokenizer(): @pytest.fixture(scope="session") def ru_tokenizer(): - pytest.importorskip("pymorphy2") + pytest.importorskip("pymorphy3") return get_lang_class("ru")().tokenizer @pytest.fixture def ru_lemmatizer(): - pytest.importorskip("pymorphy2") + pytest.importorskip("pymorphy3") return get_lang_class("ru")().add_pipe("lemmatizer") @@ -401,14 +401,14 @@ def ky_tokenizer(): @pytest.fixture(scope="session") def uk_tokenizer(): - pytest.importorskip("pymorphy2") + pytest.importorskip("pymorphy3") return get_lang_class("uk")().tokenizer @pytest.fixture def uk_lemmatizer(): - pytest.importorskip("pymorphy2") - pytest.importorskip("pymorphy2_dicts_uk") + pytest.importorskip("pymorphy3") + pytest.importorskip("pymorphy3_dicts_uk") return get_lang_class("uk")().add_pipe("lemmatizer") diff --git a/website/docs/api/lemmatizer.md b/website/docs/api/lemmatizer.md index 422f34040..905096338 100644 --- a/website/docs/api/lemmatizer.md +++ b/website/docs/api/lemmatizer.md @@ -70,7 +70,7 @@ lemmatizer is available. The lemmatizer modes `rule` and `pos_lookup` require [`token.pos`](/api/token) from a previous pipeline component (see example pipeline configurations in the [pretrained pipeline design details](/models#design-cnn)) or rely on third-party -libraries (`pymorphy2`). +libraries (`pymorphy3`). | Language | Default Mode | | -------- | ------------ | @@ -86,9 +86,9 @@ libraries (`pymorphy2`). | `nb` | `rule` | | `nl` | `rule` | | `pl` | `pos_lookup` | -| `ru` | `pymorphy2` | +| `ru` | `pymorphy3` | | `sv` | `rule` | -| `uk` | `pymorphy2` | +| `uk` | `pymorphy3` | ```python %%GITHUB_SPACY/spacy/pipeline/lemmatizer.py diff --git a/website/meta/languages.json b/website/meta/languages.json index 6bc2309ed..5305ceffc 100644 --- a/website/meta/languages.json +++ b/website/meta/languages.json @@ -369,8 +369,8 @@ "has_examples": true, "dependencies": [ { - "name": "pymorphy2", - "url": "https://github.com/kmike/pymorphy2" + "name": "pymorphy3", + "url": "https://github.com/no-plagiarism/pymorphy3" } ], "models": [ @@ -469,8 +469,8 @@ "has_examples": true, "dependencies": [ { - "name": "pymorphy2", - "url": "https://github.com/kmike/pymorphy2" + "name": "pymorphy3", + "url": "https://github.com/no-plagiarism/pymorphy3" } ] }, From aafee5e1b7c8d13d9ac14c438063621a18bec743 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Mon, 29 Aug 2022 17:32:38 +0900 Subject: [PATCH 03/24] Fix lookup usage in French/Catalan (fix #11347) (#11382) * Fix lookup usage (fix #11347) Before using the lookups table in the French (and Catalan) lemmatizers, there's a check to see if the current term is in the table. But it's checking a string against hashes, so it's always false. Also the table lookup function is designed so you don't have to do that anyway. * Use the lookup table directly * Use string, not token --- spacy/lang/ca/lemmatizer.py | 6 +++--- spacy/lang/fr/lemmatizer.py | 13 ++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/spacy/lang/ca/lemmatizer.py b/spacy/lang/ca/lemmatizer.py index 2fd012912..0f15e6e65 100644 --- a/spacy/lang/ca/lemmatizer.py +++ b/spacy/lang/ca/lemmatizer.py @@ -72,10 +72,10 @@ class CatalanLemmatizer(Lemmatizer): oov_forms.append(form) if not forms: forms.extend(oov_forms) - if not forms and string in lookup_table.keys(): - forms.append(self.lookup_lemmatize(token)[0]) + + # use lookups, and fall back to the token itself if not forms: - forms.append(string) + forms.append(lookup_table.get(string, [string])[0]) forms = list(dict.fromkeys(forms)) self.cache[cache_key] = forms return forms diff --git a/spacy/lang/fr/lemmatizer.py b/spacy/lang/fr/lemmatizer.py index c6422cf96..a7cbe0bcf 100644 --- a/spacy/lang/fr/lemmatizer.py +++ b/spacy/lang/fr/lemmatizer.py @@ -53,11 +53,16 @@ class FrenchLemmatizer(Lemmatizer): rules = rules_table.get(univ_pos, []) string = string.lower() forms = [] + # first try lookup in table based on upos if string in index: forms.append(string) self.cache[cache_key] = forms return forms + + # then add anything in the exceptions table forms.extend(exceptions.get(string, [])) + + # if nothing found yet, use the rules oov_forms = [] if not forms: for old, new in rules: @@ -69,12 +74,14 @@ class FrenchLemmatizer(Lemmatizer): forms.append(form) else: oov_forms.append(form) + + # if still nothing, add the oov forms from rules if not forms: forms.extend(oov_forms) - if not forms and string in lookup_table.keys(): - forms.append(self.lookup_lemmatize(token)[0]) + + # use lookups, which fall back to the token itself if not forms: - forms.append(string) + forms.append(lookup_table.get(string, [string])[0]) forms = list(dict.fromkeys(forms)) self.cache[cache_key] = forms return forms From 604a7c3c26bcc6737a9676c3ba1b16c9ac705be3 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Wed, 31 Aug 2022 09:03:20 +0200 Subject: [PATCH 04/24] `SpanGroup(s)`-related optimizations (#11380) * `SpanGroup`: Add support for binding copies to a new reference document * `SpanGroups`: Replace superfluous serialize-deserialize roundtrip in `copy` Instead, directly copy the in-memory representations of the constituent `SpanGroup`s. * Update `SpanGroup.copy()` signature * Rename `new_doc` param to `doc` * Fix kwdarg * Update `.pyi` file and docstrings * `mypy` fix * Update spacy/tokens/span_group.pyx * Update docs Co-authored-by: Adriane Boyd --- spacy/tokens/_dict_proxies.py | 3 ++- spacy/tokens/span_group.pyi | 4 ++-- spacy/tokens/span_group.pyx | 7 +++++-- website/docs/api/spangroup.md | 7 ++++--- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/spacy/tokens/_dict_proxies.py b/spacy/tokens/_dict_proxies.py index 9630da261..6edcce13d 100644 --- a/spacy/tokens/_dict_proxies.py +++ b/spacy/tokens/_dict_proxies.py @@ -42,7 +42,8 @@ class SpanGroups(UserDict): def copy(self, doc: Optional["Doc"] = None) -> "SpanGroups": if doc is None: doc = self._ensure_doc() - return SpanGroups(doc).from_bytes(self.to_bytes()) + data_copy = ((k, v.copy(doc=doc)) for k, v in self.items()) + return SpanGroups(doc, items=data_copy) def setdefault(self, key, default=None): if not isinstance(default, SpanGroup): diff --git a/spacy/tokens/span_group.pyi b/spacy/tokens/span_group.pyi index 245eb4dbe..21cd124ab 100644 --- a/spacy/tokens/span_group.pyi +++ b/spacy/tokens/span_group.pyi @@ -1,4 +1,4 @@ -from typing import Any, Dict, Iterable +from typing import Any, Dict, Iterable, Optional from .doc import Doc from .span import Span @@ -24,4 +24,4 @@ class SpanGroup: def __getitem__(self, i: int) -> Span: ... def to_bytes(self) -> bytes: ... def from_bytes(self, bytes_data: bytes) -> SpanGroup: ... - def copy(self) -> SpanGroup: ... + def copy(self, doc: Optional[Doc] = ...) -> SpanGroup: ... diff --git a/spacy/tokens/span_group.pyx b/spacy/tokens/span_group.pyx index bb0fab24f..1aa3c0bc8 100644 --- a/spacy/tokens/span_group.pyx +++ b/spacy/tokens/span_group.pyx @@ -241,15 +241,18 @@ cdef class SpanGroup: cdef void push_back(self, SpanC span) nogil: self.c.push_back(span) - def copy(self) -> SpanGroup: + def copy(self, doc: Optional["Doc"] = None) -> SpanGroup: """Clones the span group. + doc (Doc): New reference document to which the copy is bound. RETURNS (SpanGroup): A copy of the span group. DOCS: https://spacy.io/api/spangroup#copy """ + if doc is None: + doc = self.doc return SpanGroup( - self.doc, + doc, name=self.name, attrs=deepcopy(self.attrs), spans=list(self), diff --git a/website/docs/api/spangroup.md b/website/docs/api/spangroup.md index 8dbdefc01..2d1cf73c4 100644 --- a/website/docs/api/spangroup.md +++ b/website/docs/api/spangroup.md @@ -255,9 +255,10 @@ Return a copy of the span group. > new_group = doc.spans["errors"].copy() > ``` -| Name | Description | -| ----------- | ----------------------------------------------- | -| **RETURNS** | A copy of the `SpanGroup` object. ~~SpanGroup~~ | +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------------- | +| `doc` | The document to which the copy is bound. Defaults to `None` for the current doc. ~~Optional[Doc]~~ | +| **RETURNS** | A copy of the `SpanGroup` object. ~~SpanGroup~~ | ## SpanGroup.to_bytes {#to_bytes tag="method"} From 1f23c615d7a7326ca5a38a7d768b8b70caaa0e17 Mon Sep 17 00:00:00 2001 From: Raphael Mitsch Date: Thu, 8 Sep 2022 10:38:07 +0200 Subject: [PATCH 05/24] Refactor KB for easier customization (#11268) * Add implementation of batching + backwards compatibility fixes. Tests indicate issue with batch disambiguation for custom singular entity lookups. * Fix tests. Add distinction w.r.t. batch size. * Remove redundant and add new comments. * Adjust comments. Fix variable naming in EL prediction. * Fix mypy errors. * Remove KB entity type config option. Change return types of candidate retrieval functions to Iterable from Iterator. Fix various other issues. * Update spacy/pipeline/entity_linker.py Co-authored-by: Paul O'Leary McCann * Update spacy/pipeline/entity_linker.py Co-authored-by: Paul O'Leary McCann * Update spacy/kb_base.pyx Co-authored-by: Paul O'Leary McCann * Update spacy/kb_base.pyx Co-authored-by: Paul O'Leary McCann * Update spacy/pipeline/entity_linker.py Co-authored-by: Paul O'Leary McCann * Add error messages to NotImplementedErrors. Remove redundant comment. * Fix imports. * Remove redundant comments. * Rename KnowledgeBase to InMemoryLookupKB and BaseKnowledgeBase to KnowledgeBase. * Fix tests. * Update spacy/errors.py Co-authored-by: Sofie Van Landeghem * Move KB into subdirectory. * Adjust imports after KB move to dedicated subdirectory. * Fix config imports. * Move Candidate + retrieval functions to separate module. Fix other, small issues. * Fix docstrings and error message w.r.t. class names. Fix typing for candidate retrieval functions. * Update spacy/kb/kb_in_memory.pyx Co-authored-by: Sofie Van Landeghem * Update spacy/ml/models/entity_linker.py Co-authored-by: Sofie Van Landeghem * Fix typing. * Change typing of mentions to be Span instead of Union[Span, str]. * Update docs. * Update EntityLinker and _architecture docs. * Update website/docs/api/entitylinker.md Co-authored-by: Paul O'Leary McCann * Adjust message for E1046. * Re-add section for Candidate in kb.md, add reference to dedicated page. * Update docs and docstrings. * Re-add section + reference for KnowledgeBase.get_alias_candidates() in docs. * Update spacy/kb/candidate.pyx * Update spacy/kb/kb_in_memory.pyx * Update spacy/pipeline/legacy/entity_linker.py * Remove canididate.md. Remove mistakenly added config snippet in entity_linker.py. Co-authored-by: Paul O'Leary McCann Co-authored-by: Sofie Van Landeghem --- setup.py | 4 +- spacy/errors.py | 10 +- spacy/kb/__init__.py | 3 + spacy/kb/candidate.pxd | 12 + spacy/kb/candidate.pyx | 74 +++++ spacy/kb/kb.pxd | 10 + spacy/kb/kb.pyx | 108 +++++++ spacy/{kb.pxd => kb/kb_in_memory.pxd} | 24 +- spacy/{kb.pyx => kb/kb_in_memory.pyx} | 96 ++---- spacy/ml/models/entity_linker.py | 30 +- spacy/pipeline/entity_linker.py | 184 +++++++---- spacy/pipeline/legacy/entity_linker.py | 5 +- spacy/tests/pipeline/test_entity_linker.py | 98 +++--- .../tests/serialize/test_resource_warning.py | 8 +- spacy/tests/serialize/test_serialize_kb.py | 16 +- website/docs/api/architectures.md | 14 +- website/docs/api/entitylinker.md | 5 +- website/docs/api/kb.md | 219 +++++-------- website/docs/api/kb_in_memory.md | 302 ++++++++++++++++++ website/docs/usage/101/_architecture.md | 4 +- 20 files changed, 854 insertions(+), 372 deletions(-) create mode 100644 spacy/kb/__init__.py create mode 100644 spacy/kb/candidate.pxd create mode 100644 spacy/kb/candidate.pyx create mode 100644 spacy/kb/kb.pxd create mode 100644 spacy/kb/kb.pyx rename spacy/{kb.pxd => kb/kb_in_memory.pxd} (92%) rename spacy/{kb.pyx => kb/kb_in_memory.pyx} (90%) create mode 100644 website/docs/api/kb_in_memory.md diff --git a/setup.py b/setup.py index ec1bd35fa..899d940ed 100755 --- a/setup.py +++ b/setup.py @@ -30,7 +30,9 @@ MOD_NAMES = [ "spacy.lexeme", "spacy.vocab", "spacy.attrs", - "spacy.kb", + "spacy.kb.candidate", + "spacy.kb.kb", + "spacy.kb.kb_in_memory", "spacy.ml.parser_model", "spacy.morphology", "spacy.pipeline.dep_parser", diff --git a/spacy/errors.py b/spacy/errors.py index 5ee1476c2..e2201284f 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -709,9 +709,9 @@ class Errors(metaclass=ErrorsWithCodes): "`nlp.enable_pipe` instead.") E927 = ("Can't write to frozen list Maybe you're trying to modify a computed " "property or default function argument?") - E928 = ("A KnowledgeBase can only be serialized to/from from a directory, " + E928 = ("An InMemoryLookupKB can only be serialized to/from from a directory, " "but the provided argument {loc} points to a file.") - E929 = ("Couldn't read KnowledgeBase from {loc}. The path does not seem to exist.") + E929 = ("Couldn't read InMemoryLookupKB from {loc}. The path does not seem to exist.") E930 = ("Received invalid get_examples callback in `{method}`. " "Expected function that returns an iterable of Example objects but " "got: {obj}") @@ -941,6 +941,12 @@ class Errors(metaclass=ErrorsWithCodes): "`{arg2}`={arg2_values} but these arguments are conflicting.") E1043 = ("Expected None or a value in range [{range_start}, {range_end}] for entity linker threshold, but got " "{value}.") + E1044 = ("Expected `candidates_batch_size` to be >= 1, but got: {value}") + E1045 = ("Encountered {parent} subclass without `{parent}.{method}` " + "method in '{name}'. If you want to use this method, make " + "sure it's overwritten on the subclass.") + E1046 = ("{cls_name} is an abstract class and cannot be instantiated. If you are looking for spaCy's default " + "knowledge base, use `InMemoryLookupKB`.") # Deprecated model shortcuts, only used in errors and warnings diff --git a/spacy/kb/__init__.py b/spacy/kb/__init__.py new file mode 100644 index 000000000..1d70a9b34 --- /dev/null +++ b/spacy/kb/__init__.py @@ -0,0 +1,3 @@ +from .kb import KnowledgeBase +from .kb_in_memory import InMemoryLookupKB +from .candidate import Candidate, get_candidates, get_candidates_batch diff --git a/spacy/kb/candidate.pxd b/spacy/kb/candidate.pxd new file mode 100644 index 000000000..942ce9dd0 --- /dev/null +++ b/spacy/kb/candidate.pxd @@ -0,0 +1,12 @@ +from .kb cimport KnowledgeBase +from libcpp.vector cimport vector +from ..typedefs cimport hash_t + +# Object used by the Entity Linker that summarizes one entity-alias candidate combination. +cdef class Candidate: + cdef readonly KnowledgeBase kb + cdef hash_t entity_hash + cdef float entity_freq + cdef vector[float] entity_vector + cdef hash_t alias_hash + cdef float prior_prob diff --git a/spacy/kb/candidate.pyx b/spacy/kb/candidate.pyx new file mode 100644 index 000000000..c89efeb03 --- /dev/null +++ b/spacy/kb/candidate.pyx @@ -0,0 +1,74 @@ +# cython: infer_types=True, profile=True + +from typing import Iterable +from .kb cimport KnowledgeBase +from ..tokens import Span + +cdef class Candidate: + """A `Candidate` object refers to a textual mention (`alias`) that may or may not be resolved + to a specific `entity` from a Knowledge Base. This will be used as input for the entity linking + algorithm which will disambiguate the various candidates to the correct one. + Each candidate (alias, entity) pair is assigned a certain prior probability. + + DOCS: https://spacy.io/api/kb/#candidate-init + """ + + def __init__(self, KnowledgeBase kb, entity_hash, entity_freq, entity_vector, alias_hash, prior_prob): + self.kb = kb + self.entity_hash = entity_hash + self.entity_freq = entity_freq + self.entity_vector = entity_vector + self.alias_hash = alias_hash + self.prior_prob = prior_prob + + @property + def entity(self) -> int: + """RETURNS (uint64): hash of the entity's KB ID/name""" + return self.entity_hash + + @property + def entity_(self) -> str: + """RETURNS (str): ID/name of this entity in the KB""" + return self.kb.vocab.strings[self.entity_hash] + + @property + def alias(self) -> int: + """RETURNS (uint64): hash of the alias""" + return self.alias_hash + + @property + def alias_(self) -> str: + """RETURNS (str): ID of the original alias""" + return self.kb.vocab.strings[self.alias_hash] + + @property + def entity_freq(self) -> float: + return self.entity_freq + + @property + def entity_vector(self) -> Iterable[float]: + return self.entity_vector + + @property + def prior_prob(self) -> float: + return self.prior_prob + + +def get_candidates(kb: KnowledgeBase, mention: Span) -> Iterable[Candidate]: + """ + Return candidate entities for a given mention and fetching appropriate entries from the index. + kb (KnowledgeBase): Knowledge base to query. + mention (Span): Entity mention for which to identify candidates. + RETURNS (Iterable[Candidate]): Identified candidates. + """ + return kb.get_candidates(mention) + + +def get_candidates_batch(kb: KnowledgeBase, mentions: Iterable[Span]) -> Iterable[Iterable[Candidate]]: + """ + Return candidate entities for the given mentions and fetching appropriate entries from the index. + kb (KnowledgeBase): Knowledge base to query. + mention (Iterable[Span]): Entity mentions for which to identify candidates. + RETURNS (Iterable[Iterable[Candidate]]): Identified candidates. + """ + return kb.get_candidates_batch(mentions) diff --git a/spacy/kb/kb.pxd b/spacy/kb/kb.pxd new file mode 100644 index 000000000..1adeef8ae --- /dev/null +++ b/spacy/kb/kb.pxd @@ -0,0 +1,10 @@ +"""Knowledge-base for entity or concept linking.""" + +from cymem.cymem cimport Pool +from libc.stdint cimport int64_t +from ..vocab cimport Vocab + +cdef class KnowledgeBase: + cdef Pool mem + cdef readonly Vocab vocab + cdef readonly int64_t entity_vector_length diff --git a/spacy/kb/kb.pyx b/spacy/kb/kb.pyx new file mode 100644 index 000000000..ce4bc0138 --- /dev/null +++ b/spacy/kb/kb.pyx @@ -0,0 +1,108 @@ +# cython: infer_types=True, profile=True + +from pathlib import Path +from typing import Iterable, Tuple, Union +from cymem.cymem cimport Pool + +from .candidate import Candidate +from ..tokens import Span +from ..util import SimpleFrozenList +from ..errors import Errors + + +cdef class KnowledgeBase: + """A `KnowledgeBase` instance stores unique identifiers for entities and their textual aliases, + to support entity linking of named entities to real-world concepts. + This is an abstract class and requires its operations to be implemented. + + DOCS: https://spacy.io/api/kb + """ + + def __init__(self, vocab: Vocab, entity_vector_length: int): + """Create a KnowledgeBase.""" + # Make sure abstract KB is not instantiated. + if self.__class__ == KnowledgeBase: + raise TypeError( + Errors.E1046.format(cls_name=self.__class__.__name__) + ) + + self.vocab = vocab + self.entity_vector_length = entity_vector_length + self.mem = Pool() + + def get_candidates_batch(self, mentions: Iterable[Span]) -> Iterable[Iterable[Candidate]]: + """ + Return candidate entities for specified texts. Each candidate defines the entity, the original alias, + and the prior probability of that alias resolving to that entity. + If no candidate is found for a given text, an empty list is returned. + mentions (Iterable[Span]): Mentions for which to get candidates. + RETURNS (Iterable[Iterable[Candidate]]): Identified candidates. + """ + return [self.get_candidates(span) for span in mentions] + + def get_candidates(self, mention: Span) -> Iterable[Candidate]: + """ + Return candidate entities for specified text. Each candidate defines the entity, the original alias, + and the prior probability of that alias resolving to that entity. + If the no candidate is found for a given text, an empty list is returned. + mention (Span): Mention for which to get candidates. + RETURNS (Iterable[Candidate]): Identified candidates. + """ + raise NotImplementedError( + Errors.E1045.format(parent="KnowledgeBase", method="get_candidates", name=self.__name__) + ) + + def get_vectors(self, entities: Iterable[str]) -> Iterable[Iterable[float]]: + """ + Return vectors for entities. + entity (str): Entity name/ID. + RETURNS (Iterable[Iterable[float]]): Vectors for specified entities. + """ + return [self.get_vector(entity) for entity in entities] + + def get_vector(self, str entity) -> Iterable[float]: + """ + Return vector for entity. + entity (str): Entity name/ID. + RETURNS (Iterable[float]): Vector for specified entity. + """ + raise NotImplementedError( + Errors.E1045.format(parent="KnowledgeBase", method="get_vector", name=self.__name__) + ) + + def to_bytes(self, **kwargs) -> bytes: + """Serialize the current state to a binary string. + RETURNS (bytes): Current state as binary string. + """ + raise NotImplementedError( + Errors.E1045.format(parent="KnowledgeBase", method="to_bytes", name=self.__name__) + ) + + def from_bytes(self, bytes_data: bytes, *, exclude: Tuple[str] = tuple()): + """Load state from a binary string. + bytes_data (bytes): KB state. + exclude (Tuple[str]): Properties to exclude when restoring KB. + """ + raise NotImplementedError( + Errors.E1045.format(parent="KnowledgeBase", method="from_bytes", name=self.__name__) + ) + + def to_disk(self, path: Union[str, Path], exclude: Iterable[str] = SimpleFrozenList()) -> None: + """ + Write KnowledgeBase content to disk. + path (Union[str, Path]): Target file path. + exclude (Iterable[str]): List of components to exclude. + """ + raise NotImplementedError( + Errors.E1045.format(parent="KnowledgeBase", method="to_disk", name=self.__name__) + ) + + def from_disk(self, path: Union[str, Path], exclude: Iterable[str] = SimpleFrozenList()) -> None: + """ + Load KnowledgeBase content from disk. + path (Union[str, Path]): Target file path. + exclude (Iterable[str]): List of components to exclude. + """ + raise NotImplementedError( + Errors.E1045.format(parent="KnowledgeBase", method="from_disk", name=self.__name__) + ) diff --git a/spacy/kb.pxd b/spacy/kb/kb_in_memory.pxd similarity index 92% rename from spacy/kb.pxd rename to spacy/kb/kb_in_memory.pxd index a823dbe1e..825a6bde9 100644 --- a/spacy/kb.pxd +++ b/spacy/kb/kb_in_memory.pxd @@ -1,14 +1,12 @@ """Knowledge-base for entity or concept linking.""" -from cymem.cymem cimport Pool from preshed.maps cimport PreshMap from libcpp.vector cimport vector from libc.stdint cimport int32_t, int64_t from libc.stdio cimport FILE -from .vocab cimport Vocab -from .typedefs cimport hash_t -from .structs cimport KBEntryC, AliasC - +from ..typedefs cimport hash_t +from ..structs cimport KBEntryC, AliasC +from .kb cimport KnowledgeBase ctypedef vector[KBEntryC] entry_vec ctypedef vector[AliasC] alias_vec @@ -16,21 +14,7 @@ ctypedef vector[float] float_vec ctypedef vector[float_vec] float_matrix -# Object used by the Entity Linker that summarizes one entity-alias candidate combination. -cdef class Candidate: - cdef readonly KnowledgeBase kb - cdef hash_t entity_hash - cdef float entity_freq - cdef vector[float] entity_vector - cdef hash_t alias_hash - cdef float prior_prob - - -cdef class KnowledgeBase: - cdef Pool mem - cdef readonly Vocab vocab - cdef int64_t entity_vector_length - +cdef class InMemoryLookupKB(KnowledgeBase): # This maps 64bit keys (hash of unique entity string) # to 64bit values (position of the _KBEntryC struct in the _entries vector). # The PreshMap is pretty space efficient, as it uses open addressing. So diff --git a/spacy/kb.pyx b/spacy/kb/kb_in_memory.pyx similarity index 90% rename from spacy/kb.pyx rename to spacy/kb/kb_in_memory.pyx index ae1983a8d..485e52c2f 100644 --- a/spacy/kb.pyx +++ b/spacy/kb/kb_in_memory.pyx @@ -1,8 +1,7 @@ # cython: infer_types=True, profile=True -from typing import Iterator, Iterable, Callable, Dict, Any +from typing import Iterable, Callable, Dict, Any, Union import srsly -from cymem.cymem cimport Pool from preshed.maps cimport PreshMap from cpython.exc cimport PyErr_SetFromErrno from libc.stdio cimport fopen, fclose, fread, fwrite, feof, fseek @@ -12,85 +11,28 @@ from libcpp.vector cimport vector from pathlib import Path import warnings -from .typedefs cimport hash_t -from .errors import Errors, Warnings -from . import util -from .util import SimpleFrozenList, ensure_path - -cdef class Candidate: - """A `Candidate` object refers to a textual mention (`alias`) that may or may not be resolved - to a specific `entity` from a Knowledge Base. This will be used as input for the entity linking - algorithm which will disambiguate the various candidates to the correct one. - Each candidate (alias, entity) pair is assigned to a certain prior probability. - - DOCS: https://spacy.io/api/kb/#candidate_init - """ - - def __init__(self, KnowledgeBase kb, entity_hash, entity_freq, entity_vector, alias_hash, prior_prob): - self.kb = kb - self.entity_hash = entity_hash - self.entity_freq = entity_freq - self.entity_vector = entity_vector - self.alias_hash = alias_hash - self.prior_prob = prior_prob - - @property - def entity(self): - """RETURNS (uint64): hash of the entity's KB ID/name""" - return self.entity_hash - - @property - def entity_(self): - """RETURNS (str): ID/name of this entity in the KB""" - return self.kb.vocab.strings[self.entity_hash] - - @property - def alias(self): - """RETURNS (uint64): hash of the alias""" - return self.alias_hash - - @property - def alias_(self): - """RETURNS (str): ID of the original alias""" - return self.kb.vocab.strings[self.alias_hash] - - @property - def entity_freq(self): - return self.entity_freq - - @property - def entity_vector(self): - return self.entity_vector - - @property - def prior_prob(self): - return self.prior_prob +from ..tokens import Span +from ..typedefs cimport hash_t +from ..errors import Errors, Warnings +from .. import util +from ..util import SimpleFrozenList, ensure_path +from ..vocab cimport Vocab +from .kb cimport KnowledgeBase +from .candidate import Candidate as Candidate -def get_candidates(KnowledgeBase kb, span) -> Iterator[Candidate]: - """ - Return candidate entities for a given span by using the text of the span as the alias - and fetching appropriate entries from the index. - This particular function is optimized to work with the built-in KB functionality, - but any other custom candidate generation method can be used in combination with the KB as well. - """ - return kb.get_alias_candidates(span.text) - - -cdef class KnowledgeBase: - """A `KnowledgeBase` instance stores unique identifiers for entities and their textual aliases, +cdef class InMemoryLookupKB(KnowledgeBase): + """An `InMemoryLookupKB` instance stores unique identifiers for entities and their textual aliases, to support entity linking of named entities to real-world concepts. - DOCS: https://spacy.io/api/kb + DOCS: https://spacy.io/api/kb_in_memory """ def __init__(self, Vocab vocab, entity_vector_length): - """Create a KnowledgeBase.""" - self.mem = Pool() - self.entity_vector_length = entity_vector_length + """Create an InMemoryLookupKB.""" + super().__init__(vocab, entity_vector_length) self._entry_index = PreshMap() self._alias_index = PreshMap() - self.vocab = vocab self._create_empty_vectors(dummy_hash=self.vocab.strings[""]) def _initialize_entities(self, int64_t nr_entities): @@ -104,11 +46,6 @@ cdef class KnowledgeBase: self._alias_index = PreshMap(nr_aliases + 1) self._aliases_table = alias_vec(nr_aliases + 1) - @property - def entity_vector_length(self): - """RETURNS (uint64): length of the entity vectors""" - return self.entity_vector_length - def __len__(self): return self.get_size_entities() @@ -286,7 +223,10 @@ cdef class KnowledgeBase: alias_entry.probs = probs self._aliases_table[alias_index] = alias_entry - def get_alias_candidates(self, str alias) -> Iterator[Candidate]: + def get_candidates(self, mention: Span) -> Iterable[Candidate]: + return self.get_alias_candidates(mention.text) # type: ignore + + def get_alias_candidates(self, str alias) -> Iterable[Candidate]: """ Return candidate entities for an alias. Each candidate defines the entity, the original alias, and the prior probability of that alias resolving to that entity. diff --git a/spacy/ml/models/entity_linker.py b/spacy/ml/models/entity_linker.py index d847342a3..4d18d216a 100644 --- a/spacy/ml/models/entity_linker.py +++ b/spacy/ml/models/entity_linker.py @@ -1,11 +1,12 @@ from pathlib import Path from typing import Optional, Callable, Iterable, List, Tuple from thinc.types import Floats2d -from thinc.api import chain, clone, list2ragged, reduce_mean, residual -from thinc.api import Model, Maxout, Linear, noop, tuplify, Ragged +from thinc.api import chain, list2ragged, reduce_mean, residual +from thinc.api import Model, Maxout, Linear, tuplify, Ragged from ...util import registry -from ...kb import KnowledgeBase, Candidate, get_candidates +from ...kb import KnowledgeBase, InMemoryLookupKB +from ...kb import Candidate, get_candidates, get_candidates_batch from ...vocab import Vocab from ...tokens import Span, Doc from ..extract_spans import extract_spans @@ -78,9 +79,11 @@ def span_maker_forward(model, docs: List[Doc], is_train) -> Tuple[Ragged, Callab @registry.misc("spacy.KBFromFile.v1") -def load_kb(kb_path: Path) -> Callable[[Vocab], KnowledgeBase]: - def kb_from_file(vocab): - kb = KnowledgeBase(vocab, entity_vector_length=1) +def load_kb( + kb_path: Path, +) -> Callable[[Vocab], KnowledgeBase]: + def kb_from_file(vocab: Vocab): + kb = InMemoryLookupKB(vocab, entity_vector_length=1) kb.from_disk(kb_path) return kb @@ -88,9 +91,11 @@ def load_kb(kb_path: Path) -> Callable[[Vocab], KnowledgeBase]: @registry.misc("spacy.EmptyKB.v1") -def empty_kb(entity_vector_length: int) -> Callable[[Vocab], KnowledgeBase]: - def empty_kb_factory(vocab): - return KnowledgeBase(vocab=vocab, entity_vector_length=entity_vector_length) +def empty_kb( + entity_vector_length: int, +) -> Callable[[Vocab], KnowledgeBase]: + def empty_kb_factory(vocab: Vocab): + return InMemoryLookupKB(vocab=vocab, entity_vector_length=entity_vector_length) return empty_kb_factory @@ -98,3 +103,10 @@ def empty_kb(entity_vector_length: int) -> Callable[[Vocab], KnowledgeBase]: @registry.misc("spacy.CandidateGenerator.v1") def create_candidates() -> Callable[[KnowledgeBase, Span], Iterable[Candidate]]: return get_candidates + + +@registry.misc("spacy.CandidateBatchGenerator.v1") +def create_candidates_batch() -> Callable[ + [KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]] +]: + return get_candidates_batch diff --git a/spacy/pipeline/entity_linker.py b/spacy/pipeline/entity_linker.py index 73a90b268..62845287b 100644 --- a/spacy/pipeline/entity_linker.py +++ b/spacy/pipeline/entity_linker.py @@ -53,9 +53,11 @@ DEFAULT_NEL_MODEL = Config().from_str(default_model_config)["model"] "incl_context": True, "entity_vector_length": 64, "get_candidates": {"@misc": "spacy.CandidateGenerator.v1"}, + "get_candidates_batch": {"@misc": "spacy.CandidateBatchGenerator.v1"}, "overwrite": True, "scorer": {"@scorers": "spacy.entity_linker_scorer.v1"}, "use_gold_ents": True, + "candidates_batch_size": 1, "threshold": None, }, default_score_weights={ @@ -75,9 +77,13 @@ def make_entity_linker( incl_context: bool, entity_vector_length: int, get_candidates: Callable[[KnowledgeBase, Span], Iterable[Candidate]], + get_candidates_batch: Callable[ + [KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]] + ], overwrite: bool, scorer: Optional[Callable], use_gold_ents: bool, + candidates_batch_size: int, threshold: Optional[float] = None, ): """Construct an EntityLinker component. @@ -90,17 +96,21 @@ def make_entity_linker( incl_prior (bool): Whether or not to include prior probabilities from the KB in the model. incl_context (bool): Whether or not to include the local context in the model. entity_vector_length (int): Size of encoding vectors in the KB. - get_candidates (Callable[[KnowledgeBase, "Span"], Iterable[Candidate]]): Function that + get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that produces a list of candidates, given a certain knowledge base and a textual mention. + get_candidates_batch ( + Callable[[KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]]], Iterable[Candidate]] + ): Function that produces a list of candidates, given a certain knowledge base and several textual mentions. scorer (Optional[Callable]): The scoring method. use_gold_ents (bool): Whether to copy entities from gold docs or not. If false, another component must provide entity annotations. + candidates_batch_size (int): Size of batches for entity candidate generation. threshold (Optional[float]): Confidence threshold for entity predictions. If confidence is below the threshold, prediction is discarded. If None, predictions are not filtered by any threshold. """ if not model.attrs.get("include_span_maker", False): - # The only difference in arguments here is that use_gold_ents is not available + # The only difference in arguments here is that use_gold_ents and threshold aren't available. return EntityLinker_v1( nlp.vocab, model, @@ -124,9 +134,11 @@ def make_entity_linker( incl_context=incl_context, entity_vector_length=entity_vector_length, get_candidates=get_candidates, + get_candidates_batch=get_candidates_batch, overwrite=overwrite, scorer=scorer, use_gold_ents=use_gold_ents, + candidates_batch_size=candidates_batch_size, threshold=threshold, ) @@ -160,9 +172,13 @@ class EntityLinker(TrainablePipe): incl_context: bool, entity_vector_length: int, get_candidates: Callable[[KnowledgeBase, Span], Iterable[Candidate]], + get_candidates_batch: Callable[ + [KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]] + ], overwrite: bool = BACKWARD_OVERWRITE, scorer: Optional[Callable] = entity_linker_score, use_gold_ents: bool, + candidates_batch_size: int, threshold: Optional[float] = None, ) -> None: """Initialize an entity linker. @@ -178,10 +194,14 @@ class EntityLinker(TrainablePipe): entity_vector_length (int): Size of encoding vectors in the KB. get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that produces a list of candidates, given a certain knowledge base and a textual mention. - scorer (Optional[Callable]): The scoring method. Defaults to - Scorer.score_links. + get_candidates_batch ( + Callable[[KnowledgeBase, Iterable[Span]], Iterable[Iterable[Candidate]]], + Iterable[Candidate]] + ): Function that produces a list of candidates, given a certain knowledge base and several textual mentions. + scorer (Optional[Callable]): The scoring method. Defaults to Scorer.score_links. use_gold_ents (bool): Whether to copy entities from gold docs or not. If false, another component must provide entity annotations. + candidates_batch_size (int): Size of batches for entity candidate generation. threshold (Optional[float]): Confidence threshold for entity predictions. If confidence is below the threshold, prediction is discarded. If None, predictions are not filtered by any threshold. DOCS: https://spacy.io/api/entitylinker#init @@ -204,22 +224,27 @@ class EntityLinker(TrainablePipe): self.incl_prior = incl_prior self.incl_context = incl_context self.get_candidates = get_candidates + self.get_candidates_batch = get_candidates_batch self.cfg: Dict[str, Any] = {"overwrite": overwrite} self.distance = CosineDistance(normalize=False) # how many neighbour sentences to take into account - # create an empty KB by default. If you want to load a predefined one, specify it in 'initialize'. + # create an empty KB by default self.kb = empty_kb(entity_vector_length)(self.vocab) self.scorer = scorer self.use_gold_ents = use_gold_ents + self.candidates_batch_size = candidates_batch_size self.threshold = threshold + if candidates_batch_size < 1: + raise ValueError(Errors.E1044) + def set_kb(self, kb_loader: Callable[[Vocab], KnowledgeBase]): """Define the KB of this pipe by providing a function that will create it using this object's vocab.""" if not callable(kb_loader): raise ValueError(Errors.E885.format(arg_type=type(kb_loader))) - self.kb = kb_loader(self.vocab) + self.kb = kb_loader(self.vocab) # type: ignore def validate_kb(self) -> None: # Raise an error if the knowledge base is not initialized. @@ -241,8 +266,8 @@ class EntityLinker(TrainablePipe): get_examples (Callable[[], Iterable[Example]]): Function that returns a representative sample of gold-standard Example objects. nlp (Language): The current nlp object the component is part of. - kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates a KnowledgeBase from a Vocab instance. - Note that providing this argument, will overwrite all data accumulated in the current KB. + kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates a KnowledgeBase from a Vocab + instance. Note that providing this argument will overwrite all data accumulated in the current KB. Use this only when loading a KB as-such from file. DOCS: https://spacy.io/api/entitylinker#initialize @@ -419,66 +444,93 @@ class EntityLinker(TrainablePipe): if len(doc) == 0: continue sentences = [s for s in doc.sents] - # Looping through each entity (TODO: rewrite) - for ent in doc.ents: - sent_index = sentences.index(ent.sent) - assert sent_index >= 0 - if self.incl_context: - # get n_neighbour sentences, clipped to the length of the document - start_sentence = max(0, sent_index - self.n_sents) - end_sentence = min(len(sentences) - 1, sent_index + self.n_sents) - start_token = sentences[start_sentence].start - end_token = sentences[end_sentence].end - sent_doc = doc[start_token:end_token].as_doc() - # currently, the context is the same for each entity in a sentence (should be refined) - sentence_encoding = self.model.predict([sent_doc])[0] - sentence_encoding_t = sentence_encoding.T - sentence_norm = xp.linalg.norm(sentence_encoding_t) - entity_count += 1 - if ent.label_ in self.labels_discard: - # ignoring this entity - setting to NIL - final_kb_ids.append(self.NIL) - else: - candidates = list(self.get_candidates(self.kb, ent)) - if not candidates: - # no prediction possible for this entity - setting to NIL - final_kb_ids.append(self.NIL) - elif len(candidates) == 1 and self.threshold is None: - # shortcut for efficiency reasons: take the 1 candidate - final_kb_ids.append(candidates[0].entity_) - else: - random.shuffle(candidates) - # set all prior probabilities to 0 if incl_prior=False - prior_probs = xp.asarray([c.prior_prob for c in candidates]) - if not self.incl_prior: - prior_probs = xp.asarray([0.0 for _ in candidates]) - scores = prior_probs - # add in similarity from the context - if self.incl_context: - entity_encodings = xp.asarray( - [c.entity_vector for c in candidates] - ) - entity_norm = xp.linalg.norm(entity_encodings, axis=1) - if len(entity_encodings) != len(prior_probs): - raise RuntimeError( - Errors.E147.format( - method="predict", - msg="vectors not of equal length", - ) - ) - # cosine similarity - sims = xp.dot(entity_encodings, sentence_encoding_t) / ( - sentence_norm * entity_norm - ) - if sims.shape != prior_probs.shape: - raise ValueError(Errors.E161) - scores = prior_probs + sims - (prior_probs * sims) - final_kb_ids.append( - candidates[scores.argmax().item()].entity_ - if self.threshold is None or scores.max() >= self.threshold - else EntityLinker.NIL + # Loop over entities in batches. + for ent_idx in range(0, len(doc.ents), self.candidates_batch_size): + ent_batch = doc.ents[ent_idx : ent_idx + self.candidates_batch_size] + + # Look up candidate entities. + valid_ent_idx = [ + idx + for idx in range(len(ent_batch)) + if ent_batch[idx].label_ not in self.labels_discard + ] + + batch_candidates = list( + self.get_candidates_batch( + self.kb, [ent_batch[idx] for idx in valid_ent_idx] + ) + if self.candidates_batch_size > 1 + else [ + self.get_candidates(self.kb, ent_batch[idx]) + for idx in valid_ent_idx + ] + ) + + # Looping through each entity in batch (TODO: rewrite) + for j, ent in enumerate(ent_batch): + sent_index = sentences.index(ent.sent) + assert sent_index >= 0 + + if self.incl_context: + # get n_neighbour sentences, clipped to the length of the document + start_sentence = max(0, sent_index - self.n_sents) + end_sentence = min( + len(sentences) - 1, sent_index + self.n_sents ) + start_token = sentences[start_sentence].start + end_token = sentences[end_sentence].end + sent_doc = doc[start_token:end_token].as_doc() + # currently, the context is the same for each entity in a sentence (should be refined) + sentence_encoding = self.model.predict([sent_doc])[0] + sentence_encoding_t = sentence_encoding.T + sentence_norm = xp.linalg.norm(sentence_encoding_t) + entity_count += 1 + if ent.label_ in self.labels_discard: + # ignoring this entity - setting to NIL + final_kb_ids.append(self.NIL) + else: + candidates = list(batch_candidates[j]) + if not candidates: + # no prediction possible for this entity - setting to NIL + final_kb_ids.append(self.NIL) + elif len(candidates) == 1 and self.threshold is None: + # shortcut for efficiency reasons: take the 1 candidate + final_kb_ids.append(candidates[0].entity_) + else: + random.shuffle(candidates) + # set all prior probabilities to 0 if incl_prior=False + prior_probs = xp.asarray([c.prior_prob for c in candidates]) + if not self.incl_prior: + prior_probs = xp.asarray([0.0 for _ in candidates]) + scores = prior_probs + # add in similarity from the context + if self.incl_context: + entity_encodings = xp.asarray( + [c.entity_vector for c in candidates] + ) + entity_norm = xp.linalg.norm(entity_encodings, axis=1) + if len(entity_encodings) != len(prior_probs): + raise RuntimeError( + Errors.E147.format( + method="predict", + msg="vectors not of equal length", + ) + ) + # cosine similarity + sims = xp.dot(entity_encodings, sentence_encoding_t) / ( + sentence_norm * entity_norm + ) + if sims.shape != prior_probs.shape: + raise ValueError(Errors.E161) + scores = prior_probs + sims - (prior_probs * sims) + final_kb_ids.append( + candidates[scores.argmax().item()].entity_ + if self.threshold is None + or scores.max() >= self.threshold + else EntityLinker.NIL + ) + if not (len(final_kb_ids) == entity_count): err = Errors.E147.format( method="predict", msg="result variables not of equal length" diff --git a/spacy/pipeline/legacy/entity_linker.py b/spacy/pipeline/legacy/entity_linker.py index 2f8a1f8ea..c14dfa1db 100644 --- a/spacy/pipeline/legacy/entity_linker.py +++ b/spacy/pipeline/legacy/entity_linker.py @@ -68,8 +68,7 @@ class EntityLinker_v1(TrainablePipe): entity_vector_length (int): Size of encoding vectors in the KB. get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that produces a list of candidates, given a certain knowledge base and a textual mention. - scorer (Optional[Callable]): The scoring method. Defaults to - Scorer.score_links. + scorer (Optional[Callable]): The scoring method. Defaults to Scorer.score_links. DOCS: https://spacy.io/api/entitylinker#init """ self.vocab = vocab @@ -115,7 +114,7 @@ class EntityLinker_v1(TrainablePipe): get_examples (Callable[[], Iterable[Example]]): Function that returns a representative sample of gold-standard Example objects. nlp (Language): The current nlp object the component is part of. - kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates a KnowledgeBase from a Vocab instance. + kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates an InMemoryLookupKB from a Vocab instance. Note that providing this argument, will overwrite all data accumulated in the current KB. Use this only when loading a KB as-such from file. diff --git a/spacy/tests/pipeline/test_entity_linker.py b/spacy/tests/pipeline/test_entity_linker.py index 82bc976bb..4d683acc5 100644 --- a/spacy/tests/pipeline/test_entity_linker.py +++ b/spacy/tests/pipeline/test_entity_linker.py @@ -6,7 +6,7 @@ from numpy.testing import assert_equal from spacy import registry, util from spacy.attrs import ENT_KB_ID from spacy.compat import pickle -from spacy.kb import Candidate, KnowledgeBase, get_candidates +from spacy.kb import Candidate, InMemoryLookupKB, get_candidates, KnowledgeBase from spacy.lang.en import English from spacy.ml import load_kb from spacy.pipeline import EntityLinker @@ -34,7 +34,7 @@ def assert_almost_equal(a, b): def test_issue4674(): """Test that setting entities with overlapping identifiers does not mess up IO""" nlp = English() - kb = KnowledgeBase(nlp.vocab, entity_vector_length=3) + kb = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) vector1 = [0.9, 1.1, 1.01] vector2 = [1.8, 2.25, 2.01] with pytest.warns(UserWarning): @@ -51,7 +51,7 @@ def test_issue4674(): dir_path.mkdir() file_path = dir_path / "kb" kb.to_disk(str(file_path)) - kb2 = KnowledgeBase(nlp.vocab, entity_vector_length=3) + kb2 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) kb2.from_disk(str(file_path)) assert kb2.get_size_entities() == 1 @@ -59,9 +59,9 @@ def test_issue4674(): @pytest.mark.issue(6730) def test_issue6730(en_vocab): """Ensure that the KB does not accept empty strings, but otherwise IO works fine.""" - from spacy.kb import KnowledgeBase + from spacy.kb.kb_in_memory import InMemoryLookupKB - kb = KnowledgeBase(en_vocab, entity_vector_length=3) + kb = InMemoryLookupKB(en_vocab, entity_vector_length=3) kb.add_entity(entity="1", freq=148, entity_vector=[1, 2, 3]) with pytest.raises(ValueError): @@ -127,7 +127,7 @@ def test_issue7065_b(): def create_kb(vocab): # create artificial KB - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q270853", freq=12, entity_vector=[9, 1, -7]) mykb.add_alias( alias="No. 8", @@ -190,7 +190,7 @@ def test_no_entities(): def create_kb(vocab): # create artificial KB - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) mykb.add_alias("Russ Cochran", ["Q2146908"], [0.9]) return mykb @@ -231,7 +231,7 @@ def test_partial_links(): def create_kb(vocab): # create artificial KB - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) mykb.add_alias("Russ Cochran", ["Q2146908"], [0.9]) return mykb @@ -263,7 +263,7 @@ def test_partial_links(): def test_kb_valid_entities(nlp): """Test the valid construction of a KB with 3 entities and two aliases""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=3) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) # adding entities mykb.add_entity(entity="Q1", freq=19, entity_vector=[8, 4, 3]) @@ -292,7 +292,7 @@ def test_kb_valid_entities(nlp): def test_kb_invalid_entities(nlp): """Test the invalid construction of a KB with an alias linked to a non-existing entity""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) # adding entities mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) @@ -308,7 +308,7 @@ def test_kb_invalid_entities(nlp): def test_kb_invalid_probabilities(nlp): """Test the invalid construction of a KB with wrong prior probabilities""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) # adding entities mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) @@ -322,7 +322,7 @@ def test_kb_invalid_probabilities(nlp): def test_kb_invalid_combination(nlp): """Test the invalid construction of a KB with non-matching entity and probability lists""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) # adding entities mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) @@ -338,7 +338,7 @@ def test_kb_invalid_combination(nlp): def test_kb_invalid_entity_vector(nlp): """Test the invalid construction of a KB with non-matching entity vector lengths""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=3) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) # adding entities mykb.add_entity(entity="Q1", freq=19, entity_vector=[1, 2, 3]) @@ -376,7 +376,7 @@ def test_kb_initialize_empty(nlp): def test_kb_serialize(nlp): """Test serialization of the KB""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) with make_tempdir() as d: # normal read-write behaviour mykb.to_disk(d / "kb") @@ -393,12 +393,12 @@ def test_kb_serialize(nlp): @pytest.mark.issue(9137) def test_kb_serialize_2(nlp): v = [5, 6, 7, 8] - kb1 = KnowledgeBase(vocab=nlp.vocab, entity_vector_length=4) + kb1 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4) kb1.set_entities(["E1"], [1], [v]) assert kb1.get_vector("E1") == v with make_tempdir() as d: kb1.to_disk(d / "kb") - kb2 = KnowledgeBase(vocab=nlp.vocab, entity_vector_length=4) + kb2 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4) kb2.from_disk(d / "kb") assert kb2.get_vector("E1") == v @@ -408,7 +408,7 @@ def test_kb_set_entities(nlp): v = [5, 6, 7, 8] v1 = [1, 1, 1, 0] v2 = [2, 2, 2, 3] - kb1 = KnowledgeBase(vocab=nlp.vocab, entity_vector_length=4) + kb1 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4) kb1.set_entities(["E0"], [1], [v]) assert kb1.get_entity_strings() == ["E0"] kb1.set_entities(["E1", "E2"], [1, 9], [v1, v2]) @@ -417,7 +417,7 @@ def test_kb_set_entities(nlp): assert kb1.get_vector("E2") == v2 with make_tempdir() as d: kb1.to_disk(d / "kb") - kb2 = KnowledgeBase(vocab=nlp.vocab, entity_vector_length=4) + kb2 = InMemoryLookupKB(vocab=nlp.vocab, entity_vector_length=4) kb2.from_disk(d / "kb") assert set(kb2.get_entity_strings()) == {"E1", "E2"} assert kb2.get_vector("E1") == v1 @@ -428,7 +428,7 @@ def test_kb_serialize_vocab(nlp): """Test serialization of the KB and custom strings""" entity = "MyFunnyID" assert entity not in nlp.vocab.strings - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) assert not mykb.contains_entity(entity) mykb.add_entity(entity, freq=342, entity_vector=[3]) assert mykb.contains_entity(entity) @@ -436,14 +436,14 @@ def test_kb_serialize_vocab(nlp): with make_tempdir() as d: # normal read-write behaviour mykb.to_disk(d / "kb") - mykb_new = KnowledgeBase(Vocab(), entity_vector_length=1) + mykb_new = InMemoryLookupKB(Vocab(), entity_vector_length=1) mykb_new.from_disk(d / "kb") assert entity in mykb_new.vocab.strings def test_candidate_generation(nlp): """Test correct candidate generation""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) doc = nlp("douglas adam Adam shrubbery") douglas_ent = doc[0:1] @@ -481,7 +481,7 @@ def test_el_pipe_configuration(nlp): ruler.add_patterns([pattern]) def create_kb(vocab): - kb = KnowledgeBase(vocab, entity_vector_length=1) + kb = InMemoryLookupKB(vocab, entity_vector_length=1) kb.add_entity(entity="Q2", freq=12, entity_vector=[2]) kb.add_entity(entity="Q3", freq=5, entity_vector=[3]) kb.add_alias(alias="douglas", entities=["Q2", "Q3"], probabilities=[0.8, 0.1]) @@ -500,10 +500,21 @@ def test_el_pipe_configuration(nlp): def get_lowercased_candidates(kb, span): return kb.get_alias_candidates(span.text.lower()) + def get_lowercased_candidates_batch(kb, spans): + return [get_lowercased_candidates(kb, span) for span in spans] + @registry.misc("spacy.LowercaseCandidateGenerator.v1") - def create_candidates() -> Callable[[KnowledgeBase, "Span"], Iterable[Candidate]]: + def create_candidates() -> Callable[ + [InMemoryLookupKB, "Span"], Iterable[Candidate] + ]: return get_lowercased_candidates + @registry.misc("spacy.LowercaseCandidateBatchGenerator.v1") + def create_candidates_batch() -> Callable[ + [InMemoryLookupKB, Iterable["Span"]], Iterable[Iterable[Candidate]] + ]: + return get_lowercased_candidates_batch + # replace the pipe with a new one with with a different candidate generator entity_linker = nlp.replace_pipe( "entity_linker", @@ -511,6 +522,9 @@ def test_el_pipe_configuration(nlp): config={ "incl_context": False, "get_candidates": {"@misc": "spacy.LowercaseCandidateGenerator.v1"}, + "get_candidates_batch": { + "@misc": "spacy.LowercaseCandidateBatchGenerator.v1" + }, }, ) entity_linker.set_kb(create_kb) @@ -532,7 +546,7 @@ def test_nel_nsents(nlp): def test_vocab_serialization(nlp): """Test that string information is retained across storage""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) # adding entities mykb.add_entity(entity="Q1", freq=27, entity_vector=[1]) @@ -552,7 +566,7 @@ def test_vocab_serialization(nlp): with make_tempdir() as d: mykb.to_disk(d / "kb") - kb_new_vocab = KnowledgeBase(Vocab(), entity_vector_length=1) + kb_new_vocab = InMemoryLookupKB(Vocab(), entity_vector_length=1) kb_new_vocab.from_disk(d / "kb") candidates = kb_new_vocab.get_alias_candidates("adam") @@ -568,7 +582,7 @@ def test_vocab_serialization(nlp): def test_append_alias(nlp): """Test that we can append additional alias-entity pairs""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) # adding entities mykb.add_entity(entity="Q1", freq=27, entity_vector=[1]) @@ -599,7 +613,7 @@ def test_append_alias(nlp): @pytest.mark.filterwarnings("ignore:\\[W036") def test_append_invalid_alias(nlp): """Test that append an alias will throw an error if prior probs are exceeding 1""" - mykb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + mykb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) # adding entities mykb.add_entity(entity="Q1", freq=27, entity_vector=[1]) @@ -621,7 +635,7 @@ def test_preserving_links_asdoc(nlp): vector_length = 1 def create_kb(vocab): - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) # adding entities mykb.add_entity(entity="Q1", freq=19, entity_vector=[1]) mykb.add_entity(entity="Q2", freq=8, entity_vector=[1]) @@ -723,7 +737,7 @@ def test_overfitting_IO(): # create artificial KB - assign same prior weight to the two russ cochran's # Q2146908 (Russ Cochran): American golfer # Q7381115 (Russ Cochran): publisher - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) mykb.add_entity(entity="Q7381115", freq=12, entity_vector=[9, 1, -7]) mykb.add_alias( @@ -805,7 +819,7 @@ def test_kb_serialization(): kb_dir = tmp_dir / "kb" nlp1 = English() assert "Q2146908" not in nlp1.vocab.strings - mykb = KnowledgeBase(nlp1.vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(nlp1.vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) mykb.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) assert "Q2146908" in nlp1.vocab.strings @@ -828,7 +842,7 @@ def test_kb_serialization(): def test_kb_pickle(): # Test that the KB can be pickled nlp = English() - kb_1 = KnowledgeBase(nlp.vocab, entity_vector_length=3) + kb_1 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) kb_1.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) assert not kb_1.contains_alias("Russ Cochran") kb_1.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) @@ -842,7 +856,7 @@ def test_kb_pickle(): def test_nel_pickle(): # Test that a pipeline with an EL component can be pickled def create_kb(vocab): - kb = KnowledgeBase(vocab, entity_vector_length=3) + kb = InMemoryLookupKB(vocab, entity_vector_length=3) kb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) kb.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) return kb @@ -864,7 +878,7 @@ def test_nel_pickle(): def test_kb_to_bytes(): # Test that the KB's to_bytes method works correctly nlp = English() - kb_1 = KnowledgeBase(nlp.vocab, entity_vector_length=3) + kb_1 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) kb_1.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) kb_1.add_entity(entity="Q66", freq=9, entity_vector=[1, 2, 3]) kb_1.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) @@ -874,7 +888,7 @@ def test_kb_to_bytes(): ) assert kb_1.contains_alias("Russ Cochran") kb_bytes = kb_1.to_bytes() - kb_2 = KnowledgeBase(nlp.vocab, entity_vector_length=3) + kb_2 = InMemoryLookupKB(nlp.vocab, entity_vector_length=3) assert not kb_2.contains_alias("Russ Cochran") kb_2 = kb_2.from_bytes(kb_bytes) # check that both KBs are exactly the same @@ -897,7 +911,7 @@ def test_kb_to_bytes(): def test_nel_to_bytes(): # Test that a pipeline with an EL component can be converted to bytes def create_kb(vocab): - kb = KnowledgeBase(vocab, entity_vector_length=3) + kb = InMemoryLookupKB(vocab, entity_vector_length=3) kb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) kb.add_alias(alias="Russ Cochran", entities=["Q2146908"], probabilities=[0.8]) return kb @@ -987,7 +1001,7 @@ def test_legacy_architectures(name, config): train_examples.append(Example.from_dict(doc, annotation)) def create_kb(vocab): - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q2146908", freq=12, entity_vector=[6, -4, 3]) mykb.add_entity(entity="Q7381115", freq=12, entity_vector=[9, 1, -7]) mykb.add_alias( @@ -1054,7 +1068,7 @@ def test_no_gold_ents(patterns): def create_kb(vocab): # create artificial KB - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q613241", freq=12, entity_vector=[6, -4, 3]) mykb.add_alias("Kirby", ["Q613241"], [0.9]) # Placeholder @@ -1104,7 +1118,7 @@ def test_tokenization_mismatch(): def create_kb(vocab): # create placeholder KB - mykb = KnowledgeBase(vocab, entity_vector_length=vector_length) + mykb = InMemoryLookupKB(vocab, entity_vector_length=vector_length) mykb.add_entity(entity="Q613241", freq=12, entity_vector=[6, -4, 3]) mykb.add_alias("Kirby", ["Q613241"], [0.9]) return mykb @@ -1121,6 +1135,12 @@ def test_tokenization_mismatch(): nlp.evaluate(train_examples) +def test_abstract_kb_instantiation(): + """Test whether instantiation of abstract KB base class fails.""" + with pytest.raises(TypeError): + KnowledgeBase(None, 3) + + # fmt: off @pytest.mark.parametrize( "meet_threshold,config", @@ -1151,7 +1171,7 @@ def test_threshold(meet_threshold: bool, config: Dict[str, Any]): def create_kb(vocab): # create artificial KB - mykb = KnowledgeBase(vocab, entity_vector_length=3) + mykb = InMemoryLookupKB(vocab, entity_vector_length=3) mykb.add_entity(entity=entity_id, freq=12, entity_vector=[6, -4, 3]) mykb.add_alias( alias="Mahler", diff --git a/spacy/tests/serialize/test_resource_warning.py b/spacy/tests/serialize/test_resource_warning.py index a00b2a688..38701c6d9 100644 --- a/spacy/tests/serialize/test_resource_warning.py +++ b/spacy/tests/serialize/test_resource_warning.py @@ -3,7 +3,7 @@ from unittest import TestCase import pytest import srsly from numpy import zeros -from spacy.kb import KnowledgeBase, Writer +from spacy.kb.kb_in_memory import InMemoryLookupKB, Writer from spacy.vectors import Vectors from spacy.language import Language from spacy.pipeline import TrainablePipe @@ -71,7 +71,7 @@ def entity_linker(): nlp = Language() def create_kb(vocab): - kb = KnowledgeBase(vocab, entity_vector_length=1) + kb = InMemoryLookupKB(vocab, entity_vector_length=1) kb.add_entity("test", 0.0, zeros((1, 1), dtype="f")) return kb @@ -120,7 +120,7 @@ def test_writer_with_path_py35(): def test_save_and_load_knowledge_base(): nlp = Language() - kb = KnowledgeBase(nlp.vocab, entity_vector_length=1) + kb = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) with make_tempdir() as d: path = d / "kb" try: @@ -129,7 +129,7 @@ def test_save_and_load_knowledge_base(): pytest.fail(str(e)) try: - kb_loaded = KnowledgeBase(nlp.vocab, entity_vector_length=1) + kb_loaded = InMemoryLookupKB(nlp.vocab, entity_vector_length=1) kb_loaded.from_disk(path) except Exception as e: pytest.fail(str(e)) diff --git a/spacy/tests/serialize/test_serialize_kb.py b/spacy/tests/serialize/test_serialize_kb.py index 1e0ae3c76..8d3653ab1 100644 --- a/spacy/tests/serialize/test_serialize_kb.py +++ b/spacy/tests/serialize/test_serialize_kb.py @@ -2,7 +2,7 @@ from typing import Callable from spacy import util from spacy.util import ensure_path, registry, load_model_from_config -from spacy.kb import KnowledgeBase +from spacy.kb.kb_in_memory import InMemoryLookupKB from spacy.vocab import Vocab from thinc.api import Config @@ -22,7 +22,7 @@ def test_serialize_kb_disk(en_vocab): dir_path.mkdir() file_path = dir_path / "kb" kb1.to_disk(str(file_path)) - kb2 = KnowledgeBase(vocab=en_vocab, entity_vector_length=3) + kb2 = InMemoryLookupKB(vocab=en_vocab, entity_vector_length=3) kb2.from_disk(str(file_path)) # final assertions @@ -30,7 +30,7 @@ def test_serialize_kb_disk(en_vocab): def _get_dummy_kb(vocab): - kb = KnowledgeBase(vocab, entity_vector_length=3) + kb = InMemoryLookupKB(vocab, entity_vector_length=3) kb.add_entity(entity="Q53", freq=33, entity_vector=[0, 5, 3]) kb.add_entity(entity="Q17", freq=2, entity_vector=[7, 1, 0]) kb.add_entity(entity="Q007", freq=7, entity_vector=[0, 0, 7]) @@ -104,7 +104,7 @@ def test_serialize_subclassed_kb(): custom_field = 666 """ - class SubKnowledgeBase(KnowledgeBase): + class SubInMemoryLookupKB(InMemoryLookupKB): def __init__(self, vocab, entity_vector_length, custom_field): super().__init__(vocab, entity_vector_length) self.custom_field = custom_field @@ -112,9 +112,9 @@ def test_serialize_subclassed_kb(): @registry.misc("spacy.CustomKB.v1") def custom_kb( entity_vector_length: int, custom_field: int - ) -> Callable[[Vocab], KnowledgeBase]: + ) -> Callable[[Vocab], InMemoryLookupKB]: def custom_kb_factory(vocab): - kb = SubKnowledgeBase( + kb = SubInMemoryLookupKB( vocab=vocab, entity_vector_length=entity_vector_length, custom_field=custom_field, @@ -129,7 +129,7 @@ def test_serialize_subclassed_kb(): nlp.initialize() entity_linker = nlp.get_pipe("entity_linker") - assert type(entity_linker.kb) == SubKnowledgeBase + assert type(entity_linker.kb) == SubInMemoryLookupKB assert entity_linker.kb.entity_vector_length == 342 assert entity_linker.kb.custom_field == 666 @@ -139,6 +139,6 @@ def test_serialize_subclassed_kb(): nlp2 = util.load_model_from_path(tmp_dir) entity_linker2 = nlp2.get_pipe("entity_linker") # After IO, the KB is the standard one - assert type(entity_linker2.kb) == KnowledgeBase + assert type(entity_linker2.kb) == InMemoryLookupKB assert entity_linker2.kb.entity_vector_length == 342 assert not hasattr(entity_linker2.kb, "custom_field") diff --git a/website/docs/api/architectures.md b/website/docs/api/architectures.md index 2537faff6..a3cb07b44 100644 --- a/website/docs/api/architectures.md +++ b/website/docs/api/architectures.md @@ -587,8 +587,8 @@ consists of either two or three subnetworks: run once for each batch. - **lower**: Construct a feature-specific vector for each `(token, feature)` pair. This is also run once for each batch. Constructing the state - representation is then a matter of summing the component features and - applying the non-linearity. + representation is then a matter of summing the component features and applying + the non-linearity. - **upper** (optional): A feed-forward network that predicts scores from the state representation. If not present, the output from the lower model is used as action scores directly. @@ -628,8 +628,8 @@ same signature, but the `use_upper` argument was `True` by default. > ``` Build a tagger model, using a provided token-to-vector component. The tagger -model adds a linear layer with softmax activation to predict scores given -the token vectors. +model adds a linear layer with softmax activation to predict scores given the +token vectors. | Name | Description | | ----------- | ------------------------------------------------------------------------------------------ | @@ -919,6 +919,6 @@ A function that reads an existing `KnowledgeBase` from file. A function that takes as input a [`KnowledgeBase`](/api/kb) and a [`Span`](/api/span) object denoting a named entity, and returns a list of -plausible [`Candidate`](/api/kb/#candidate) objects. The default -`CandidateGenerator` uses the text of a mention to find its potential -aliases in the `KnowledgeBase`. Note that this function is case-dependent. +plausible [`Candidate`](/api/kb#candidate) objects. The default +`CandidateGenerator` uses the text of a mention to find its potential aliases in +the `KnowledgeBase`. Note that this function is case-dependent. diff --git a/website/docs/api/entitylinker.md b/website/docs/api/entitylinker.md index 43e08a39c..40ec8afb5 100644 --- a/website/docs/api/entitylinker.md +++ b/website/docs/api/entitylinker.md @@ -14,7 +14,8 @@ entities) to unique identifiers, grounding the named entities into the "real world". It requires a `KnowledgeBase`, as well as a function to generate plausible candidates from that `KnowledgeBase` given a certain textual mention, and a machine learning model to pick the right candidate, given the local -context of the mention. +context of the mention. `EntityLinker` defaults to using the +[`InMemoryLookupKB`](/api/kb_in_memory) implementation. ## Assigned Attributes {#assigned-attributes} @@ -170,7 +171,7 @@ with the current vocab. > > ```python > def create_kb(vocab): -> kb = KnowledgeBase(vocab, entity_vector_length=128) +> kb = InMemoryLookupKB(vocab, entity_vector_length=128) > kb.add_entity(...) > kb.add_alias(...) > return kb diff --git a/website/docs/api/kb.md b/website/docs/api/kb.md index e7a8fcd6f..b217a1678 100644 --- a/website/docs/api/kb.md +++ b/website/docs/api/kb.md @@ -4,27 +4,45 @@ teaser: A storage class for entities and aliases of a specific knowledge base (ontology) tag: class -source: spacy/kb.pyx +source: spacy/kb/kb.pyx new: 2.2 --- -The `KnowledgeBase` object provides a method to generate -[`Candidate`](/api/kb/#candidate) objects, which are plausible external +The `KnowledgeBase` object is an abstract class providing a method to generate +[`Candidate`](/api/kb#candidate) objects, which are plausible external identifiers given a certain textual mention. Each such `Candidate` holds information from the relevant KB entities, such as its frequency in text and possible aliases. Each entity in the knowledge base also has a pretrained entity vector of a fixed size. +Beyond that, `KnowledgeBase` classes have to implement a number of utility +functions called by the [`EntityLinker`](/api/entitylinker) component. + + + +This class was not abstract up to spaCy version 3.5. The `KnowledgeBase` +implementation up to that point is available as `InMemoryLookupKB` from 3.5 +onwards. + + + ## KnowledgeBase.\_\_init\_\_ {#init tag="method"} -Create the knowledge base. +`KnowledgeBase` is an abstract class and cannot be instantiated. Its child +classes should call `__init__()` to set up some necessary attributes. > #### Example > > ```python > from spacy.kb import KnowledgeBase +> from spacy.vocab import Vocab +> +> class FullyImplementedKB(KnowledgeBase): +> def __init__(self, vocab: Vocab, entity_vector_length: int): +> super().__init__(vocab, entity_vector_length) +> ... > vocab = nlp.vocab -> kb = KnowledgeBase(vocab=vocab, entity_vector_length=64) +> kb = FullyImplementedKB(vocab=vocab, entity_vector_length=64) > ``` | Name | Description | @@ -40,133 +58,66 @@ The length of the fixed-size entity vectors in the knowledge base. | ----------- | ------------------------------------------------ | | **RETURNS** | Length of the fixed-size entity vectors. ~~int~~ | -## KnowledgeBase.add_entity {#add_entity tag="method"} +## KnowledgeBase.get_candidates {#get_candidates tag="method"} -Add an entity to the knowledge base, specifying its corpus frequency and entity -vector, which should be of length -[`entity_vector_length`](/api/kb#entity_vector_length). +Given a certain textual mention as input, retrieve a list of candidate entities +of type [`Candidate`](/api/kb#candidate). > #### Example > > ```python -> kb.add_entity(entity="Q42", freq=32, entity_vector=vector1) -> kb.add_entity(entity="Q463035", freq=111, entity_vector=vector2) +> from spacy.lang.en import English +> nlp = English() +> doc = nlp("Douglas Adams wrote 'The Hitchhiker's Guide to the Galaxy'.") +> candidates = kb.get_candidates(doc[0:2]) > ``` -| Name | Description | -| --------------- | ---------------------------------------------------------- | -| `entity` | The unique entity identifier. ~~str~~ | -| `freq` | The frequency of the entity in a typical corpus. ~~float~~ | -| `entity_vector` | The pretrained vector of the entity. ~~numpy.ndarray~~ | +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `mention` | The textual mention or alias. ~~Span~~ | +| **RETURNS** | An iterable of relevant `Candidate` objects. ~~Iterable[Candidate]~~ | -## KnowledgeBase.set_entities {#set_entities tag="method"} +## KnowledgeBase.get_candidates_batch {#get_candidates_batch tag="method"} -Define the full list of entities in the knowledge base, specifying the corpus -frequency and entity vector for each entity. +Same as [`get_candidates()`](/api/kb#get_candidates), but for an arbitrary +number of mentions. The [`EntityLinker`](/api/entitylinker) component will call +`get_candidates_batch()` instead of `get_candidates()`, if the config parameter +`candidates_batch_size` is greater or equal than 1. + +The default implementation of `get_candidates_batch()` executes +`get_candidates()` in a loop. We recommend implementing a more efficient way to +retrieve candidates for multiple mentions at once, if performance is of concern +to you. > #### Example > > ```python -> kb.set_entities(entity_list=["Q42", "Q463035"], freq_list=[32, 111], vector_list=[vector1, vector2]) +> from spacy.lang.en import English +> nlp = English() +> doc = nlp("Douglas Adams wrote 'The Hitchhiker's Guide to the Galaxy'.") +> candidates = kb.get_candidates((doc[0:2], doc[3:])) > ``` -| Name | Description | -| ------------- | ---------------------------------------------------------------- | -| `entity_list` | List of unique entity identifiers. ~~Iterable[Union[str, int]]~~ | -| `freq_list` | List of entity frequencies. ~~Iterable[int]~~ | -| `vector_list` | List of entity vectors. ~~Iterable[numpy.ndarray]~~ | - -## KnowledgeBase.add_alias {#add_alias tag="method"} - -Add an alias or mention to the knowledge base, specifying its potential KB -identifiers and their prior probabilities. The entity identifiers should refer -to entities previously added with [`add_entity`](/api/kb#add_entity) or -[`set_entities`](/api/kb#set_entities). The sum of the prior probabilities -should not exceed 1. Note that an empty string can not be used as alias. - -> #### Example -> -> ```python -> kb.add_alias(alias="Douglas", entities=["Q42", "Q463035"], probabilities=[0.6, 0.3]) -> ``` - -| Name | Description | -| --------------- | --------------------------------------------------------------------------------- | -| `alias` | The textual mention or alias. Can not be the empty string. ~~str~~ | -| `entities` | The potential entities that the alias may refer to. ~~Iterable[Union[str, int]]~~ | -| `probabilities` | The prior probabilities of each entity. ~~Iterable[float]~~ | - -## KnowledgeBase.\_\_len\_\_ {#len tag="method"} - -Get the total number of entities in the knowledge base. - -> #### Example -> -> ```python -> total_entities = len(kb) -> ``` - -| Name | Description | -| ----------- | ----------------------------------------------------- | -| **RETURNS** | The number of entities in the knowledge base. ~~int~~ | - -## KnowledgeBase.get_entity_strings {#get_entity_strings tag="method"} - -Get a list of all entity IDs in the knowledge base. - -> #### Example -> -> ```python -> all_entities = kb.get_entity_strings() -> ``` - -| Name | Description | -| ----------- | --------------------------------------------------------- | -| **RETURNS** | The list of entities in the knowledge base. ~~List[str]~~ | - -## KnowledgeBase.get_size_aliases {#get_size_aliases tag="method"} - -Get the total number of aliases in the knowledge base. - -> #### Example -> -> ```python -> total_aliases = kb.get_size_aliases() -> ``` - -| Name | Description | -| ----------- | ---------------------------------------------------- | -| **RETURNS** | The number of aliases in the knowledge base. ~~int~~ | - -## KnowledgeBase.get_alias_strings {#get_alias_strings tag="method"} - -Get a list of all aliases in the knowledge base. - -> #### Example -> -> ```python -> all_aliases = kb.get_alias_strings() -> ``` - -| Name | Description | -| ----------- | -------------------------------------------------------- | -| **RETURNS** | The list of aliases in the knowledge base. ~~List[str]~~ | +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------- | +| `mentions` | The textual mention or alias. ~~Iterable[Span]~~ | +| **RETURNS** | An iterable of iterable with relevant `Candidate` objects. ~~Iterable[Iterable[Candidate]]~~ | ## KnowledgeBase.get_alias_candidates {#get_alias_candidates tag="method"} -Given a certain textual mention as input, retrieve a list of candidate entities -of type [`Candidate`](/api/kb/#candidate). + +This method is _not_ available from spaCy 3.5 onwards. + -> #### Example -> -> ```python -> candidates = kb.get_alias_candidates("Douglas") -> ``` - -| Name | Description | -| ----------- | ------------------------------------------------------------- | -| `alias` | The textual mention or alias. ~~str~~ | -| **RETURNS** | The list of relevant `Candidate` objects. ~~List[Candidate]~~ | +From spaCy 3.5 on `KnowledgeBase` is an abstract class (with +[`InMemoryLookupKB`](/api/kb_in_memory) being a drop-in replacement) to allow +more flexibility in customizing knowledge bases. Some of its methods were moved +to [`InMemoryLookupKB`](/api/kb_in_memory) during this refactoring, one of those +being `get_alias_candidates()`. This method is now available as +[`InMemoryLookupKB.get_alias_candidates()`](/api/kb_in_memory#get_alias_candidates). +Note: [`InMemoryLookupKB.get_candidates()`](/api/kb_in_memory#get_candidates) +defaults to +[`InMemoryLookupKB.get_alias_candidates()`](/api/kb_in_memory#get_alias_candidates). ## KnowledgeBase.get_vector {#get_vector tag="method"} @@ -178,27 +129,30 @@ Given a certain entity ID, retrieve its pretrained entity vector. > vector = kb.get_vector("Q42") > ``` -| Name | Description | -| ----------- | ------------------------------------ | -| `entity` | The entity ID. ~~str~~ | -| **RETURNS** | The entity vector. ~~numpy.ndarray~~ | +| Name | Description | +| ----------- | -------------------------------------- | +| `entity` | The entity ID. ~~str~~ | +| **RETURNS** | The entity vector. ~~Iterable[float]~~ | -## KnowledgeBase.get_prior_prob {#get_prior_prob tag="method"} +## KnowledgeBase.get_vectors {#get_vectors tag="method"} -Given a certain entity ID and a certain textual mention, retrieve the prior -probability of the fact that the mention links to the entity ID. +Same as [`get_vector()`](/api/kb#get_vector), but for an arbitrary number of +entity IDs. + +The default implementation of `get_vectors()` executes `get_vector()` in a loop. +We recommend implementing a more efficient way to retrieve vectors for multiple +entities at once, if performance is of concern to you. > #### Example > > ```python -> probability = kb.get_prior_prob("Q42", "Douglas") +> vectors = kb.get_vectors(("Q42", "Q3107329")) > ``` -| Name | Description | -| ----------- | ------------------------------------------------------------------------- | -| `entity` | The entity ID. ~~str~~ | -| `alias` | The textual mention or alias. ~~str~~ | -| **RETURNS** | The prior probability of the `alias` referring to the `entity`. ~~float~~ | +| Name | Description | +| ----------- | --------------------------------------------------------- | +| `entities` | The entity IDs. ~~Iterable[str]~~ | +| **RETURNS** | The entity vectors. ~~Iterable[Iterable[numpy.ndarray]]~~ | ## KnowledgeBase.to_disk {#to_disk tag="method"} @@ -207,12 +161,13 @@ Save the current state of the knowledge base to a directory. > #### Example > > ```python -> kb.to_disk(loc) +> kb.to_disk(path) > ``` -| Name | Description | -| ----- | ------------------------------------------------------------------------------------------------------------------------------------------ | -| `loc` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| Name | Description | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| `exclude` | List of components to exclude. ~~Iterable[str]~~ | ## KnowledgeBase.from_disk {#from_disk tag="method"} @@ -222,16 +177,16 @@ Restore the state of the knowledge base from a given directory. Note that the > #### Example > > ```python -> from spacy.kb import KnowledgeBase > from spacy.vocab import Vocab > vocab = Vocab().from_disk("/path/to/vocab") -> kb = KnowledgeBase(vocab=vocab, entity_vector_length=64) +> kb = FullyImplementedKB(vocab=vocab, entity_vector_length=64) > kb.from_disk("/path/to/kb") > ``` | Name | Description | | ----------- | ----------------------------------------------------------------------------------------------- | | `loc` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| `exclude` | List of components to exclude. ~~Iterable[str]~~ | | **RETURNS** | The modified `KnowledgeBase` object. ~~KnowledgeBase~~ | ## Candidate {#candidate tag="class"} diff --git a/website/docs/api/kb_in_memory.md b/website/docs/api/kb_in_memory.md new file mode 100644 index 000000000..c9ce624f0 --- /dev/null +++ b/website/docs/api/kb_in_memory.md @@ -0,0 +1,302 @@ +--- +title: InMemoryLookupKB +teaser: + The default implementation of the KnowledgeBase interface. Stores all + information in-memory. +tag: class +source: spacy/kb/kb_in_memory.pyx +new: 3.5 +--- + +The `InMemoryLookupKB` class inherits from [`KnowledgeBase`](/api/kb) and +implements all of its methods. It stores all KB data in-memory and generates +[`Candidate`](/api/kb#candidate) objects by exactly matching mentions with +entity names. It's highly optimized for both a low memory footprint and speed of +retrieval. + +## InMemoryLookupKB.\_\_init\_\_ {#init tag="method"} + +Create the knowledge base. + +> #### Example +> +> ```python +> from spacy.kb import KnowledgeBase +> vocab = nlp.vocab +> kb = KnowledgeBase(vocab=vocab, entity_vector_length=64) +> ``` + +| Name | Description | +| ---------------------- | ------------------------------------------------ | +| `vocab` | The shared vocabulary. ~~Vocab~~ | +| `entity_vector_length` | Length of the fixed-size entity vectors. ~~int~~ | + +## InMemoryLookupKB.entity_vector_length {#entity_vector_length tag="property"} + +The length of the fixed-size entity vectors in the knowledge base. + +| Name | Description | +| ----------- | ------------------------------------------------ | +| **RETURNS** | Length of the fixed-size entity vectors. ~~int~~ | + +## InMemoryLookupKB.add_entity {#add_entity tag="method"} + +Add an entity to the knowledge base, specifying its corpus frequency and entity +vector, which should be of length +[`entity_vector_length`](/api/kb_in_memory#entity_vector_length). + +> #### Example +> +> ```python +> kb.add_entity(entity="Q42", freq=32, entity_vector=vector1) +> kb.add_entity(entity="Q463035", freq=111, entity_vector=vector2) +> ``` + +| Name | Description | +| --------------- | ---------------------------------------------------------- | +| `entity` | The unique entity identifier. ~~str~~ | +| `freq` | The frequency of the entity in a typical corpus. ~~float~~ | +| `entity_vector` | The pretrained vector of the entity. ~~numpy.ndarray~~ | + +## InMemoryLookupKB.set_entities {#set_entities tag="method"} + +Define the full list of entities in the knowledge base, specifying the corpus +frequency and entity vector for each entity. + +> #### Example +> +> ```python +> kb.set_entities(entity_list=["Q42", "Q463035"], freq_list=[32, 111], vector_list=[vector1, vector2]) +> ``` + +| Name | Description | +| ------------- | ---------------------------------------------------------------- | +| `entity_list` | List of unique entity identifiers. ~~Iterable[Union[str, int]]~~ | +| `freq_list` | List of entity frequencies. ~~Iterable[int]~~ | +| `vector_list` | List of entity vectors. ~~Iterable[numpy.ndarray]~~ | + +## InMemoryLookupKB.add_alias {#add_alias tag="method"} + +Add an alias or mention to the knowledge base, specifying its potential KB +identifiers and their prior probabilities. The entity identifiers should refer +to entities previously added with [`add_entity`](/api/kb_in_memory#add_entity) +or [`set_entities`](/api/kb_in_memory#set_entities). The sum of the prior +probabilities should not exceed 1. Note that an empty string can not be used as +alias. + +> #### Example +> +> ```python +> kb.add_alias(alias="Douglas", entities=["Q42", "Q463035"], probabilities=[0.6, 0.3]) +> ``` + +| Name | Description | +| --------------- | --------------------------------------------------------------------------------- | +| `alias` | The textual mention or alias. Can not be the empty string. ~~str~~ | +| `entities` | The potential entities that the alias may refer to. ~~Iterable[Union[str, int]]~~ | +| `probabilities` | The prior probabilities of each entity. ~~Iterable[float]~~ | + +## InMemoryLookupKB.\_\_len\_\_ {#len tag="method"} + +Get the total number of entities in the knowledge base. + +> #### Example +> +> ```python +> total_entities = len(kb) +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------- | +| **RETURNS** | The number of entities in the knowledge base. ~~int~~ | + +## InMemoryLookupKB.get_entity_strings {#get_entity_strings tag="method"} + +Get a list of all entity IDs in the knowledge base. + +> #### Example +> +> ```python +> all_entities = kb.get_entity_strings() +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------- | +| **RETURNS** | The list of entities in the knowledge base. ~~List[str]~~ | + +## InMemoryLookupKB.get_size_aliases {#get_size_aliases tag="method"} + +Get the total number of aliases in the knowledge base. + +> #### Example +> +> ```python +> total_aliases = kb.get_size_aliases() +> ``` + +| Name | Description | +| ----------- | ---------------------------------------------------- | +| **RETURNS** | The number of aliases in the knowledge base. ~~int~~ | + +## InMemoryLookupKB.get_alias_strings {#get_alias_strings tag="method"} + +Get a list of all aliases in the knowledge base. + +> #### Example +> +> ```python +> all_aliases = kb.get_alias_strings() +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------- | +| **RETURNS** | The list of aliases in the knowledge base. ~~List[str]~~ | + +## InMemoryLookupKB.get_candidates {#get_candidates tag="method"} + +Given a certain textual mention as input, retrieve a list of candidate entities +of type [`Candidate`](/api/kb#candidate). Wraps +[`get_alias_candidates()`](/api/kb_in_memory#get_alias_candidates). + +> #### Example +> +> ```python +> from spacy.lang.en import English +> nlp = English() +> doc = nlp("Douglas Adams wrote 'The Hitchhiker's Guide to the Galaxy'.") +> candidates = kb.get_candidates(doc[0:2]) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------- | +| `mention` | The textual mention or alias. ~~Span~~ | +| **RETURNS** | An iterable of relevant `Candidate` objects. ~~Iterable[Candidate]~~ | + +## InMemoryLookupKB.get_candidates_batch {#get_candidates_batch tag="method"} + +Same as [`get_candidates()`](/api/kb_in_memory#get_candidates), but for an +arbitrary number of mentions. The [`EntityLinker`](/api/entitylinker) component +will call `get_candidates_batch()` instead of `get_candidates()`, if the config +parameter `candidates_batch_size` is greater or equal than 1. + +The default implementation of `get_candidates_batch()` executes +`get_candidates()` in a loop. We recommend implementing a more efficient way to +retrieve candidates for multiple mentions at once, if performance is of concern +to you. + +> #### Example +> +> ```python +> from spacy.lang.en import English +> nlp = English() +> doc = nlp("Douglas Adams wrote 'The Hitchhiker's Guide to the Galaxy'.") +> candidates = kb.get_candidates((doc[0:2], doc[3:])) +> ``` + +| Name | Description | +| ----------- | -------------------------------------------------------------------------------------------- | +| `mentions` | The textual mention or alias. ~~Iterable[Span]~~ | +| **RETURNS** | An iterable of iterable with relevant `Candidate` objects. ~~Iterable[Iterable[Candidate]]~~ | + +## InMemoryLookupKB.get_alias_candidates {#get_alias_candidates tag="method"} + +Given a certain textual mention as input, retrieve a list of candidate entities +of type [`Candidate`](/api/kb#candidate). + +> #### Example +> +> ```python +> candidates = kb.get_alias_candidates("Douglas") +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------- | +| `alias` | The textual mention or alias. ~~str~~ | +| **RETURNS** | The list of relevant `Candidate` objects. ~~List[Candidate]~~ | + +## InMemoryLookupKB.get_vector {#get_vector tag="method"} + +Given a certain entity ID, retrieve its pretrained entity vector. + +> #### Example +> +> ```python +> vector = kb.get_vector("Q42") +> ``` + +| Name | Description | +| ----------- | ------------------------------------ | +| `entity` | The entity ID. ~~str~~ | +| **RETURNS** | The entity vector. ~~numpy.ndarray~~ | + +## InMemoryLookupKB.get_vectors {#get_vectors tag="method"} + +Same as [`get_vector()`](/api/kb_in_memory#get_vector), but for an arbitrary +number of entity IDs. + +The default implementation of `get_vectors()` executes `get_vector()` in a loop. +We recommend implementing a more efficient way to retrieve vectors for multiple +entities at once, if performance is of concern to you. + +> #### Example +> +> ```python +> vectors = kb.get_vectors(("Q42", "Q3107329")) +> ``` + +| Name | Description | +| ----------- | --------------------------------------------------------- | +| `entities` | The entity IDs. ~~Iterable[str]~~ | +| **RETURNS** | The entity vectors. ~~Iterable[Iterable[numpy.ndarray]]~~ | + +## InMemoryLookupKB.get_prior_prob {#get_prior_prob tag="method"} + +Given a certain entity ID and a certain textual mention, retrieve the prior +probability of the fact that the mention links to the entity ID. + +> #### Example +> +> ```python +> probability = kb.get_prior_prob("Q42", "Douglas") +> ``` + +| Name | Description | +| ----------- | ------------------------------------------------------------------------- | +| `entity` | The entity ID. ~~str~~ | +| `alias` | The textual mention or alias. ~~str~~ | +| **RETURNS** | The prior probability of the `alias` referring to the `entity`. ~~float~~ | + +## InMemoryLookupKB.to_disk {#to_disk tag="method"} + +Save the current state of the knowledge base to a directory. + +> #### Example +> +> ```python +> kb.to_disk(path) +> ``` + +| Name | Description | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| `path` | A path to a directory, which will be created if it doesn't exist. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| `exclude` | List of components to exclude. ~~Iterable[str]~~ | + +## InMemoryLookupKB.from_disk {#from_disk tag="method"} + +Restore the state of the knowledge base from a given directory. Note that the +[`Vocab`](/api/vocab) should also be the same as the one used to create the KB. + +> #### Example +> +> ```python +> from spacy.vocab import Vocab +> vocab = Vocab().from_disk("/path/to/vocab") +> kb = FullyImplementedKB(vocab=vocab, entity_vector_length=64) +> kb.from_disk("/path/to/kb") +> ``` + +| Name | Description | +| ----------- | ----------------------------------------------------------------------------------------------- | +| `loc` | A path to a directory. Paths may be either strings or `Path`-like objects. ~~Union[str, Path]~~ | +| `exclude` | List of components to exclude. ~~Iterable[str]~~ | +| **RETURNS** | The modified `KnowledgeBase` object. ~~KnowledgeBase~~ | diff --git a/website/docs/usage/101/_architecture.md b/website/docs/usage/101/_architecture.md index 22e2b961e..4ebca2756 100644 --- a/website/docs/usage/101/_architecture.md +++ b/website/docs/usage/101/_architecture.md @@ -78,7 +78,9 @@ operates on a `Doc` and gives you access to the matched tokens **in context**. | Name | Description | | ------------------------------------------------ | -------------------------------------------------------------------------------------------------- | | [`Corpus`](/api/corpus) | Class for managing annotated corpora for training and evaluation data. | -| [`KnowledgeBase`](/api/kb) | Storage for entities and aliases of a knowledge base for entity linking. | +| [`KnowledgeBase`](/api/kb) | Abstract base class for storage and retrieval of data for entity linking. | +| [`InMemoryLookupKB`](/api/kb_in_memory) | Implementation of `KnowledgeBase` storing all data in memory. | +| [`Candidate`](/api/kb#candidate) | Object associating a textual mention with a specific entity contained in a `KnowledgeBase`. | | [`Lookups`](/api/lookups) | Container for convenient access to large lookup tables and dictionaries. | | [`MorphAnalysis`](/api/morphology#morphanalysis) | A morphological analysis. | | [`Morphology`](/api/morphology) | Store morphological analyses and map them to and from hash values. | From cc10a27c59a3e5fe3c2d08667534fcbf22908f06 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Mon, 12 Sep 2022 15:36:48 +0200 Subject: [PATCH 06/24] Prevent tok2vec to broadcast to listeners when predicting (#11385) * replicate bug with tok2vec in annotating components * add overfitting test with a frozen tok2vec * remove broadcast from predict and check doc.tensor instead * remove broadcast * proper error * slight rephrase of documentation --- spacy/errors.py | 2 + spacy/pipeline/tok2vec.py | 20 ++++--- spacy/tests/pipeline/test_tok2vec.py | 81 ++++++++++++++++++++++++++++ website/docs/usage/training.md | 2 +- 4 files changed, 98 insertions(+), 7 deletions(-) diff --git a/spacy/errors.py b/spacy/errors.py index e2201284f..7e63dc76c 100644 --- a/spacy/errors.py +++ b/spacy/errors.py @@ -538,6 +538,8 @@ class Errors(metaclass=ErrorsWithCodes): E199 = ("Unable to merge 0-length span at `doc[{start}:{end}]`.") E200 = ("Can't set {attr} from Span.") E202 = ("Unsupported {name} mode '{mode}'. Supported modes: {modes}.") + E203 = ("If the {name} embedding layer is not updated " + "during training, make sure to include it in 'annotating components'") # New errors added in v3.x E853 = ("Unsupported component factory name '{name}'. The character '.' is " diff --git a/spacy/pipeline/tok2vec.py b/spacy/pipeline/tok2vec.py index 2e3dde3cb..c742aaeaa 100644 --- a/spacy/pipeline/tok2vec.py +++ b/spacy/pipeline/tok2vec.py @@ -123,9 +123,6 @@ class Tok2Vec(TrainablePipe): width = self.model.get_dim("nO") return [self.model.ops.alloc((0, width)) for doc in docs] tokvecs = self.model.predict(docs) - batch_id = Tok2VecListener.get_batch_id(docs) - for listener in self.listeners: - listener.receive(batch_id, tokvecs, _empty_backprop) return tokvecs def set_annotations(self, docs: Sequence[Doc], tokvecses) -> None: @@ -286,8 +283,19 @@ class Tok2VecListener(Model): def forward(model: Tok2VecListener, inputs, is_train: bool): """Supply the outputs from the upstream Tok2Vec component.""" if is_train: - model.verify_inputs(inputs) - return model._outputs, model._backprop + # This might occur during training when the tok2vec layer is frozen / hasn't been updated. + # In that case, it should be set to "annotating" so we can retrieve the embeddings from the doc. + if model._batch_id is None: + outputs = [] + for doc in inputs: + if doc.tensor.size == 0: + raise ValueError(Errors.E203.format(name="tok2vec")) + else: + outputs.append(doc.tensor) + return outputs, _empty_backprop + else: + model.verify_inputs(inputs) + return model._outputs, model._backprop else: # This is pretty grim, but it's hard to do better :(. # It's hard to avoid relying on the doc.tensor attribute, because the @@ -306,7 +314,7 @@ def forward(model: Tok2VecListener, inputs, is_train: bool): outputs.append(model.ops.alloc2f(len(doc), width)) else: outputs.append(doc.tensor) - return outputs, lambda dX: [] + return outputs, _empty_backprop def _empty_backprop(dX): # for pickling diff --git a/spacy/tests/pipeline/test_tok2vec.py b/spacy/tests/pipeline/test_tok2vec.py index 64faf133d..659274db9 100644 --- a/spacy/tests/pipeline/test_tok2vec.py +++ b/spacy/tests/pipeline/test_tok2vec.py @@ -230,6 +230,87 @@ def test_tok2vec_listener_callback(): assert get_dX(Y) is not None +def test_tok2vec_listener_overfitting(): + """ Test that a pipeline with a listener properly overfits, even if 'tok2vec' is in the annotating components """ + orig_config = Config().from_str(cfg_string) + nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + + for i in range(50): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses, annotates=["tok2vec"]) + assert losses["tagger"] < 0.00001 + + # test the trained model + test_text = "I like blue eggs" + doc = nlp(test_text) + assert doc[0].tag_ == "N" + assert doc[1].tag_ == "V" + assert doc[2].tag_ == "J" + assert doc[3].tag_ == "N" + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + assert doc2[0].tag_ == "N" + assert doc2[1].tag_ == "V" + assert doc2[2].tag_ == "J" + assert doc2[3].tag_ == "N" + + +def test_tok2vec_frozen_not_annotating(): + """ Test that a pipeline with a frozen tok2vec raises an error when the tok2vec is not annotating """ + orig_config = Config().from_str(cfg_string) + nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + + for i in range(2): + losses = {} + with pytest.raises(ValueError, match=r"the tok2vec embedding layer is not updated"): + nlp.update(train_examples, sgd=optimizer, losses=losses, exclude=["tok2vec"]) + + +def test_tok2vec_frozen_overfitting(): + """ Test that a pipeline with a frozen & annotating tok2vec can still overfit """ + orig_config = Config().from_str(cfg_string) + nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) + train_examples = [] + for t in TRAIN_DATA: + train_examples.append(Example.from_dict(nlp.make_doc(t[0]), t[1])) + optimizer = nlp.initialize(get_examples=lambda: train_examples) + + for i in range(100): + losses = {} + nlp.update(train_examples, sgd=optimizer, losses=losses, exclude=["tok2vec"], annotates=["tok2vec"]) + assert losses["tagger"] < 0.0001 + + # test the trained model + test_text = "I like blue eggs" + doc = nlp(test_text) + assert doc[0].tag_ == "N" + assert doc[1].tag_ == "V" + assert doc[2].tag_ == "J" + assert doc[3].tag_ == "N" + + # Also test the results are still the same after IO + with make_tempdir() as tmp_dir: + nlp.to_disk(tmp_dir) + nlp2 = util.load_model_from_path(tmp_dir) + doc2 = nlp2(test_text) + assert doc2[0].tag_ == "N" + assert doc2[1].tag_ == "V" + assert doc2[2].tag_ == "J" + assert doc2[3].tag_ == "N" + + def test_replace_listeners(): orig_config = Config().from_str(cfg_string) nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) diff --git a/website/docs/usage/training.md b/website/docs/usage/training.md index 5e064b269..27a8bbca7 100644 --- a/website/docs/usage/training.md +++ b/website/docs/usage/training.md @@ -480,7 +480,7 @@ as-is. They are also excluded when calling > parse. So the evaluation results should always reflect what your pipeline will > produce at runtime. If you want a frozen component to run (without updating) > during training as well, so that downstream components can use its -> **predictions**, you can add it to the list of +> **predictions**, you should add it to the list of > [`annotating_components`](/usage/training#annotating-components). ```ini From d5c8498f2f8c26fdfb1f18aafeeebbe94e6126bb Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Thu, 15 Sep 2022 17:41:25 +0200 Subject: [PATCH 07/24] disable mypy run for Python 3.10 (#11508) (#11511) --- .github/azure-steps.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml index 18224ba8c..c7722391f 100644 --- a/.github/azure-steps.yml +++ b/.github/azure-steps.yml @@ -27,6 +27,7 @@ steps: - script: python -m mypy spacy displayName: 'Run mypy' + condition: ne(variables['python_version'], '3.10') - task: DeleteFiles@1 inputs: From 3e8bc1272f95c89e0aa9e5a19f51e286a7934ffa Mon Sep 17 00:00:00 2001 From: Jacobo Myerston <43222279+jmyerston@users.noreply.github.com> Date: Tue, 27 Sep 2022 02:38:56 -0700 Subject: [PATCH 08/24] add punctuation to grc (#11426) * add punctuation to grc Add support for special editorial punctuation that is common in ancient Greek texts. Ancient Greek texts, as found in digital and print form, have been largely edited by scholars. Restorations and improvements are normally marked with special characters that need to be handled properly by the tokenizer. * add unit tests * simplify regex * move generic quotes to char classes * rename unit test * fix regex Co-authored-by: Adriane Boyd Co-authored-by: svlandeg Co-authored-by: Sofie Van Landeghem Co-authored-by: Adriane Boyd --- spacy/lang/char_classes.py | 2 +- spacy/lang/grc/__init__.py | 4 +++ spacy/lang/grc/punctuation.py | 46 ++++++++++++++++++++++++++ spacy/tests/lang/grc/test_tokenizer.py | 18 ++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 spacy/lang/grc/punctuation.py create mode 100644 spacy/tests/lang/grc/test_tokenizer.py diff --git a/spacy/lang/char_classes.py b/spacy/lang/char_classes.py index 1d204c46c..37c58c85f 100644 --- a/spacy/lang/char_classes.py +++ b/spacy/lang/char_classes.py @@ -280,7 +280,7 @@ _currency = ( _punct = ( r"… …… , : ; \! \? ¿ ؟ ¡ \( \) \[ \] \{ \} < > _ # \* & 。 ？ ！ ， 、 ； ： ～ · । ، ۔ ؛ ٪" ) -_quotes = r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉' +_quotes = r'\' " ” “ ` ‘ ´ ’ ‚ , „ » « 「 」 『 』 （ ） 〔 〕 【 】 《 》 〈 〉 〈 〉 ⟦ ⟧' _hyphens = "- – — -- --- —— ~" # Various symbols like dingbats, but also emoji diff --git a/spacy/lang/grc/__init__.py b/spacy/lang/grc/__init__.py index e83f0c5a5..019b3802e 100644 --- a/spacy/lang/grc/__init__.py +++ b/spacy/lang/grc/__init__.py @@ -1,11 +1,15 @@ from .tokenizer_exceptions import TOKENIZER_EXCEPTIONS from .stop_words import STOP_WORDS from .lex_attrs import LEX_ATTRS +from .punctuation import TOKENIZER_PREFIXES, TOKENIZER_SUFFIXES, TOKENIZER_INFIXES from ...language import Language, BaseDefaults class AncientGreekDefaults(BaseDefaults): tokenizer_exceptions = TOKENIZER_EXCEPTIONS + prefixes = TOKENIZER_PREFIXES + suffixes = TOKENIZER_SUFFIXES + infixes = TOKENIZER_INFIXES lex_attr_getters = LEX_ATTRS stop_words = STOP_WORDS diff --git a/spacy/lang/grc/punctuation.py b/spacy/lang/grc/punctuation.py new file mode 100644 index 000000000..8f3589e9a --- /dev/null +++ b/spacy/lang/grc/punctuation.py @@ -0,0 +1,46 @@ +from ..char_classes import LIST_PUNCT, LIST_ELLIPSES, LIST_QUOTES, LIST_CURRENCY +from ..char_classes import LIST_ICONS, ALPHA_LOWER, ALPHA_UPPER, ALPHA, HYPHENS +from ..char_classes import CONCAT_QUOTES + +_prefixes = ( + [ + "†", + "⸏", + ] + + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_CURRENCY + + LIST_ICONS +) + +_suffixes = ( + LIST_PUNCT + + LIST_ELLIPSES + + LIST_QUOTES + + LIST_ICONS + + [ + "†", + "⸎", + r"(?<=[\u1F00-\u1FFF\u0370-\u03FF])[\-\.⸏]", + ] +) + +_infixes = ( + LIST_ELLIPSES + + LIST_ICONS + + [ + r"(?<=[0-9])[+\-\*^](?=[0-9-])", + r"(?<=[{al}{q}])\.(?=[{au}{q}])".format( + al=ALPHA_LOWER, au=ALPHA_UPPER, q=CONCAT_QUOTES + ), + r"(?<=[{a}]),(?=[{a}])".format(a=ALPHA), + r"(?<=[{a}0-9])(?:{h})(?=[{a}])".format(a=ALPHA, h=HYPHENS), + r"(?<=[{a}0-9])[:<>=/](?=[{a}])".format(a=ALPHA), + r"(?<=[\u1F00-\u1FFF\u0370-\u03FF])—", + ] +) + +TOKENIZER_PREFIXES = _prefixes +TOKENIZER_SUFFIXES = _suffixes +TOKENIZER_INFIXES = _infixes diff --git a/spacy/tests/lang/grc/test_tokenizer.py b/spacy/tests/lang/grc/test_tokenizer.py new file mode 100644 index 000000000..3df5b546b --- /dev/null +++ b/spacy/tests/lang/grc/test_tokenizer.py @@ -0,0 +1,18 @@ +import pytest + + +# fmt: off +GRC_TOKEN_EXCEPTION_TESTS = [ + ("τὸ 〈τῆς〉 φιλοσοφίας ἔργον ἔνιοί φασιν ἀπὸ ⟦βαρβάρων⟧ ἄρξαι.", ["τὸ", "〈", "τῆς", "〉", "φιλοσοφίας", "ἔργον", "ἔνιοί", "φασιν", "ἀπὸ", "⟦", "βαρβάρων", "⟧", "ἄρξαι", "."]), + ("τὴν δὲ τῶν Αἰγυπτίων φιλοσοφίαν εἶναι τοιαύτην περί τε †θεῶν† καὶ ὑπὲρ δικαιοσύνης.", ["τὴν", "δὲ", "τῶν", "Αἰγυπτίων", "φιλοσοφίαν", "εἶναι", "τοιαύτην", "περί", "τε", "†", "θεῶν", "†", "καὶ", "ὑπὲρ", "δικαιοσύνης", "."]), + ("⸏πόσις δ' Ἐρεχθεύς ἐστί μοι σεσωσμένος⸏", ["⸏", "πόσις", "δ'", "Ἐρεχθεύς", "ἐστί", "μοι", "σεσωσμένος", "⸏"]), + ("⸏ὔπνον ἴδωμεν⸎", ["⸏", "ὔπνον", "ἴδωμεν", "⸎"]), +] +# fmt: on + + +@pytest.mark.parametrize("text,expected_tokens", GRC_TOKEN_EXCEPTION_TESTS) +def test_grc_tokenizer(grc_tokenizer, text, expected_tokens): + tokens = grc_tokenizer(text) + token_list = [token.text for token in tokens if not token.is_space] + assert expected_tokens == token_list From b187076a2dd0f034c1a8918c9b332711688b5dc2 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Mon, 3 Oct 2022 17:01:04 +0200 Subject: [PATCH 09/24] fix docs (#11573) --- website/docs/api/kb_in_memory.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/api/kb_in_memory.md b/website/docs/api/kb_in_memory.md index c9ce624f0..9e3279e6a 100644 --- a/website/docs/api/kb_in_memory.md +++ b/website/docs/api/kb_in_memory.md @@ -21,9 +21,9 @@ Create the knowledge base. > #### Example > > ```python -> from spacy.kb import KnowledgeBase +> from spacy.kb import InMemoryLookupKB > vocab = nlp.vocab -> kb = KnowledgeBase(vocab=vocab, entity_vector_length=64) +> kb = InMemoryLookupKB(vocab=vocab, entity_vector_length=64) > ``` | Name | Description | From ef74f8f5e447dec10ab69d2a7e94f0e09165db75 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Tue, 11 Oct 2022 14:15:22 +0200 Subject: [PATCH 10/24] Fix mypy error in edittree lemmatizer (#11612) * cleanup imports * try limiting Thinc to previous release * remove Model specification * fix code and revert Thinc constraint --- spacy/pipeline/edit_tree_lemmatizer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spacy/pipeline/edit_tree_lemmatizer.py b/spacy/pipeline/edit_tree_lemmatizer.py index b7d615f6d..7f6367c75 100644 --- a/spacy/pipeline/edit_tree_lemmatizer.py +++ b/spacy/pipeline/edit_tree_lemmatizer.py @@ -1,7 +1,6 @@ from typing import cast, Any, Callable, Dict, Iterable, List, Optional -from typing import Sequence, Tuple, Union +from typing import Tuple from collections import Counter -from copy import deepcopy from itertools import islice import numpy as np @@ -150,7 +149,7 @@ class EditTreeLemmatizer(TrainablePipe): # Handle cases where there are no tokens in any docs. n_labels = len(self.cfg["labels"]) guesses: List[Ints2d] = [ - self.model.ops.alloc((0, n_labels), dtype="i") for doc in docs + self.model.ops.alloc2i(0, n_labels, dtype="i") for _ in docs ] assert len(guesses) == n_docs return guesses From 29649589fc889a58c8b631d569d4ae378a10aa2b Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Tue, 11 Oct 2022 15:25:05 +0200 Subject: [PATCH 11/24] remove dtype (#11615) --- spacy/pipeline/edit_tree_lemmatizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/pipeline/edit_tree_lemmatizer.py b/spacy/pipeline/edit_tree_lemmatizer.py index 7f6367c75..76b0e0bc9 100644 --- a/spacy/pipeline/edit_tree_lemmatizer.py +++ b/spacy/pipeline/edit_tree_lemmatizer.py @@ -149,7 +149,7 @@ class EditTreeLemmatizer(TrainablePipe): # Handle cases where there are no tokens in any docs. n_labels = len(self.cfg["labels"]) guesses: List[Ints2d] = [ - self.model.ops.alloc2i(0, n_labels, dtype="i") for _ in docs + self.model.ops.alloc2i(0, n_labels) for _ in docs ] assert len(guesses) == n_docs return guesses From 2e52479eec987367117d27fb4f049df2efb2518d Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Tue, 11 Oct 2022 23:45:05 +0900 Subject: [PATCH 12/24] Fix example code for spacy-wordnet (#11593) * Fix example code for spacy-wordnet It looks like in the most recent version, 0.1.0, it's no longer possible to pass the lang parameter to the component separately. Doing so will raise an error. * Apply suggestions from code review Co-authored-by: Sofie Van Landeghem * Cleanup * More cleanup Co-authored-by: Sofie Van Landeghem --- website/meta/universe.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/website/meta/universe.json b/website/meta/universe.json index a6a1a0fc7..637e9d6ce 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -2460,20 +2460,20 @@ "import spacy", "from spacy_wordnet.wordnet_annotator import WordnetAnnotator ", "", - "# Load an spacy model (supported models are \"es\" and \"en\") ", - "nlp = spacy.load('en')", - "# Spacy 3.x", - "nlp.add_pipe(\"spacy_wordnet\", after='tagger', config={'lang': nlp.lang})", - "# Spacy 2.x", + "# Load a spaCy model (supported languages are \"es\" and \"en\") ", + "nlp = spacy.load('en_core_web_sm')", + "# spaCy 3.x", + "nlp.add_pipe(\"spacy_wordnet\", after='tagger')", + "# spaCy 2.x", "# nlp.add_pipe(WordnetAnnotator(nlp.lang), after='tagger')", "token = nlp('prices')[0]", "", - "# wordnet object link spacy token with nltk wordnet interface by giving acces to", + "# WordNet object links spaCy token with NLTK WordNet interface by giving access to", "# synsets and lemmas ", "token._.wordnet.synsets()", "token._.wordnet.lemmas()", "", - "# And automatically tags with wordnet domains", + "# And automatically add info about WordNet domains", "token._.wordnet.wordnet_domains()" ], "author": "recognai", From fe06e037bcd733708401bce082863994b1fc48bd Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 12 Oct 2022 12:18:39 +0200 Subject: [PATCH 13/24] Fix init for pymorphy2_lookup lemmatizer mode (#11631) --- spacy/lang/ru/lemmatizer.py | 2 +- spacy/lang/uk/lemmatizer.py | 2 +- spacy/tests/conftest.py | 17 +++++++++++++++++ spacy/tests/lang/ru/test_lemmatizer.py | 14 ++++++++++++++ spacy/tests/lang/uk/test_lemmatizer.py | 8 ++++++++ 5 files changed, 41 insertions(+), 2 deletions(-) diff --git a/spacy/lang/ru/lemmatizer.py b/spacy/lang/ru/lemmatizer.py index 85180b1e4..5bf685d44 100644 --- a/spacy/lang/ru/lemmatizer.py +++ b/spacy/lang/ru/lemmatizer.py @@ -23,7 +23,7 @@ class RussianLemmatizer(Lemmatizer): overwrite: bool = False, scorer: Optional[Callable] = lemmatizer_score, ) -> None: - if mode == "pymorphy2": + if mode in {"pymorphy2", "pymorphy2_lookup"}: try: from pymorphy2 import MorphAnalyzer except ImportError: diff --git a/spacy/lang/uk/lemmatizer.py b/spacy/lang/uk/lemmatizer.py index a8bc56057..d4f8cc9e5 100644 --- a/spacy/lang/uk/lemmatizer.py +++ b/spacy/lang/uk/lemmatizer.py @@ -18,7 +18,7 @@ class UkrainianLemmatizer(RussianLemmatizer): overwrite: bool = False, scorer: Optional[Callable] = lemmatizer_score, ) -> None: - if mode == "pymorphy2": + if mode in {"pymorphy2", "pymorphy2_lookup"}: try: from pymorphy2 import MorphAnalyzer except ImportError: diff --git a/spacy/tests/conftest.py b/spacy/tests/conftest.py index 742bfcc6a..394ef00d3 100644 --- a/spacy/tests/conftest.py +++ b/spacy/tests/conftest.py @@ -343,6 +343,14 @@ def ru_lemmatizer(): return get_lang_class("ru")().add_pipe("lemmatizer") +@pytest.fixture +def ru_lookup_lemmatizer(): + pytest.importorskip("pymorphy2") + return get_lang_class("ru")().add_pipe( + "lemmatizer", config={"mode": "pymorphy2_lookup"} + ) + + @pytest.fixture(scope="session") def sa_tokenizer(): return get_lang_class("sa")().tokenizer @@ -422,6 +430,15 @@ def uk_lemmatizer(): return get_lang_class("uk")().add_pipe("lemmatizer") +@pytest.fixture +def uk_lookup_lemmatizer(): + pytest.importorskip("pymorphy2") + pytest.importorskip("pymorphy2_dicts_uk") + return get_lang_class("uk")().add_pipe( + "lemmatizer", config={"mode": "pymorphy2_lookup"} + ) + + @pytest.fixture(scope="session") def ur_tokenizer(): return get_lang_class("ur")().tokenizer diff --git a/spacy/tests/lang/ru/test_lemmatizer.py b/spacy/tests/lang/ru/test_lemmatizer.py index 9ca7f441b..e82fd4f8c 100644 --- a/spacy/tests/lang/ru/test_lemmatizer.py +++ b/spacy/tests/lang/ru/test_lemmatizer.py @@ -78,3 +78,17 @@ def test_ru_lemmatizer_punct(ru_lemmatizer): assert ru_lemmatizer.pymorphy2_lemmatize(doc[0]) == ['"'] doc = Doc(ru_lemmatizer.vocab, words=["»"], pos=["PUNCT"]) assert ru_lemmatizer.pymorphy2_lemmatize(doc[0]) == ['"'] + + +def test_ru_doc_lookup_lemmatization(ru_lookup_lemmatizer): + words = ["мама", "мыла", "раму"] + pos = ["NOUN", "VERB", "NOUN"] + morphs = [ + "Animacy=Anim|Case=Nom|Gender=Fem|Number=Sing", + "Aspect=Imp|Gender=Fem|Mood=Ind|Number=Sing|Tense=Past|VerbForm=Fin|Voice=Act", + "Animacy=Anim|Case=Acc|Gender=Fem|Number=Sing", + ] + doc = Doc(ru_lookup_lemmatizer.vocab, words=words, pos=pos, morphs=morphs) + doc = ru_lookup_lemmatizer(doc) + lemmas = [token.lemma_ for token in doc] + assert lemmas == ["мама", "мыла", "раму"] diff --git a/spacy/tests/lang/uk/test_lemmatizer.py b/spacy/tests/lang/uk/test_lemmatizer.py index 57dd4198a..788744aa1 100644 --- a/spacy/tests/lang/uk/test_lemmatizer.py +++ b/spacy/tests/lang/uk/test_lemmatizer.py @@ -9,3 +9,11 @@ def test_uk_lemmatizer(uk_lemmatizer): """Check that the default uk lemmatizer runs.""" doc = Doc(uk_lemmatizer.vocab, words=["a", "b", "c"]) uk_lemmatizer(doc) + assert [token.lemma for token in doc] + + +def test_uk_lookup_lemmatizer(uk_lookup_lemmatizer): + """Check that the lookup uk lemmatizer runs.""" + doc = Doc(uk_lookup_lemmatizer.vocab, words=["a", "b", "c"]) + uk_lookup_lemmatizer(doc) + assert [token.lemma for token in doc] From 4d869fcc111151bcefa08ee1a2b7b49dc5ecd677 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Wed, 12 Oct 2022 15:17:40 +0200 Subject: [PATCH 14/24] Small fixes to docstrings (#11610) * add missing scorer arg to docstring * fix class names in textcat_multilabel * add missing scorer to docstrings --- spacy/pipeline/spancat.py | 3 +++ spacy/pipeline/textcat_multilabel.py | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/spacy/pipeline/spancat.py b/spacy/pipeline/spancat.py index 1b7a9eecb..ca9f1dab0 100644 --- a/spacy/pipeline/spancat.py +++ b/spacy/pipeline/spancat.py @@ -133,6 +133,9 @@ def make_spancat( spans_key (str): Key of the doc.spans dict to save the spans under. During initialization and training, the component will look for spans on the reference document under the same key. + scorer (Optional[Callable]): The scoring method. Defaults to + Scorer.score_spans for the Doc.spans[spans_key] with overlapping + spans allowed. threshold (float): Minimum probability to consider a prediction positive. Spans with a positive prediction will be saved on the Doc. Defaults to 0.5. diff --git a/spacy/pipeline/textcat_multilabel.py b/spacy/pipeline/textcat_multilabel.py index e33a885f8..119ae3310 100644 --- a/spacy/pipeline/textcat_multilabel.py +++ b/spacy/pipeline/textcat_multilabel.py @@ -96,8 +96,8 @@ def make_multilabel_textcat( model: Model[List[Doc], List[Floats2d]], threshold: float, scorer: Optional[Callable], -) -> "TextCategorizer": - """Create a TextCategorizer component. The text categorizer predicts categories +) -> "MultiLabel_TextCategorizer": + """Create a MultiLabel_TextCategorizer component. The text categorizer predicts categories over a whole document. It can learn one or more labels, and the labels are considered to be non-mutually exclusive, which means that there can be zero or more labels per doc). @@ -105,6 +105,7 @@ def make_multilabel_textcat( model (Model[List[Doc], List[Floats2d]]): A model instance that predicts scores for each category. threshold (float): Cutoff to consider a prediction "positive". + scorer (Optional[Callable]): The scoring method. """ return MultiLabel_TextCategorizer( nlp.vocab, model, name, threshold=threshold, scorer=scorer @@ -147,6 +148,7 @@ class MultiLabel_TextCategorizer(TextCategorizer): name (str): The component instance name, used to add entries to the losses during training. threshold (float): Cutoff to consider a prediction "positive". + scorer (Optional[Callable]): The scoring method. DOCS: https://spacy.io/api/textcategorizer#init """ From 6b5a3e72198aa9735587b0712e3eb2c24234b463 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Fri, 14 Oct 2022 08:16:49 +0200 Subject: [PATCH 15/24] Extend to pydantic v1.10 (#11635) * Update types in `spacy.schemas` for updated pydantic+mypy --- requirements.txt | 2 +- setup.cfg | 2 +- spacy/schemas.py | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/requirements.txt b/requirements.txt index 14847ff21..9d6bbb2c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ pathy>=0.3.5 numpy>=1.15.0 requests>=2.13.0,<3.0.0 tqdm>=4.38.0,<5.0.0 -pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0 +pydantic>=1.7.4,!=1.8,!=1.8.1,<1.11.0 jinja2 langcodes>=3.2.0,<4.0.0 # Official Python utilities diff --git a/setup.cfg b/setup.cfg index 2dc5e7042..c2653feba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,7 +56,7 @@ install_requires = tqdm>=4.38.0,<5.0.0 numpy>=1.15.0 requests>=2.13.0,<3.0.0 - pydantic>=1.7.4,!=1.8,!=1.8.1,<1.10.0 + pydantic>=1.7.4,!=1.8,!=1.8.1,<1.11.0 jinja2 # Official Python utilities setuptools diff --git a/spacy/schemas.py b/spacy/schemas.py index 048082134..ab71b2016 100644 --- a/spacy/schemas.py +++ b/spacy/schemas.py @@ -181,12 +181,12 @@ class TokenPatternNumber(BaseModel): IS_SUBSET: Optional[List[StrictInt]] = Field(None, alias="is_subset") IS_SUPERSET: Optional[List[StrictInt]] = Field(None, alias="is_superset") INTERSECTS: Optional[List[StrictInt]] = Field(None, alias="intersects") - EQ: Union[StrictInt, StrictFloat] = Field(None, alias="==") - NEQ: Union[StrictInt, StrictFloat] = Field(None, alias="!=") - GEQ: Union[StrictInt, StrictFloat] = Field(None, alias=">=") - LEQ: Union[StrictInt, StrictFloat] = Field(None, alias="<=") - GT: Union[StrictInt, StrictFloat] = Field(None, alias=">") - LT: Union[StrictInt, StrictFloat] = Field(None, alias="<") + EQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="==") + NEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="!=") + GEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias=">=") + LEQ: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="<=") + GT: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias=">") + LT: Optional[Union[StrictInt, StrictFloat]] = Field(None, alias="<") class Config: extra = "forbid" @@ -430,7 +430,7 @@ class ProjectConfigAssetURL(BaseModel): # fmt: off dest: StrictStr = Field(..., title="Destination of downloaded asset") url: Optional[StrictStr] = Field(None, title="URL of asset") - checksum: str = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})") + checksum: Optional[str] = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})") description: StrictStr = Field("", title="Description of asset") # fmt: on @@ -438,7 +438,7 @@ class ProjectConfigAssetURL(BaseModel): class ProjectConfigAssetGit(BaseModel): # fmt: off git: ProjectConfigAssetGitItem = Field(..., title="Git repo information") - checksum: str = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})") + checksum: Optional[str] = Field(None, title="MD5 hash of file", regex=r"([a-fA-F\d]{32})") description: Optional[StrictStr] = Field(None, title="Description of asset") # fmt: on @@ -508,7 +508,7 @@ class DocJSONSchema(BaseModel): None, title="Indices of sentences' start and end indices" ) text: StrictStr = Field(..., title="Document text") - spans: Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]] = Field( + spans: Optional[Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]]] = Field( None, title="Span information - end/start indices, label, KB ID" ) tokens: List[Dict[StrictStr, Union[StrictStr, StrictInt]]] = Field( From ceb62352bfcad49b3ad63e3e65ef12dabab645b3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 14 Oct 2022 18:04:55 +0900 Subject: [PATCH 16/24] Auto-format code with black (#11649) Co-authored-by: explosion-bot --- spacy/pipeline/edit_tree_lemmatizer.py | 4 +--- spacy/schemas.py | 6 +++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/spacy/pipeline/edit_tree_lemmatizer.py b/spacy/pipeline/edit_tree_lemmatizer.py index 76b0e0bc9..12f9b73a3 100644 --- a/spacy/pipeline/edit_tree_lemmatizer.py +++ b/spacy/pipeline/edit_tree_lemmatizer.py @@ -148,9 +148,7 @@ class EditTreeLemmatizer(TrainablePipe): if not any(len(doc) for doc in docs): # Handle cases where there are no tokens in any docs. n_labels = len(self.cfg["labels"]) - guesses: List[Ints2d] = [ - self.model.ops.alloc2i(0, n_labels) for _ in docs - ] + guesses: List[Ints2d] = [self.model.ops.alloc2i(0, n_labels) for _ in docs] assert len(guesses) == n_docs return guesses scores = self.model.predict(docs) diff --git a/spacy/schemas.py b/spacy/schemas.py index ab71b2016..a67d96d9d 100644 --- a/spacy/schemas.py +++ b/spacy/schemas.py @@ -508,9 +508,9 @@ class DocJSONSchema(BaseModel): None, title="Indices of sentences' start and end indices" ) text: StrictStr = Field(..., title="Document text") - spans: Optional[Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]]] = Field( - None, title="Span information - end/start indices, label, KB ID" - ) + spans: Optional[ + Dict[StrictStr, List[Dict[StrictStr, Union[StrictStr, StrictInt]]]] + ] = Field(None, title="Span information - end/start indices, label, KB ID") tokens: List[Dict[StrictStr, Union[StrictStr, StrictInt]]] = Field( ..., title="Token information - ID, start, annotations" ) From 2ce6aadda2d455cf2f2a1aef494b2bafe3e07119 Mon Sep 17 00:00:00 2001 From: Sofie Van Landeghem Date: Mon, 17 Oct 2022 12:10:03 +0200 Subject: [PATCH 17/24] update default configs to recent versions (#11618) --- spacy/pipeline/spancat.py | 6 +++--- spacy/pipeline/textcat_multilabel.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/spacy/pipeline/spancat.py b/spacy/pipeline/spancat.py index 1b7a9eecb..5ede622c2 100644 --- a/spacy/pipeline/spancat.py +++ b/spacy/pipeline/spancat.py @@ -26,17 +26,17 @@ scorer = {"@layers": "spacy.LinearLogistic.v1"} hidden_size = 128 [model.tok2vec] -@architectures = "spacy.Tok2Vec.v1" +@architectures = "spacy.Tok2Vec.v2" [model.tok2vec.embed] -@architectures = "spacy.MultiHashEmbed.v1" +@architectures = "spacy.MultiHashEmbed.v2" width = 96 rows = [5000, 2000, 1000, 1000] attrs = ["ORTH", "PREFIX", "SUFFIX", "SHAPE"] include_static_vectors = false [model.tok2vec.encode] -@architectures = "spacy.MaxoutWindowEncoder.v1" +@architectures = "spacy.MaxoutWindowEncoder.v2" width = ${model.tok2vec.embed.width} window_size = 1 maxout_pieces = 3 diff --git a/spacy/pipeline/textcat_multilabel.py b/spacy/pipeline/textcat_multilabel.py index e33a885f8..10aef46aa 100644 --- a/spacy/pipeline/textcat_multilabel.py +++ b/spacy/pipeline/textcat_multilabel.py @@ -19,7 +19,7 @@ multi_label_default_config = """ @architectures = "spacy.TextCatEnsemble.v2" [model.tok2vec] -@architectures = "spacy.Tok2Vec.v1" +@architectures = "spacy.Tok2Vec.v2" [model.tok2vec.embed] @architectures = "spacy.MultiHashEmbed.v2" @@ -29,7 +29,7 @@ attrs = ["ORTH", "LOWER", "PREFIX", "SUFFIX", "SHAPE", "ID"] include_static_vectors = false [model.tok2vec.encode] -@architectures = "spacy.MaxoutWindowEncoder.v1" +@architectures = "spacy.MaxoutWindowEncoder.v2" width = ${model.tok2vec.embed.width} window_size = 1 maxout_pieces = 3 From 858565a5671de61334443d6a2348164bc39216e1 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Tue, 18 Oct 2022 15:11:39 +0900 Subject: [PATCH 18/24] Fix issues with DVC commands (#11592) * Fix flag handling in dvc Prior to this commit, if a flag (--verbose or --quiet) was passed to DVC, it would be added to the end of the generated dvc command line. This would result in the command being interpreted as part of the actual command to run, rather than an argument to dvc. This would result in command lines like: spacy project run preprocess --verbose That would fail with an error that there's no such directory as `--verbose`. This change puts the flags at the front of the dvc command so that they are interpreted correctly. It removes the `run_dvc_commands` function, which had been reduced to just a for loop and wasn't used elsewhere. A separate problem is that there's no way to specify the quiet behaviour to dvc from the command line, though it's unclear if that's a bug. * Add dvc quiet flag to docs * Handle case in DVC where no commands are appropriate If only have commands with no deps or outputs (admittedly unlikely), you get a weird error about the dvc file not existing. This gives explicit output instead. * Add support for quiet flag * Fix command execution Commands are strings now because they're joined further up. --- spacy/cli/project/dvc.py | 57 +++++++++++++++++++++------------------- website/docs/api/cli.md | 3 ++- 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/spacy/cli/project/dvc.py b/spacy/cli/project/dvc.py index 83dc5efbf..a15353855 100644 --- a/spacy/cli/project/dvc.py +++ b/spacy/cli/project/dvc.py @@ -25,6 +25,7 @@ def project_update_dvc_cli( project_dir: Path = Arg(Path.cwd(), help="Location of project directory. Defaults to current working directory.", exists=True, file_okay=False), workflow: Optional[str] = Arg(None, help=f"Name of workflow defined in {PROJECT_FILE}. Defaults to first workflow if not set."), verbose: bool = Opt(False, "--verbose", "-V", help="Print more info"), + quiet: bool = Opt(False, "--quiet", "-q", help="Print less info"), force: bool = Opt(False, "--force", "-F", help="Force update DVC config"), # fmt: on ): @@ -36,7 +37,7 @@ def project_update_dvc_cli( DOCS: https://spacy.io/api/cli#project-dvc """ - project_update_dvc(project_dir, workflow, verbose=verbose, force=force) + project_update_dvc(project_dir, workflow, verbose=verbose, quiet=quiet, force=force) def project_update_dvc( @@ -44,6 +45,7 @@ def project_update_dvc( workflow: Optional[str] = None, *, verbose: bool = False, + quiet: bool = False, force: bool = False, ) -> None: """Update the auto-generated Data Version Control (DVC) config file. A DVC @@ -54,11 +56,12 @@ def project_update_dvc( workflow (Optional[str]): Optional name of workflow defined in project.yml. If not set, the first workflow will be used. verbose (bool): Print more info. + quiet (bool): Print less info. force (bool): Force update DVC config. """ config = load_project_config(project_dir) updated = update_dvc_config( - project_dir, config, workflow, verbose=verbose, force=force + project_dir, config, workflow, verbose=verbose, quiet=quiet, force=force ) help_msg = "To execute the workflow with DVC, run: dvc repro" if updated: @@ -72,7 +75,7 @@ def update_dvc_config( config: Dict[str, Any], workflow: Optional[str] = None, verbose: bool = False, - silent: bool = False, + quiet: bool = False, force: bool = False, ) -> bool: """Re-run the DVC commands in dry mode and update dvc.yaml file in the @@ -83,7 +86,7 @@ def update_dvc_config( path (Path): The path to the project directory. config (Dict[str, Any]): The loaded project.yml. verbose (bool): Whether to print additional info (via DVC). - silent (bool): Don't output anything (via DVC). + quiet (bool): Don't output anything (via DVC). force (bool): Force update, even if hashes match. RETURNS (bool): Whether the DVC config file was updated. """ @@ -105,6 +108,14 @@ def update_dvc_config( dvc_config_path.unlink() dvc_commands = [] config_commands = {cmd["name"]: cmd for cmd in config.get("commands", [])} + + # some flags that apply to every command + flags = [] + if verbose: + flags.append("--verbose") + if quiet: + flags.append("--quiet") + for name in workflows[workflow]: command = config_commands[name] deps = command.get("deps", []) @@ -118,14 +129,26 @@ def update_dvc_config( deps_cmd = [c for cl in [["-d", p] for p in deps] for c in cl] outputs_cmd = [c for cl in [["-o", p] for p in outputs] for c in cl] outputs_nc_cmd = [c for cl in [["-O", p] for p in outputs_no_cache] for c in cl] - dvc_cmd = ["run", "-n", name, "-w", str(path), "--no-exec"] + + dvc_cmd = ["run", *flags, "-n", name, "-w", str(path), "--no-exec"] if command.get("no_skip"): dvc_cmd.append("--always-changed") full_cmd = [*dvc_cmd, *deps_cmd, *outputs_cmd, *outputs_nc_cmd, *project_cmd] dvc_commands.append(join_command(full_cmd)) + + if not dvc_commands: + # If we don't check for this, then there will be an error when reading the + # config, since DVC wouldn't create it. + msg.fail( + "No usable commands for DVC found. This can happen if none of your " + "commands have dependencies or outputs.", + exits=1, + ) + with working_dir(path): - dvc_flags = {"--verbose": verbose, "--quiet": silent} - run_dvc_commands(dvc_commands, flags=dvc_flags) + for c in dvc_commands: + dvc_command = "dvc " + c + run_command(dvc_command) with dvc_config_path.open("r+", encoding="utf8") as f: content = f.read() f.seek(0, 0) @@ -133,26 +156,6 @@ def update_dvc_config( return True -def run_dvc_commands( - commands: Iterable[str] = SimpleFrozenList(), flags: Dict[str, bool] = {} -) -> None: - """Run a sequence of DVC commands in a subprocess, in order. - - commands (List[str]): The string commands without the leading "dvc". - flags (Dict[str, bool]): Conditional flags to be added to command. Makes it - easier to pass flags like --quiet that depend on a variable or - command-line setting while avoiding lots of nested conditionals. - """ - for c in commands: - command = split_command(c) - dvc_command = ["dvc", *command] - # Add the flags if they are set to True - for flag, is_active in flags.items(): - if is_active: - dvc_command.append(flag) - run_command(dvc_command) - - def check_workflows(workflows: List[str], workflow: Optional[str] = None) -> None: """Validate workflows provided in project.yml and check that a given workflow can be used to generate a DVC config. diff --git a/website/docs/api/cli.md b/website/docs/api/cli.md index e5cd3089b..fc2c46022 100644 --- a/website/docs/api/cli.md +++ b/website/docs/api/cli.md @@ -1482,7 +1482,7 @@ You'll also need to add the assets you want to track with ```cli -$ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] +$ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] [--quiet] ``` > #### Example @@ -1499,6 +1499,7 @@ $ python -m spacy project dvc [project_dir] [workflow] [--force] [--verbose] | `workflow` | Name of workflow defined in `project.yml`. Defaults to first workflow if not set. ~~Optional[str] \(option)~~ | | `--force`, `-F` | Force-updating config file. ~~bool (flag)~~ | | `--verbose`, `-V` | Print more output generated by DVC. ~~bool (flag)~~ | +| `--quiet`, `-q` | Print no output generated by DVC. ~~bool (flag)~~ | | `--help`, `-h` | Show help message and available arguments. ~~bool (flag)~~ | | **CREATES** | A `dvc.yaml` file in the project directory, based on the steps defined in the given workflow. | From a1eacaa8db055322d4a066a08b730243a2f5b969 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Tue, 18 Oct 2022 14:36:06 +0200 Subject: [PATCH 19/24] Add python 3.11.0rc2 to CI (#11667) --- .github/azure-steps.yml | 1 + azure-pipelines.yml | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.github/azure-steps.yml b/.github/azure-steps.yml index 9d57219ca..cc0247b3a 100644 --- a/.github/azure-steps.yml +++ b/.github/azure-steps.yml @@ -10,6 +10,7 @@ steps: inputs: versionSpec: ${{ parameters.python_version }} architecture: ${{ parameters.architecture }} + allowUnstable: true - bash: | echo "##vso[task.setvariable variable=python_version]${{ parameters.python_version }}" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 2f5201614..357cce835 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -85,6 +85,15 @@ jobs: Python310Mac: imageName: "macos-latest" python.version: "3.10" + Python311Linux: + imageName: 'ubuntu-latest' + python.version: '3.11.0-rc.2' + Python311Windows: + imageName: 'windows-latest' + python.version: '3.11.0-rc.2' + Python311Mac: + imageName: 'macos-latest' + python.version: '3.11.0-rc.2' maxParallel: 4 pool: vmImage: $(imageName) From d66ccb8eb08cd515904045de84351546065fb3ed Mon Sep 17 00:00:00 2001 From: Edward <43848523+thomashacker@users.noreply.github.com> Date: Wed, 19 Oct 2022 15:52:47 +0200 Subject: [PATCH 20/24] Fix multiple entries per custom extension in doc json (#11551) * Fix multiple extensions and character offset * Rename token_start/end to start/end * Refactor Doc.from_json based on review * Iterate over user_data items * Only add non-empty underscore entries Co-authored-by: Adriane Boyd --- spacy/schemas.py | 4 +- spacy/tests/doc/test_json_doc_conversion.py | 25 +++++++---- spacy/tokens/doc.pyx | 48 ++++++++++----------- 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/spacy/schemas.py b/spacy/schemas.py index a67d96d9d..c824d76b9 100644 --- a/spacy/schemas.py +++ b/spacy/schemas.py @@ -519,9 +519,9 @@ class DocJSONSchema(BaseModel): title="Any custom data stored in the document's _ attribute", alias="_", ) - underscore_token: Optional[Dict[StrictStr, Dict[StrictStr, Any]]] = Field( + underscore_token: Optional[Dict[StrictStr, List[Dict[StrictStr, Any]]]] = Field( None, title="Any custom data stored in the token's _ attribute" ) - underscore_span: Optional[Dict[StrictStr, Dict[StrictStr, Any]]] = Field( + underscore_span: Optional[Dict[StrictStr, List[Dict[StrictStr, Any]]]] = Field( None, title="Any custom data stored in the span's _ attribute" ) diff --git a/spacy/tests/doc/test_json_doc_conversion.py b/spacy/tests/doc/test_json_doc_conversion.py index 0d7c061c9..19698cfb2 100644 --- a/spacy/tests/doc/test_json_doc_conversion.py +++ b/spacy/tests/doc/test_json_doc_conversion.py @@ -128,7 +128,9 @@ def test_doc_to_json_with_token_span_attributes(doc): doc._.json_test1 = "hello world" doc._.json_test2 = [1, 2, 3] doc[0:1]._.span_test = "span_attribute" + doc[0:2]._.span_test = "span_attribute_2" doc[0]._.token_test = 117 + doc[1]._.token_test = 118 doc.spans["span_group"] = [doc[0:1]] json_doc = doc.to_json( underscore=["json_test1", "json_test2", "token_test", "span_test"] @@ -139,8 +141,10 @@ def test_doc_to_json_with_token_span_attributes(doc): assert json_doc["_"]["json_test2"] == [1, 2, 3] assert "underscore_token" in json_doc assert "underscore_span" in json_doc - assert json_doc["underscore_token"]["token_test"]["value"] == 117 - assert json_doc["underscore_span"]["span_test"]["value"] == "span_attribute" + assert json_doc["underscore_token"]["token_test"][0]["value"] == 117 + assert json_doc["underscore_token"]["token_test"][1]["value"] == 118 + assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute" + assert json_doc["underscore_span"]["span_test"][1]["value"] == "span_attribute_2" assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc @@ -161,8 +165,8 @@ def test_doc_to_json_with_custom_user_data(doc): assert json_doc["_"]["json_test"] == "hello world" assert "underscore_token" in json_doc assert "underscore_span" in json_doc - assert json_doc["underscore_token"]["token_test"]["value"] == 117 - assert json_doc["underscore_span"]["span_test"]["value"] == "span_attribute" + assert json_doc["underscore_token"]["token_test"][0]["value"] == 117 + assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute" assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc @@ -181,8 +185,8 @@ def test_doc_to_json_with_token_span_same_identifier(doc): assert json_doc["_"]["my_ext"] == "hello world" assert "underscore_token" in json_doc assert "underscore_span" in json_doc - assert json_doc["underscore_token"]["my_ext"]["value"] == 117 - assert json_doc["underscore_span"]["my_ext"]["value"] == "span_attribute" + assert json_doc["underscore_token"]["my_ext"][0]["value"] == 117 + assert json_doc["underscore_span"]["my_ext"][0]["value"] == "span_attribute" assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 assert srsly.json_loads(srsly.json_dumps(json_doc)) == json_doc @@ -195,10 +199,9 @@ def test_doc_to_json_with_token_attributes_missing(doc): doc[0]._.token_test = 117 json_doc = doc.to_json(underscore=["span_test"]) - assert "underscore_token" in json_doc assert "underscore_span" in json_doc - assert json_doc["underscore_span"]["span_test"]["value"] == "span_attribute" - assert "token_test" not in json_doc["underscore_token"] + assert json_doc["underscore_span"]["span_test"][0]["value"] == "span_attribute" + assert "underscore_token" not in json_doc assert len(schemas.validate(schemas.DocJSONSchema, json_doc)) == 0 @@ -283,7 +286,9 @@ def test_json_to_doc_with_token_span_attributes(doc): doc._.json_test1 = "hello world" doc._.json_test2 = [1, 2, 3] doc[0:1]._.span_test = "span_attribute" + doc[0:2]._.span_test = "span_attribute_2" doc[0]._.token_test = 117 + doc[1]._.token_test = 118 json_doc = doc.to_json( underscore=["json_test1", "json_test2", "token_test", "span_test"] @@ -295,7 +300,9 @@ def test_json_to_doc_with_token_span_attributes(doc): assert new_doc._.json_test1 == "hello world" assert new_doc._.json_test2 == [1, 2, 3] assert new_doc[0]._.token_test == 117 + assert new_doc[1]._.token_test == 118 assert new_doc[0:1]._.span_test == "span_attribute" + assert new_doc[0:2]._.span_test == "span_attribute_2" assert new_doc.user_data == doc.user_data assert new_doc.to_bytes(exclude=["user_data"]) == doc.to_bytes( exclude=["user_data"] diff --git a/spacy/tokens/doc.pyx b/spacy/tokens/doc.pyx index d7d2fd8e6..295f91c28 100644 --- a/spacy/tokens/doc.pyx +++ b/spacy/tokens/doc.pyx @@ -1608,24 +1608,20 @@ cdef class Doc: Doc.set_extension(attr) self._.set(attr, doc_json["_"][attr]) - if doc_json.get("underscore_token", {}): - for token_attr in doc_json["underscore_token"]: - token_start = doc_json["underscore_token"][token_attr]["token_start"] - value = doc_json["underscore_token"][token_attr]["value"] - - if not Token.has_extension(token_attr): - Token.set_extension(token_attr) - self[token_start]._.set(token_attr, value) + for token_attr in doc_json.get("underscore_token", {}): + if not Token.has_extension(token_attr): + Token.set_extension(token_attr) + for token_data in doc_json["underscore_token"][token_attr]: + start = token_by_char(self.c, self.length, token_data["start"]) + value = token_data["value"] + self[start]._.set(token_attr, value) - if doc_json.get("underscore_span", {}): - for span_attr in doc_json["underscore_span"]: - token_start = doc_json["underscore_span"][span_attr]["token_start"] - token_end = doc_json["underscore_span"][span_attr]["token_end"] - value = doc_json["underscore_span"][span_attr]["value"] - - if not Span.has_extension(span_attr): - Span.set_extension(span_attr) - self[token_start:token_end]._.set(span_attr, value) + for span_attr in doc_json.get("underscore_span", {}): + if not Span.has_extension(span_attr): + Span.set_extension(span_attr) + for span_data in doc_json["underscore_span"][span_attr]: + value = span_data["value"] + self.char_span(span_data["start"], span_data["end"])._.set(span_attr, value) return self def to_json(self, underscore=None): @@ -1673,30 +1669,34 @@ cdef class Doc: if underscore: user_keys = set() if self.user_data: - data["_"] = {} - data["underscore_token"] = {} - data["underscore_span"] = {} - for data_key in self.user_data: + for data_key, value in self.user_data.copy().items(): if type(data_key) == tuple and len(data_key) >= 4 and data_key[0] == "._.": attr = data_key[1] start = data_key[2] end = data_key[3] if attr in underscore: user_keys.add(attr) - value = self.user_data[data_key] if not srsly.is_json_serializable(value): raise ValueError(Errors.E107.format(attr=attr, value=repr(value))) # Check if doc attribute if start is None: + if "_" not in data: + data["_"] = {} data["_"][attr] = value # Check if token attribute elif end is None: + if "underscore_token" not in data: + data["underscore_token"] = {} if attr not in data["underscore_token"]: - data["underscore_token"][attr] = {"token_start": start, "value": value} + data["underscore_token"][attr] = [] + data["underscore_token"][attr].append({"start": start, "value": value}) # Else span attribute else: + if "underscore_span" not in data: + data["underscore_span"] = {} if attr not in data["underscore_span"]: - data["underscore_span"][attr] = {"token_start": start, "token_end": end, "value": value} + data["underscore_span"][attr] = [] + data["underscore_span"][attr].append({"start": start, "end": end, "value": value}) for attr in underscore: if attr not in user_keys: From 3d0e895363921d4acb7f89a5b708472681e6fc1b Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Wed, 19 Oct 2022 17:33:55 +0200 Subject: [PATCH 21/24] Set version to v3.4.2 (#11672) --- spacy/about.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/about.py b/spacy/about.py index 843c15aba..ce86e6294 100644 --- a/spacy/about.py +++ b/spacy/about.py @@ -1,6 +1,6 @@ # fmt: off __title__ = "spacy" -__version__ = "3.4.1" +__version__ = "3.4.2" __download_url__ = "https://github.com/explosion/spacy-models/releases/download" __compatibility__ = "https://raw.githubusercontent.com/explosion/spacy-models/master/compatibility.json" __projects__ = "https://github.com/explosion/projects" From bf83f6872a55e307da289fb901db3c16dd35e8d1 Mon Sep 17 00:00:00 2001 From: Paul O'Leary McCann Date: Thu, 20 Oct 2022 20:35:03 +0900 Subject: [PATCH 22/24] Add detailed example of env dict usage (#11677) * Add detailed example of env dict usage * Mark code blocks as yaml --- website/docs/usage/projects.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/website/docs/usage/projects.md b/website/docs/usage/projects.md index 4797bbfe3..90b612358 100644 --- a/website/docs/usage/projects.md +++ b/website/docs/usage/projects.md @@ -243,6 +243,27 @@ pipelines. > python -m spacy project run test . --vars.foo bar > ``` +> #### Tip: Environment Variables +> +> Commands in a project file are not executed in a shell, so they don't have +> direct access to environment variables. But you can insert environment +> variables using the `env` dictionary to make values available for +> interpolation, just like values in `vars`. Here's an example `env` dict that +> makes `$PATH` available as `ENV_PATH`: +> +> ```yaml +> env: +> ENV_PATH: PATH +> ``` +> +> This can be used in a project command like so: +> +> ```yaml +> - name: "echo-path" +> script: +> - "echo ${env.ENV_PATH}" +> ``` + | Section | Description | | --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `title` | An optional project title used in `--help` message and [auto-generated docs](#custom-docs). | From b69d249a223fa4e633e11babc0830f3b68df57e2 Mon Sep 17 00:00:00 2001 From: Cellan Hall <60790416+Ce11an@users.noreply.github.com> Date: Thu, 20 Oct 2022 12:38:29 +0100 Subject: [PATCH 23/24] Adding `spacy-cleaner` to the spaCy universe (#11674) * added spacy-cleaner to the spaCy universe * Move data to righ section of universe.json * Cleanup - fix typo ("replacers") - spaCy doesn't need to be marked as code - lemma of "Hello" is lower case Co-authored-by: Paul O'Leary McCann --- website/meta/universe.json | 41 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/website/meta/universe.json b/website/meta/universe.json index 637e9d6ce..d7c99956b 100644 --- a/website/meta/universe.json +++ b/website/meta/universe.json @@ -1,5 +1,46 @@ { "resources": [ + { + "id": "spacy-cleaner", + "title": "spacy-cleaner", + "slogan": "Easily clean text with spaCy!", + "description": "**spacy-cleaner** utilises spaCy `Language` models to replace, remove, and \n mutate spaCy tokens. Cleaning actions available are:\n\n* Remove/replace stopwords.\n* Remove/replace punctuation.\n* Remove/replace numbers.\n* Remove/replace emails.\n* Remove/replace URLs.\n* Perform lemmatisation.\n\nSee our [docs](https://ce11an.github.io/spacy-cleaner/) for more information.", + "github": "Ce11an/spacy-cleaner", + "pip": "spacy-cleaner", + "code_example": [ + "import spacy", + "import spacy_cleaner", + "from spacy_cleaner.processing import removers, replacers, mutators", + "", + "model = spacy.load(\"en_core_web_sm\")", + "pipeline = spacy_cleaner.Pipeline(", + " model,", + " removers.remove_stopword_token,", + " replacers.replace_punctuation_token,", + " mutators.mutate_lemma_token,", + ")", + "", + "texts = [\"Hello, my name is Cellan! I love to swim!\"]", + "", + "pipeline.clean(texts)", + "# ['hello _IS_PUNCT_ Cellan _IS_PUNCT_ love swim _IS_PUNCT_']" + ], + "code_language": "python", + "url": "https://ce11an.github.io/spacy-cleaner/", + "image": "https://raw.githubusercontent.com/Ce11an/spacy-cleaner/main/docs/assets/images/spacemen.png", + "author": "Cellan Hall", + "author_links": { + "twitter": "Ce11an", + "github": "Ce11an", + "website": "https://www.linkedin.com/in/cellan-hall/" + }, + "category": [ + "extension" + ], + "tags": [ + "text-processing" + ] + }, { "id": "Zshot", "title": "Zshot", From 84d9cb6b387572293c8bcf26b0e71b508104b165 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 21 Oct 2022 11:54:17 +0200 Subject: [PATCH 24/24] Auto-format code with black (#11687) Co-authored-by: explosion-bot --- spacy/tests/pipeline/test_tok2vec.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/spacy/tests/pipeline/test_tok2vec.py b/spacy/tests/pipeline/test_tok2vec.py index 659274db9..e423d9a19 100644 --- a/spacy/tests/pipeline/test_tok2vec.py +++ b/spacy/tests/pipeline/test_tok2vec.py @@ -231,7 +231,7 @@ def test_tok2vec_listener_callback(): def test_tok2vec_listener_overfitting(): - """ Test that a pipeline with a listener properly overfits, even if 'tok2vec' is in the annotating components """ + """Test that a pipeline with a listener properly overfits, even if 'tok2vec' is in the annotating components""" orig_config = Config().from_str(cfg_string) nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) train_examples = [] @@ -264,7 +264,7 @@ def test_tok2vec_listener_overfitting(): def test_tok2vec_frozen_not_annotating(): - """ Test that a pipeline with a frozen tok2vec raises an error when the tok2vec is not annotating """ + """Test that a pipeline with a frozen tok2vec raises an error when the tok2vec is not annotating""" orig_config = Config().from_str(cfg_string) nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) train_examples = [] @@ -274,12 +274,16 @@ def test_tok2vec_frozen_not_annotating(): for i in range(2): losses = {} - with pytest.raises(ValueError, match=r"the tok2vec embedding layer is not updated"): - nlp.update(train_examples, sgd=optimizer, losses=losses, exclude=["tok2vec"]) + with pytest.raises( + ValueError, match=r"the tok2vec embedding layer is not updated" + ): + nlp.update( + train_examples, sgd=optimizer, losses=losses, exclude=["tok2vec"] + ) def test_tok2vec_frozen_overfitting(): - """ Test that a pipeline with a frozen & annotating tok2vec can still overfit """ + """Test that a pipeline with a frozen & annotating tok2vec can still overfit""" orig_config = Config().from_str(cfg_string) nlp = util.load_model_from_config(orig_config, auto_fill=True, validate=True) train_examples = [] @@ -289,7 +293,13 @@ def test_tok2vec_frozen_overfitting(): for i in range(100): losses = {} - nlp.update(train_examples, sgd=optimizer, losses=losses, exclude=["tok2vec"], annotates=["tok2vec"]) + nlp.update( + train_examples, + sgd=optimizer, + losses=losses, + exclude=["tok2vec"], + annotates=["tok2vec"], + ) assert losses["tagger"] < 0.0001 # test the trained model