explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-06-30 01:43:21 +03:00
Code Issues Packages Projects Releases Wiki Activity

Auto-fill [nlp] on load from config and from bytes/disk

Browse Source
This commit is contained in:
Adriane Boyd 2023-04-02 11:56:21 +02:00
parent 7b36e7c9ec
commit 431c2ecd78
2 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
spacy/language.py
View File

@ -2101,6 +2101,13 @@ class Language:
DOCS: https://spacy.io/api/language#from_disk DOCS: https://spacy.io/api/language#from_disk
""" """
def deserialize_config(path: Path) -> None:
if path.exists():
config = Config().from_disk(
path, interpolate=False, overrides=overrides
)
self.config.merge(config)
def deserialize_meta(path: Path) -> None: def deserialize_meta(path: Path) -> None:
if path.exists(): if path.exists():
data = srsly.read_json(path) data = srsly.read_json(path)
@ -2115,12 +2122,9 @@ class Language:
path = util.ensure_path(path) path = util.ensure_path(path)
deserializers = {} deserializers = {}
if Path(path / "config.cfg").exists(): # type: ignore[operator] deserializers["config.cfg"] = deserialize_config
deserializers["config.cfg"] = lambda p: self.config.from_disk( deserializers["meta.json"] = deserialize_meta
p, interpolate=False, overrides=overrides deserializers["vocab"] = deserialize_vocab
)
deserializers["meta.json"] = deserialize_meta # type: ignore[assignment]
deserializers["vocab"] = deserialize_vocab # type: ignore[assignment]
deserializers["tokenizer"] = lambda p: self.tokenizer.from_disk( # type: ignore[union-attr] deserializers["tokenizer"] = lambda p: self.tokenizer.from_disk( # type: ignore[union-attr]
p, exclude=["vocab"] p, exclude=["vocab"]
) )
@ -2173,6 +2177,10 @@ class Language:
DOCS: https://spacy.io/api/language#from_bytes DOCS: https://spacy.io/api/language#from_bytes
""" """
def deserialize_config(b):
config = Config().from_bytes(b, interpolate=False)
self.config.merge(config)
def deserialize_meta(b): def deserialize_meta(b):
data = srsly.json_loads(b) data = srsly.json_loads(b)
self.meta.update(data) self.meta.update(data)
@ -2181,9 +2189,7 @@ class Language:
self.vocab.vectors.name = data.get("vectors", {}).get("name") self.vocab.vectors.name = data.get("vectors", {}).get("name")
deserializers: Dict[str, Callable[[bytes], Any]] = {} deserializers: Dict[str, Callable[[bytes], Any]] = {}
deserializers["config.cfg"] = lambda b: self.config.from_bytes( deserializers["config.cfg"] = deserialize_config
b, interpolate=False
)
deserializers["meta.json"] = deserialize_meta deserializers["meta.json"] = deserialize_meta
deserializers["vocab"] = lambda b: self.vocab.from_bytes(b, exclude=exclude) deserializers["vocab"] = lambda b: self.vocab.from_bytes(b, exclude=exclude)
deserializers["tokenizer"] = lambda b: self.tokenizer.from_bytes( # type: ignore[union-attr] deserializers["tokenizer"] = lambda b: self.tokenizer.from_bytes( # type: ignore[union-attr]

2
spacy/util.py
View File

@ -532,7 +532,7 @@ def load_model_from_config(
disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, disable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, enable: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES, exclude: Union[str, Iterable[str]] = _DEFAULT_EMPTY_PIPES,
auto_fill: bool = False, auto_fill: bool = True,
validate: bool = True, validate: bool = True,
) -> "Language": ) -> "Language":
"""Create an nlp object from a config. Expects the full config file including """Create an nlp object from a config. Expects the full config file including