explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2024-11-10 19:57:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

Always serialize lookups and vectors to disk

Browse Source
This commit is contained in:
Adriane Boyd 2020-10-05 09:33:05 +02:00
parent 549758f67d
commit 03cfb2d2f4
2 changed files with 8 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
spacy/lookups.py
View File

@ -289,13 +289,12 @@ class Lookups:
DOCS: https://nightly.spacy.io/api/lookups#to_disk DOCS: https://nightly.spacy.io/api/lookups#to_disk
""" """
if len(self._tables): path = ensure_path(path)
path = ensure_path(path) if not path.exists():
if not path.exists(): path.mkdir()
path.mkdir() filepath = path / filename
filepath = path / filename with filepath.open("wb") as file_:
with filepath.open("wb") as file_: file_.write(self.to_bytes())
file_.write(self.to_bytes())
def from_disk( def from_disk(
self, path: Union[str, Path], filename: str = "lookups.bin", **kwargs self, path: Union[str, Path], filename: str = "lookups.bin", **kwargs

4
spacy/vocab.pyx
View File

@ -445,9 +445,9 @@ cdef class Vocab:
setters = ["strings", "vectors"] setters = ["strings", "vectors"]
if "strings" not in exclude: if "strings" not in exclude:
self.strings.to_disk(path / "strings.json") self.strings.to_disk(path / "strings.json")
if "vectors" not in "exclude" and self.vectors is not None: if "vectors" not in "exclude":
self.vectors.to_disk(path) self.vectors.to_disk(path)
if "lookups" not in "exclude" and self.lookups is not None: if "lookups" not in "exclude":
self.lookups.to_disk(path) self.lookups.to_disk(path)
def from_disk(self, path, *, exclude=tuple()): def from_disk(self, path, *, exclude=tuple()):