explosion/spaCy
1
1
Fork 0
mirror of https://github.com/explosion/spaCy.git synced 2025-10-31 07:57:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix up/download of http and local paths

Browse Source
This commit is contained in:
Matthew Honnibal 2020-08-24 20:11:33 +02:00
parent f232d8db96
commit 99d31e7662
1 changed files with 31 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
spacy/cli/_util.py
View File

@ -260,10 +260,22 @@ def upload_file(src: Path, dest: Union[str, "Pathy"]) -> None:
src (Path): The source path. src (Path): The source path.
url (str): The destination URL to upload to. url (str): The destination URL to upload to.
""" """
dest = ensure_pathy(dest) import smart_open
with dest.open(mode="wb") as output_file: # This logic is pretty hacky. We'd like pathy to do this probably?
with src.open(mode="rb") as input_file: if ":/" not in str(dest):
output_file.write(input_file.read()) # Local path
with Path(dest).open(mode="wb") as output_file:
with src.open(mode="rb") as input_file:
output_file.write(input_file.read())
elif str(dest).startswith("http") or str(dest).startswith("https"):
with smart_open.open(str(dest), mode="wb") as output_file:
with src.open(mode="rb") as input_file:
output_file.write(input_file.read())
else:
dest = ensure_pathy(dest)
with dest.open(mode="wb") as output_file:
with src.open(mode="rb") as input_file:
output_file.write(input_file.read())
def download_file(src: Union[str, "Pathy"], dest: Path, *, force: bool = False) -> None: def download_file(src: Union[str, "Pathy"], dest: Path, *, force: bool = False) -> None:
@ -274,12 +286,23 @@ def download_file(src: Union[str, "Pathy"], dest: Path, *, force: bool = False)
force (bool): Whether to force download even if file exists. force (bool): Whether to force download even if file exists.
If False, the download will be skipped. If False, the download will be skipped.
""" """
import smart_open
# This logic is pretty hacky. We'd like pathy to do this probably?
if dest.exists() and not force: if dest.exists() and not force:
return None return None
src = ensure_pathy(src) if src.startswith("http"):
with src.open(mode="rb") as input_file: with smart_open.open(src, mode="rb") as input_file:
with dest.open(mode="wb") as output_file: with dest.open(mode="wb") as output_file:
output_file.write(input_file.read()) output_file.write(input_file.read())
elif ":/" not in src:
with open(src, mode="rb") as input_file:
with dest.open(mode="wb") as output_file:
output_file.write(input_file.read())
else:
src = ensure_pathy(src)
with src.open(mode="rb") as input_file:
with dest.open(mode="wb") as output_file:
output_file.write(input_file.read())
def ensure_pathy(path): def ensure_pathy(path):