Rooba
/
spaCy
mirror of https://github.com/explosion/spaCy.git
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update asset logic and add import-url

This commit is contained in:
Ines Montani 2020-06-28 13:07:31 +02:00
parent d6aa4cb478
commit f385344286
1 changed files with 12 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
spacy/cli/project.py
View File

@ -177,20 +177,27 @@ def fetch_asset(
dest_path = (project_path / dest).resolve() dest_path = (project_path / dest).resolve()
if dest_path.exists() and checksum: if dest_path.exists() and checksum:
# If there's already a file, check for checksum # If there's already a file, check for checksum
# TODO: add support for caches # TODO: add support for caches (dvc import-url with local path)
if checksum == get_checksum(dest_path): if checksum == get_checksum(dest_path):
msg.good(f"Skipping download with matching checksum: {dest}") msg.good(f"Skipping download with matching checksum: {dest}")
return return
with working_dir(project_path): with working_dir(project_path):
try: try:
# If these fail, we don't want to output an error or info message.
# Try with tracking the source first, then just downloading with
# DVC, then a regular non-DVC download.
dvc_cmd = ["dvc", "import-url", url, str(dest_path)]
print(subprocess.check_output(dvc_cmd, stderr=subprocess.DEVNULL))
except subprocess.CalledProcessError:
dvc_cmd = ["dvc", "get-url", url, str(dest_path)] dvc_cmd = ["dvc", "get-url", url, str(dest_path)]
# If this fails, we don't want to output an error or info message print(subprocess.check_output(dvc_cmd, stderr=subprocess.DEVNULL))
out = subprocess.check_output(dvc_cmd, stderr=subprocess.DEVNULL) run_command(["dvc", "add", str(dest_path)])
print(out)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
# TODO: replace curl # TODO: replace curl
run_command(["curl", url, "--output", str(dest_path), "--progress-bar"]) run_command(["curl", url, "--output", str(dest_path), "--progress-bar"])
run_command(["dvc", "add", str(dest_path)]) run_command(["dvc", "add", str(dest_path)])
if checksum and checksum != get_checksum(dest_path):
msg.warn(f"Checksum doesn't match value defined in {CONFIG_FILE}: {dest}")
msg.good(f"Fetched asset {dest}") msg.good(f"Fetched asset {dest}")