2020-08-23 16:32:09 +00:00
|
|
|
from typing import Dict, Any, Union, List, Optional, TYPE_CHECKING
|
|
|
|
import sys
|
2020-08-24 22:30:52 +00:00
|
|
|
import shutil
|
2020-07-08 23:42:51 +00:00
|
|
|
from pathlib import Path
|
|
|
|
from wasabi import msg
|
|
|
|
import srsly
|
2020-07-09 21:51:18 +00:00
|
|
|
import hashlib
|
2020-07-10 15:57:40 +00:00
|
|
|
import typer
|
|
|
|
from typer.main import get_command
|
2020-07-10 21:34:17 +00:00
|
|
|
from contextlib import contextmanager
|
2020-08-04 21:39:19 +00:00
|
|
|
from thinc.config import Config, ConfigValidationError
|
2020-07-10 21:34:17 +00:00
|
|
|
from configparser import InterpolationError
|
2020-07-08 23:42:51 +00:00
|
|
|
|
2020-07-10 15:57:40 +00:00
|
|
|
from ..schemas import ProjectConfigSchema, validate
|
2020-08-24 22:30:52 +00:00
|
|
|
from ..util import import_file, run_command, make_tempdir
|
2020-07-08 23:42:51 +00:00
|
|
|
|
2020-08-23 16:32:09 +00:00
|
|
|
if TYPE_CHECKING:
|
|
|
|
from pathy import Pathy # noqa: F401
|
|
|
|
|
2020-07-08 23:42:51 +00:00
|
|
|
|
|
|
|
PROJECT_FILE = "project.yml"
|
|
|
|
PROJECT_LOCK = "project.lock"
|
2020-07-10 15:57:40 +00:00
|
|
|
COMMAND = "python -m spacy"
|
|
|
|
NAME = "spacy"
|
|
|
|
HELP = """spaCy Command-line Interface
|
|
|
|
|
|
|
|
DOCS: https://spacy.io/api/cli
|
|
|
|
"""
|
2020-07-12 11:53:41 +00:00
|
|
|
PROJECT_HELP = f"""Command-line interface for spaCy projects and templates.
|
|
|
|
You'd typically start by cloning a project template to a local directory and
|
|
|
|
fetching its assets like datasets etc. See the project's {PROJECT_FILE} for the
|
|
|
|
available commands.
|
|
|
|
"""
|
|
|
|
DEBUG_HELP = """Suite of helpful commands for debugging and profiling. Includes
|
|
|
|
commands to check and validate your config files, training and evaluation data,
|
|
|
|
and custom model implementations.
|
2020-07-10 15:57:40 +00:00
|
|
|
"""
|
2020-08-02 13:18:30 +00:00
|
|
|
INIT_HELP = """Commands for initializing configs and models."""
|
2020-07-10 15:57:40 +00:00
|
|
|
|
|
|
|
# Wrappers for Typer's annotations. Initially created to set defaults and to
|
|
|
|
# keep the names short, but not needed at the moment.
|
|
|
|
Arg = typer.Argument
|
|
|
|
Opt = typer.Option
|
|
|
|
|
|
|
|
app = typer.Typer(name=NAME, help=HELP)
|
|
|
|
project_cli = typer.Typer(name="project", help=PROJECT_HELP, no_args_is_help=True)
|
2020-07-12 11:53:41 +00:00
|
|
|
debug_cli = typer.Typer(name="debug", help=DEBUG_HELP, no_args_is_help=True)
|
2020-08-02 13:18:30 +00:00
|
|
|
init_cli = typer.Typer(name="init", help=INIT_HELP, no_args_is_help=True)
|
2020-07-12 11:53:41 +00:00
|
|
|
|
2020-07-10 15:57:40 +00:00
|
|
|
app.add_typer(project_cli)
|
2020-07-12 11:53:41 +00:00
|
|
|
app.add_typer(debug_cli)
|
2020-08-02 13:18:30 +00:00
|
|
|
app.add_typer(init_cli)
|
2020-07-10 15:57:40 +00:00
|
|
|
|
|
|
|
|
|
|
|
def setup_cli() -> None:
|
|
|
|
# Ensure that the help messages always display the correct prompt
|
|
|
|
command = get_command(app)
|
|
|
|
command(prog_name=COMMAND)
|
|
|
|
|
|
|
|
|
|
|
|
def parse_config_overrides(args: List[str]) -> Dict[str, Any]:
|
|
|
|
"""Generate a dictionary of config overrides based on the extra arguments
|
|
|
|
provided on the CLI, e.g. --training.batch_size to override
|
|
|
|
"training.batch_size". Arguments without a "." are considered invalid,
|
|
|
|
since the config only allows top-level sections to exist.
|
|
|
|
|
|
|
|
args (List[str]): The extra arguments from the command line.
|
|
|
|
RETURNS (Dict[str, Any]): The parsed dict, keyed by nested config setting.
|
|
|
|
"""
|
|
|
|
result = {}
|
|
|
|
while args:
|
|
|
|
opt = args.pop(0)
|
2020-07-22 11:42:59 +00:00
|
|
|
err = f"Invalid CLI argument '{opt}'"
|
2020-07-10 15:57:40 +00:00
|
|
|
if opt.startswith("--"): # new argument
|
2020-08-18 14:06:37 +00:00
|
|
|
opt = opt.replace("--", "")
|
2020-07-10 15:57:40 +00:00
|
|
|
if "." not in opt:
|
|
|
|
msg.fail(f"{err}: can't override top-level section", exits=1)
|
2020-07-28 11:43:15 +00:00
|
|
|
if "=" in opt: # we have --opt=value
|
|
|
|
opt, value = opt.split("=", 1)
|
2020-08-18 14:06:37 +00:00
|
|
|
opt = opt.replace("-", "_")
|
2020-07-10 15:57:40 +00:00
|
|
|
else:
|
2020-07-28 11:43:15 +00:00
|
|
|
if not args or args[0].startswith("--"): # flag with no value
|
|
|
|
value = "true"
|
|
|
|
else:
|
|
|
|
value = args.pop(0)
|
2020-07-10 15:57:40 +00:00
|
|
|
# Just like we do in the config, we're calling json.loads on the
|
2020-07-22 11:42:59 +00:00
|
|
|
# values. But since they come from the CLI, it'd be unintuitive to
|
2020-07-10 15:57:40 +00:00
|
|
|
# explicitly mark strings with escaped quotes. So we're working
|
|
|
|
# around that here by falling back to a string if parsing fails.
|
2020-07-10 16:20:52 +00:00
|
|
|
# TODO: improve logic to handle simple types like list of strings?
|
2020-07-10 15:57:40 +00:00
|
|
|
try:
|
|
|
|
result[opt] = srsly.json_loads(value)
|
|
|
|
except ValueError:
|
|
|
|
result[opt] = str(value)
|
|
|
|
else:
|
2020-07-22 11:42:59 +00:00
|
|
|
msg.fail(f"{err}: override option should start with --", exits=1)
|
2020-07-10 15:57:40 +00:00
|
|
|
return result
|
2020-07-08 23:42:51 +00:00
|
|
|
|
|
|
|
|
2020-08-23 16:32:09 +00:00
|
|
|
def load_project_config(path: Path, interpolate: bool = True) -> Dict[str, Any]:
|
2020-07-09 21:51:18 +00:00
|
|
|
"""Load the project.yml file from a directory and validate it. Also make
|
|
|
|
sure that all directories defined in the config exist.
|
2020-07-08 23:42:51 +00:00
|
|
|
|
|
|
|
path (Path): The path to the project directory.
|
2020-08-23 16:32:09 +00:00
|
|
|
interpolate (bool): Whether to substitute project variables.
|
2020-07-08 23:42:51 +00:00
|
|
|
RETURNS (Dict[str, Any]): The loaded project.yml.
|
|
|
|
"""
|
|
|
|
config_path = path / PROJECT_FILE
|
|
|
|
if not config_path.exists():
|
|
|
|
msg.fail(f"Can't find {PROJECT_FILE}", config_path, exits=1)
|
|
|
|
invalid_err = f"Invalid {PROJECT_FILE}. Double-check that the YAML is correct."
|
|
|
|
try:
|
|
|
|
config = srsly.read_yaml(config_path)
|
|
|
|
except ValueError as e:
|
|
|
|
msg.fail(invalid_err, e, exits=1)
|
|
|
|
errors = validate(ProjectConfigSchema, config)
|
|
|
|
if errors:
|
2020-08-23 10:14:02 +00:00
|
|
|
msg.fail(invalid_err)
|
|
|
|
print("\n".join(errors))
|
|
|
|
sys.exit(1)
|
2020-07-08 23:42:51 +00:00
|
|
|
validate_project_commands(config)
|
2020-07-09 21:51:18 +00:00
|
|
|
# Make sure directories defined in config exist
|
|
|
|
for subdir in config.get("directories", []):
|
|
|
|
dir_path = path / subdir
|
|
|
|
if not dir_path.exists():
|
|
|
|
dir_path.mkdir(parents=True)
|
2020-08-23 16:32:09 +00:00
|
|
|
if interpolate:
|
|
|
|
err = "project.yml validation error"
|
|
|
|
with show_validation_error(title=err, hint_fill=False):
|
|
|
|
config = substitute_project_variables(config)
|
2020-07-08 23:42:51 +00:00
|
|
|
return config
|
|
|
|
|
|
|
|
|
2020-08-23 16:32:09 +00:00
|
|
|
def substitute_project_variables(config: Dict[str, Any], overrides: Dict = {}):
|
|
|
|
key = "vars"
|
|
|
|
config.setdefault(key, {})
|
|
|
|
config[key].update(overrides)
|
|
|
|
# Need to put variables in the top scope again so we can have a top-level
|
|
|
|
# section "project" (otherwise, a list of commands in the top scope wouldn't)
|
|
|
|
# be allowed by Thinc's config system
|
|
|
|
cfg = Config({"project": config, key: config[key]})
|
|
|
|
interpolated = cfg.interpolate()
|
|
|
|
return dict(interpolated["project"])
|
|
|
|
|
|
|
|
|
2020-07-08 23:42:51 +00:00
|
|
|
def validate_project_commands(config: Dict[str, Any]) -> None:
|
|
|
|
"""Check that project commands and workflows are valid, don't contain
|
|
|
|
duplicates, don't clash and only refer to commands that exist.
|
|
|
|
|
|
|
|
config (Dict[str, Any]): The loaded config.
|
|
|
|
"""
|
|
|
|
command_names = [cmd["name"] for cmd in config.get("commands", [])]
|
|
|
|
workflows = config.get("workflows", {})
|
|
|
|
duplicates = set([cmd for cmd in command_names if command_names.count(cmd) > 1])
|
|
|
|
if duplicates:
|
|
|
|
err = f"Duplicate commands defined in {PROJECT_FILE}: {', '.join(duplicates)}"
|
|
|
|
msg.fail(err, exits=1)
|
|
|
|
for workflow_name, workflow_steps in workflows.items():
|
|
|
|
if workflow_name in command_names:
|
|
|
|
err = f"Can't use workflow name '{workflow_name}': name already exists as a command"
|
|
|
|
msg.fail(err, exits=1)
|
|
|
|
for step in workflow_steps:
|
|
|
|
if step not in command_names:
|
|
|
|
msg.fail(
|
|
|
|
f"Unknown command specified in workflow '{workflow_name}': {step}",
|
|
|
|
f"Workflows can only refer to commands defined in the 'commands' "
|
|
|
|
f"section of the {PROJECT_FILE}.",
|
|
|
|
exits=1,
|
|
|
|
)
|
2020-07-09 21:51:18 +00:00
|
|
|
|
|
|
|
|
|
|
|
def get_hash(data) -> str:
|
|
|
|
"""Get the hash for a JSON-serializable object.
|
|
|
|
|
|
|
|
data: The data to hash.
|
|
|
|
RETURNS (str): The hash.
|
|
|
|
"""
|
|
|
|
data_str = srsly.json_dumps(data, sort_keys=True).encode("utf8")
|
|
|
|
return hashlib.md5(data_str).hexdigest()
|
|
|
|
|
|
|
|
|
|
|
|
def get_checksum(path: Union[Path, str]) -> str:
|
|
|
|
"""Get the checksum for a file or directory given its file path. If a
|
|
|
|
directory path is provided, this uses all files in that directory.
|
|
|
|
|
|
|
|
path (Union[Path, str]): The file or directory path.
|
|
|
|
RETURNS (str): The checksum.
|
|
|
|
"""
|
|
|
|
path = Path(path)
|
|
|
|
if path.is_file():
|
|
|
|
return hashlib.md5(Path(path).read_bytes()).hexdigest()
|
|
|
|
if path.is_dir():
|
|
|
|
# TODO: this is currently pretty slow
|
|
|
|
dir_checksum = hashlib.md5()
|
|
|
|
for sub_file in sorted(fp for fp in path.rglob("*") if fp.is_file()):
|
|
|
|
dir_checksum.update(sub_file.read_bytes())
|
|
|
|
return dir_checksum.hexdigest()
|
2020-08-25 09:54:53 +00:00
|
|
|
msg.fail(f"Can't get checksum for {path}: not a file or directory", exits=1)
|
2020-07-10 21:34:17 +00:00
|
|
|
|
|
|
|
|
|
|
|
@contextmanager
|
2020-08-02 13:18:30 +00:00
|
|
|
def show_validation_error(
|
|
|
|
file_path: Optional[Union[str, Path]] = None,
|
|
|
|
*,
|
|
|
|
title: str = "Config validation error",
|
2020-08-14 14:49:26 +00:00
|
|
|
hint_fill: bool = True,
|
2020-08-02 13:18:30 +00:00
|
|
|
):
|
2020-07-10 21:34:17 +00:00
|
|
|
"""Helper to show custom config validation errors on the CLI.
|
|
|
|
|
2020-08-02 13:18:30 +00:00
|
|
|
file_path (str / Path): Optional file path of config file, used in hints.
|
2020-07-10 21:34:17 +00:00
|
|
|
title (str): Title of the custom formatted error.
|
2020-08-14 14:49:26 +00:00
|
|
|
hint_fill (bool): Show hint about filling config.
|
2020-07-10 21:34:17 +00:00
|
|
|
"""
|
|
|
|
try:
|
|
|
|
yield
|
|
|
|
except (ConfigValidationError, InterpolationError) as e:
|
|
|
|
msg.fail(title, spaced=True)
|
2020-08-02 13:18:30 +00:00
|
|
|
# TODO: This is kinda hacky and we should probably provide a better
|
|
|
|
# helper for this in Thinc
|
|
|
|
err_text = str(e).replace("Config validation error", "").strip()
|
|
|
|
print(err_text)
|
2020-08-14 14:49:26 +00:00
|
|
|
if hint_fill and "field required" in err_text:
|
2020-08-02 13:18:30 +00:00
|
|
|
config_path = file_path if file_path is not None else "config.cfg"
|
|
|
|
msg.text(
|
|
|
|
"If your config contains missing values, you can run the 'init "
|
2020-08-14 14:49:26 +00:00
|
|
|
"fill-config' command to fill in all the defaults, if possible:",
|
2020-08-02 13:18:30 +00:00
|
|
|
spaced=True,
|
|
|
|
)
|
2020-08-14 14:49:26 +00:00
|
|
|
print(f"{COMMAND} init fill-config {config_path} --base {config_path}\n")
|
2020-07-10 21:34:17 +00:00
|
|
|
sys.exit(1)
|
2020-07-11 11:03:53 +00:00
|
|
|
|
|
|
|
|
|
|
|
def import_code(code_path: Optional[Union[Path, str]]) -> None:
|
|
|
|
"""Helper to import Python file provided in training commands / commands
|
|
|
|
using the config. This makes custom registered functions available.
|
|
|
|
"""
|
|
|
|
if code_path is not None:
|
|
|
|
if not Path(code_path).exists():
|
|
|
|
msg.fail("Path to Python code not found", code_path, exits=1)
|
|
|
|
try:
|
|
|
|
import_file("python_code", code_path)
|
|
|
|
except Exception as e:
|
|
|
|
msg.fail(f"Couldn't load Python code: {code_path}", e, exits=1)
|
2020-08-04 21:39:19 +00:00
|
|
|
|
|
|
|
|
|
|
|
def get_sourced_components(config: Union[Dict[str, Any], Config]) -> List[str]:
|
|
|
|
"""RETURNS (List[str]): All sourced components in the original config,
|
|
|
|
e.g. {"source": "en_core_web_sm"}. If the config contains a key
|
|
|
|
"factory", we assume it refers to a component factory.
|
|
|
|
"""
|
|
|
|
return [
|
|
|
|
name
|
|
|
|
for name, cfg in config.get("components", {}).items()
|
|
|
|
if "factory" not in cfg and "source" in cfg
|
|
|
|
]
|
2020-08-23 16:32:09 +00:00
|
|
|
|
|
|
|
|
|
|
|
def upload_file(src: Path, dest: Union[str, "Pathy"]) -> None:
|
|
|
|
"""Upload a file.
|
|
|
|
|
|
|
|
src (Path): The source path.
|
|
|
|
url (str): The destination URL to upload to.
|
|
|
|
"""
|
2020-08-24 22:30:52 +00:00
|
|
|
import smart_open
|
|
|
|
|
|
|
|
# This logic is pretty hacky. We'd like pathy to do this probably?
|
|
|
|
if ":/" not in str(dest):
|
|
|
|
# Local path
|
|
|
|
with Path(dest).open(mode="wb") as output_file:
|
|
|
|
with src.open(mode="rb") as input_file:
|
|
|
|
output_file.write(input_file.read())
|
|
|
|
elif str(dest).startswith("http") or str(dest).startswith("https"):
|
|
|
|
with smart_open.open(str(dest), mode="wb") as output_file:
|
|
|
|
with src.open(mode="rb") as input_file:
|
|
|
|
output_file.write(input_file.read())
|
|
|
|
else:
|
|
|
|
dest = ensure_pathy(dest)
|
|
|
|
with dest.open(mode="wb") as output_file:
|
|
|
|
with src.open(mode="rb") as input_file:
|
|
|
|
output_file.write(input_file.read())
|
2020-08-23 16:32:09 +00:00
|
|
|
|
|
|
|
|
|
|
|
def download_file(src: Union[str, "Pathy"], dest: Path, *, force: bool = False) -> None:
|
|
|
|
"""Download a file using smart_open.
|
|
|
|
|
|
|
|
url (str): The URL of the file.
|
|
|
|
dest (Path): The destination path.
|
|
|
|
force (bool): Whether to force download even if file exists.
|
|
|
|
If False, the download will be skipped.
|
|
|
|
"""
|
2020-08-24 22:30:52 +00:00
|
|
|
import smart_open
|
|
|
|
|
|
|
|
# This logic is pretty hacky. We'd like pathy to do this probably?
|
2020-08-23 16:32:09 +00:00
|
|
|
if dest.exists() and not force:
|
|
|
|
return None
|
2020-08-24 22:30:52 +00:00
|
|
|
if src.startswith("http"):
|
|
|
|
with smart_open.open(src, mode="rb") as input_file:
|
|
|
|
with dest.open(mode="wb") as output_file:
|
|
|
|
output_file.write(input_file.read())
|
|
|
|
elif ":/" not in src:
|
|
|
|
with open(src, mode="rb") as input_file:
|
|
|
|
with dest.open(mode="wb") as output_file:
|
|
|
|
output_file.write(input_file.read())
|
|
|
|
else:
|
|
|
|
src = ensure_pathy(src)
|
|
|
|
with src.open(mode="rb") as input_file:
|
|
|
|
with dest.open(mode="wb") as output_file:
|
|
|
|
output_file.write(input_file.read())
|
2020-08-23 16:32:09 +00:00
|
|
|
|
|
|
|
|
|
|
|
def ensure_pathy(path):
|
|
|
|
"""Temporary helper to prevent importing Pathy globally (which can cause
|
|
|
|
slow and annoying Google Cloud warning)."""
|
|
|
|
from pathy import Pathy # noqa: F811
|
|
|
|
|
|
|
|
return Pathy(path)
|
2020-08-24 22:30:52 +00:00
|
|
|
|
|
|
|
|
|
|
|
def git_sparse_checkout(
|
|
|
|
repo: str, subpath: str, dest: Path, *, branch: Optional[str] = None
|
|
|
|
):
|
|
|
|
if dest.exists():
|
2020-08-25 09:54:53 +00:00
|
|
|
msg.fail("Destination of checkout must not exist", exits=1)
|
2020-08-24 22:30:52 +00:00
|
|
|
if not dest.parent.exists():
|
2020-08-26 02:00:14 +00:00
|
|
|
raise IOError("Parent of destination of checkout must exist")
|
|
|
|
# We're using Git, partial clone and sparse checkout to
|
|
|
|
# only clone the files we need
|
|
|
|
# This ends up being RIDICULOUS. omg.
|
|
|
|
# So, every tutorial and SO post talks about 'sparse checkout'...But they
|
|
|
|
# go and *clone* the whole repo. Worthless. And cloning part of a repo
|
|
|
|
# turns out to be completely broken. The only way to specify a "path" is..
|
|
|
|
# a path *on the server*? The contents of which, specifies the paths. Wat.
|
|
|
|
# Obviously this is hopelessly broken and insecure, because you can query
|
|
|
|
# arbitrary paths on the server! So nobody enables this.
|
|
|
|
# What we have to do is disable *all* files. We could then just checkout
|
|
|
|
# the path, and it'd "work", but be hopelessly slow...Because it goes and
|
|
|
|
# transfers every missing object one-by-one. So the final piece is that we
|
|
|
|
# need to use some weird git internals to fetch the missings in bulk, and
|
|
|
|
# *that* we can do by path.
|
2020-08-24 22:30:52 +00:00
|
|
|
# We're using Git and sparse checkout to only clone the files we need
|
|
|
|
with make_tempdir() as tmp_dir:
|
2020-08-26 02:00:14 +00:00
|
|
|
# This is the "clone, but don't download anything" part.
|
2020-08-24 22:30:52 +00:00
|
|
|
cmd = (
|
2020-08-26 02:00:14 +00:00
|
|
|
f"git clone {repo} {tmp_dir} --no-checkout --depth 1 "
|
|
|
|
"--filter=blob:none" # <-- The key bit
|
2020-08-24 22:30:52 +00:00
|
|
|
)
|
|
|
|
if branch is not None:
|
|
|
|
cmd = f"{cmd} -b {branch}"
|
2020-08-26 02:00:14 +00:00
|
|
|
run_command(cmd, capture=True)
|
|
|
|
# Now we need to find the missing filenames for the subpath we want.
|
|
|
|
# Looking for this 'rev-list' command in the git --help? Hah.
|
|
|
|
cmd = f"git -C {tmp_dir} rev-list --objects --all --missing=print -- {subpath}"
|
|
|
|
ret = run_command(cmd, capture=True)
|
|
|
|
missings = "\n".join([x[1:] for x in ret.stdout.split() if x.startswith("?")])
|
|
|
|
# Now pass those missings into another bit of git internals
|
|
|
|
run_command(
|
|
|
|
f"git -C {tmp_dir} fetch-pack --stdin {repo}", capture=True, stdin=missings
|
|
|
|
)
|
|
|
|
# And finally, we can checkout our subpath
|
|
|
|
run_command(f"git -C {tmp_dir} checkout {branch} {subpath}")
|
2020-08-24 22:30:52 +00:00
|
|
|
# We need Path(name) to make sure we also support subdirectories
|
|
|
|
shutil.move(str(tmp_dir / Path(subpath)), str(dest))
|