2022-09-16 14:54:31 +00:00
|
|
|
import os.path
|
2023-06-14 15:48:41 +00:00
|
|
|
import sys
|
2020-07-08 23:42:51 +00:00
|
|
|
from pathlib import Path
|
2023-06-14 15:48:41 +00:00
|
|
|
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
|
2022-09-16 14:54:31 +00:00
|
|
|
|
2020-07-08 23:42:51 +00:00
|
|
|
import srsly
|
2021-02-10 02:45:27 +00:00
|
|
|
import typer
|
2023-06-14 15:48:41 +00:00
|
|
|
from wasabi import msg
|
|
|
|
from wasabi.util import locale_escape
|
2020-07-08 23:42:51 +00:00
|
|
|
|
2020-10-05 11:53:07 +00:00
|
|
|
from ... import about
|
|
|
|
from ...git_info import GIT_VERSION
|
2023-06-14 15:48:41 +00:00
|
|
|
from ...util import (
|
|
|
|
ENV_VARS,
|
|
|
|
SimpleFrozenDict,
|
|
|
|
SimpleFrozenList,
|
|
|
|
check_bool_env_var,
|
|
|
|
is_cwd,
|
|
|
|
is_minor_version_match,
|
|
|
|
join_command,
|
|
|
|
run_command,
|
|
|
|
split_command,
|
|
|
|
working_dir,
|
|
|
|
)
|
|
|
|
from .._util import (
|
|
|
|
COMMAND,
|
|
|
|
PROJECT_FILE,
|
|
|
|
PROJECT_LOCK,
|
|
|
|
Arg,
|
|
|
|
Opt,
|
|
|
|
get_checksum,
|
|
|
|
get_hash,
|
|
|
|
load_project_config,
|
|
|
|
parse_config_overrides,
|
|
|
|
project_cli,
|
|
|
|
)
|
2020-07-08 23:42:51 +00:00
|
|
|
|
|
|
|
|
2021-02-10 02:45:27 +00:00
|
|
|
@project_cli.command(
|
|
|
|
"run", context_settings={"allow_extra_args": True, "ignore_unknown_options": True}
|
|
|
|
)
|
2020-07-08 23:42:51 +00:00
|
|
|
def project_run_cli(
|
|
|
|
# fmt: off
|
2021-02-10 02:45:27 +00:00
|
|
|
ctx: typer.Context, # This is only used to read additional arguments
|
2020-07-08 23:42:51 +00:00
|
|
|
subcommand: str = Arg(None, help=f"Name of command defined in the {PROJECT_FILE}"),
|
|
|
|
project_dir: Path = Arg(Path.cwd(), help="Location of project directory. Defaults to current working directory.", exists=True, file_okay=False),
|
|
|
|
force: bool = Opt(False, "--force", "-F", help="Force re-running steps, even if nothing changed"),
|
2020-07-09 17:44:28 +00:00
|
|
|
dry: bool = Opt(False, "--dry", "-D", help="Perform a dry run and don't execute scripts"),
|
2020-07-08 23:42:51 +00:00
|
|
|
show_help: bool = Opt(False, "--help", help="Show help message and available subcommands")
|
|
|
|
# fmt: on
|
|
|
|
):
|
2020-07-09 17:44:28 +00:00
|
|
|
"""Run a named command or workflow defined in the project.yml. If a workflow
|
2020-07-08 23:42:51 +00:00
|
|
|
name is specified, all commands in the workflow are run, in order. If
|
2020-07-09 17:44:28 +00:00
|
|
|
commands define dependencies and/or outputs, they will only be re-run if
|
|
|
|
state has changed.
|
2020-09-04 10:58:50 +00:00
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
DOCS: https://spacy.io/api/cli#project-run
|
2020-07-08 23:42:51 +00:00
|
|
|
"""
|
|
|
|
if show_help or not subcommand:
|
|
|
|
print_run_help(project_dir, subcommand)
|
|
|
|
else:
|
2021-02-10 02:45:27 +00:00
|
|
|
overrides = parse_config_overrides(ctx.args)
|
|
|
|
project_run(project_dir, subcommand, overrides=overrides, force=force, dry=dry)
|
2020-07-08 23:42:51 +00:00
|
|
|
|
|
|
|
|
|
|
|
def project_run(
|
2021-02-02 02:11:15 +00:00
|
|
|
project_dir: Path,
|
|
|
|
subcommand: str,
|
|
|
|
*,
|
2021-02-10 02:45:27 +00:00
|
|
|
overrides: Dict[str, Any] = SimpleFrozenDict(),
|
2021-02-02 02:11:15 +00:00
|
|
|
force: bool = False,
|
|
|
|
dry: bool = False,
|
|
|
|
capture: bool = False,
|
2022-11-07 13:46:08 +00:00
|
|
|
skip_requirements_check: bool = False,
|
2020-07-08 23:42:51 +00:00
|
|
|
) -> None:
|
|
|
|
"""Run a named script defined in the project.yml. If the script is part
|
|
|
|
of the default pipeline (defined in the "run" section), DVC is used to
|
|
|
|
execute the command, so it can determine whether to rerun it. It then
|
|
|
|
calls into "exec" to execute it.
|
|
|
|
|
|
|
|
project_dir (Path): Path to project directory.
|
|
|
|
subcommand (str): Name of command to run.
|
2021-08-30 07:23:54 +00:00
|
|
|
overrides (Dict[str, Any]): Optional config overrides.
|
2020-07-08 23:42:51 +00:00
|
|
|
force (bool): Force re-running, even if nothing changed.
|
|
|
|
dry (bool): Perform a dry run and don't execute commands.
|
2021-02-02 02:11:15 +00:00
|
|
|
capture (bool): Whether to capture the output and errors of individual commands.
|
|
|
|
If False, the stdout and stderr will not be redirected, and if there's an error,
|
|
|
|
sys.exit will be called with the return code. You should use capture=False
|
|
|
|
when you want to turn over execution to the command, and capture=True
|
|
|
|
when you want to run the command more like a function.
|
2022-11-07 13:46:08 +00:00
|
|
|
skip_requirements_check (bool): Whether to skip the requirements check.
|
2020-07-08 23:42:51 +00:00
|
|
|
"""
|
2021-02-10 02:45:27 +00:00
|
|
|
config = load_project_config(project_dir, overrides=overrides)
|
2020-07-08 23:42:51 +00:00
|
|
|
commands = {cmd["name"]: cmd for cmd in config.get("commands", [])}
|
|
|
|
workflows = config.get("workflows", {})
|
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
|
|
|
validate_subcommand(list(commands.keys()), list(workflows.keys()), subcommand)
|
2022-09-16 14:54:31 +00:00
|
|
|
|
|
|
|
req_path = project_dir / "requirements.txt"
|
2022-11-07 13:46:08 +00:00
|
|
|
if not skip_requirements_check:
|
|
|
|
if config.get("check_requirements", True) and os.path.exists(req_path):
|
|
|
|
with req_path.open() as requirements_file:
|
|
|
|
_check_requirements([req.strip() for req in requirements_file])
|
2022-09-16 14:54:31 +00:00
|
|
|
|
2020-07-08 23:42:51 +00:00
|
|
|
if subcommand in workflows:
|
|
|
|
msg.info(f"Running workflow '{subcommand}'")
|
|
|
|
for cmd in workflows[subcommand]:
|
2021-09-03 08:47:03 +00:00
|
|
|
project_run(
|
|
|
|
project_dir,
|
|
|
|
cmd,
|
|
|
|
overrides=overrides,
|
|
|
|
force=force,
|
|
|
|
dry=dry,
|
|
|
|
capture=capture,
|
2022-11-07 13:46:08 +00:00
|
|
|
skip_requirements_check=True,
|
2021-09-03 08:47:03 +00:00
|
|
|
)
|
2020-07-08 23:42:51 +00:00
|
|
|
else:
|
|
|
|
cmd = commands[subcommand]
|
|
|
|
for dep in cmd.get("deps", []):
|
|
|
|
if not (project_dir / dep).exists():
|
|
|
|
err = f"Missing dependency specified by command '{subcommand}': {dep}"
|
2020-09-22 07:45:34 +00:00
|
|
|
err_help = "Maybe you forgot to run the 'project assets' command or a previous step?"
|
2022-12-05 09:00:00 +00:00
|
|
|
err_exits = 1 if not dry else None
|
|
|
|
msg.fail(err, err_help, exits=err_exits)
|
2020-10-05 18:51:15 +00:00
|
|
|
check_spacy_commit = check_bool_env_var(ENV_VARS.PROJECT_USE_GIT_VERSION)
|
2020-07-08 23:42:51 +00:00
|
|
|
with working_dir(project_dir) as current_dir:
|
2020-10-05 11:53:07 +00:00
|
|
|
msg.divider(subcommand)
|
2020-10-05 18:51:15 +00:00
|
|
|
rerun = check_rerun(current_dir, cmd, check_spacy_commit=check_spacy_commit)
|
2020-07-08 23:42:51 +00:00
|
|
|
if not rerun and not force:
|
|
|
|
msg.info(f"Skipping '{cmd['name']}': nothing changed")
|
|
|
|
else:
|
2021-02-02 02:11:15 +00:00
|
|
|
run_commands(cmd["script"], dry=dry, capture=capture)
|
2020-07-09 21:51:18 +00:00
|
|
|
if not dry:
|
2020-08-23 16:32:09 +00:00
|
|
|
update_lockfile(current_dir, cmd)
|
2020-07-08 23:42:51 +00:00
|
|
|
|
|
|
|
|
|
|
|
def print_run_help(project_dir: Path, subcommand: Optional[str] = None) -> None:
|
|
|
|
"""Simulate a CLI help prompt using the info available in the project.yml.
|
|
|
|
|
|
|
|
project_dir (Path): The project directory.
|
|
|
|
subcommand (Optional[str]): The subcommand or None. If a subcommand is
|
|
|
|
provided, the subcommand help is shown. Otherwise, the top-level help
|
|
|
|
and a list of available commands is printed.
|
|
|
|
"""
|
|
|
|
config = load_project_config(project_dir)
|
|
|
|
config_commands = config.get("commands", [])
|
|
|
|
commands = {cmd["name"]: cmd for cmd in config_commands}
|
2020-07-09 21:51:18 +00:00
|
|
|
workflows = config.get("workflows", {})
|
2020-07-08 23:42:51 +00:00
|
|
|
project_loc = "" if is_cwd(project_dir) else project_dir
|
|
|
|
if subcommand:
|
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
|
|
|
validate_subcommand(list(commands.keys()), list(workflows.keys()), subcommand)
|
2020-07-08 23:42:51 +00:00
|
|
|
print(f"Usage: {COMMAND} project run {subcommand} {project_loc}")
|
2020-07-09 21:51:18 +00:00
|
|
|
if subcommand in commands:
|
|
|
|
help_text = commands[subcommand].get("help")
|
|
|
|
if help_text:
|
|
|
|
print(f"\n{help_text}\n")
|
|
|
|
elif subcommand in workflows:
|
|
|
|
steps = workflows[subcommand]
|
|
|
|
print(f"\nWorkflow consisting of {len(steps)} commands:")
|
|
|
|
steps_data = [
|
|
|
|
(f"{i + 1}. {step}", commands[step].get("help", ""))
|
|
|
|
for i, step in enumerate(steps)
|
|
|
|
]
|
|
|
|
msg.table(steps_data)
|
|
|
|
help_cmd = f"{COMMAND} project run [COMMAND] {project_loc} --help"
|
|
|
|
print(f"For command details, run: {help_cmd}")
|
2020-07-08 23:42:51 +00:00
|
|
|
else:
|
2020-07-09 21:51:18 +00:00
|
|
|
print("")
|
2020-08-26 08:26:57 +00:00
|
|
|
title = config.get("title")
|
|
|
|
if title:
|
2021-09-22 19:01:34 +00:00
|
|
|
print(f"{locale_escape(title)}\n")
|
2020-07-09 21:51:18 +00:00
|
|
|
if config_commands:
|
|
|
|
print(f"Available commands in {PROJECT_FILE}")
|
|
|
|
print(f"Usage: {COMMAND} project run [COMMAND] {project_loc}")
|
|
|
|
msg.table([(cmd["name"], cmd.get("help", "")) for cmd in config_commands])
|
|
|
|
if workflows:
|
|
|
|
print(f"Available workflows in {PROJECT_FILE}")
|
|
|
|
print(f"Usage: {COMMAND} project run [WORKFLOW] {project_loc}")
|
|
|
|
msg.table([(name, " -> ".join(steps)) for name, steps in workflows.items()])
|
2020-07-08 23:42:51 +00:00
|
|
|
|
|
|
|
|
|
|
|
def run_commands(
|
2020-08-29 13:20:11 +00:00
|
|
|
commands: Iterable[str] = SimpleFrozenList(),
|
|
|
|
silent: bool = False,
|
|
|
|
dry: bool = False,
|
2021-02-02 02:11:15 +00:00
|
|
|
capture: bool = False,
|
2020-07-08 23:42:51 +00:00
|
|
|
) -> None:
|
|
|
|
"""Run a sequence of commands in a subprocess, in order.
|
|
|
|
|
|
|
|
commands (List[str]): The string commands.
|
|
|
|
silent (bool): Don't print the commands.
|
|
|
|
dry (bool): Perform a dry run and don't execut anything.
|
2021-02-02 02:11:15 +00:00
|
|
|
capture (bool): Whether to capture the output and errors of individual commands.
|
|
|
|
If False, the stdout and stderr will not be redirected, and if there's an error,
|
|
|
|
sys.exit will be called with the return code. You should use capture=False
|
|
|
|
when you want to turn over execution to the command, and capture=True
|
|
|
|
when you want to run the command more like a function.
|
2020-07-08 23:42:51 +00:00
|
|
|
"""
|
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
|
|
|
for c in commands:
|
|
|
|
command = split_command(c)
|
2020-07-08 23:42:51 +00:00
|
|
|
# Not sure if this is needed or a good idea. Motivation: users may often
|
|
|
|
# use commands in their config that reference "python" and we want to
|
|
|
|
# make sure that it's always executing the same Python that spaCy is
|
|
|
|
# executed with and the pip in the same env, not some other Python/pip.
|
|
|
|
# Also ensures cross-compatibility if user 1 writes "python3" (because
|
|
|
|
# that's how it's set up on their system), and user 2 without the
|
|
|
|
# shortcut tries to re-run the command.
|
|
|
|
if len(command) and command[0] in ("python", "python3"):
|
|
|
|
command[0] = sys.executable
|
|
|
|
elif len(command) and command[0] in ("pip", "pip3"):
|
|
|
|
command = [sys.executable, "-m", "pip", *command[1:]]
|
|
|
|
if not silent:
|
|
|
|
print(f"Running command: {join_command(command)}")
|
|
|
|
if not dry:
|
2021-02-02 02:11:15 +00:00
|
|
|
run_command(command, capture=capture)
|
2020-07-08 23:42:51 +00:00
|
|
|
|
|
|
|
|
|
|
|
def validate_subcommand(
|
|
|
|
commands: Sequence[str], workflows: Sequence[str], subcommand: str
|
|
|
|
) -> None:
|
|
|
|
"""Check that a subcommand is valid and defined. Raises an error otherwise.
|
|
|
|
|
|
|
|
commands (Sequence[str]): The available commands.
|
|
|
|
subcommand (str): The subcommand.
|
|
|
|
"""
|
|
|
|
if not commands and not workflows:
|
|
|
|
msg.fail(f"No commands or workflows defined in {PROJECT_FILE}", exits=1)
|
|
|
|
if subcommand not in commands and subcommand not in workflows:
|
|
|
|
help_msg = []
|
2022-09-09 15:17:10 +00:00
|
|
|
if subcommand in ["assets", "asset"]:
|
|
|
|
help_msg.append("Did you mean to run: python -m spacy project assets?")
|
2020-07-08 23:42:51 +00:00
|
|
|
if commands:
|
|
|
|
help_msg.append(f"Available commands: {', '.join(commands)}")
|
|
|
|
if workflows:
|
|
|
|
help_msg.append(f"Available workflows: {', '.join(workflows)}")
|
|
|
|
msg.fail(
|
|
|
|
f"Can't find command or workflow '{subcommand}' in {PROJECT_FILE}",
|
|
|
|
". ".join(help_msg),
|
|
|
|
exits=1,
|
|
|
|
)
|
|
|
|
|
|
|
|
|
2020-10-05 11:53:07 +00:00
|
|
|
def check_rerun(
|
|
|
|
project_dir: Path,
|
|
|
|
command: Dict[str, Any],
|
2020-10-05 15:07:35 +00:00
|
|
|
*,
|
2020-10-05 11:53:07 +00:00
|
|
|
check_spacy_version: bool = True,
|
|
|
|
check_spacy_commit: bool = False,
|
|
|
|
) -> bool:
|
2020-07-08 23:42:51 +00:00
|
|
|
"""Check if a command should be rerun because its settings or inputs/outputs
|
|
|
|
changed.
|
|
|
|
|
|
|
|
project_dir (Path): The current project directory.
|
|
|
|
command (Dict[str, Any]): The command, as defined in the project.yml.
|
2020-10-05 11:53:07 +00:00
|
|
|
strict_version (bool):
|
2020-07-08 23:42:51 +00:00
|
|
|
RETURNS (bool): Whether to re-run the command.
|
|
|
|
"""
|
2021-07-24 06:31:17 +00:00
|
|
|
# Always rerun if no-skip is set
|
|
|
|
if command.get("no_skip", False):
|
|
|
|
return True
|
2020-07-08 23:42:51 +00:00
|
|
|
lock_path = project_dir / PROJECT_LOCK
|
|
|
|
if not lock_path.exists(): # We don't have a lockfile, run command
|
|
|
|
return True
|
|
|
|
data = srsly.read_yaml(lock_path)
|
|
|
|
if command["name"] not in data: # We don't have info about this command
|
|
|
|
return True
|
|
|
|
entry = data[command["name"]]
|
2020-07-09 21:51:18 +00:00
|
|
|
# Always run commands with no outputs (otherwise they'd always be skipped)
|
|
|
|
if not entry.get("outs", []):
|
|
|
|
return True
|
2020-10-05 11:53:07 +00:00
|
|
|
# Always rerun if spaCy version or commit hash changed
|
|
|
|
spacy_v = entry.get("spacy_version")
|
|
|
|
commit = entry.get("spacy_git_version")
|
|
|
|
if check_spacy_version and not is_minor_version_match(spacy_v, about.__version__):
|
|
|
|
info = f"({spacy_v} in {PROJECT_LOCK}, {about.__version__} current)"
|
|
|
|
msg.info(f"Re-running '{command['name']}': spaCy minor version changed {info}")
|
|
|
|
return True
|
|
|
|
if check_spacy_commit and commit != GIT_VERSION:
|
|
|
|
info = f"({commit} in {PROJECT_LOCK}, {GIT_VERSION} current)"
|
|
|
|
msg.info(f"Re-running '{command['name']}': spaCy commit changed {info}")
|
|
|
|
return True
|
2020-07-08 23:42:51 +00:00
|
|
|
# If the entry in the lockfile matches the lockfile entry that would be
|
|
|
|
# generated from the current command, we don't rerun because it means that
|
|
|
|
# all inputs/outputs, hashes and scripts are the same and nothing changed
|
2020-10-05 11:53:07 +00:00
|
|
|
lock_entry = get_lock_entry(project_dir, command)
|
|
|
|
exclude = ["spacy_version", "spacy_git_version"]
|
|
|
|
return get_hash(lock_entry, exclude=exclude) != get_hash(entry, exclude=exclude)
|
2020-07-08 23:42:51 +00:00
|
|
|
|
|
|
|
|
2020-08-23 16:32:09 +00:00
|
|
|
def update_lockfile(project_dir: Path, command: Dict[str, Any]) -> None:
|
2020-07-08 23:42:51 +00:00
|
|
|
"""Update the lockfile after running a command. Will create a lockfile if
|
|
|
|
it doesn't yet exist and will add an entry for the current command, its
|
|
|
|
script and dependencies/outputs.
|
|
|
|
|
|
|
|
project_dir (Path): The current project directory.
|
|
|
|
command (Dict[str, Any]): The command, as defined in the project.yml.
|
|
|
|
"""
|
|
|
|
lock_path = project_dir / PROJECT_LOCK
|
|
|
|
if not lock_path.exists():
|
|
|
|
srsly.write_yaml(lock_path, {})
|
|
|
|
data = {}
|
|
|
|
else:
|
|
|
|
data = srsly.read_yaml(lock_path)
|
2020-08-23 16:32:09 +00:00
|
|
|
data[command["name"]] = get_lock_entry(project_dir, command)
|
2020-07-08 23:42:51 +00:00
|
|
|
srsly.write_yaml(lock_path, data)
|
|
|
|
|
|
|
|
|
2020-08-23 16:32:09 +00:00
|
|
|
def get_lock_entry(project_dir: Path, command: Dict[str, Any]) -> Dict[str, Any]:
|
2020-07-08 23:42:51 +00:00
|
|
|
"""Get a lockfile entry for a given command. An entry includes the command,
|
|
|
|
the script (command steps) and a list of dependencies and outputs with
|
|
|
|
their paths and file hashes, if available. The format is based on the
|
|
|
|
dvc.lock files, to keep things consistent.
|
|
|
|
|
|
|
|
project_dir (Path): The current project directory.
|
|
|
|
command (Dict[str, Any]): The command, as defined in the project.yml.
|
|
|
|
RETURNS (Dict[str, Any]): The lockfile entry.
|
|
|
|
"""
|
2020-08-23 16:32:09 +00:00
|
|
|
deps = get_fileinfo(project_dir, command.get("deps", []))
|
|
|
|
outs = get_fileinfo(project_dir, command.get("outputs", []))
|
|
|
|
outs_nc = get_fileinfo(project_dir, command.get("outputs_no_cache", []))
|
2020-07-08 23:42:51 +00:00
|
|
|
return {
|
|
|
|
"cmd": f"{COMMAND} run {command['name']}",
|
|
|
|
"script": command["script"],
|
|
|
|
"deps": deps,
|
|
|
|
"outs": [*outs, *outs_nc],
|
2020-10-05 11:53:07 +00:00
|
|
|
"spacy_version": about.__version__,
|
|
|
|
"spacy_git_version": GIT_VERSION,
|
2020-07-08 23:42:51 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
|
|
|
def get_fileinfo(project_dir: Path, paths: List[str]) -> List[Dict[str, Optional[str]]]:
|
2020-07-08 23:42:51 +00:00
|
|
|
"""Generate the file information for a list of paths (dependencies, outputs).
|
|
|
|
Includes the file path and the file's checksum.
|
|
|
|
|
|
|
|
project_dir (Path): The current project directory.
|
|
|
|
paths (List[str]): The file paths.
|
|
|
|
RETURNS (List[Dict[str, str]]): The lockfile entry for a file.
|
|
|
|
"""
|
|
|
|
data = []
|
|
|
|
for path in paths:
|
|
|
|
file_path = project_dir / path
|
|
|
|
md5 = get_checksum(file_path) if file_path.exists() else None
|
|
|
|
data.append({"path": path, "md5": md5})
|
|
|
|
return data
|
2022-09-16 14:54:31 +00:00
|
|
|
|
|
|
|
|
|
|
|
def _check_requirements(requirements: List[str]) -> Tuple[bool, bool]:
|
|
|
|
"""Checks whether requirements are installed and free of version conflicts.
|
|
|
|
requirements (List[str]): List of requirements.
|
|
|
|
RETURNS (Tuple[bool, bool]): Whether (1) any packages couldn't be imported, (2) any packages with version conflicts
|
|
|
|
exist.
|
|
|
|
"""
|
2023-03-06 13:48:57 +00:00
|
|
|
import pkg_resources
|
2022-09-16 14:54:31 +00:00
|
|
|
|
|
|
|
failed_pkgs_msgs: List[str] = []
|
|
|
|
conflicting_pkgs_msgs: List[str] = []
|
|
|
|
|
|
|
|
for req in requirements:
|
|
|
|
try:
|
|
|
|
pkg_resources.require(req)
|
|
|
|
except pkg_resources.DistributionNotFound as dnf:
|
|
|
|
failed_pkgs_msgs.append(dnf.report())
|
|
|
|
except pkg_resources.VersionConflict as vc:
|
|
|
|
conflicting_pkgs_msgs.append(vc.report())
|
2022-11-07 13:46:08 +00:00
|
|
|
except Exception:
|
2022-11-11 08:58:31 +00:00
|
|
|
msg.warn(
|
|
|
|
f"Unable to check requirement: {req} "
|
|
|
|
"Checks are currently limited to requirement specifiers "
|
|
|
|
"(PEP 508)"
|
|
|
|
)
|
2022-09-16 14:54:31 +00:00
|
|
|
|
|
|
|
if len(failed_pkgs_msgs) or len(conflicting_pkgs_msgs):
|
|
|
|
msg.warn(
|
|
|
|
title="Missing requirements or requirement conflicts detected. Make sure your Python environment is set up "
|
|
|
|
"correctly and you installed all requirements specified in your project's requirements.txt: "
|
|
|
|
)
|
|
|
|
for pgk_msg in failed_pkgs_msgs + conflicting_pkgs_msgs:
|
|
|
|
msg.text(pgk_msg)
|
|
|
|
|
|
|
|
return len(failed_pkgs_msgs) > 0, len(conflicting_pkgs_msgs) > 0
|