spaCy/fabfile.py

import contextlib
from pathlib import Path
from fabric.api import local, lcd
from os import path, environ
import shutil
import sys


PWD = path.dirname(__file__)
ENV = environ["VENV_DIR"] if "VENV_DIR" in environ else ".env"
VENV_DIR = Path(PWD) / ENV


@contextlib.contextmanager
def virtualenv(name, create=False, python="/usr/bin/python3.6"):
    python = Path(python).resolve()
    env_path = VENV_DIR
    if create:
        if env_path.exists():
            shutil.rmtree(str(env_path))
        local("{python} -m venv {env_path}".format(python=python, env_path=VENV_DIR))

    def wrapped_local(cmd, env_vars=[], capture=False, direct=False):
        return local(
            "source {}/bin/activate && {}".format(env_path, cmd),
            shell="/bin/bash",
            capture=False,
        )

    yield wrapped_local


def env(lang="python3.6"):
    if VENV_DIR.exists():
        local("rm -rf {env}".format(env=VENV_DIR))
    if lang.startswith("python3"):
        local("{lang} -m venv {env}".format(lang=lang, env=VENV_DIR))
    else:
        local("{lang} -m pip install virtualenv --no-cache-dir".format(lang=lang))
        local(
            "{lang} -m virtualenv {env} --no-cache-dir".format(lang=lang, env=VENV_DIR)
        )
    with virtualenv(VENV_DIR) as venv_local:
        print(venv_local("python --version", capture=True))
        venv_local("pip install --upgrade setuptools --no-cache-dir")
        venv_local("pip install pytest --no-cache-dir")
        venv_local("pip install wheel --no-cache-dir")
        venv_local("pip install -r requirements.txt --no-cache-dir")
        venv_local("pip install pex --no-cache-dir")


def install():
    with virtualenv(VENV_DIR) as venv_local:
        venv_local("pip install dist/*.tar.gz")


def make():
    with lcd(path.dirname(__file__)):
        local(
            "export PYTHONPATH=`pwd` && source .env/bin/activate && python setup.py build_ext --inplace",
            shell="/bin/bash",
        )


def sdist():
    with virtualenv(VENV_DIR) as venv_local:
        with lcd(path.dirname(__file__)):
            venv_local("python -m pip install -U setuptools srsly")
            venv_local("python setup.py sdist")


def wheel():
    with virtualenv(VENV_DIR) as venv_local:
        with lcd(path.dirname(__file__)):
            venv_local("python setup.py bdist_wheel")


def pex():
    with virtualenv(VENV_DIR) as venv_local:
        with lcd(path.dirname(__file__)):
            sha = local("git rev-parse --short HEAD", capture=True)
            venv_local(f"pex dist/*.whl -e spacy -o dist/spacy-{sha}.pex", direct=True)


def clean():
    with lcd(path.dirname(__file__)):
        local("rm -f dist/*.whl")
        local("rm -f dist/*.pex")
        with virtualenv(VENV_DIR) as venv_local:
            venv_local("python setup.py clean --all")


def test():
    with virtualenv(VENV_DIR) as venv_local:
        with lcd(path.dirname(__file__)):
            venv_local("pytest -x spacy/tests")


def train():
    args = environ.get("SPACY_TRAIN_ARGS", "")
    with virtualenv(VENV_DIR) as venv_local:
        venv_local("spacy train {args}".format(args=args))


def conll17(treebank_dir, experiment_dir, vectors_dir, config, corpus=""):
    is_not_clean = local("git status --porcelain", capture=True)
    if is_not_clean:
        print("Repository is not clean")
        print(is_not_clean)
        sys.exit(1)
    git_sha = local("git rev-parse --short HEAD", capture=True)
    config_checksum = local("sha256sum {config}".format(config=config), capture=True)
    experiment_dir = Path(experiment_dir) / "{}--{}".format(
        config_checksum[:6], git_sha
    )
    if not experiment_dir.exists():
        experiment_dir.mkdir()
    test_data_dir = Path(treebank_dir) / "ud-test-v2.0-conll2017"
    assert test_data_dir.exists()
    assert test_data_dir.is_dir()
    if corpus:
        corpora = [corpus]
    else:
        corpora = ["UD_English", "UD_Chinese", "UD_Japanese", "UD_Vietnamese"]

    local(
        "cp {config} {experiment_dir}/config.json".format(
            config=config, experiment_dir=experiment_dir
        )
    )
    with virtualenv(VENV_DIR) as venv_local:
        for corpus in corpora:
            venv_local(
                "spacy ud-train {treebank_dir} {experiment_dir} {config} {corpus} -v {vectors_dir}".format(
                    treebank_dir=treebank_dir,
                    experiment_dir=experiment_dir,
                    config=config,
                    corpus=corpus,
                    vectors_dir=vectors_dir,
                )
            )
            venv_local(
                "spacy ud-run-test {test_data_dir} {experiment_dir} {corpus}".format(
                    test_data_dir=test_data_dir,
                    experiment_dir=experiment_dir,
                    config=config,
                    corpus=corpus,
                )
            )