From 998b607f659687d56023ae3433371605f1f5bf8c Mon Sep 17 00:00:00 2001 From: Matthew Honnibal Date: Fri, 30 Jan 2015 18:04:01 +1100 Subject: [PATCH] * Upd download script, having it download all data if there's no data/ directory, allowing easier compilation from source --- spacy/en/download.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/spacy/en/download.py b/spacy/en/download.py index 9997c90b6..88a39a1cb 100644 --- a/spacy/en/download.py +++ b/spacy/en/download.py @@ -4,19 +4,27 @@ import tarfile import shutil import wget -PARSER_URL = 'http://s3-us-west-1.amazonaws.com/media.spacynlp.com/en_deps-0.30.tgz' +DATA_DIR_URL = 'http://s3-us-west-1.amazonaws.com/media.spacynlp.com/en_data_all-0.4.tgz' + +PARSER_URL = 'http://s3-us-west-1.amazonaws.com/media.spacynlp.com/en_deps-0.30.tgz' DEP_VECTORS_URL = 'http://s3-us-west-1.amazonaws.com/media.spacynlp.com/vec.bin' -DEST_DIR = path.join(path.dirname(__file__), 'data') +DEST_DIR = path.join(path.dirname(__file__), 'tmp_data') def download_file(url, out): wget.download(url, out=out) + +def install_all_data(url, dest_dir): + filename = download_file(url, dest_dir) + t = tarfile.open(path.join(dest_dir, filename), mode=":gz") + t.extractall(dest_dir) + def install_parser_model(url, dest_dir): - download_file(url, dest_dir) - t = tarfile.open(path.join(DEST_DIR, 'en_deps-0.30.tgz'), mode=":gz") + filename = download_file(url, dest_dir) + t = tarfile.open(path.join(dest_dir, filename), mode=":gz") t.extractall(dest_dir) @@ -28,8 +36,11 @@ def install_dep_vectors(url, dest_dir): def main(): - install_parser_model(PARSER_URL, DEST_DIR) - install_dep_vectors(DEP_VECTORS_URL, path.join(DEST_DIR, 'vocab')) + if not path.exists(DEST_DIR): + install_all_data(DATA_DIR_URL, DEST_DIR) + else: + install_parser_model(PARSER_URL, DEST_DIR) + install_dep_vectors(DEP_VECTORS_URL, path.join(DEST_DIR, 'vocab')) if __name__ == '__main__':