From d85117f88c689c8914be5386d15b63a3330a9124 Mon Sep 17 00:00:00 2001 From: Adriane Boyd Date: Thu, 24 Mar 2022 11:47:05 +0100 Subject: [PATCH] Stream large assets on download (#10521) Stream large assets on download rather than reading the whole file at once and potentially running into `urllib3` limits on single read sizes. --- spacy/cli/_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spacy/cli/_util.py b/spacy/cli/_util.py index fb680d888..df98e711f 100644 --- a/spacy/cli/_util.py +++ b/spacy/cli/_util.py @@ -360,7 +360,7 @@ def download_file(src: Union[str, "Pathy"], dest: Path, *, force: bool = False) src = str(src) with smart_open.open(src, mode="rb", ignore_ext=True) as input_file: with dest.open(mode="wb") as output_file: - output_file.write(input_file.read()) + shutil.copyfileobj(input_file, output_file) def ensure_pathy(path):