2020-07-25 13:01:15 +00:00
|
|
|
from typing import Optional, Any, List, Union
|
2020-06-21 11:44:00 +00:00
|
|
|
from enum import Enum
|
2017-04-13 11:51:54 +00:00
|
|
|
from pathlib import Path
|
2018-11-30 19:16:14 +00:00
|
|
|
from wasabi import Printer
|
💫 Replace ujson, msgpack and dill/pickle/cloudpickle with srsly (#3003)
Remove hacks and wrappers, keep code in sync across our libraries and move spaCy a few steps closer to only depending on packages with binary wheels 🎉
See here: https://github.com/explosion/srsly
Serialization is hard, especially across Python versions and multiple platforms. After dealing with many subtle bugs over the years (encodings, locales, large files) our libraries like spaCy and Prodigy have steadily grown a number of utility functions to wrap the multiple serialization formats we need to support (especially json, msgpack and pickle). These wrapping functions ended up duplicated across our codebases, so we wanted to put them in one place.
At the same time, we noticed that having a lot of small dependencies was making maintainence harder, and making installation slower. To solve this, we've made srsly standalone, by including the component packages directly within it. This way we can provide all the serialization utilities we need in a single binary wheel.
srsly currently includes forks of the following packages:
ujson
msgpack
msgpack-numpy
cloudpickle
* WIP: replace json/ujson with srsly
* Replace ujson in examples
Use regular json instead of srsly to make code easier to read and follow
* Update requirements
* Fix imports
* Fix typos
* Replace msgpack with srsly
* Fix warning
2018-12-03 00:28:22 +00:00
|
|
|
import srsly
|
2019-08-29 10:04:01 +00:00
|
|
|
import re
|
2020-06-26 17:34:12 +00:00
|
|
|
import sys
|
2020-12-15 08:47:16 +00:00
|
|
|
import itertools
|
2017-04-07 11:04:17 +00:00
|
|
|
|
2020-07-10 15:57:40 +00:00
|
|
|
from ._util import app, Arg, Opt
|
2020-09-09 08:31:03 +00:00
|
|
|
from ..training import docs_to_json
|
2020-06-26 17:34:12 +00:00
|
|
|
from ..tokens import DocBin
|
2020-09-29 19:39:28 +00:00
|
|
|
from ..training.converters import iob_to_docs, conll_ner_to_docs, json_to_docs
|
|
|
|
from ..training.converters import conllu_to_docs
|
2018-11-30 19:16:14 +00:00
|
|
|
|
2017-04-07 11:04:17 +00:00
|
|
|
|
2019-08-29 10:04:01 +00:00
|
|
|
# Converters are matched by file extension except for ner/iob, which are
|
|
|
|
# matched by file extension and content. To add a converter, add a new
|
2017-10-27 12:38:39 +00:00
|
|
|
# entry to this dict with the file extension mapped to the converter function
|
|
|
|
# imported from /converters.
|
2020-06-26 17:34:12 +00:00
|
|
|
|
2017-04-07 11:04:17 +00:00
|
|
|
CONVERTERS = {
|
2020-09-22 09:50:19 +00:00
|
|
|
"conllubio": conllu_to_docs,
|
|
|
|
"conllu": conllu_to_docs,
|
|
|
|
"conll": conllu_to_docs,
|
|
|
|
"ner": conll_ner_to_docs,
|
|
|
|
"iob": iob_to_docs,
|
|
|
|
"json": json_to_docs,
|
2017-04-07 11:04:17 +00:00
|
|
|
}
|
|
|
|
|
2020-06-26 17:34:12 +00:00
|
|
|
|
|
|
|
# File types that can be written to stdout
|
2020-07-03 13:20:10 +00:00
|
|
|
FILE_TYPES_STDOUT = ("json",)
|
2018-11-30 19:16:14 +00:00
|
|
|
|
2017-04-07 11:04:17 +00:00
|
|
|
|
2020-06-21 11:44:00 +00:00
|
|
|
class FileTypes(str, Enum):
|
|
|
|
json = "json"
|
2020-06-26 17:34:12 +00:00
|
|
|
spacy = "spacy"
|
2020-06-21 11:44:00 +00:00
|
|
|
|
|
|
|
|
|
|
|
@app.command("convert")
|
2020-06-21 19:35:01 +00:00
|
|
|
def convert_cli(
|
2019-12-22 00:53:56 +00:00
|
|
|
# fmt: off
|
2020-06-26 17:34:12 +00:00
|
|
|
input_path: str = Arg(..., help="Input file or directory", exists=True),
|
2020-06-21 19:35:01 +00:00
|
|
|
output_dir: Path = Arg("-", help="Output directory. '-' for stdout.", allow_dash=True, exists=True),
|
2020-06-26 17:34:12 +00:00
|
|
|
file_type: FileTypes = Opt("spacy", "--file-type", "-t", help="Type of data to produce"),
|
2020-06-21 11:44:00 +00:00
|
|
|
n_sents: int = Opt(1, "--n-sents", "-n", help="Number of sentences per doc (0 to disable)"),
|
|
|
|
seg_sents: bool = Opt(False, "--seg-sents", "-s", help="Segment sentences (for -c ner)"),
|
2020-09-03 15:12:24 +00:00
|
|
|
model: Optional[str] = Opt(None, "--model", "--base", "-b", help="Trained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)"),
|
2020-06-21 11:44:00 +00:00
|
|
|
morphology: bool = Opt(False, "--morphology", "-m", help="Enable appending morphology to tags"),
|
|
|
|
merge_subtokens: bool = Opt(False, "--merge-subtokens", "-T", help="Merge CoNLL-U subtokens"),
|
|
|
|
converter: str = Opt("auto", "--converter", "-c", help=f"Converter: {tuple(CONVERTERS.keys())}"),
|
2020-07-04 12:23:32 +00:00
|
|
|
ner_map: Optional[Path] = Opt(None, "--ner-map", "-nm", help="NER tag mapping (as JSON-encoded dict of entity types)", exists=True),
|
2020-06-21 11:44:00 +00:00
|
|
|
lang: Optional[str] = Opt(None, "--lang", "-l", help="Language (if tokenizer required)"),
|
2020-08-24 22:30:52 +00:00
|
|
|
concatenate: bool = Opt(None, "--concatenate", "-C", help="Concatenate output to a single file"),
|
2019-12-22 00:53:56 +00:00
|
|
|
# fmt: on
|
2018-11-30 19:16:14 +00:00
|
|
|
):
|
2017-05-27 18:01:46 +00:00
|
|
|
"""
|
2020-07-12 11:53:49 +00:00
|
|
|
Convert files into json or DocBin format for training. The resulting .spacy
|
|
|
|
file can be used with the train command and other experiment management
|
|
|
|
functions.
|
|
|
|
|
|
|
|
If no output_dir is specified and the output format is JSON, the data
|
2019-04-12 09:31:23 +00:00
|
|
|
is written to stdout, so you can pipe them forward to a JSON file:
|
2020-07-12 11:53:49 +00:00
|
|
|
$ spacy convert some_file.conllu --file-type json > some_file.json
|
2020-09-04 10:58:50 +00:00
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
DOCS: https://spacy.io/api/cli#convert
|
2017-05-22 10:28:58 +00:00
|
|
|
"""
|
2020-06-21 11:44:00 +00:00
|
|
|
if isinstance(file_type, FileTypes):
|
|
|
|
# We get an instance of the FileTypes from the CLI so we need its string value
|
|
|
|
file_type = file_type.value
|
2020-06-26 17:34:12 +00:00
|
|
|
input_path = Path(input_path)
|
|
|
|
output_dir = "-" if output_dir == Path("-") else output_dir
|
2020-06-21 19:35:01 +00:00
|
|
|
silent = output_dir == "-"
|
2020-06-26 17:34:12 +00:00
|
|
|
msg = Printer(no_print=silent)
|
2020-07-25 13:01:15 +00:00
|
|
|
verify_cli_args(msg, input_path, output_dir, file_type, converter, ner_map)
|
2020-06-26 17:34:12 +00:00
|
|
|
converter = _get_converter(msg, converter, input_path)
|
2020-06-21 19:35:01 +00:00
|
|
|
convert(
|
2020-06-26 17:34:12 +00:00
|
|
|
input_path,
|
2020-06-21 19:35:01 +00:00
|
|
|
output_dir,
|
|
|
|
file_type=file_type,
|
|
|
|
n_sents=n_sents,
|
|
|
|
seg_sents=seg_sents,
|
|
|
|
model=model,
|
|
|
|
morphology=morphology,
|
|
|
|
merge_subtokens=merge_subtokens,
|
|
|
|
converter=converter,
|
2020-06-26 17:34:12 +00:00
|
|
|
ner_map=ner_map,
|
2020-06-21 19:35:01 +00:00
|
|
|
lang=lang,
|
2020-08-24 22:30:52 +00:00
|
|
|
concatenate=concatenate,
|
2020-06-21 19:35:01 +00:00
|
|
|
silent=silent,
|
2020-06-26 17:34:12 +00:00
|
|
|
msg=msg,
|
2020-06-21 19:35:01 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
def convert(
|
2020-07-25 13:01:15 +00:00
|
|
|
input_path: Union[str, Path],
|
|
|
|
output_dir: Union[str, Path],
|
2020-07-03 13:20:10 +00:00
|
|
|
*,
|
|
|
|
file_type: str = "json",
|
|
|
|
n_sents: int = 1,
|
|
|
|
seg_sents: bool = False,
|
|
|
|
model: Optional[str] = None,
|
|
|
|
morphology: bool = False,
|
|
|
|
merge_subtokens: bool = False,
|
|
|
|
converter: str = "auto",
|
|
|
|
ner_map: Optional[Path] = None,
|
|
|
|
lang: Optional[str] = None,
|
2020-08-24 22:30:52 +00:00
|
|
|
concatenate: bool = False,
|
2020-07-03 13:20:10 +00:00
|
|
|
silent: bool = True,
|
2020-07-25 13:01:15 +00:00
|
|
|
msg: Optional[Printer],
|
2020-06-21 19:35:01 +00:00
|
|
|
) -> None:
|
2020-06-26 17:34:12 +00:00
|
|
|
if not msg:
|
|
|
|
msg = Printer(no_print=silent)
|
|
|
|
ner_map = srsly.read_json(ner_map) if ner_map is not None else None
|
2020-08-24 22:30:52 +00:00
|
|
|
doc_files = []
|
|
|
|
for input_loc in walk_directory(Path(input_path), converter):
|
2020-06-26 17:34:12 +00:00
|
|
|
input_data = input_loc.open("r", encoding="utf-8").read()
|
|
|
|
# Use converter function to convert data
|
|
|
|
func = CONVERTERS[converter]
|
|
|
|
docs = func(
|
|
|
|
input_data,
|
|
|
|
n_sents=n_sents,
|
|
|
|
seg_sents=seg_sents,
|
|
|
|
append_morphology=morphology,
|
|
|
|
merge_subtokens=merge_subtokens,
|
|
|
|
lang=lang,
|
|
|
|
model=model,
|
|
|
|
no_print=silent,
|
|
|
|
ner_map=ner_map,
|
|
|
|
)
|
2020-08-24 22:30:52 +00:00
|
|
|
doc_files.append((input_loc, docs))
|
|
|
|
if concatenate:
|
2020-12-15 08:47:16 +00:00
|
|
|
all_docs = itertools.chain.from_iterable([docs for _, docs in doc_files])
|
2020-08-24 22:30:52 +00:00
|
|
|
doc_files = [(input_path, all_docs)]
|
|
|
|
for input_loc, docs in doc_files:
|
2020-07-09 17:42:32 +00:00
|
|
|
if file_type == "json":
|
|
|
|
data = [docs_to_json(docs)]
|
2020-12-15 08:47:16 +00:00
|
|
|
len_docs = len(data)
|
2020-07-09 17:42:32 +00:00
|
|
|
else:
|
2020-12-15 08:47:16 +00:00
|
|
|
db = DocBin(docs=docs, store_user_data=True)
|
|
|
|
len_docs = len(db)
|
|
|
|
data = db.to_bytes()
|
2020-06-26 17:34:12 +00:00
|
|
|
if output_dir == "-":
|
2020-07-09 17:42:32 +00:00
|
|
|
_print_docs_to_stdout(data, file_type)
|
2020-06-26 17:34:12 +00:00
|
|
|
else:
|
|
|
|
if input_loc != input_path:
|
|
|
|
subpath = input_loc.relative_to(input_path)
|
|
|
|
output_file = Path(output_dir) / subpath.with_suffix(f".{file_type}")
|
|
|
|
else:
|
|
|
|
output_file = Path(output_dir) / input_loc.parts[-1]
|
|
|
|
output_file = output_file.with_suffix(f".{file_type}")
|
2020-07-09 17:42:32 +00:00
|
|
|
_write_docs_to_file(data, output_file, file_type)
|
2020-12-15 08:47:16 +00:00
|
|
|
msg.good(f"Generated output file ({len_docs} documents): {output_file}")
|
2020-06-26 17:34:12 +00:00
|
|
|
|
|
|
|
|
2020-07-25 13:01:15 +00:00
|
|
|
def _print_docs_to_stdout(data: Any, output_type: str) -> None:
|
2020-06-26 17:34:12 +00:00
|
|
|
if output_type == "json":
|
2020-07-09 17:42:32 +00:00
|
|
|
srsly.write_json("-", data)
|
2020-06-26 17:34:12 +00:00
|
|
|
else:
|
2020-07-09 17:42:32 +00:00
|
|
|
sys.stdout.buffer.write(data)
|
2020-06-26 17:34:12 +00:00
|
|
|
|
|
|
|
|
2020-07-25 13:01:15 +00:00
|
|
|
def _write_docs_to_file(data: Any, output_file: Path, output_type: str) -> None:
|
2020-06-26 17:34:12 +00:00
|
|
|
if not output_file.parent.exists():
|
|
|
|
output_file.parent.mkdir(parents=True)
|
|
|
|
if output_type == "json":
|
2020-07-09 17:42:32 +00:00
|
|
|
srsly.write_json(output_file, data)
|
2020-06-26 17:34:12 +00:00
|
|
|
else:
|
|
|
|
with output_file.open("wb") as file_:
|
|
|
|
file_.write(data)
|
2020-07-03 13:20:10 +00:00
|
|
|
|
2020-06-26 17:34:12 +00:00
|
|
|
|
2020-07-25 13:01:15 +00:00
|
|
|
def autodetect_ner_format(input_data: str) -> Optional[str]:
|
2020-06-26 17:34:12 +00:00
|
|
|
# guess format from the first 20 lines
|
|
|
|
lines = input_data.split("\n")[:20]
|
|
|
|
format_guesses = {"ner": 0, "iob": 0}
|
|
|
|
iob_re = re.compile(r"\S+\|(O|[IB]-\S+)")
|
|
|
|
ner_re = re.compile(r"\S+\s+(O|[IB]-\S+)$")
|
|
|
|
for line in lines:
|
|
|
|
line = line.strip()
|
|
|
|
if iob_re.search(line):
|
|
|
|
format_guesses["iob"] += 1
|
|
|
|
if ner_re.search(line):
|
|
|
|
format_guesses["ner"] += 1
|
|
|
|
if format_guesses["iob"] == 0 and format_guesses["ner"] > 0:
|
|
|
|
return "ner"
|
|
|
|
if format_guesses["ner"] == 0 and format_guesses["iob"] > 0:
|
|
|
|
return "iob"
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
2020-08-24 22:30:52 +00:00
|
|
|
def walk_directory(path: Path, converter: str) -> List[Path]:
|
2020-06-26 17:34:12 +00:00
|
|
|
if not path.is_dir():
|
|
|
|
return [path]
|
|
|
|
paths = [path]
|
|
|
|
locs = []
|
|
|
|
seen = set()
|
|
|
|
for path in paths:
|
|
|
|
if str(path) in seen:
|
|
|
|
continue
|
|
|
|
seen.add(str(path))
|
|
|
|
if path.parts[-1].startswith("."):
|
|
|
|
continue
|
|
|
|
elif path.is_dir():
|
|
|
|
paths.extend(path.iterdir())
|
2020-08-24 22:30:52 +00:00
|
|
|
elif converter == "json" and not path.parts[-1].endswith("json"):
|
|
|
|
continue
|
|
|
|
elif converter == "conll" and not path.parts[-1].endswith("conll"):
|
|
|
|
continue
|
|
|
|
elif converter == "iob" and not path.parts[-1].endswith("iob"):
|
|
|
|
continue
|
2020-06-26 17:34:12 +00:00
|
|
|
else:
|
|
|
|
locs.append(path)
|
2020-09-25 17:07:26 +00:00
|
|
|
# It's good to sort these, in case the ordering messes up cache.
|
|
|
|
locs.sort()
|
2020-06-26 17:34:12 +00:00
|
|
|
return locs
|
|
|
|
|
|
|
|
|
|
|
|
def verify_cli_args(
|
2020-07-25 13:01:15 +00:00
|
|
|
msg: Printer,
|
|
|
|
input_path: Union[str, Path],
|
|
|
|
output_dir: Union[str, Path],
|
|
|
|
file_type: FileTypes,
|
|
|
|
converter: str,
|
|
|
|
ner_map: Optional[Path],
|
2020-06-26 17:34:12 +00:00
|
|
|
):
|
|
|
|
input_path = Path(input_path)
|
2019-03-08 22:15:23 +00:00
|
|
|
if file_type not in FILE_TYPES_STDOUT and output_dir == "-":
|
|
|
|
msg.fail(
|
2020-07-25 13:01:15 +00:00
|
|
|
f"Can't write .{file_type} data to stdout. Please specify an output directory.",
|
2019-03-08 22:15:23 +00:00
|
|
|
exits=1,
|
|
|
|
)
|
2017-05-07 21:25:29 +00:00
|
|
|
if not input_path.exists():
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.fail("Input file not found", input_path, exits=1)
|
2018-11-30 19:16:14 +00:00
|
|
|
if output_dir != "-" and not Path(output_dir).exists():
|
2018-12-08 10:49:43 +00:00
|
|
|
msg.fail("Output directory not found", output_dir, exits=1)
|
2020-07-25 13:01:15 +00:00
|
|
|
if ner_map is not None and not Path(ner_map).exists():
|
|
|
|
msg.fail("NER map not found", ner_map, exits=1)
|
2020-06-26 17:34:12 +00:00
|
|
|
if input_path.is_dir():
|
2020-08-24 22:30:52 +00:00
|
|
|
input_locs = walk_directory(input_path, converter)
|
2020-06-26 17:34:12 +00:00
|
|
|
if len(input_locs) == 0:
|
|
|
|
msg.fail("No input files in directory", input_path, exits=1)
|
|
|
|
file_types = list(set([loc.suffix[1:] for loc in input_locs]))
|
2020-08-24 22:30:52 +00:00
|
|
|
if converter == "auto" and len(file_types) >= 2:
|
2020-06-26 17:34:12 +00:00
|
|
|
file_types = ",".join(file_types)
|
|
|
|
msg.fail("All input files must be same type", file_types, exits=1)
|
2020-07-25 13:01:15 +00:00
|
|
|
if converter != "auto" and converter not in CONVERTERS:
|
2020-06-26 17:34:12 +00:00
|
|
|
msg.fail(f"Can't find converter for {converter}", exits=1)
|
|
|
|
|
|
|
|
|
|
|
|
def _get_converter(msg, converter, input_path):
|
|
|
|
if input_path.is_dir():
|
2020-08-24 22:30:52 +00:00
|
|
|
input_path = walk_directory(input_path, converter)[0]
|
2018-11-30 19:16:14 +00:00
|
|
|
if converter == "auto":
|
2017-10-10 01:06:28 +00:00
|
|
|
converter = input_path.suffix[1:]
|
2019-08-29 10:04:01 +00:00
|
|
|
if converter == "ner" or converter == "iob":
|
2020-10-09 14:03:14 +00:00
|
|
|
with input_path.open(encoding="utf8") as file_:
|
2020-06-26 17:34:12 +00:00
|
|
|
input_data = file_.read()
|
2019-08-29 10:04:01 +00:00
|
|
|
converter_autodetect = autodetect_ner_format(input_data)
|
|
|
|
if converter_autodetect == "ner":
|
|
|
|
msg.info("Auto-detected token-per-line NER format")
|
|
|
|
converter = converter_autodetect
|
|
|
|
elif converter_autodetect == "iob":
|
|
|
|
msg.info("Auto-detected sentence-per-line NER format")
|
|
|
|
converter = converter_autodetect
|
|
|
|
else:
|
2019-08-31 11:39:06 +00:00
|
|
|
msg.warn(
|
2020-06-26 17:34:12 +00:00
|
|
|
"Can't automatically detect NER format. "
|
|
|
|
"Conversion may not succeed. "
|
2021-01-30 09:09:38 +00:00
|
|
|
"See https://spacy.io/api/cli#convert"
|
2019-08-31 11:39:06 +00:00
|
|
|
)
|
2020-06-26 17:34:12 +00:00
|
|
|
return converter
|