2018-07-25 20:21:31 +00:00
|
|
|
import re
|
2017-04-07 11:05:12 +00:00
|
|
|
|
2019-11-11 16:35:27 +00:00
|
|
|
from spacy.gold import Example
|
2018-11-30 19:16:14 +00:00
|
|
|
from ...gold import iob_to_biluo
|
2017-04-07 11:05:12 +00:00
|
|
|
|
2018-07-25 20:21:31 +00:00
|
|
|
|
2019-12-21 17:55:03 +00:00
|
|
|
def conllu2json(
|
|
|
|
input_data, n_sents=10, use_morphology=False, lang=None, ner_map=None, **_
|
|
|
|
):
|
2017-04-15 09:59:21 +00:00
|
|
|
"""
|
|
|
|
Convert conllu files into JSON format for use with train cli.
|
2017-04-07 11:05:12 +00:00
|
|
|
use_morphology parameter enables appending morphology to tags, which is
|
|
|
|
useful for languages such as Spanish, where UD tags are not so rich.
|
|
|
|
|
2018-07-25 20:21:31 +00:00
|
|
|
Extract NER tags if available and convert them so that they follow
|
|
|
|
BILUO and the Wikipedia scheme
|
|
|
|
"""
|
2018-11-30 19:16:14 +00:00
|
|
|
# by @dvsrepo, via #11 explosion/spacy-dev-resources
|
2018-07-25 20:21:31 +00:00
|
|
|
# by @katarkor
|
2019-11-26 15:10:08 +00:00
|
|
|
# name=NER is to handle NorNE
|
|
|
|
MISC_NER_PATTERN = "\|?(?:name=)?(([A-Z_]+)-([A-Z_]+)|O)\|?"
|
2017-04-07 11:05:12 +00:00
|
|
|
docs = []
|
2019-11-26 15:10:08 +00:00
|
|
|
raw = ""
|
2017-04-07 11:05:12 +00:00
|
|
|
sentences = []
|
2019-11-11 16:35:27 +00:00
|
|
|
conll_data = read_conllx(input_data, use_morphology=use_morphology)
|
2018-07-25 20:21:31 +00:00
|
|
|
checked_for_ner = False
|
|
|
|
has_ner_tags = False
|
2019-11-11 16:35:27 +00:00
|
|
|
for i, example in enumerate(conll_data):
|
2019-11-25 15:03:28 +00:00
|
|
|
if not checked_for_ner:
|
2019-12-21 17:55:03 +00:00
|
|
|
has_ner_tags = is_ner(
|
|
|
|
example.token_annotation.entities[0], MISC_NER_PATTERN
|
|
|
|
)
|
2019-11-25 15:03:28 +00:00
|
|
|
checked_for_ner = True
|
2019-11-26 15:10:08 +00:00
|
|
|
raw += example.text
|
2019-12-21 17:55:03 +00:00
|
|
|
sentences.append(
|
|
|
|
generate_sentence(
|
|
|
|
example.token_annotation,
|
|
|
|
has_ner_tags,
|
|
|
|
MISC_NER_PATTERN,
|
|
|
|
ner_map=ner_map,
|
|
|
|
)
|
|
|
|
)
|
2019-11-25 15:03:28 +00:00
|
|
|
# Real-sized documents could be extracted using the comments on the
|
|
|
|
# conllu document
|
|
|
|
if len(sentences) % n_sents == 0:
|
2019-11-26 15:10:08 +00:00
|
|
|
doc = create_doc(raw, sentences, i)
|
2019-11-25 15:03:28 +00:00
|
|
|
docs.append(doc)
|
2019-11-26 15:10:08 +00:00
|
|
|
raw = ""
|
2019-11-25 15:03:28 +00:00
|
|
|
sentences = []
|
2019-11-26 15:05:17 +00:00
|
|
|
if sentences:
|
2019-11-29 09:22:03 +00:00
|
|
|
doc = create_doc(raw, sentences, i)
|
2019-11-26 15:05:17 +00:00
|
|
|
docs.append(doc)
|
2018-11-30 19:16:14 +00:00
|
|
|
return docs
|
2017-04-07 11:05:12 +00:00
|
|
|
|
|
|
|
|
2019-11-26 15:10:08 +00:00
|
|
|
def is_ner(tag, tag_pattern):
|
2018-11-30 19:16:14 +00:00
|
|
|
"""
|
2018-07-25 20:21:31 +00:00
|
|
|
Check the 10th column of the first token to determine if the file contains
|
2018-11-30 19:16:14 +00:00
|
|
|
NER tags
|
2018-07-25 20:21:31 +00:00
|
|
|
"""
|
2019-11-26 15:10:08 +00:00
|
|
|
tag_match = re.search(tag_pattern, tag)
|
2018-07-25 20:21:31 +00:00
|
|
|
if tag_match:
|
|
|
|
return True
|
|
|
|
elif tag == "O":
|
|
|
|
return True
|
|
|
|
else:
|
|
|
|
return False
|
|
|
|
|
2018-11-30 19:16:14 +00:00
|
|
|
|
|
|
|
def read_conllx(input_data, use_morphology=False, n=0):
|
2019-11-11 16:35:27 +00:00
|
|
|
""" Yield example data points, one for each sentence """
|
2017-04-07 11:05:12 +00:00
|
|
|
i = 0
|
2018-11-30 19:16:14 +00:00
|
|
|
for sent in input_data.strip().split("\n\n"):
|
|
|
|
lines = sent.strip().split("\n")
|
2017-04-07 11:05:12 +00:00
|
|
|
if lines:
|
2018-11-30 19:16:14 +00:00
|
|
|
while lines[0].startswith("#"):
|
2017-04-07 11:05:12 +00:00
|
|
|
lines.pop(0)
|
2019-11-11 16:35:27 +00:00
|
|
|
ids, words, tags, heads, deps, ents = [], [], [], [], [], []
|
2019-11-26 15:10:08 +00:00
|
|
|
spaces = []
|
2017-04-07 11:05:12 +00:00
|
|
|
for line in lines:
|
2018-11-30 19:16:14 +00:00
|
|
|
parts = line.split("\t")
|
2019-11-26 15:10:08 +00:00
|
|
|
id_, word, lemma, pos, tag, morph, head, dep, _1, misc = parts
|
2018-11-30 19:16:14 +00:00
|
|
|
if "-" in id_ or "." in id_:
|
2017-04-07 11:05:12 +00:00
|
|
|
continue
|
|
|
|
try:
|
|
|
|
id_ = int(id_) - 1
|
2018-11-30 19:16:14 +00:00
|
|
|
head = (int(head) - 1) if head != "0" else id_
|
|
|
|
dep = "ROOT" if dep == "root" else dep
|
|
|
|
tag = pos if tag == "_" else tag
|
|
|
|
tag = tag + "__" + morph if use_morphology else tag
|
2019-11-26 15:10:08 +00:00
|
|
|
ent = misc if misc else "O"
|
2019-11-11 16:35:27 +00:00
|
|
|
|
|
|
|
ids.append(id_)
|
|
|
|
words.append(word)
|
|
|
|
tags.append(tag)
|
|
|
|
heads.append(head)
|
|
|
|
deps.append(dep)
|
2019-11-26 15:10:08 +00:00
|
|
|
ents.append(ent)
|
|
|
|
if "SpaceAfter=No" in misc:
|
|
|
|
spaces.append(False)
|
|
|
|
else:
|
|
|
|
spaces.append(True)
|
2018-11-30 19:16:14 +00:00
|
|
|
except: # noqa: E722
|
2017-04-07 11:05:12 +00:00
|
|
|
print(line)
|
|
|
|
raise
|
2019-11-26 15:10:08 +00:00
|
|
|
raw = ""
|
|
|
|
for word, space in zip(words, spaces):
|
|
|
|
raw += word
|
|
|
|
if space:
|
|
|
|
raw += " "
|
|
|
|
example = Example(doc=raw)
|
2019-12-21 17:55:03 +00:00
|
|
|
example.set_token_annotation(
|
|
|
|
ids=ids, words=words, tags=tags, heads=heads, deps=deps, entities=ents
|
|
|
|
)
|
2019-11-11 16:35:27 +00:00
|
|
|
yield example
|
2017-04-07 11:05:12 +00:00
|
|
|
i += 1
|
2019-11-11 16:35:27 +00:00
|
|
|
if 1 <= n <= i:
|
2017-04-07 11:05:12 +00:00
|
|
|
break
|
|
|
|
|
2018-11-30 19:16:14 +00:00
|
|
|
|
2019-12-11 17:20:49 +00:00
|
|
|
def extract_tags(iob, tag_pattern, ner_map=None):
|
2018-07-25 20:21:31 +00:00
|
|
|
"""
|
2019-12-11 17:20:49 +00:00
|
|
|
Extract tag from MISC column according to `tag_pattern` and map to final
|
|
|
|
entity type with `ner_map` if mapping present.
|
|
|
|
|
|
|
|
For NorNE:
|
2018-07-25 20:21:31 +00:00
|
|
|
Simplify tags obtained from the dataset in order to follow Wikipedia
|
|
|
|
scheme (PER, LOC, ORG, MISC). 'PER', 'LOC' and 'ORG' keep their tags, while
|
|
|
|
'GPE_LOC' is simplified to 'LOC', 'GPE_ORG' to 'ORG' and all remaining tags to
|
2018-11-30 19:16:14 +00:00
|
|
|
'MISC'.
|
2018-07-25 20:21:31 +00:00
|
|
|
"""
|
|
|
|
new_iob = []
|
|
|
|
for tag in iob:
|
2019-11-26 15:10:08 +00:00
|
|
|
tag_match = re.search(tag_pattern, tag)
|
|
|
|
new_tag = "O"
|
2018-07-25 20:21:31 +00:00
|
|
|
if tag_match:
|
2019-11-26 15:10:08 +00:00
|
|
|
prefix = tag_match.group(2)
|
|
|
|
suffix = tag_match.group(3)
|
|
|
|
if prefix and suffix:
|
|
|
|
new_tag = prefix + "-" + suffix
|
2019-12-11 17:20:49 +00:00
|
|
|
if ner_map:
|
|
|
|
suffix = ner_map.get(suffix, suffix)
|
|
|
|
if suffix == "":
|
|
|
|
new_tag = "O"
|
|
|
|
else:
|
|
|
|
new_tag = prefix + "-" + suffix
|
2019-11-26 15:10:08 +00:00
|
|
|
new_iob.append(new_tag)
|
2018-07-25 20:21:31 +00:00
|
|
|
return new_iob
|
2017-04-07 11:05:12 +00:00
|
|
|
|
2018-11-30 19:16:14 +00:00
|
|
|
|
2019-12-21 17:55:03 +00:00
|
|
|
def generate_sentence(token_annotation, has_ner_tags, tag_pattern, ner_map=None):
|
2017-04-07 11:05:12 +00:00
|
|
|
sentence = {}
|
|
|
|
tokens = []
|
2018-07-25 20:21:31 +00:00
|
|
|
if has_ner_tags:
|
2019-12-21 17:55:03 +00:00
|
|
|
iob = extract_tags(token_annotation.entities, tag_pattern, ner_map=ner_map)
|
2018-07-25 20:21:31 +00:00
|
|
|
biluo = iob_to_biluo(iob)
|
2019-11-11 16:35:27 +00:00
|
|
|
for i, id in enumerate(token_annotation.ids):
|
2017-04-07 11:05:12 +00:00
|
|
|
token = {}
|
2018-07-25 20:21:31 +00:00
|
|
|
token["id"] = id
|
2019-11-11 16:35:27 +00:00
|
|
|
token["orth"] = token_annotation.words[i]
|
|
|
|
token["tag"] = token_annotation.tags[i]
|
|
|
|
token["head"] = token_annotation.heads[i] - id
|
|
|
|
token["dep"] = token_annotation.deps[i]
|
2018-07-25 20:21:31 +00:00
|
|
|
if has_ner_tags:
|
|
|
|
token["ner"] = biluo[i]
|
2017-04-07 11:05:12 +00:00
|
|
|
tokens.append(token)
|
|
|
|
sentence["tokens"] = tokens
|
|
|
|
return sentence
|
|
|
|
|
|
|
|
|
2019-11-26 15:10:08 +00:00
|
|
|
def create_doc(raw, sentences, id):
|
2017-04-07 11:05:12 +00:00
|
|
|
doc = {}
|
|
|
|
paragraph = {}
|
|
|
|
doc["id"] = id
|
|
|
|
doc["paragraphs"] = []
|
2019-11-26 15:10:08 +00:00
|
|
|
paragraph["raw"] = raw.strip()
|
2017-04-07 11:05:12 +00:00
|
|
|
paragraph["sentences"] = sentences
|
|
|
|
doc["paragraphs"].append(paragraph)
|
|
|
|
return doc
|