mirror of https://github.com/explosion/spaCy.git
Move old examples
This commit is contained in:
Matthew Honnibal
parent
7887ab3b36
commit
c36e8676aa
|
|
@ -1,64 +0,0 @@
|
||||||
from __future__ import unicode_literals, print_function
|
|
||||||
import json
|
|
||||||
import pathlib
|
|
||||||
import random
|
|
||||||
|
|
||||||
import spacy
|
|
||||||
from spacy.pipeline import EntityRecognizer
|
|
||||||
from spacy.gold import GoldParse
|
|
||||||
|
|
||||||
|
|
||||||
def train_ner(nlp, train_data, entity_types):
|
|
||||||
ner = EntityRecognizer.blank(nlp.vocab, entity_types=entity_types,
|
|
||||||
features=nlp.Defaults.entity_features)
|
|
||||||
for itn in range(5):
|
|
||||||
random.shuffle(train_data)
|
|
||||||
for raw_text, entity_offsets in train_data:
|
|
||||||
doc = nlp.make_doc(raw_text)
|
|
||||||
gold = GoldParse(doc, entities=entity_offsets)
|
|
||||||
ner.update(doc, gold)
|
|
||||||
ner.model.end_training()
|
|
||||||
return ner
|
|
||||||
|
|
||||||
|
|
||||||
def main(model_dir=None):
|
|
||||||
if model_dir is not None:
|
|
||||||
model_dir = pathlb.Path(model_dir)
|
|
||||||
if not model_dir.exists():
|
|
||||||
model_dir.mkdir()
|
|
||||||
assert model_dir.isdir()
|
|
||||||
|
|
||||||
nlp = spacy.load('en', parser=False, entity=False, vectors=False)
|
|
||||||
|
|
||||||
train_data = [
|
|
||||||
(
|
|
||||||
'Who is Shaka Khan?',
|
|
||||||
[(len('Who is '), len('Who is Shaka Khan'), 'PERSON')]
|
|
||||||
),
|
|
||||||
(
|
|
||||||
'I like London and Berlin.',
|
|
||||||
[(len('I like '), len('I like London'), 'LOC'),
|
|
||||||
(len('I like London and '), len('I like London and Berlin'), 'LOC')]
|
|
||||||
)
|
|
||||||
]
|
|
||||||
ner = train_ner(nlp, train_data, ['PERSON', 'LOC'])
|
|
||||||
|
|
||||||
doc = nlp.make_doc('Who is Shaka Khan?')
|
|
||||||
nlp.tagger(doc)
|
|
||||||
ner(doc)
|
|
||||||
for word in doc:
|
|
||||||
print(word.text, word.tag_, word.ent_type_, word.ent_iob)
|
|
||||||
|
|
||||||
if model_dir is not None:
|
|
||||||
with (model_dir / 'config.json').open('wb') as file_:
|
|
||||||
json.dump(ner.cfg, file_)
|
|
||||||
ner.model.dump(str(model_dir / 'model'))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
# Who "" 2
|
|
||||||
# is "" 2
|
|
||||||
# Shaka "" PERSON 3
|
|
||||||
# Khan "" PERSON 1
|
|
||||||
# ? "" 2
|
|
||||||
|
|
@ -1,72 +0,0 @@
|
||||||
"""A quick example for training a part-of-speech tagger, without worrying
|
|
||||||
about the tokenization, or other language-specific customizations."""
|
|
||||||
|
|
||||||
from __future__ import unicode_literals
|
|
||||||
from __future__ import print_function
|
|
||||||
|
|
||||||
import plac
|
|
||||||
from os import path
|
|
||||||
import os
|
|
||||||
|
|
||||||
from spacy.vocab import Vocab
|
|
||||||
from spacy.tokenizer import Tokenizer
|
|
||||||
from spacy.tagger import Tagger
|
|
||||||
import random
|
|
||||||
|
|
||||||
|
|
||||||
# You need to define a mapping from your data's part-of-speech tag names to the
|
|
||||||
# Universal Part-of-Speech tag set, as spaCy includes an enum of these tags.
|
|
||||||
# See here for the Universal Tag Set:
|
|
||||||
# http://universaldependencies.github.io/docs/u/pos/index.html
|
|
||||||
# You may also specify morphological features for your tags, from the universal
|
|
||||||
# scheme.
|
|
||||||
TAG_MAP = {
|
|
||||||
'N': {"pos": "NOUN"},
|
|
||||||
'V': {"pos": "VERB"},
|
|
||||||
'J': {"pos": "ADJ"}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Usually you'll read this in, of course. Data formats vary.
|
|
||||||
# Ensure your strings are unicode.
|
|
||||||
DATA = [
|
|
||||||
(
|
|
||||||
["I", "like", "green", "eggs"],
|
|
||||||
["N", "V", "J", "N"]
|
|
||||||
),
|
|
||||||
(
|
|
||||||
["Eat", "blue", "ham"],
|
|
||||||
["V", "J", "N"]
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
def ensure_dir(*parts):
|
|
||||||
path_ = path.join(*parts)
|
|
||||||
if not path.exists(path_):
|
|
||||||
os.mkdir(path_)
|
|
||||||
return path_
|
|
||||||
|
|
||||||
|
|
||||||
def main(output_dir):
|
|
||||||
ensure_dir(output_dir)
|
|
||||||
ensure_dir(output_dir, "pos")
|
|
||||||
ensure_dir(output_dir, "vocab")
|
|
||||||
|
|
||||||
vocab = Vocab(tag_map=TAG_MAP)
|
|
||||||
tokenizer = Tokenizer(vocab, {}, None, None, None)
|
|
||||||
# The default_templates argument is where features are specified. See
|
|
||||||
# spacy/tagger.pyx for the defaults.
|
|
||||||
tagger = Tagger.blank(vocab, Tagger.default_templates())
|
|
||||||
|
|
||||||
for i in range(5):
|
|
||||||
for words, tags in DATA:
|
|
||||||
tokens = tokenizer.tokens_from_list(words)
|
|
||||||
tagger.train(tokens, tags)
|
|
||||||
random.shuffle(DATA)
|
|
||||||
tagger.model.end_training()
|
|
||||||
tagger.model.dump(path.join(output_dir, 'pos', 'model'))
|
|
||||||
with io.open(output_dir, 'vocab', 'strings.json') as file_:
|
|
||||||
tagger.vocab.strings.dump(file_)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
plac.call(main)
|
|
||||||
Loading…
Reference in New Issue