From 547973b92a43c0244c7e0ed7268c8b0dffc93988 Mon Sep 17 00:00:00 2001 From: Jim Geovedi Date: Thu, 27 Jul 2017 10:51:34 +0700 Subject: [PATCH] wip syntax iterators --- spacy/lang/id/syntax_iterators.py | 42 +++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 spacy/lang/id/syntax_iterators.py diff --git a/spacy/lang/id/syntax_iterators.py b/spacy/lang/id/syntax_iterators.py new file mode 100644 index 000000000..c9de4f084 --- /dev/null +++ b/spacy/lang/id/syntax_iterators.py @@ -0,0 +1,42 @@ +# coding: utf8 +from __future__ import unicode_literals + +from ...symbols import NOUN, PROPN, PRON + + +def noun_chunks(obj): + """ + Detect base noun phrases from a dependency parse. Works on both Doc and Span. + """ + labels = ['nsubj', 'nsubj:pass', 'obj', 'iobj', 'ROOT', 'appos', 'nmod', 'nmod:poss'] + doc = obj.doc # Ensure works on both Doc and Span. + np_deps = [doc.vocab.strings[label] for label in labels] + conj = doc.vocab.strings.add('conj') + np_label = doc.vocab.strings.add('NP') + seen = set() + for i, word in enumerate(obj): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.i in seen: + continue + if word.dep in np_deps: + if any(w.i in seen for w in word.subtree): + continue + seen.update(j for j in range(word.left_edge.i, word.right_edge.i+1)) + yield word.left_edge.i, word.right_edge.i+1, np_label + elif word.dep == conj: + head = word.head + while head.dep == conj and head.head.i < head.i: + head = head.head + # If the head is an NP, and we're coordinated to it, we're an NP + if head.dep in np_deps: + if any(w.i in seen for w in word.subtree): + continue + seen.update(j for j in range(word.left_edge.i, word.right_edge.i+1)) + yield word.left_edge.i, word.right_edge.i+1, np_label + + +SYNTAX_ITERATORS = { + 'noun_chunks': noun_chunks +}