From e40465487c045bb19adf65d965e93b23a35e3b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ole=20Henrik=20Skogstr=C3=B8m?= Date: Tue, 30 Jan 2018 15:44:29 +0100 Subject: [PATCH 1/2] Added french syntax iterator with explenation --- spacy/lang/nb/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/spacy/lang/nb/__init__.py b/spacy/lang/nb/__init__.py index 900e59626..b6ec65e1e 100644 --- a/spacy/lang/nb/__init__.py +++ b/spacy/lang/nb/__init__.py @@ -13,6 +13,12 @@ from ...language import Language from ...attrs import LANG, NORM from ...util import update_exc, add_lookups +# Borrowing french syntax parser because both languages use +# universal dependencies for tagging/parsing. +# Read here for more: +# https://github.com/explosion/spaCy/pull/1882#issuecomment-361409573 +from ..fr.syntax_iterators import SYNTAX_ITERATORS + class NorwegianDefaults(Language.Defaults): lex_attr_getters = dict(Language.Defaults.lex_attr_getters) @@ -22,6 +28,7 @@ class NorwegianDefaults(Language.Defaults): stop_words = STOP_WORDS tag_map = TAG_MAP lemma_lookup = LOOKUP + syntax_iterators = SYNTAX_ITERATORS class Norwegian(Language): From 251a7805fe1d64f4c7c3890648b46776c5c6a5b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ole=20Henrik=20Skogstr=C3=B8m?= Date: Mon, 5 Feb 2018 14:45:05 +0100 Subject: [PATCH 2/2] Copied French syntax iterator to simplify future changes --- spacy/lang/nb/__init__.py | 2 +- spacy/lang/nb/syntax_iterators.py | 42 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 spacy/lang/nb/syntax_iterators.py diff --git a/spacy/lang/nb/__init__.py b/spacy/lang/nb/__init__.py index b6ec65e1e..629f4d6d8 100644 --- a/spacy/lang/nb/__init__.py +++ b/spacy/lang/nb/__init__.py @@ -17,7 +17,7 @@ from ...util import update_exc, add_lookups # universal dependencies for tagging/parsing. # Read here for more: # https://github.com/explosion/spaCy/pull/1882#issuecomment-361409573 -from ..fr.syntax_iterators import SYNTAX_ITERATORS +from .syntax_iterators import SYNTAX_ITERATORS class NorwegianDefaults(Language.Defaults): diff --git a/spacy/lang/nb/syntax_iterators.py b/spacy/lang/nb/syntax_iterators.py new file mode 100644 index 000000000..c9de4f084 --- /dev/null +++ b/spacy/lang/nb/syntax_iterators.py @@ -0,0 +1,42 @@ +# coding: utf8 +from __future__ import unicode_literals + +from ...symbols import NOUN, PROPN, PRON + + +def noun_chunks(obj): + """ + Detect base noun phrases from a dependency parse. Works on both Doc and Span. + """ + labels = ['nsubj', 'nsubj:pass', 'obj', 'iobj', 'ROOT', 'appos', 'nmod', 'nmod:poss'] + doc = obj.doc # Ensure works on both Doc and Span. + np_deps = [doc.vocab.strings[label] for label in labels] + conj = doc.vocab.strings.add('conj') + np_label = doc.vocab.strings.add('NP') + seen = set() + for i, word in enumerate(obj): + if word.pos not in (NOUN, PROPN, PRON): + continue + # Prevent nested chunks from being produced + if word.i in seen: + continue + if word.dep in np_deps: + if any(w.i in seen for w in word.subtree): + continue + seen.update(j for j in range(word.left_edge.i, word.right_edge.i+1)) + yield word.left_edge.i, word.right_edge.i+1, np_label + elif word.dep == conj: + head = word.head + while head.dep == conj and head.head.i < head.i: + head = head.head + # If the head is an NP, and we're coordinated to it, we're an NP + if head.dep in np_deps: + if any(w.i in seen for w in word.subtree): + continue + seen.update(j for j in range(word.left_edge.i, word.right_edge.i+1)) + yield word.left_edge.i, word.right_edge.i+1, np_label + + +SYNTAX_ITERATORS = { + 'noun_chunks': noun_chunks +}