* Filter out phrases that consist of common, lower-case words.

2015-10-09 12:47:43 +11:00 · 2015-10-09 12:47:43 +11:00 · 5af4b62fe7
parent 4bbc8f45c6
commit 5af4b62fe7
1 changed files with 2 additions and 0 deletions
--- a/examples/multi_word_matches.py
+++ b/examples/multi_word_matches.py
@ -45,6 +45,8 @@ def read_gazetteer(tokenizer, loc, n=-1):
        if i >= n:
            break
        phrase = tokenizer(phrase)
+        if all((t.is_lower and t.prob >= -10) for t in phrase):
+            continue
        if len(phrase) >= 2:
            yield phrase