2023-01-10 09:36:17 +00:00
|
|
|
# cython: binding=True, infer_types=True, profile=True
|
2023-06-14 15:48:41 +00:00
|
|
|
from typing import Iterable, List
|
2019-02-07 08:42:25 +00:00
|
|
|
|
2018-03-27 17:23:02 +00:00
|
|
|
from cymem.cymem cimport Pool
|
2023-06-14 15:48:41 +00:00
|
|
|
from libc.stdint cimport int8_t, int32_t
|
|
|
|
from libc.string cimport memcmp, memset
|
|
|
|
from libcpp.vector cimport vector
|
2015-10-08 15:00:45 +00:00
|
|
|
from murmurhash.mrmr cimport hash64
|
2018-03-27 17:23:02 +00:00
|
|
|
|
2019-02-07 08:42:25 +00:00
|
|
|
import re
|
2020-04-28 11:37:37 +00:00
|
|
|
import warnings
|
2018-03-27 17:23:02 +00:00
|
|
|
|
2023-06-14 15:48:41 +00:00
|
|
|
import srsly
|
|
|
|
|
2023-07-19 10:03:31 +00:00
|
|
|
from ..attrs cimport DEP, ENT_IOB, ID, LEMMA, MORPH, NULL_ATTR, POS, TAG
|
2019-02-07 08:42:25 +00:00
|
|
|
from ..structs cimport TokenC
|
2020-04-29 10:57:30 +00:00
|
|
|
from ..tokens.doc cimport Doc, get_token_attr_for_matcher
|
2023-06-14 15:48:41 +00:00
|
|
|
from ..tokens.morphanalysis cimport MorphAnalysis
|
2020-04-15 11:51:33 +00:00
|
|
|
from ..tokens.span cimport Span
|
2019-02-07 08:42:25 +00:00
|
|
|
from ..tokens.token cimport Token
|
2023-06-14 15:48:41 +00:00
|
|
|
from ..typedefs cimport attr_t
|
2018-03-27 17:23:02 +00:00
|
|
|
|
2023-06-14 15:48:41 +00:00
|
|
|
from ..attrs import IDS
|
2020-04-28 11:37:37 +00:00
|
|
|
from ..errors import Errors, MatchPatternError, Warnings
|
2023-06-14 15:48:41 +00:00
|
|
|
from ..schemas import validate_token_pattern
|
2019-02-07 08:42:25 +00:00
|
|
|
from ..strings import get_string_id
|
2023-06-14 15:48:41 +00:00
|
|
|
from .levenshtein import levenshtein_compare
|
2018-03-27 17:23:02 +00:00
|
|
|
|
2019-03-08 10:42:26 +00:00
|
|
|
DEF PADDING = 5
|
|
|
|
|
|
|
|
|
|
|
|
cdef class Matcher:
|
|
|
|
"""Match sequences of tokens, based on pattern rules.
|
|
|
|
|
2021-01-30 09:09:38 +00:00
|
|
|
DOCS: https://spacy.io/api/matcher
|
|
|
|
USAGE: https://spacy.io/usage/rule-based-matching
|
2019-03-08 10:42:26 +00:00
|
|
|
"""
|
|
|
|
|
2023-01-10 09:36:17 +00:00
|
|
|
def __init__(self, vocab, validate=True, *, fuzzy_compare=levenshtein_compare):
|
2019-03-08 10:42:26 +00:00
|
|
|
"""Create the Matcher.
|
|
|
|
|
|
|
|
vocab (Vocab): The vocabulary object, which must be shared with the
|
2023-01-10 09:36:17 +00:00
|
|
|
validate (bool): Validate all patterns added to this matcher.
|
|
|
|
fuzzy_compare (Callable[[str, str, int], bool]): The comparison method
|
|
|
|
for the FUZZY operators.
|
2019-03-08 10:42:26 +00:00
|
|
|
"""
|
|
|
|
self._extra_predicates = []
|
|
|
|
self._patterns = {}
|
|
|
|
self._callbacks = {}
|
2020-07-29 09:04:43 +00:00
|
|
|
self._filter = {}
|
2019-03-08 10:42:26 +00:00
|
|
|
self._extensions = {}
|
2019-08-21 18:52:36 +00:00
|
|
|
self._seen_attrs = set()
|
2019-03-08 10:42:26 +00:00
|
|
|
self.vocab = vocab
|
|
|
|
self.mem = Pool()
|
2019-12-25 11:39:49 +00:00
|
|
|
self.validate = validate
|
2023-01-10 09:36:17 +00:00
|
|
|
self._fuzzy_compare = fuzzy_compare
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
|
|
def __reduce__(self):
|
2023-01-10 09:36:17 +00:00
|
|
|
data = (self.vocab, self._patterns, self._callbacks, self.validate, self._fuzzy_compare)
|
2019-03-08 10:42:26 +00:00
|
|
|
return (unpickle_matcher, data, None, None)
|
|
|
|
|
|
|
|
def __len__(self):
|
|
|
|
"""Get the number of rules added to the matcher. Note that this only
|
|
|
|
returns the number of rules (identical with the number of IDs), not the
|
|
|
|
number of individual patterns.
|
|
|
|
|
|
|
|
RETURNS (int): The number of rules.
|
|
|
|
"""
|
|
|
|
return len(self._patterns)
|
|
|
|
|
|
|
|
def __contains__(self, key):
|
|
|
|
"""Check whether the matcher contains rules for a match ID.
|
|
|
|
|
2020-05-24 15:20:58 +00:00
|
|
|
key (str): The match ID.
|
2019-03-08 10:42:26 +00:00
|
|
|
RETURNS (bool): Whether the matcher contains rules for this match ID.
|
|
|
|
"""
|
2023-07-19 10:03:31 +00:00
|
|
|
return self.has_key(key) # no-cython-lint: W601
|
2019-03-08 10:42:26 +00:00
|
|
|
|
2023-07-19 10:03:31 +00:00
|
|
|
def add(self, key, patterns, *, on_match=None, greedy: str = None):
|
2019-03-08 10:42:26 +00:00
|
|
|
"""Add a match-rule to the matcher. A match-rule consists of: an ID
|
|
|
|
key, an on_match callback, and one or more patterns.
|
|
|
|
|
|
|
|
If the key exists, the patterns are appended to the previous ones, and
|
|
|
|
the previous on_match callback is replaced. The `on_match` callback
|
|
|
|
will receive the arguments `(matcher, doc, i, matches)`. You can also
|
|
|
|
set `on_match` to `None` to not perform any actions.
|
|
|
|
|
|
|
|
A pattern consists of one or more `token_specs`, where a `token_spec`
|
|
|
|
is a dictionary mapping attribute IDs to values, and optionally a
|
|
|
|
quantifier operator under the key "op". The available quantifiers are:
|
|
|
|
|
2022-06-30 09:01:58 +00:00
|
|
|
'!': Negate the pattern, by requiring it to match exactly 0 times.
|
|
|
|
'?': Make the pattern optional, by allowing it to match 0 or 1 times.
|
|
|
|
'+': Require the pattern to match 1 or more times.
|
|
|
|
'*': Allow the pattern to zero or more times.
|
|
|
|
'{n}': Require the pattern to match exactly _n_ times.
|
|
|
|
'{n,m}': Require the pattern to match at least _n_ but not more than _m_ times.
|
|
|
|
'{n,}': Require the pattern to match at least _n_ times.
|
|
|
|
'{,m}': Require the pattern to match at most _m_ times.
|
2019-03-08 10:42:26 +00:00
|
|
|
|
2020-07-29 09:04:43 +00:00
|
|
|
The + and * operators return all possible matches (not just the greedy
|
|
|
|
ones). However, the "greedy" argument can filter the final matches
|
|
|
|
by returning a non-overlapping set per key, either taking preference to
|
|
|
|
the first greedy match ("FIRST"), or the longest ("LONGEST").
|
2019-03-08 10:42:26 +00:00
|
|
|
|
2021-10-21 09:17:59 +00:00
|
|
|
Since spaCy v2.2.2, Matcher.add takes a list of patterns as the second
|
|
|
|
argument, and the on_match callback is an optional keyword argument.
|
2019-10-25 20:21:08 +00:00
|
|
|
|
🏷 Add Mypy check to CI and ignore all existing Mypy errors (#9167)
* 🚨 Ignore all existing Mypy errors
* 🏗 Add Mypy check to CI
* Add types-mock and types-requests as dev requirements
* Add additional type ignore directives
* Add types packages to dev-only list in reqs test
* Add types-dataclasses for python 3.6
* Add ignore to pretrain
* 🏷 Improve type annotation on `run_command` helper
The `run_command` helper previously declared that it returned an
`Optional[subprocess.CompletedProcess]`, but it isn't actually possible
for the function to return `None`. These changes modify the type
annotation of the `run_command` helper and remove all now-unnecessary
`# type: ignore` directives.
* 🔧 Allow variable type redefinition in limited contexts
These changes modify how Mypy is configured to allow variables to have
their type automatically redefined under certain conditions. The Mypy
documentation contains the following example:
```python
def process(items: List[str]) -> None:
# 'items' has type List[str]
items = [item.split() for item in items]
# 'items' now has type List[List[str]]
...
```
This configuration change is especially helpful in reducing the number
of `# type: ignore` directives needed to handle the common pattern of:
* Accepting a filepath as a string
* Overwriting the variable using `filepath = ensure_path(filepath)`
These changes enable redefinition and remove all `# type: ignore`
directives rendered redundant by this change.
* 🏷 Add type annotation to converters mapping
* 🚨 Fix Mypy error in convert CLI argument verification
* 🏷 Improve type annotation on `resolve_dot_names` helper
* 🏷 Add type annotations for `Vocab` attributes `strings` and `vectors`
* 🏷 Add type annotations for more `Vocab` attributes
* 🏷 Add loose type annotation for gold data compilation
* 🏷 Improve `_format_labels` type annotation
* 🏷 Fix `get_lang_class` type annotation
* 🏷 Loosen return type of `Language.evaluate`
* 🏷 Don't accept `Scorer` in `handle_scores_per_type`
* 🏷 Add `string_to_list` overloads
* 🏷 Fix non-Optional command-line options
* 🙈 Ignore redefinition of `wandb_logger` in `loggers.py`
* ➕ Install `typing_extensions` in Python 3.8+
The `typing_extensions` package states that it should be used when
"writing code that must be compatible with multiple Python versions".
Since SpaCy needs to support multiple Python versions, it should be used
when newer `typing` module members are required. One example of this is
`Literal`, which is available starting with Python 3.8.
Previously SpaCy tried to import `Literal` from `typing`, falling back
to `typing_extensions` if the import failed. However, Mypy doesn't seem
to be able to understand what `Literal` means when the initial import
means. Therefore, these changes modify how `compat` imports `Literal` by
always importing it from `typing_extensions`.
These changes also modify how `typing_extensions` is installed, so that
it is a requirement for all Python versions, including those greater
than or equal to 3.8.
* 🏷 Improve type annotation for `Language.pipe`
These changes add a missing overload variant to the type signature of
`Language.pipe`. Additionally, the type signature is enhanced to allow
type checkers to differentiate between the two overload variants based
on the `as_tuple` parameter.
Fixes #8772
* ➖ Don't install `typing-extensions` in Python 3.8+
After more detailed analysis of how to implement Python version-specific
type annotations using SpaCy, it has been determined that by branching
on a comparison against `sys.version_info` can be statically analyzed by
Mypy well enough to enable us to conditionally use
`typing_extensions.Literal`. This means that we no longer need to
install `typing_extensions` for Python versions greater than or equal to
3.8! 🎉
These changes revert previous changes installing `typing-extensions`
regardless of Python version and modify how we import the `Literal` type
to ensure that Mypy treats it properly.
* resolve mypy errors for Strict pydantic types
* refactor code to avoid missing return statement
* fix types of convert CLI command
* avoid list-set confustion in debug_data
* fix typo and formatting
* small fixes to avoid type ignores
* fix types in profile CLI command and make it more efficient
* type fixes in projects CLI
* put one ignore back
* type fixes for render
* fix render types - the sequel
* fix BaseDefault in language definitions
* fix type of noun_chunks iterator - yields tuple instead of span
* fix types in language-specific modules
* 🏷 Expand accepted inputs of `get_string_id`
`get_string_id` accepts either a string (in which case it returns its
ID) or an ID (in which case it immediately returns the ID). These
changes extend the type annotation of `get_string_id` to indicate that
it can accept either strings or IDs.
* 🏷 Handle override types in `combine_score_weights`
The `combine_score_weights` function allows users to pass an `overrides`
mapping to override data extracted from the `weights` argument. Since it
allows `Optional` dictionary values, the return value may also include
`Optional` dictionary values.
These changes update the type annotations for `combine_score_weights` to
reflect this fact.
* 🏷 Fix tokenizer serialization method signatures in `DummyTokenizer`
* 🏷 Fix redefinition of `wandb_logger`
These changes fix the redefinition of `wandb_logger` by giving a
separate name to each `WandbLogger` version. For
backwards-compatibility, `spacy.train` still exports `wandb_logger_v3`
as `wandb_logger` for now.
* more fixes for typing in language
* type fixes in model definitions
* 🏷 Annotate `_RandomWords.probs` as `NDArray`
* 🏷 Annotate `tok2vec` layers to help Mypy
* 🐛 Fix `_RandomWords.probs` type annotations for Python 3.6
Also remove an import that I forgot to move to the top of the module 😅
* more fixes for matchers and other pipeline components
* quick fix for entity linker
* fixing types for spancat, textcat, etc
* bugfix for tok2vec
* type annotations for scorer
* add runtime_checkable for Protocol
* type and import fixes in tests
* mypy fixes for training utilities
* few fixes in util
* fix import
* 🐵 Remove unused `# type: ignore` directives
* 🏷 Annotate `Language._components`
* 🏷 Annotate `spacy.pipeline.Pipe`
* add doc as property to span.pyi
* small fixes and cleanup
* explicit type annotations instead of via comment
Co-authored-by: Adriane Boyd <adrianeboyd@gmail.com>
Co-authored-by: svlandeg <sofie.vanlandeghem@gmail.com>
Co-authored-by: svlandeg <svlandeg@github.com>
2021-10-14 13:21:40 +00:00
|
|
|
key (Union[str, int]): The match ID.
|
2019-10-25 20:21:08 +00:00
|
|
|
patterns (list): The patterns to add for the given key.
|
|
|
|
on_match (callable): Optional callback executed on match.
|
2020-07-29 09:04:43 +00:00
|
|
|
greedy (str): Optional filter: "FIRST" or "LONGEST".
|
2019-03-08 10:42:26 +00:00
|
|
|
"""
|
|
|
|
errors = {}
|
2019-09-24 21:06:24 +00:00
|
|
|
if on_match is not None and not hasattr(on_match, "__call__"):
|
|
|
|
raise ValueError(Errors.E171.format(arg_type=type(on_match)))
|
2020-07-29 09:04:43 +00:00
|
|
|
if patterns is None or not isinstance(patterns, List): # old API
|
|
|
|
raise ValueError(Errors.E948.format(arg_type=type(patterns)))
|
|
|
|
if greedy is not None and greedy not in ["FIRST", "LONGEST"]:
|
|
|
|
raise ValueError(Errors.E947.format(expected=["FIRST", "LONGEST"], arg=greedy))
|
2019-03-08 10:42:26 +00:00
|
|
|
for i, pattern in enumerate(patterns):
|
|
|
|
if len(pattern) == 0:
|
|
|
|
raise ValueError(Errors.E012.format(key=key))
|
2019-10-25 20:21:08 +00:00
|
|
|
if not isinstance(pattern, list):
|
|
|
|
raise ValueError(Errors.E178.format(pat=pattern, key=key))
|
2019-12-25 11:39:49 +00:00
|
|
|
if self.validate:
|
|
|
|
errors[i] = validate_token_pattern(pattern)
|
2019-04-09 10:50:43 +00:00
|
|
|
if any(err for err in errors.values()):
|
2019-03-08 10:42:26 +00:00
|
|
|
raise MatchPatternError(key, errors)
|
|
|
|
key = self._normalize_key(key)
|
|
|
|
for pattern in patterns:
|
2019-08-21 12:00:37 +00:00
|
|
|
try:
|
2023-07-19 10:03:31 +00:00
|
|
|
specs = _preprocess_pattern(
|
|
|
|
pattern,
|
|
|
|
self.vocab,
|
|
|
|
self._extensions,
|
|
|
|
self._extra_predicates,
|
|
|
|
self._fuzzy_compare
|
|
|
|
)
|
2019-08-21 12:00:37 +00:00
|
|
|
self.patterns.push_back(init_pattern(self.mem, key, specs))
|
2019-08-21 18:52:36 +00:00
|
|
|
for spec in specs:
|
|
|
|
for attr, _ in spec[1]:
|
|
|
|
self._seen_attrs.add(attr)
|
2019-08-21 12:00:37 +00:00
|
|
|
except OverflowError, AttributeError:
|
2020-08-05 21:53:21 +00:00
|
|
|
raise ValueError(Errors.E154.format()) from None
|
2019-03-08 10:42:26 +00:00
|
|
|
self._patterns.setdefault(key, [])
|
|
|
|
self._callbacks[key] = on_match
|
2020-07-29 09:04:43 +00:00
|
|
|
self._filter[key] = greedy
|
2019-03-08 10:42:26 +00:00
|
|
|
self._patterns[key].extend(patterns)
|
|
|
|
|
2021-05-31 08:20:27 +00:00
|
|
|
def _require_patterns(self) -> None:
|
|
|
|
"""Raise a warning if this component has no patterns defined."""
|
|
|
|
if len(self) == 0:
|
|
|
|
warnings.warn(Warnings.W036.format(name="matcher"))
|
|
|
|
|
2019-03-08 10:42:26 +00:00
|
|
|
def remove(self, key):
|
|
|
|
"""Remove a rule from the matcher. A KeyError is raised if the key does
|
|
|
|
not exist.
|
|
|
|
|
2020-05-24 15:20:58 +00:00
|
|
|
key (str): The ID of the match rule.
|
2019-03-08 10:42:26 +00:00
|
|
|
"""
|
2019-10-10 12:01:53 +00:00
|
|
|
norm_key = self._normalize_key(key)
|
2023-07-19 10:03:31 +00:00
|
|
|
if norm_key not in self._patterns:
|
2019-10-10 12:01:53 +00:00
|
|
|
raise ValueError(Errors.E175.format(key=key))
|
|
|
|
self._patterns.pop(norm_key)
|
|
|
|
self._callbacks.pop(norm_key)
|
2019-03-08 10:42:26 +00:00
|
|
|
cdef int i = 0
|
|
|
|
while i < self.patterns.size():
|
2019-10-09 13:26:31 +00:00
|
|
|
pattern_key = get_ent_id(self.patterns.at(i))
|
2019-10-16 11:34:58 +00:00
|
|
|
if pattern_key == norm_key:
|
2019-03-08 10:42:26 +00:00
|
|
|
self.patterns.erase(self.patterns.begin()+i)
|
|
|
|
else:
|
|
|
|
i += 1
|
|
|
|
|
|
|
|
def has_key(self, key):
|
|
|
|
"""Check whether the matcher has a rule with a given key.
|
|
|
|
|
|
|
|
key (string or int): The key to check.
|
|
|
|
RETURNS (bool): Whether the matcher has the rule.
|
|
|
|
"""
|
2020-08-09 20:31:52 +00:00
|
|
|
return self._normalize_key(key) in self._patterns
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
|
|
def get(self, key, default=None):
|
|
|
|
"""Retrieve the pattern stored for a key.
|
|
|
|
|
2020-05-24 16:51:10 +00:00
|
|
|
key (str / int): The key to retrieve.
|
2019-03-08 10:42:26 +00:00
|
|
|
RETURNS (tuple): The rule, as an (on_match, patterns) tuple.
|
|
|
|
"""
|
|
|
|
key = self._normalize_key(key)
|
|
|
|
if key not in self._patterns:
|
|
|
|
return default
|
|
|
|
return (self._callbacks[key], self._patterns[key])
|
|
|
|
|
2020-07-06 11:06:25 +00:00
|
|
|
def pipe(self, docs, batch_size=1000, return_matches=False, as_tuples=False):
|
2020-08-31 15:01:24 +00:00
|
|
|
"""Match a stream of documents, yielding them in turn. Deprecated as of
|
|
|
|
spaCy v3.0.
|
2019-03-08 10:42:26 +00:00
|
|
|
"""
|
2020-08-31 15:01:24 +00:00
|
|
|
warnings.warn(Warnings.W105.format(matcher="Matcher"), DeprecationWarning)
|
2019-09-18 20:00:33 +00:00
|
|
|
if as_tuples:
|
|
|
|
for doc, context in docs:
|
|
|
|
matches = self(doc)
|
|
|
|
if return_matches:
|
|
|
|
yield ((doc, matches), context)
|
|
|
|
else:
|
|
|
|
yield (doc, context)
|
|
|
|
else:
|
|
|
|
for doc in docs:
|
|
|
|
matches = self(doc)
|
|
|
|
if return_matches:
|
|
|
|
yield (doc, matches)
|
|
|
|
else:
|
|
|
|
yield doc
|
2019-03-08 10:42:26 +00:00
|
|
|
|
2021-04-08 08:10:14 +00:00
|
|
|
def __call__(self, object doclike, *, as_spans=False, allow_missing=False, with_alignments=False):
|
2019-03-08 10:42:26 +00:00
|
|
|
"""Find all token sequences matching the supplied pattern.
|
|
|
|
|
2020-05-21 13:17:39 +00:00
|
|
|
doclike (Doc or Span): The document to match over.
|
2020-08-31 12:53:22 +00:00
|
|
|
as_spans (bool): Return Span objects with labels instead of (match_id,
|
|
|
|
start, end) tuples.
|
2021-03-19 09:11:10 +00:00
|
|
|
allow_missing (bool): Whether to skip checks for missing annotation for
|
|
|
|
attributes included in patterns. Defaults to False.
|
2021-04-08 08:10:14 +00:00
|
|
|
with_alignments (bool): Return match alignment information, which is
|
|
|
|
`List[int]` with length of matched span. Each entry denotes the
|
|
|
|
corresponding index of token pattern. If as_spans is set to True,
|
|
|
|
this setting is ignored.
|
2020-08-31 12:53:22 +00:00
|
|
|
RETURNS (list): A list of `(match_id, start, end)` tuples,
|
2019-03-08 10:42:26 +00:00
|
|
|
describing the matches. A match tuple describes a span
|
2020-08-31 12:53:22 +00:00
|
|
|
`doc[start:end]`. The `match_id` is an integer. If as_spans is set
|
|
|
|
to True, a list of Span objects is returned.
|
2021-04-08 08:10:14 +00:00
|
|
|
If with_alignments is set to True and as_spans is set to False,
|
|
|
|
A list of `(match_id, start, end, alignments)` tuples is returned.
|
2019-03-08 10:42:26 +00:00
|
|
|
"""
|
2021-05-31 08:20:27 +00:00
|
|
|
self._require_patterns()
|
2020-05-21 13:17:39 +00:00
|
|
|
if isinstance(doclike, Doc):
|
|
|
|
doc = doclike
|
2020-04-15 11:51:33 +00:00
|
|
|
length = len(doc)
|
2020-05-21 13:17:39 +00:00
|
|
|
elif isinstance(doclike, Span):
|
|
|
|
doc = doclike.doc
|
|
|
|
length = doclike.end - doclike.start
|
2020-04-15 11:51:33 +00:00
|
|
|
else:
|
2020-05-21 13:17:39 +00:00
|
|
|
raise ValueError(Errors.E195.format(good="Doc or Span", got=type(doclike).__name__))
|
2021-04-08 08:10:14 +00:00
|
|
|
# Skip alignments calculations if as_spans is set
|
|
|
|
if as_spans:
|
|
|
|
with_alignments = False
|
2020-07-29 09:04:43 +00:00
|
|
|
cdef Pool tmp_pool = Pool()
|
2020-09-24 14:54:39 +00:00
|
|
|
if not allow_missing:
|
|
|
|
for attr in (TAG, POS, MORPH, LEMMA, DEP):
|
|
|
|
if attr in self._seen_attrs and not doc.has_annotation(attr):
|
|
|
|
if attr == TAG:
|
|
|
|
pipe = "tagger"
|
|
|
|
elif attr in (POS, MORPH):
|
2021-03-19 09:11:10 +00:00
|
|
|
pipe = "morphologizer or tagger+attribute_ruler"
|
2020-09-24 14:54:39 +00:00
|
|
|
elif attr == LEMMA:
|
|
|
|
pipe = "lemmatizer"
|
|
|
|
elif attr == DEP:
|
|
|
|
pipe = "parser"
|
|
|
|
error_msg = Errors.E155.format(pipe=pipe, attr=self.vocab.strings.as_string(attr))
|
|
|
|
raise ValueError(error_msg)
|
2022-03-24 10:48:22 +00:00
|
|
|
|
|
|
|
if self.patterns.empty():
|
|
|
|
matches = []
|
|
|
|
else:
|
2023-07-19 10:03:31 +00:00
|
|
|
matches = find_matches(
|
|
|
|
&self.patterns[0],
|
|
|
|
self.patterns.size(),
|
|
|
|
doclike,
|
|
|
|
length,
|
|
|
|
extensions=self._extensions,
|
|
|
|
predicates=self._extra_predicates,
|
|
|
|
with_alignments=with_alignments
|
|
|
|
)
|
2020-07-29 09:04:43 +00:00
|
|
|
final_matches = []
|
|
|
|
pairs_by_id = {}
|
2021-04-08 08:10:14 +00:00
|
|
|
# For each key, either add all matches, or only the filtered,
|
|
|
|
# non-overlapping ones this `match` can be either (start, end) or
|
|
|
|
# (start, end, alignments) depending on `with_alignments=` option.
|
|
|
|
for key, *match in matches:
|
2020-07-29 09:04:43 +00:00
|
|
|
span_filter = self._filter.get(key)
|
|
|
|
if span_filter is not None:
|
|
|
|
pairs = pairs_by_id.get(key, [])
|
2021-04-08 08:10:14 +00:00
|
|
|
pairs.append(match)
|
2020-07-29 09:04:43 +00:00
|
|
|
pairs_by_id[key] = pairs
|
|
|
|
else:
|
2021-04-08 08:10:14 +00:00
|
|
|
final_matches.append((key, *match))
|
2020-07-29 09:04:43 +00:00
|
|
|
matched = <char*>tmp_pool.alloc(length, sizeof(char))
|
|
|
|
empty = <char*>tmp_pool.alloc(length, sizeof(char))
|
|
|
|
for key, pairs in pairs_by_id.items():
|
|
|
|
memset(matched, 0, length * sizeof(matched[0]))
|
|
|
|
span_filter = self._filter.get(key)
|
|
|
|
if span_filter == "FIRST":
|
2023-07-19 10:03:31 +00:00
|
|
|
sorted_pairs = sorted(pairs, key=lambda x: (x[0], -x[1]), reverse=False) # sort by start
|
2020-07-29 09:04:43 +00:00
|
|
|
elif span_filter == "LONGEST":
|
2023-07-19 10:03:31 +00:00
|
|
|
sorted_pairs = sorted(pairs, key=lambda x: (x[1]-x[0], -x[0]), reverse=True) # reverse sort by length
|
2020-07-29 09:04:43 +00:00
|
|
|
else:
|
|
|
|
raise ValueError(Errors.E947.format(expected=["FIRST", "LONGEST"], arg=span_filter))
|
2021-04-08 08:10:14 +00:00
|
|
|
for match in sorted_pairs:
|
|
|
|
start, end = match[:2]
|
2020-07-29 09:04:43 +00:00
|
|
|
assert 0 <= start < end # Defend against segfaults
|
|
|
|
span_len = end-start
|
|
|
|
# If no tokens in the span have matched
|
|
|
|
if memcmp(&matched[start], &empty[start], span_len * sizeof(matched[0])) == 0:
|
2021-04-08 08:10:14 +00:00
|
|
|
final_matches.append((key, *match))
|
2020-07-29 09:04:43 +00:00
|
|
|
# Mark tokens that have matched
|
|
|
|
memset(&matched[start], 1, span_len * sizeof(matched[0]))
|
2020-08-31 12:53:22 +00:00
|
|
|
if as_spans:
|
2021-09-02 10:58:05 +00:00
|
|
|
final_results = []
|
|
|
|
for key, start, end, *_ in final_matches:
|
2021-05-06 08:42:44 +00:00
|
|
|
if isinstance(doclike, Span):
|
|
|
|
start += doclike.start
|
|
|
|
end += doclike.start
|
2021-09-02 10:58:05 +00:00
|
|
|
final_results.append(Span(doc, start, end, label=key))
|
2021-04-08 08:10:14 +00:00
|
|
|
elif with_alignments:
|
|
|
|
# convert alignments List[Dict[str, int]] --> List[int]
|
|
|
|
# when multiple alignment (belongs to the same length) is found,
|
|
|
|
# keeps the alignment that has largest token_idx
|
2021-09-02 10:58:05 +00:00
|
|
|
final_results = []
|
|
|
|
for key, start, end, alignments in final_matches:
|
2021-04-08 08:10:14 +00:00
|
|
|
sorted_alignments = sorted(alignments, key=lambda x: (x['length'], x['token_idx']), reverse=False)
|
|
|
|
alignments = [0] * (end-start)
|
|
|
|
for align in sorted_alignments:
|
|
|
|
if align['length'] >= end-start:
|
|
|
|
continue
|
|
|
|
# Since alignments are sorted in order of (length, token_idx)
|
|
|
|
# this overwrites smaller token_idx when they have same length.
|
|
|
|
alignments[align['length']] = align['token_idx']
|
2021-09-02 10:58:05 +00:00
|
|
|
final_results.append((key, start, end, alignments))
|
|
|
|
final_matches = final_results # for callbacks
|
2020-08-31 12:53:22 +00:00
|
|
|
else:
|
2021-09-02 10:58:05 +00:00
|
|
|
final_results = final_matches
|
|
|
|
# perform the callbacks on the filtered set of results
|
|
|
|
for i, (key, *_) in enumerate(final_matches):
|
|
|
|
on_match = self._callbacks.get(key, None)
|
|
|
|
if on_match is not None:
|
|
|
|
on_match(self, doc, i, final_matches)
|
|
|
|
return final_results
|
2019-03-08 10:42:26 +00:00
|
|
|
|
|
|
|
def _normalize_key(self, key):
|
2021-09-13 15:02:17 +00:00
|
|
|
if isinstance(key, str):
|
2019-03-08 10:42:26 +00:00
|
|
|
return self.vocab.strings.add(key)
|
|
|
|
else:
|
|
|
|
return key
|
|
|
|
|
|
|
|
|
2023-01-10 09:36:17 +00:00
|
|
|
def unpickle_matcher(vocab, patterns, callbacks, validate, fuzzy_compare):
|
|
|
|
matcher = Matcher(vocab, validate=validate, fuzzy_compare=fuzzy_compare)
|
2020-07-29 09:04:43 +00:00
|
|
|
for key, pattern in patterns.items():
|
2019-03-08 10:42:26 +00:00
|
|
|
callback = callbacks.get(key, None)
|
2020-07-29 09:04:43 +00:00
|
|
|
matcher.add(key, pattern, on_match=callback)
|
2019-03-08 10:42:26 +00:00
|
|
|
return matcher
|
|
|
|
|
|
|
|
|
2021-04-08 08:10:14 +00:00
|
|
|
cdef find_matches(TokenPatternC** patterns, int n, object doclike, int length, extensions=None, predicates=tuple(), bint with_alignments=0):
|
2019-03-08 10:42:26 +00:00
|
|
|
"""Find matches in a doc, with a compiled array of patterns. Matches are
|
2021-04-08 08:10:14 +00:00
|
|
|
returned as a list of (id, start, end) tuples or (id, start, end, alignments) tuples (if with_alignments != 0)
|
2019-01-21 12:23:15 +00:00
|
|
|
|
|
|
|
To augment the compiled patterns, we optionally also take two Python lists.
|
2019-02-07 08:42:25 +00:00
|
|
|
|
2019-01-21 12:23:15 +00:00
|
|
|
The "predicates" list contains functions that take a Python list and return a
|
|
|
|
boolean value. It's mostly used for regular expressions.
|
2019-02-07 08:42:25 +00:00
|
|
|
|
2021-09-02 07:26:33 +00:00
|
|
|
The "extensions" list contains functions that take a Python list and return
|
2019-01-21 12:23:15 +00:00
|
|
|
an attr ID. It's mostly used for extension attributes.
|
2019-03-08 10:42:26 +00:00
|
|
|
"""
|
2018-03-27 17:23:02 +00:00
|
|
|
cdef vector[PatternStateC] states
|
|
|
|
cdef vector[MatchC] matches
|
2021-04-08 08:10:14 +00:00
|
|
|
cdef vector[vector[MatchAlignmentC]] align_states
|
|
|
|
cdef vector[vector[MatchAlignmentC]] align_matches
|
2019-01-21 12:23:15 +00:00
|
|
|
cdef int i, j, nr_extra_attr
|
2018-03-27 17:23:02 +00:00
|
|
|
cdef Pool mem = Pool()
|
2019-10-22 14:54:33 +00:00
|
|
|
output = []
|
2020-04-15 11:51:33 +00:00
|
|
|
if length == 0:
|
2019-10-22 14:54:33 +00:00
|
|
|
# avoid any processing or mem alloc if the document is empty
|
|
|
|
return output
|
|
|
|
if len(predicates) > 0:
|
2020-11-23 09:26:47 +00:00
|
|
|
predicate_cache = <int8_t*>mem.alloc(length * len(predicates), sizeof(int8_t))
|
2019-01-21 12:23:15 +00:00
|
|
|
if extensions is not None and len(extensions) >= 1:
|
2019-02-20 20:30:39 +00:00
|
|
|
nr_extra_attr = max(extensions.values()) + 1
|
2020-04-15 11:51:33 +00:00
|
|
|
extra_attr_values = <attr_t*>mem.alloc(length * nr_extra_attr, sizeof(attr_t))
|
2019-01-21 12:23:15 +00:00
|
|
|
else:
|
|
|
|
nr_extra_attr = 0
|
2020-04-15 11:51:33 +00:00
|
|
|
extra_attr_values = <attr_t*>mem.alloc(length, sizeof(attr_t))
|
2020-05-21 13:17:39 +00:00
|
|
|
for i, token in enumerate(doclike):
|
2019-01-21 12:23:15 +00:00
|
|
|
for name, index in extensions.items():
|
|
|
|
value = token._.get(name)
|
2021-09-13 15:02:17 +00:00
|
|
|
if isinstance(value, str):
|
2019-01-21 12:23:15 +00:00
|
|
|
value = token.vocab.strings[value]
|
|
|
|
extra_attr_values[i * nr_extra_attr + index] = value
|
2018-03-27 17:23:02 +00:00
|
|
|
# Main loop
|
2020-04-15 11:51:33 +00:00
|
|
|
for i in range(length):
|
2018-03-27 17:23:02 +00:00
|
|
|
for j in range(n):
|
|
|
|
states.push_back(PatternStateC(patterns[j], i, 0))
|
2021-04-08 08:10:14 +00:00
|
|
|
if with_alignments != 0:
|
|
|
|
align_states.resize(states.size())
|
2023-07-19 10:03:31 +00:00
|
|
|
transition_states(
|
|
|
|
states,
|
|
|
|
matches,
|
|
|
|
align_states,
|
|
|
|
align_matches,
|
|
|
|
predicate_cache,
|
|
|
|
doclike[i],
|
|
|
|
extra_attr_values,
|
|
|
|
predicates,
|
|
|
|
with_alignments
|
|
|
|
)
|
2019-01-21 12:23:15 +00:00
|
|
|
extra_attr_values += nr_extra_attr
|
2019-02-27 09:25:39 +00:00
|
|
|
predicate_cache += len(predicates)
|
2018-03-27 17:23:02 +00:00
|
|
|
# Handle matches that end in 0-width patterns
|
2021-04-08 08:10:14 +00:00
|
|
|
finish_states(matches, states, align_matches, align_states, with_alignments)
|
Fix behaviour of Matcher's ? quantifier for v2.1 (#3105)
* Add failing test for matcher bug #3009
* Deduplicate matches from Matcher
* Update matcher ? quantifier test
* Fix bug with ? quantifier in Matcher
The ? quantifier indicates a token may occur zero or one times. If the
token pattern fit, the matcher would fail to consider valid matches
where the token pattern did not fit. Consider a simple regex like:
.?b
If we have the string 'b', the .? part will fit --- but then the 'b' in
the pattern will not fit, leaving us with no match. The same bug left us
with too few matches in some cases. For instance, consider:
.?.?
If we have a string of length two, like 'ab', we actually have three
possible matches here: [a, b, ab]. We were only recovering 'ab'. This
should now be fixed. Note that the fix also uncovered another bug, where
we weren't deduplicating the matches. There are actually two ways we
might match 'a' and two ways we might match 'b': as the second token of the pattern,
or as the first token of the pattern. This ambiguity is spurious, so we
need to deduplicate.
Closes #2464 and #3009
* Fix Python2
2018-12-29 15:18:09 +00:00
|
|
|
seen = set()
|
|
|
|
for i in range(matches.size()):
|
|
|
|
match = (
|
|
|
|
matches[i].pattern_id,
|
|
|
|
matches[i].start,
|
|
|
|
matches[i].start+matches[i].length
|
|
|
|
)
|
|
|
|
# We need to deduplicate, because we could otherwise arrive at the same
|
|
|
|
# match through two paths, e.g. .?.? matching 'a'. Are we matching the
|
|
|
|
# first .?, or the second .? -- it doesn't matter, it's just one match.
|
2021-01-26 03:52:45 +00:00
|
|
|
# Skip 0-length matches. (TODO: fix algorithm)
|
|
|
|
if match not in seen and matches[i].length > 0:
|
2021-04-08 08:10:14 +00:00
|
|
|
if with_alignments != 0:
|
|
|
|
# since the length of align_matches equals to that of match, we can share same 'i'
|
|
|
|
output.append(match + (align_matches[i],))
|
|
|
|
else:
|
|
|
|
output.append(match)
|
Fix behaviour of Matcher's ? quantifier for v2.1 (#3105)
* Add failing test for matcher bug #3009
* Deduplicate matches from Matcher
* Update matcher ? quantifier test
* Fix bug with ? quantifier in Matcher
The ? quantifier indicates a token may occur zero or one times. If the
token pattern fit, the matcher would fail to consider valid matches
where the token pattern did not fit. Consider a simple regex like:
.?b
If we have the string 'b', the .? part will fit --- but then the 'b' in
the pattern will not fit, leaving us with no match. The same bug left us
with too few matches in some cases. For instance, consider:
.?.?
If we have a string of length two, like 'ab', we actually have three
possible matches here: [a, b, ab]. We were only recovering 'ab'. This
should now be fixed. Note that the fix also uncovered another bug, where
we weren't deduplicating the matches. There are actually two ways we
might match 'a' and two ways we might match 'b': as the second token of the pattern,
or as the first token of the pattern. This ambiguity is spurious, so we
need to deduplicate.
Closes #2464 and #3009
* Fix Python2
2018-12-29 15:18:09 +00:00
|
|
|
seen.add(match)
|
|
|
|
return output
|
2018-03-27 17:23:02 +00:00
|
|
|
|
|
|
|
|
2023-07-19 10:03:31 +00:00
|
|
|
cdef void transition_states(
|
|
|
|
vector[PatternStateC]& states,
|
|
|
|
vector[MatchC]& matches,
|
|
|
|
vector[vector[MatchAlignmentC]]& align_states,
|
|
|
|
vector[vector[MatchAlignmentC]]& align_matches,
|
|
|
|
int8_t* cached_py_predicates,
|
|
|
|
Token token,
|
|
|
|
const attr_t* extra_attrs,
|
|
|
|
py_predicates,
|
|
|
|
bint with_alignments
|
|
|
|
) except *:
|
2018-03-27 17:23:02 +00:00
|
|
|
cdef int q = 0
|
|
|
|
cdef vector[PatternStateC] new_states
|
2021-04-08 08:10:14 +00:00
|
|
|
cdef vector[vector[MatchAlignmentC]] align_new_states
|
2018-03-27 17:23:02 +00:00
|
|
|
for i in range(states.size()):
|
2019-02-20 20:30:39 +00:00
|
|
|
if states[i].pattern.nr_py >= 1:
|
2023-07-19 10:03:31 +00:00
|
|
|
update_predicate_cache(
|
|
|
|
cached_py_predicates,
|
|
|
|
states[i].pattern,
|
|
|
|
token,
|
|
|
|
py_predicates
|
|
|
|
)
|
2019-01-21 12:23:15 +00:00
|
|
|
action = get_action(states[i], token.c, extra_attrs,
|
2019-02-20 20:30:39 +00:00
|
|
|
cached_py_predicates)
|
2018-03-27 17:23:02 +00:00
|
|
|
if action == REJECT:
|
|
|
|
continue
|
2019-02-20 20:30:39 +00:00
|
|
|
# Keep only a subset of states (the active ones). Index q is the
|
|
|
|
# states which are still alive. If we reject a state, we overwrite
|
|
|
|
# it in the states list, because q doesn't advance.
|
2018-03-27 17:23:02 +00:00
|
|
|
state = states[i]
|
|
|
|
states[q] = state
|
2021-04-08 08:10:14 +00:00
|
|
|
# Separate from states, performance is guaranteed for users who only need basic options (without alignments).
|
|
|
|
# `align_states` always corresponds to `states` 1:1.
|
|
|
|
if with_alignments != 0:
|
|
|
|
align_state = align_states[i]
|
|
|
|
align_states[q] = align_state
|
Fix behaviour of Matcher's ? quantifier for v2.1 (#3105)
* Add failing test for matcher bug #3009
* Deduplicate matches from Matcher
* Update matcher ? quantifier test
* Fix bug with ? quantifier in Matcher
The ? quantifier indicates a token may occur zero or one times. If the
token pattern fit, the matcher would fail to consider valid matches
where the token pattern did not fit. Consider a simple regex like:
.?b
If we have the string 'b', the .? part will fit --- but then the 'b' in
the pattern will not fit, leaving us with no match. The same bug left us
with too few matches in some cases. For instance, consider:
.?.?
If we have a string of length two, like 'ab', we actually have three
possible matches here: [a, b, ab]. We were only recovering 'ab'. This
should now be fixed. Note that the fix also uncovered another bug, where
we weren't deduplicating the matches. There are actually two ways we
might match 'a' and two ways we might match 'b': as the second token of the pattern,
or as the first token of the pattern. This ambiguity is spurious, so we
need to deduplicate.
Closes #2464 and #3009
* Fix Python2
2018-12-29 15:18:09 +00:00
|
|
|
while action in (RETRY, RETRY_ADVANCE, RETRY_EXTEND):
|
2021-04-08 08:10:14 +00:00
|
|
|
# Update alignment before the transition of current state
|
|
|
|
# 'MatchAlignmentC' maps 'original token index of current pattern' to 'current matching length'
|
|
|
|
if with_alignments != 0:
|
|
|
|
align_states[q].push_back(MatchAlignmentC(states[q].pattern.token_idx, states[q].length))
|
2018-03-27 17:23:02 +00:00
|
|
|
if action == RETRY_EXTEND:
|
Fix behaviour of Matcher's ? quantifier for v2.1 (#3105)
* Add failing test for matcher bug #3009
* Deduplicate matches from Matcher
* Update matcher ? quantifier test
* Fix bug with ? quantifier in Matcher
The ? quantifier indicates a token may occur zero or one times. If the
token pattern fit, the matcher would fail to consider valid matches
where the token pattern did not fit. Consider a simple regex like:
.?b
If we have the string 'b', the .? part will fit --- but then the 'b' in
the pattern will not fit, leaving us with no match. The same bug left us
with too few matches in some cases. For instance, consider:
.?.?
If we have a string of length two, like 'ab', we actually have three
possible matches here: [a, b, ab]. We were only recovering 'ab'. This
should now be fixed. Note that the fix also uncovered another bug, where
we weren't deduplicating the matches. There are actually two ways we
might match 'a' and two ways we might match 'b': as the second token of the pattern,
or as the first token of the pattern. This ambiguity is spurious, so we
need to deduplicate.
Closes #2464 and #3009
* Fix Python2
2018-12-29 15:18:09 +00:00
|
|
|
# This handles the 'extend'
|
2018-03-27 17:23:02 +00:00
|
|
|
new_states.push_back(
|
2019-08-21 19:58:04 +00:00
|
|
|
PatternStateC(pattern=states[q].pattern, start=state.start,
|
2018-03-27 17:23:02 +00:00
|
|
|
length=state.length+1))
|
2021-04-08 08:10:14 +00:00
|
|
|
if with_alignments != 0:
|
|
|
|
align_new_states.push_back(align_states[q])
|
Fix behaviour of Matcher's ? quantifier for v2.1 (#3105)
* Add failing test for matcher bug #3009
* Deduplicate matches from Matcher
* Update matcher ? quantifier test
* Fix bug with ? quantifier in Matcher
The ? quantifier indicates a token may occur zero or one times. If the
token pattern fit, the matcher would fail to consider valid matches
where the token pattern did not fit. Consider a simple regex like:
.?b
If we have the string 'b', the .? part will fit --- but then the 'b' in
the pattern will not fit, leaving us with no match. The same bug left us
with too few matches in some cases. For instance, consider:
.?.?
If we have a string of length two, like 'ab', we actually have three
possible matches here: [a, b, ab]. We were only recovering 'ab'. This
should now be fixed. Note that the fix also uncovered another bug, where
we weren't deduplicating the matches. There are actually two ways we
might match 'a' and two ways we might match 'b': as the second token of the pattern,
or as the first token of the pattern. This ambiguity is spurious, so we
need to deduplicate.
Closes #2464 and #3009
* Fix Python2
2018-12-29 15:18:09 +00:00
|
|
|
if action == RETRY_ADVANCE:
|
|
|
|
# This handles the 'advance'
|
|
|
|
new_states.push_back(
|
2019-08-21 19:58:04 +00:00
|
|
|
PatternStateC(pattern=states[q].pattern+1, start=state.start,
|
Fix behaviour of Matcher's ? quantifier for v2.1 (#3105)
* Add failing test for matcher bug #3009
* Deduplicate matches from Matcher
* Update matcher ? quantifier test
* Fix bug with ? quantifier in Matcher
The ? quantifier indicates a token may occur zero or one times. If the
token pattern fit, the matcher would fail to consider valid matches
where the token pattern did not fit. Consider a simple regex like:
.?b
If we have the string 'b', the .? part will fit --- but then the 'b' in
the pattern will not fit, leaving us with no match. The same bug left us
with too few matches in some cases. For instance, consider:
.?.?
If we have a string of length two, like 'ab', we actually have three
possible matches here: [a, b, ab]. We were only recovering 'ab'. This
should now be fixed. Note that the fix also uncovered another bug, where
we weren't deduplicating the matches. There are actually two ways we
might match 'a' and two ways we might match 'b': as the second token of the pattern,
or as the first token of the pattern. This ambiguity is spurious, so we
need to deduplicate.
Closes #2464 and #3009
* Fix Python2
2018-12-29 15:18:09 +00:00
|
|
|
length=state.length+1))
|
2021-04-08 08:10:14 +00:00
|
|
|
if with_alignments != 0:
|
|
|
|
align_new_states.push_back(align_states[q])
|
2018-03-27 17:23:02 +00:00
|
|
|
states[q].pattern += 1
|
2019-01-21 12:23:15 +00:00
|
|
|
if states[q].pattern.nr_py != 0:
|
2023-07-19 10:03:31 +00:00
|
|
|
update_predicate_cache(
|
|
|
|
cached_py_predicates,
|
|
|
|
states[q].pattern,
|
|
|
|
token,
|
|
|
|
py_predicates
|
|
|
|
)
|
2019-01-21 12:23:15 +00:00
|
|
|
action = get_action(states[q], token.c, extra_attrs,
|
2019-02-20 20:30:39 +00:00
|
|
|
cached_py_predicates)
|
2021-04-08 08:10:14 +00:00
|
|
|
# Update alignment before the transition of current state
|
|
|
|
if with_alignments != 0:
|
|
|
|
align_states[q].push_back(MatchAlignmentC(states[q].pattern.token_idx, states[q].length))
|
2018-03-27 17:23:02 +00:00
|
|
|
if action == REJECT:
|
|
|
|
pass
|
|
|
|
elif action == ADVANCE:
|
|
|
|
states[q].pattern += 1
|
|
|
|
states[q].length += 1
|
|
|
|
q += 1
|
|
|
|
else:
|
2019-08-22 15:17:07 +00:00
|
|
|
ent_id = get_ent_id(state.pattern)
|
2018-03-27 17:23:02 +00:00
|
|
|
if action == MATCH:
|
|
|
|
matches.push_back(
|
2023-07-19 10:03:31 +00:00
|
|
|
MatchC(
|
|
|
|
pattern_id=ent_id,
|
|
|
|
start=state.start,
|
|
|
|
length=state.length+1
|
|
|
|
)
|
|
|
|
)
|
2021-04-08 08:10:14 +00:00
|
|
|
# `align_matches` always corresponds to `matches` 1:1
|
|
|
|
if with_alignments != 0:
|
|
|
|
align_matches.push_back(align_states[q])
|
2019-08-21 20:46:56 +00:00
|
|
|
elif action == MATCH_DOUBLE:
|
|
|
|
# push match without last token if length > 0
|
|
|
|
if state.length > 0:
|
|
|
|
matches.push_back(
|
2023-07-19 10:03:31 +00:00
|
|
|
MatchC(
|
|
|
|
pattern_id=ent_id,
|
|
|
|
start=state.start,
|
|
|
|
length=state.length
|
|
|
|
)
|
|
|
|
)
|
2021-04-08 08:10:14 +00:00
|
|
|
# MATCH_DOUBLE emits matches twice,
|
|
|
|
# add one more to align_matches in order to keep 1:1 relationship
|
|
|
|
if with_alignments != 0:
|
|
|
|
align_matches.push_back(align_states[q])
|
2019-08-21 20:46:56 +00:00
|
|
|
# push match with last token
|
|
|
|
matches.push_back(
|
2023-07-19 10:03:31 +00:00
|
|
|
MatchC(
|
|
|
|
pattern_id=ent_id,
|
|
|
|
start=state.start,
|
|
|
|
length=state.length + 1
|
|
|
|
)
|
|
|
|
)
|
2021-04-08 08:10:14 +00:00
|
|
|
# `align_matches` always corresponds to `matches` 1:1
|
|
|
|
if with_alignments != 0:
|
|
|
|
align_matches.push_back(align_states[q])
|
2018-03-27 17:23:02 +00:00
|
|
|
elif action == MATCH_REJECT:
|
|
|
|
matches.push_back(
|
2023-07-19 10:03:31 +00:00
|
|
|
MatchC(
|
|
|
|
pattern_id=ent_id,
|
|
|
|
start=state.start,
|
|
|
|
length=state.length
|
|
|
|
)
|
|
|
|
)
|
2021-04-08 08:10:14 +00:00
|
|
|
# `align_matches` always corresponds to `matches` 1:1
|
|
|
|
if with_alignments != 0:
|
|
|
|
align_matches.push_back(align_states[q])
|
2018-03-27 17:23:02 +00:00
|
|
|
elif action == MATCH_EXTEND:
|
|
|
|
matches.push_back(
|
|
|
|
MatchC(pattern_id=ent_id, start=state.start,
|
|
|
|
length=state.length))
|
2021-04-08 08:10:14 +00:00
|
|
|
# `align_matches` always corresponds to `matches` 1:1
|
|
|
|
if with_alignments != 0:
|
|
|
|
align_matches.push_back(align_states[q])
|
2018-03-27 17:23:02 +00:00
|
|
|
states[q].length += 1
|
|
|
|
q += 1
|
|
|
|
states.resize(q)
|
|
|
|
for i in range(new_states.size()):
|
|
|
|
states.push_back(new_states[i])
|
2021-04-08 08:10:14 +00:00
|
|
|
# `align_states` always corresponds to `states` 1:1
|
|
|
|
if with_alignments != 0:
|
|
|
|
align_states.resize(q)
|
|
|
|
for i in range(align_new_states.size()):
|
|
|
|
align_states.push_back(align_new_states[i])
|
2018-03-27 17:23:02 +00:00
|
|
|
|
|
|
|
|
2023-07-19 10:03:31 +00:00
|
|
|
cdef int update_predicate_cache(
|
|
|
|
int8_t* cache,
|
|
|
|
const TokenPatternC* pattern,
|
|
|
|
Token token,
|
|
|
|
predicates
|
|
|
|
) except -1:
|
2019-01-21 12:23:15 +00:00
|
|
|
# If the state references any extra predicates, check whether they match.
|
|
|
|
# These are cached, so that we don't call these potentially expensive
|
|
|
|
# Python functions more than we need to.
|
|
|
|
for i in range(pattern.nr_py):
|
|
|
|
index = pattern.py_predicates[i]
|
|
|
|
if cache[index] == 0:
|
|
|
|
predicate = predicates[index]
|
|
|
|
result = predicate(token)
|
|
|
|
if result is True:
|
|
|
|
cache[index] = 1
|
|
|
|
elif result is False:
|
|
|
|
cache[index] = -1
|
|
|
|
elif result is None:
|
|
|
|
pass
|
|
|
|
else:
|
2019-03-08 10:42:26 +00:00
|
|
|
raise ValueError(Errors.E125.format(value=result))
|
2019-01-21 12:23:15 +00:00
|
|
|
|
2019-02-07 08:42:25 +00:00
|
|
|
|
2021-04-08 08:10:14 +00:00
|
|
|
cdef void finish_states(vector[MatchC]& matches, vector[PatternStateC]& states,
|
|
|
|
vector[vector[MatchAlignmentC]]& align_matches,
|
|
|
|
vector[vector[MatchAlignmentC]]& align_states,
|
|
|
|
bint with_alignments) except *:
|
2019-03-08 10:42:26 +00:00
|
|
|
"""Handle states that end in zero-width patterns."""
|
2018-03-27 17:23:02 +00:00
|
|
|
cdef PatternStateC state
|
2021-04-08 08:10:14 +00:00
|
|
|
cdef vector[MatchAlignmentC] align_state
|
2018-03-27 17:23:02 +00:00
|
|
|
for i in range(states.size()):
|
|
|
|
state = states[i]
|
2021-04-08 08:10:14 +00:00
|
|
|
if with_alignments != 0:
|
|
|
|
align_state = align_states[i]
|
2018-03-27 17:23:02 +00:00
|
|
|
while get_quantifier(state) in (ZERO_PLUS, ZERO_ONE):
|
2021-04-08 08:10:14 +00:00
|
|
|
# Update alignment before the transition of current state
|
|
|
|
if with_alignments != 0:
|
|
|
|
align_state.push_back(MatchAlignmentC(state.pattern.token_idx, state.length))
|
2018-03-27 17:23:02 +00:00
|
|
|
is_final = get_is_final(state)
|
|
|
|
if is_final:
|
2018-08-15 14:19:08 +00:00
|
|
|
ent_id = get_ent_id(state.pattern)
|
2021-04-08 08:10:14 +00:00
|
|
|
# `align_matches` always corresponds to `matches` 1:1
|
|
|
|
if with_alignments != 0:
|
|
|
|
align_matches.push_back(align_state)
|
2018-03-27 17:23:02 +00:00
|
|
|
matches.push_back(
|
|
|
|
MatchC(pattern_id=ent_id, start=state.start, length=state.length))
|
|
|
|
break
|
|
|
|
else:
|
|
|
|
state.pattern += 1
|
|
|
|
|
2023-07-19 10:03:31 +00:00
|
|
|
cdef action_t get_action(
|
|
|
|
PatternStateC state,
|
|
|
|
const TokenC * token,
|
|
|
|
const attr_t * extra_attrs,
|
|
|
|
const int8_t * predicate_matches
|
|
|
|
) nogil:
|
2019-03-08 10:42:26 +00:00
|
|
|
"""We need to consider:
|
2018-03-27 17:23:02 +00:00
|
|
|
a) Does the token match the specification? [Yes, No]
|
|
|
|
b) What's the quantifier? [1, 0+, ?]
|
|
|
|
c) Is this the last specification? [final, non-final]
|
|
|
|
|
|
|
|
We can transition in the following ways:
|
|
|
|
a) Do we emit a match?
|
|
|
|
b) Do we add a state with (next state, next token)?
|
|
|
|
c) Do we add a state with (next state, same token)?
|
|
|
|
d) Do we add a state with (same state, next token)?
|
|
|
|
|
|
|
|
We'll code the actions as boolean strings, so 0000 means no to all 4,
|
|
|
|
1000 means match but no states added, etc.
|
2018-11-14 18:10:21 +00:00
|
|
|
|
2018-03-27 17:23:02 +00:00
|
|
|
1:
|
|
|
|
Yes, final:
|
|
|
|
1000
|
|
|
|
Yes, non-final:
|
|
|
|
0100
|
|
|
|
No, final:
|
|
|
|
0000
|
|
|
|
No, non-final
|
|
|
|
0000
|
|
|
|
0+:
|
|
|
|
Yes, final:
|
|
|
|
1001
|
|
|
|
Yes, non-final:
|
|
|
|
0011
|
|
|
|
No, final:
|
|
|
|
1000 (note: Don't include last token!)
|
|
|
|
No, non-final:
|
|
|
|
0010
|
|
|
|
?:
|
|
|
|
Yes, final:
|
|
|
|
1000
|
|
|
|
Yes, non-final:
|
|
|
|
0100
|
|
|
|
No, final:
|
|
|
|
1000 (note: Don't include last token!)
|
|
|
|
No, non-final:
|
|
|
|
0010
|
|
|
|
|
Fix behaviour of Matcher's ? quantifier for v2.1 (#3105)
* Add failing test for matcher bug #3009
* Deduplicate matches from Matcher
* Update matcher ? quantifier test
* Fix bug with ? quantifier in Matcher
The ? quantifier indicates a token may occur zero or one times. If the
token pattern fit, the matcher would fail to consider valid matches
where the token pattern did not fit. Consider a simple regex like:
.?b
If we have the string 'b', the .? part will fit --- but then the 'b' in
the pattern will not fit, leaving us with no match. The same bug left us
with too few matches in some cases. For instance, consider:
.?.?
If we have a string of length two, like 'ab', we actually have three
possible matches here: [a, b, ab]. We were only recovering 'ab'. This
should now be fixed. Note that the fix also uncovered another bug, where
we weren't deduplicating the matches. There are actually two ways we
might match 'a' and two ways we might match 'b': as the second token of the pattern,
or as the first token of the pattern. This ambiguity is spurious, so we
need to deduplicate.
Closes #2464 and #3009
* Fix Python2
2018-12-29 15:18:09 +00:00
|
|
|
Possible combinations: 1000, 0100, 0000, 1001, 0110, 0011, 0010,
|
2018-11-14 18:10:21 +00:00
|
|
|
|
2018-03-27 17:23:02 +00:00
|
|
|
We'll name the bits "match", "advance", "retry", "extend"
|
|
|
|
REJECT = 0000
|
|
|
|
MATCH = 1000
|
|
|
|
ADVANCE = 0100
|
|
|
|
RETRY = 0010
|
|
|
|
MATCH_EXTEND = 1001
|
Fix behaviour of Matcher's ? quantifier for v2.1 (#3105)
* Add failing test for matcher bug #3009
* Deduplicate matches from Matcher
* Update matcher ? quantifier test
* Fix bug with ? quantifier in Matcher
The ? quantifier indicates a token may occur zero or one times. If the
token pattern fit, the matcher would fail to consider valid matches
where the token pattern did not fit. Consider a simple regex like:
.?b
If we have the string 'b', the .? part will fit --- but then the 'b' in
the pattern will not fit, leaving us with no match. The same bug left us
with too few matches in some cases. For instance, consider:
.?.?
If we have a string of length two, like 'ab', we actually have three
possible matches here: [a, b, ab]. We were only recovering 'ab'. This
should now be fixed. Note that the fix also uncovered another bug, where
we weren't deduplicating the matches. There are actually two ways we
might match 'a' and two ways we might match 'b': as the second token of the pattern,
or as the first token of the pattern. This ambiguity is spurious, so we
need to deduplicate.
Closes #2464 and #3009
* Fix Python2
2018-12-29 15:18:09 +00:00
|
|
|
RETRY_ADVANCE = 0110
|
2018-03-27 17:23:02 +00:00
|
|
|
RETRY_EXTEND = 0011
|
|
|
|
MATCH_REJECT = 2000 # Match, but don't include last token
|
2019-08-21 20:46:56 +00:00
|
|
|
MATCH_DOUBLE = 3000 # Match both with and without last token
|
2018-03-27 17:23:02 +00:00
|
|
|
|
|
|
|
Problem: If a quantifier is matching, we're adding a lot of open partials
|
2019-03-08 10:42:26 +00:00
|
|
|
"""
|
2020-11-23 09:26:47 +00:00
|
|
|
cdef int8_t is_match
|
2019-02-20 20:30:39 +00:00
|
|
|
is_match = get_is_match(state, token, extra_attrs, predicate_matches)
|
2018-03-27 17:23:02 +00:00
|
|
|
quantifier = get_quantifier(state)
|
|
|
|
is_final = get_is_final(state)
|
|
|
|
if quantifier == ZERO:
|
|
|
|
is_match = not is_match
|
|
|
|
quantifier = ONE
|
|
|
|
if quantifier == ONE:
|
2023-07-19 10:03:31 +00:00
|
|
|
if is_match and is_final:
|
|
|
|
# Yes, final: 1000
|
|
|
|
return MATCH
|
|
|
|
elif is_match and not is_final:
|
|
|
|
# Yes, non-final: 0100
|
|
|
|
return ADVANCE
|
|
|
|
elif not is_match and is_final:
|
|
|
|
# No, final: 0000
|
|
|
|
return REJECT
|
|
|
|
else:
|
|
|
|
return REJECT
|
2018-03-27 17:23:02 +00:00
|
|
|
elif quantifier == ZERO_PLUS:
|
2023-07-19 10:03:31 +00:00
|
|
|
if is_match and is_final:
|
|
|
|
# Yes, final: 1001
|
|
|
|
return MATCH_EXTEND
|
|
|
|
elif is_match and not is_final:
|
|
|
|
# Yes, non-final: 0011
|
|
|
|
return RETRY_EXTEND
|
|
|
|
elif not is_match and is_final:
|
|
|
|
# No, final 2000 (note: Don't include last token!)
|
|
|
|
return MATCH_REJECT
|
|
|
|
else:
|
|
|
|
# No, non-final 0010
|
|
|
|
return RETRY
|
2018-03-27 17:23:02 +00:00
|
|
|
elif quantifier == ZERO_ONE:
|
2023-07-19 10:03:31 +00:00
|
|
|
if is_match and is_final:
|
|
|
|
# Yes, final: 3000
|
|
|
|
# To cater for a pattern ending in "?", we need to add
|
|
|
|
# a match both with and without the last token
|
|
|
|
return MATCH_DOUBLE
|
|
|
|
elif is_match and not is_final:
|
|
|
|
# Yes, non-final: 0110
|
|
|
|
# We need both branches here, consider a pair like:
|
|
|
|
# pattern: .?b string: b
|
|
|
|
# If we 'ADVANCE' on the .?, we miss the match.
|
|
|
|
return RETRY_ADVANCE
|
|
|
|
elif not is_match and is_final:
|
|
|
|
# No, final 2000 (note: Don't include last token!)
|
|
|
|
return MATCH_REJECT
|
|
|
|
else:
|
|
|
|
# No, non-final 0010
|
|
|
|
return RETRY
|
|
|
|
|
|
|
|
|
|
|
|
cdef int8_t get_is_match(
|
|
|
|
PatternStateC state,
|
|
|
|
const TokenC* token,
|
|
|
|
const attr_t* extra_attrs,
|
|
|
|
const int8_t* predicate_matches
|
|
|
|
) nogil:
|
2019-02-20 20:30:39 +00:00
|
|
|
for i in range(state.pattern.nr_py):
|
|
|
|
if predicate_matches[state.pattern.py_predicates[i]] == -1:
|
2019-01-21 12:23:15 +00:00
|
|
|
return 0
|
2018-03-27 17:23:02 +00:00
|
|
|
spec = state.pattern
|
2019-10-10 13:20:59 +00:00
|
|
|
if spec.nr_attr > 0:
|
|
|
|
for attr in spec.attrs[:spec.nr_attr]:
|
2020-04-29 10:57:30 +00:00
|
|
|
if get_token_attr_for_matcher(token, attr.attr) != attr.value:
|
2019-10-10 13:20:59 +00:00
|
|
|
return 0
|
2019-01-21 12:23:15 +00:00
|
|
|
for i in range(spec.nr_extra_attr):
|
|
|
|
if spec.extra_attrs[i].value != extra_attrs[spec.extra_attrs[i].index]:
|
|
|
|
return 0
|
|
|
|
return True
|
2018-03-27 17:23:02 +00:00
|
|
|
|
|
|
|
|
2022-04-18 10:59:34 +00:00
|
|
|
cdef inline int8_t get_is_final(PatternStateC state) nogil:
|
2020-10-31 11:18:48 +00:00
|
|
|
if state.pattern[1].quantifier == FINAL_ID:
|
2018-03-27 17:23:02 +00:00
|
|
|
return 1
|
|
|
|
else:
|
|
|
|
return 0
|
2015-08-04 13:55:28 +00:00
|
|
|
|
|
|
|
|
2022-04-18 10:59:34 +00:00
|
|
|
cdef inline int8_t get_quantifier(PatternStateC state) nogil:
|
2018-03-27 17:23:02 +00:00
|
|
|
return state.pattern.quantifier
|
2016-09-21 12:54:55 +00:00
|
|
|
|
💫 Port master changes over to develop (#2979)
* Create aryaprabhudesai.md (#2681)
* Update _install.jade (#2688)
Typo fix: "models" -> "model"
* Add FAC to spacy.explain (resolves #2706)
* Remove docstrings for deprecated arguments (see #2703)
* When calling getoption() in conftest.py, pass a default option (#2709)
* When calling getoption() in conftest.py, pass a default option
This is necessary to allow testing an installed spacy by running:
pytest --pyargs spacy
* Add contributor agreement
* update bengali token rules for hyphen and digits (#2731)
* Less norm computations in token similarity (#2730)
* Less norm computations in token similarity
* Contributor agreement
* Remove ')' for clarity (#2737)
Sorry, don't mean to be nitpicky, I just noticed this when going through the CLI and thought it was a quick fix. That said, if this was intention than please let me know.
* added contributor agreement for mbkupfer (#2738)
* Basic support for Telugu language (#2751)
* Lex _attrs for polish language (#2750)
* Signed spaCy contributor agreement
* Added polish version of english lex_attrs
* Introduces a bulk merge function, in order to solve issue #653 (#2696)
* Fix comment
* Introduce bulk merge to increase performance on many span merges
* Sign contributor agreement
* Implement pull request suggestions
* Describe converters more explicitly (see #2643)
* Add multi-threading note to Language.pipe (resolves #2582) [ci skip]
* Fix formatting
* Fix dependency scheme docs (closes #2705) [ci skip]
* Don't set stop word in example (closes #2657) [ci skip]
* Add words to portuguese language _num_words (#2759)
* Add words to portuguese language _num_words
* Add words to portuguese language _num_words
* Update Indonesian model (#2752)
* adding e-KTP in tokenizer exceptions list
* add exception token
* removing lines with containing space as it won't matter since we use .split() method in the end, added new tokens in exception
* add tokenizer exceptions list
* combining base_norms with norm_exceptions
* adding norm_exception
* fix double key in lemmatizer
* remove unused import on punctuation.py
* reformat stop_words to reduce number of lines, improve readibility
* updating tokenizer exception
* implement is_currency for lang/id
* adding orth_first_upper in tokenizer_exceptions
* update the norm_exception list
* remove bunch of abbreviations
* adding contributors file
* Fixed spaCy+Keras example (#2763)
* bug fixes in keras example
* created contributor agreement
* Adding French hyphenated first name (#2786)
* Fix typo (closes #2784)
* Fix typo (#2795) [ci skip]
Fixed typo on line 6 "regcognizer --> recognizer"
* Adding basic support for Sinhala language. (#2788)
* adding Sinhala language package, stop words, examples and lex_attrs.
* Adding contributor agreement
* Updating contributor agreement
* Also include lowercase norm exceptions
* Fix error (#2802)
* Fix error
ValueError: cannot resize an array that references or is referenced
by another array in this way. Use the resize function
* added spaCy Contributor Agreement
* Add charlax's contributor agreement (#2805)
* agreement of contributor, may I introduce a tiny pl languge contribution (#2799)
* Contributors agreement
* Contributors agreement
* Contributors agreement
* Add jupyter=True to displacy.render in documentation (#2806)
* Revert "Also include lowercase norm exceptions"
This reverts commit 70f4e8adf37cfcfab60be2b97d6deae949b30e9e.
* Remove deprecated encoding argument to msgpack
* Set up dependency tree pattern matching skeleton (#2732)
* Fix bug when too many entity types. Fixes #2800
* Fix Python 2 test failure
* Require older msgpack-numpy
* Restore encoding arg on msgpack-numpy
* Try to fix version pin for msgpack-numpy
* Update Portuguese Language (#2790)
* Add words to portuguese language _num_words
* Add words to portuguese language _num_words
* Portuguese - Add/remove stopwords, fix tokenizer, add currency symbols
* Extended punctuation and norm_exceptions in the Portuguese language
* Correct error in spacy universe docs concerning spacy-lookup (#2814)
* Update Keras Example for (Parikh et al, 2016) implementation (#2803)
* bug fixes in keras example
* created contributor agreement
* baseline for Parikh model
* initial version of parikh 2016 implemented
* tested asymmetric models
* fixed grevious error in normalization
* use standard SNLI test file
* begin to rework parikh example
* initial version of running example
* start to document the new version
* start to document the new version
* Update Decompositional Attention.ipynb
* fixed calls to similarity
* updated the README
* import sys package duh
* simplified indexing on mapping word to IDs
* stupid python indent error
* added code from https://github.com/tensorflow/tensorflow/issues/3388 for tf bug workaround
* Fix typo (closes #2815) [ci skip]
* Update regex version dependency
* Set version to 2.0.13.dev3
* Skip seemingly problematic test
* Remove problematic test
* Try previous version of regex
* Revert "Remove problematic test"
This reverts commit bdebbef45552d698d390aa430b527ee27830f11b.
* Unskip test
* Try older version of regex
* 💫 Update training examples and use minibatching (#2830)
<!--- Provide a general summary of your changes in the title. -->
## Description
Update the training examples in `/examples/training` to show usage of spaCy's `minibatch` and `compounding` helpers ([see here](https://spacy.io/usage/training#tips-batch-size) for details). The lack of batching in the examples has caused some confusion in the past, especially for beginners who would copy-paste the examples, update them with large training sets and experienced slow and unsatisfying results.
### Types of change
enhancements
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Visual C++ link updated (#2842) (closes #2841) [ci skip]
* New landing page
* Add contribution agreement
* Correcting lang/ru/examples.py (#2845)
* Correct some grammatical inaccuracies in lang\ru\examples.py; filled Contributor Agreement
* Correct some grammatical inaccuracies in lang\ru\examples.py
* Move contributor agreement to separate file
* Set version to 2.0.13.dev4
* Add Persian(Farsi) language support (#2797)
* Also include lowercase norm exceptions
* Remove in favour of https://github.com/explosion/spaCy/graphs/contributors
* Rule-based French Lemmatizer (#2818)
<!--- Provide a general summary of your changes in the title. -->
## Description
<!--- Use this section to describe your changes. If your changes required
testing, include information about the testing environment and the tests you
ran. If your test fixes a bug reported in an issue, don't forget to include the
issue number. If your PR is still a work in progress, that's totally fine – just
include a note to let us know. -->
Add a rule-based French Lemmatizer following the english one and the excellent PR for [greek language optimizations](https://github.com/explosion/spaCy/pull/2558) to adapt the Lemmatizer class.
### Types of change
<!-- What type of change does your PR cover? Is it a bug fix, an enhancement
or new feature, or a change to the documentation? -->
- Lemma dictionary used can be found [here](http://infolingu.univ-mlv.fr/DonneesLinguistiques/Dictionnaires/telechargement.html), I used the XML version.
- Add several files containing exhaustive list of words for each part of speech
- Add some lemma rules
- Add POS that are not checked in the standard Lemmatizer, i.e PRON, DET, ADV and AUX
- Modify the Lemmatizer class to check in lookup table as a last resort if POS not mentionned
- Modify the lemmatize function to check in lookup table as a last resort
- Init files are updated so the model can support all the functionalities mentioned above
- Add words to tokenizer_exceptions_list.py in respect to regex used in tokenizer_exceptions.py
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [X] I have submitted the spaCy Contributor Agreement.
- [X] I ran the tests, and all new and existing tests passed.
- [X] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Set version to 2.0.13
* Fix formatting and consistency
* Update docs for new version [ci skip]
* Increment version [ci skip]
* Add info on wheels [ci skip]
* Adding "This is a sentence" example to Sinhala (#2846)
* Add wheels badge
* Update badge [ci skip]
* Update README.rst [ci skip]
* Update murmurhash pin
* Increment version to 2.0.14.dev0
* Update GPU docs for v2.0.14
* Add wheel to setup_requires
* Import prefer_gpu and require_gpu functions from Thinc
* Add tests for prefer_gpu() and require_gpu()
* Update requirements and setup.py
* Workaround bug in thinc require_gpu
* Set version to v2.0.14
* Update push-tag script
* Unhack prefer_gpu
* Require thinc 6.10.6
* Update prefer_gpu and require_gpu docs [ci skip]
* Fix specifiers for GPU
* Set version to 2.0.14.dev1
* Set version to 2.0.14
* Update Thinc version pin
* Increment version
* Fix msgpack-numpy version pin
* Increment version
* Update version to 2.0.16
* Update version [ci skip]
* Redundant ')' in the Stop words' example (#2856)
<!--- Provide a general summary of your changes in the title. -->
## Description
<!--- Use this section to describe your changes. If your changes required
testing, include information about the testing environment and the tests you
ran. If your test fixes a bug reported in an issue, don't forget to include the
issue number. If your PR is still a work in progress, that's totally fine – just
include a note to let us know. -->
### Types of change
<!-- What type of change does your PR cover? Is it a bug fix, an enhancement
or new feature, or a change to the documentation? -->
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [ ] I have submitted the spaCy Contributor Agreement.
- [ ] I ran the tests, and all new and existing tests passed.
- [ ] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Documentation improvement regarding joblib and SO (#2867)
Some documentation improvements
## Description
1. Fixed the dead URL to joblib
2. Fixed Stack Overflow brand name (with space)
### Types of change
Documentation
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
* raise error when setting overlapping entities as doc.ents (#2880)
* Fix out-of-bounds access in NER training
The helper method state.B(1) gets the index of the first token of the
buffer, or -1 if no such token exists. Normally this is safe because we
pass this to functions like state.safe_get(), which returns an empty
token. Here we used it directly as an array index, which is not okay!
This error may have been the cause of out-of-bounds access errors during
training. Similar errors may still be around, so much be hunted down.
Hunting this one down took a long time...I printed out values across
training runs and diffed, looking for points of divergence between
runs, when no randomness should be allowed.
* Change PyThaiNLP Url (#2876)
* Fix missing comma
* Add example showing a fix-up rule for space entities
* Set version to 2.0.17.dev0
* Update regex version
* Revert "Update regex version"
This reverts commit 62358dd867d15bc6a475942dff34effba69dd70a.
* Try setting older regex version, to align with conda
* Set version to 2.0.17
* Add spacy-js to universe [ci-skip]
* Add spacy-raspberry to universe (closes #2889)
* Add script to validate universe json [ci skip]
* Removed space in docs + added contributor indo (#2909)
* - removed unneeded space in documentation
* - added contributor info
* Allow input text of length up to max_length, inclusive (#2922)
* Include universe spec for spacy-wordnet component (#2919)
* feat: include universe spec for spacy-wordnet component
* chore: include spaCy contributor agreement
* Minor formatting changes [ci skip]
* Fix image [ci skip]
Twitter URL doesn't work on live site
* Check if the word is in one of the regular lists specific to each POS (#2886)
* 💫 Create random IDs for SVGs to prevent ID clashes (#2927)
Resolves #2924.
## Description
Fixes problem where multiple visualizations in Jupyter notebooks would have clashing arc IDs, resulting in weirdly positioned arc labels. Generating a random ID prefix so even identical parses won't receive the same IDs for consistency (even if effect of ID clash isn't noticable here.)
### Types of change
bug fix
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Fix typo [ci skip]
* fixes symbolic link on py3 and windows (#2949)
* fixes symbolic link on py3 and windows
during setup of spacy using command
python -m spacy link en_core_web_sm en
closes #2948
* Update spacy/compat.py
Co-Authored-By: cicorias <cicorias@users.noreply.github.com>
* Fix formatting
* Update universe [ci skip]
* Catalan Language Support (#2940)
* Catalan language Support
* Ddding Catalan to documentation
* Sort languages alphabetically [ci skip]
* Update tests for pytest 4.x (#2965)
<!--- Provide a general summary of your changes in the title. -->
## Description
- [x] Replace marks in params for pytest 4.0 compat ([see here](https://docs.pytest.org/en/latest/deprecations.html#marks-in-pytest-mark-parametrize))
- [x] Un-xfail passing tests (some fixes in a recent update resolved a bunch of issues, but tests were apparently never updated here)
### Types of change
<!-- What type of change does your PR cover? Is it a bug fix, an enhancement
or new feature, or a change to the documentation? -->
## Checklist
<!--- Before you submit the PR, go over this checklist and make sure you can
tick off all the boxes. [] -> [x] -->
- [x] I have submitted the spaCy Contributor Agreement.
- [x] I ran the tests, and all new and existing tests passed.
- [x] My changes don't require a change to the documentation, or if they do, I've added all required information.
* Fix regex pin to harmonize with conda (#2964)
* Update README.rst
* Fix bug where Vocab.prune_vector did not use 'batch_size' (#2977)
Fixes #2976
* Fix typo
* Fix typo
* Remove duplicate file
* Require thinc 7.0.0.dev2
Fixes bug in gpu_ops that would use cupy instead of numpy on CPU
* Add missing import
* Fix error IDs
* Fix tests
2018-11-29 15:30:29 +00:00
|
|
|
|
2019-01-21 12:23:15 +00:00
|
|
|
cdef TokenPatternC* init_pattern(Pool mem, attr_t entity_id, object token_specs) except NULL:
|
2016-09-21 12:54:55 +00:00
|
|
|
pattern = <TokenPatternC*>mem.alloc(len(token_specs) + 1, sizeof(TokenPatternC))
|
2019-02-20 20:30:39 +00:00
|
|
|
cdef int i, index
|
2021-04-08 08:10:14 +00:00
|
|
|
for i, (quantifier, spec, extensions, predicates, token_idx) in enumerate(token_specs):
|
2016-09-21 12:54:55 +00:00
|
|
|
pattern[i].quantifier = quantifier
|
2019-10-10 13:20:59 +00:00
|
|
|
# Ensure attrs refers to a null pointer if nr_attr == 0
|
|
|
|
if len(spec) > 0:
|
|
|
|
pattern[i].attrs = <AttrValueC*>mem.alloc(len(spec), sizeof(AttrValueC))
|
2016-09-21 12:54:55 +00:00
|
|
|
pattern[i].nr_attr = len(spec)
|
2015-08-04 23:05:54 +00:00
|
|
|
for j, (attr, value) in enumerate(spec):
|
2016-09-21 12:54:55 +00:00
|
|
|
pattern[i].attrs[j].attr = attr
|
|
|
|
pattern[i].attrs[j].value = value
|
2019-10-22 14:54:33 +00:00
|
|
|
if len(extensions) > 0:
|
|
|
|
pattern[i].extra_attrs = <IndexValueC*>mem.alloc(len(extensions), sizeof(IndexValueC))
|
2019-01-21 12:23:15 +00:00
|
|
|
for j, (index, value) in enumerate(extensions):
|
|
|
|
pattern[i].extra_attrs[j].index = index
|
|
|
|
pattern[i].extra_attrs[j].value = value
|
|
|
|
pattern[i].nr_extra_attr = len(extensions)
|
2019-10-22 14:54:33 +00:00
|
|
|
if len(predicates) > 0:
|
|
|
|
pattern[i].py_predicates = <int32_t*>mem.alloc(len(predicates), sizeof(int32_t))
|
2019-01-21 12:23:15 +00:00
|
|
|
for j, index in enumerate(predicates):
|
|
|
|
pattern[i].py_predicates[j] = index
|
|
|
|
pattern[i].nr_py = len(predicates)
|
2018-03-27 17:23:02 +00:00
|
|
|
pattern[i].key = hash64(pattern[i].attrs, pattern[i].nr_attr * sizeof(AttrValueC), 0)
|
2021-04-08 08:10:14 +00:00
|
|
|
pattern[i].token_idx = token_idx
|
2015-08-04 23:05:54 +00:00
|
|
|
i = len(token_specs)
|
2020-10-31 11:18:48 +00:00
|
|
|
# Use quantifier to identify final ID pattern node (rather than previous
|
|
|
|
# uninitialized quantifier == 0/ZERO + nr_attr == 0 + non-zero-length attrs)
|
|
|
|
pattern[i].quantifier = FINAL_ID
|
|
|
|
pattern[i].attrs = <AttrValueC*>mem.alloc(1, sizeof(AttrValueC))
|
2016-09-21 12:54:55 +00:00
|
|
|
pattern[i].attrs[0].attr = ID
|
|
|
|
pattern[i].attrs[0].value = entity_id
|
2020-10-31 11:18:48 +00:00
|
|
|
pattern[i].nr_attr = 1
|
2019-02-20 20:30:39 +00:00
|
|
|
pattern[i].nr_extra_attr = 0
|
|
|
|
pattern[i].nr_py = 0
|
2021-04-08 08:10:14 +00:00
|
|
|
pattern[i].token_idx = -1
|
2015-08-04 23:05:54 +00:00
|
|
|
return pattern
|
|
|
|
|
|
|
|
|
2019-10-09 13:26:31 +00:00
|
|
|
cdef attr_t get_ent_id(const TokenPatternC* pattern) nogil:
|
2020-10-31 11:18:48 +00:00
|
|
|
while pattern.quantifier != FINAL_ID:
|
2017-05-20 11:54:53 +00:00
|
|
|
pattern += 1
|
|
|
|
id_attr = pattern[0].attrs[0]
|
2018-04-03 13:50:31 +00:00
|
|
|
if id_attr.attr != ID:
|
2018-04-29 13:48:34 +00:00
|
|
|
with gil:
|
|
|
|
raise ValueError(Errors.E074.format(attr=ID, bad_attr=id_attr.attr))
|
2017-05-20 11:54:53 +00:00
|
|
|
return id_attr.value
|
|
|
|
|
2019-01-21 12:23:15 +00:00
|
|
|
|
2023-01-10 09:36:17 +00:00
|
|
|
def _preprocess_pattern(token_specs, vocab, extensions_table, extra_predicates, fuzzy_compare):
|
2019-01-21 12:23:15 +00:00
|
|
|
"""This function interprets the pattern, converting the various bits of
|
|
|
|
syntactic sugar before we compile it into a struct with init_pattern.
|
|
|
|
|
2021-04-08 08:10:14 +00:00
|
|
|
We need to split the pattern up into four parts:
|
2019-01-21 12:23:15 +00:00
|
|
|
* Normal attribute/value pairs, which are stored on either the token or lexeme,
|
|
|
|
can be handled directly.
|
|
|
|
* Extension attributes are handled specially, as we need to prefetch the
|
|
|
|
values from Python for the doc before we begin matching.
|
|
|
|
* Extra predicates also call Python functions, so we have to create the
|
|
|
|
functions and store them. So we store these specially as well.
|
|
|
|
* Extension attributes that have extra predicates are stored within the
|
|
|
|
extra_predicates.
|
2021-04-08 08:10:14 +00:00
|
|
|
* Token index that this pattern belongs to.
|
2019-01-21 12:23:15 +00:00
|
|
|
"""
|
2016-09-21 12:54:55 +00:00
|
|
|
tokens = []
|
2020-09-24 14:55:09 +00:00
|
|
|
string_store = vocab.strings
|
2021-04-08 08:10:14 +00:00
|
|
|
for token_idx, spec in enumerate(token_specs):
|
2017-10-07 01:36:15 +00:00
|
|
|
if not spec:
|
|
|
|
# Signifier for 'any token'
|
2021-04-08 08:10:14 +00:00
|
|
|
tokens.append((ONE, [(NULL_ATTR, 0)], [], [], token_idx))
|
2017-10-07 01:36:15 +00:00
|
|
|
continue
|
2019-08-21 12:00:37 +00:00
|
|
|
if not isinstance(spec, dict):
|
|
|
|
raise ValueError(Errors.E154.format())
|
2019-01-21 12:23:15 +00:00
|
|
|
ops = _get_operators(spec)
|
|
|
|
attr_values = _get_attr_values(spec, string_store)
|
|
|
|
extensions = _get_extensions(spec, string_store, extensions_table)
|
2023-01-10 09:36:17 +00:00
|
|
|
predicates = _get_extra_predicates(spec, extra_predicates, vocab, fuzzy_compare)
|
2016-09-21 12:54:55 +00:00
|
|
|
for op in ops:
|
2021-04-08 08:10:14 +00:00
|
|
|
tokens.append((op, list(attr_values), list(extensions), list(predicates), token_idx))
|
2016-09-21 12:54:55 +00:00
|
|
|
return tokens
|
2015-10-08 15:00:45 +00:00
|
|
|
|
|
|
|
|
2019-01-21 12:23:15 +00:00
|
|
|
def _get_attr_values(spec, string_store):
|
|
|
|
attr_values = []
|
|
|
|
for attr, value in spec.items():
|
2022-05-25 09:12:29 +00:00
|
|
|
input_attr = attr
|
2021-09-13 15:02:17 +00:00
|
|
|
if isinstance(attr, str):
|
2019-08-21 12:00:37 +00:00
|
|
|
attr = attr.upper()
|
2019-01-21 12:23:15 +00:00
|
|
|
if attr == '_':
|
|
|
|
continue
|
2019-08-21 12:00:37 +00:00
|
|
|
elif attr == "OP":
|
2019-01-21 12:23:15 +00:00
|
|
|
continue
|
2019-08-21 12:00:37 +00:00
|
|
|
if attr == "TEXT":
|
2019-03-08 10:42:26 +00:00
|
|
|
attr = "ORTH"
|
2020-03-03 11:22:39 +00:00
|
|
|
if attr == "IS_SENT_START":
|
|
|
|
attr = "SENT_START"
|
2019-08-21 12:00:37 +00:00
|
|
|
attr = IDS.get(attr)
|
2021-09-13 15:02:17 +00:00
|
|
|
if isinstance(value, str):
|
2022-01-20 12:18:39 +00:00
|
|
|
if attr == ENT_IOB and value in Token.iob_strings():
|
|
|
|
value = Token.iob_strings().index(value)
|
|
|
|
else:
|
|
|
|
value = string_store.add(value)
|
2019-01-21 12:23:15 +00:00
|
|
|
elif isinstance(value, bool):
|
|
|
|
value = int(value)
|
2019-12-06 18:22:57 +00:00
|
|
|
elif isinstance(value, int):
|
|
|
|
pass
|
|
|
|
elif isinstance(value, dict):
|
2019-01-21 12:23:15 +00:00
|
|
|
continue
|
2019-08-21 12:00:37 +00:00
|
|
|
else:
|
|
|
|
raise ValueError(Errors.E153.format(vtype=type(value).__name__))
|
2019-01-21 12:23:15 +00:00
|
|
|
if attr is not None:
|
|
|
|
attr_values.append((attr, value))
|
2019-08-21 12:00:37 +00:00
|
|
|
else:
|
2019-12-25 11:39:49 +00:00
|
|
|
# should be caught in validation
|
2022-05-25 09:12:29 +00:00
|
|
|
raise ValueError(Errors.E152.format(attr=input_attr))
|
2019-01-21 12:23:15 +00:00
|
|
|
return attr_values
|
|
|
|
|
2019-03-08 10:42:26 +00:00
|
|
|
|
2023-02-27 07:35:08 +00:00
|
|
|
def _predicate_cache_key(attr, predicate, value, *, regex=False, fuzzy=None):
|
|
|
|
# tuple order affects performance
|
|
|
|
return (attr, regex, fuzzy, predicate, srsly.json_dumps(value, sort_keys=True))
|
|
|
|
|
|
|
|
|
2019-01-21 12:23:15 +00:00
|
|
|
# These predicate helper classes are used to match the REGEX, IN, >= etc
|
|
|
|
# extensions to the matcher introduced in #3173.
|
|
|
|
|
2023-01-10 09:36:17 +00:00
|
|
|
class _FuzzyPredicate:
|
|
|
|
operators = ("FUZZY", "FUZZY1", "FUZZY2", "FUZZY3", "FUZZY4", "FUZZY5",
|
|
|
|
"FUZZY6", "FUZZY7", "FUZZY8", "FUZZY9")
|
|
|
|
|
|
|
|
def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None,
|
|
|
|
regex=False, fuzzy=None, fuzzy_compare=None):
|
|
|
|
self.i = i
|
|
|
|
self.attr = attr
|
|
|
|
self.value = value
|
|
|
|
self.predicate = predicate
|
|
|
|
self.is_extension = is_extension
|
|
|
|
if self.predicate not in self.operators:
|
|
|
|
raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate))
|
2023-07-19 10:03:31 +00:00
|
|
|
fuzz = self.predicate[len("FUZZY"):] # number after prefix
|
2023-01-10 09:36:17 +00:00
|
|
|
self.fuzzy = int(fuzz) if fuzz else -1
|
|
|
|
self.fuzzy_compare = fuzzy_compare
|
2023-02-27 07:35:08 +00:00
|
|
|
self.key = _predicate_cache_key(self.attr, self.predicate, value, fuzzy=self.fuzzy)
|
2023-01-10 09:36:17 +00:00
|
|
|
|
|
|
|
def __call__(self, Token token):
|
|
|
|
if self.is_extension:
|
|
|
|
value = token._.get(self.attr)
|
|
|
|
else:
|
|
|
|
value = token.vocab.strings[get_token_attr_for_matcher(token.c, self.attr)]
|
|
|
|
if self.value == value:
|
|
|
|
return True
|
|
|
|
return self.fuzzy_compare(value, self.value, self.fuzzy)
|
|
|
|
|
|
|
|
|
2020-07-12 12:03:23 +00:00
|
|
|
class _RegexPredicate:
|
2019-03-08 10:42:26 +00:00
|
|
|
operators = ("REGEX",)
|
|
|
|
|
2023-01-10 09:36:17 +00:00
|
|
|
def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None,
|
|
|
|
regex=False, fuzzy=None, fuzzy_compare=None):
|
2019-01-21 12:23:15 +00:00
|
|
|
self.i = i
|
|
|
|
self.attr = attr
|
|
|
|
self.value = re.compile(value)
|
|
|
|
self.predicate = predicate
|
|
|
|
self.is_extension = is_extension
|
2023-02-27 07:35:08 +00:00
|
|
|
self.key = _predicate_cache_key(self.attr, self.predicate, value)
|
2019-03-08 10:42:26 +00:00
|
|
|
if self.predicate not in self.operators:
|
|
|
|
raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate))
|
2019-01-21 12:23:15 +00:00
|
|
|
|
|
|
|
def __call__(self, Token token):
|
|
|
|
if self.is_extension:
|
|
|
|
value = token._.get(self.attr)
|
|
|
|
else:
|
2020-04-29 10:57:30 +00:00
|
|
|
value = token.vocab.strings[get_token_attr_for_matcher(token.c, self.attr)]
|
2019-01-21 12:23:15 +00:00
|
|
|
return bool(self.value.search(value))
|
|
|
|
|
|
|
|
|
2020-09-24 14:55:09 +00:00
|
|
|
class _SetPredicate:
|
2021-08-02 17:39:26 +00:00
|
|
|
operators = ("IN", "NOT_IN", "IS_SUBSET", "IS_SUPERSET", "INTERSECTS")
|
2019-03-08 10:42:26 +00:00
|
|
|
|
2023-01-10 09:36:17 +00:00
|
|
|
def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None,
|
|
|
|
regex=False, fuzzy=None, fuzzy_compare=None):
|
2019-01-21 12:23:15 +00:00
|
|
|
self.i = i
|
|
|
|
self.attr = attr
|
2020-09-24 14:55:09 +00:00
|
|
|
self.vocab = vocab
|
2023-01-10 09:36:17 +00:00
|
|
|
self.regex = regex
|
|
|
|
self.fuzzy = fuzzy
|
|
|
|
self.fuzzy_compare = fuzzy_compare
|
2020-09-24 14:55:09 +00:00
|
|
|
if self.attr == MORPH:
|
|
|
|
# normalize morph strings
|
|
|
|
self.value = set(self.vocab.morphology.add(v) for v in value)
|
|
|
|
else:
|
2023-01-10 09:36:17 +00:00
|
|
|
if self.regex:
|
|
|
|
self.value = set(re.compile(v) for v in value)
|
|
|
|
elif self.fuzzy is not None:
|
|
|
|
# add to string store
|
|
|
|
self.value = set(self.vocab.strings.add(v) for v in value)
|
|
|
|
else:
|
|
|
|
self.value = set(get_string_id(v) for v in value)
|
2019-01-21 12:23:15 +00:00
|
|
|
self.predicate = predicate
|
|
|
|
self.is_extension = is_extension
|
2023-02-27 07:35:08 +00:00
|
|
|
self.key = _predicate_cache_key(self.attr, self.predicate, value, regex=self.regex, fuzzy=self.fuzzy)
|
2019-03-08 10:42:26 +00:00
|
|
|
if self.predicate not in self.operators:
|
|
|
|
raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate))
|
2019-01-21 12:23:15 +00:00
|
|
|
|
|
|
|
def __call__(self, Token token):
|
|
|
|
if self.is_extension:
|
2022-09-02 07:09:48 +00:00
|
|
|
value = token._.get(self.attr)
|
2019-01-21 12:23:15 +00:00
|
|
|
else:
|
2020-04-29 10:57:30 +00:00
|
|
|
value = get_token_attr_for_matcher(token.c, self.attr)
|
2020-09-24 14:55:09 +00:00
|
|
|
|
2022-09-02 07:09:48 +00:00
|
|
|
if self.predicate in ("IN", "NOT_IN"):
|
|
|
|
if isinstance(value, (str, int)):
|
|
|
|
value = get_string_id(value)
|
|
|
|
else:
|
|
|
|
return False
|
|
|
|
elif self.predicate in ("IS_SUBSET", "IS_SUPERSET", "INTERSECTS"):
|
|
|
|
# ensure that all values are enclosed in a set
|
2020-09-24 14:55:09 +00:00
|
|
|
if self.attr == MORPH:
|
|
|
|
# break up MORPH into individual Feat=Val values
|
|
|
|
value = set(get_string_id(v) for v in MorphAnalysis.from_id(self.vocab, value))
|
2022-09-02 07:09:48 +00:00
|
|
|
elif isinstance(value, (str, int)):
|
|
|
|
value = set((get_string_id(value),))
|
|
|
|
elif isinstance(value, Iterable) and all(isinstance(v, (str, int)) for v in value):
|
|
|
|
value = set(get_string_id(v) for v in value)
|
2020-09-24 14:55:09 +00:00
|
|
|
else:
|
2022-09-02 07:09:48 +00:00
|
|
|
return False
|
|
|
|
|
2019-03-08 10:42:26 +00:00
|
|
|
if self.predicate == "IN":
|
2023-01-10 09:36:17 +00:00
|
|
|
if self.regex:
|
|
|
|
value = self.vocab.strings[value]
|
|
|
|
return any(bool(v.search(value)) for v in self.value)
|
|
|
|
elif self.fuzzy is not None:
|
|
|
|
value = self.vocab.strings[value]
|
|
|
|
return any(self.fuzzy_compare(value, self.vocab.strings[v], self.fuzzy)
|
|
|
|
for v in self.value)
|
|
|
|
elif value in self.value:
|
|
|
|
return True
|
|
|
|
else:
|
|
|
|
return False
|
2020-09-24 14:55:09 +00:00
|
|
|
elif self.predicate == "NOT_IN":
|
2023-01-10 09:36:17 +00:00
|
|
|
if self.regex:
|
|
|
|
value = self.vocab.strings[value]
|
|
|
|
return not any(bool(v.search(value)) for v in self.value)
|
|
|
|
elif self.fuzzy is not None:
|
|
|
|
value = self.vocab.strings[value]
|
|
|
|
return not any(self.fuzzy_compare(value, self.vocab.strings[v], self.fuzzy)
|
|
|
|
for v in self.value)
|
|
|
|
elif value in self.value:
|
|
|
|
return False
|
|
|
|
else:
|
|
|
|
return True
|
2020-09-24 14:55:09 +00:00
|
|
|
elif self.predicate == "IS_SUBSET":
|
|
|
|
return value <= self.value
|
|
|
|
elif self.predicate == "IS_SUPERSET":
|
|
|
|
return value >= self.value
|
2021-08-02 17:39:26 +00:00
|
|
|
elif self.predicate == "INTERSECTS":
|
|
|
|
return bool(value & self.value)
|
2019-01-21 12:23:15 +00:00
|
|
|
|
2019-02-20 20:30:39 +00:00
|
|
|
def __repr__(self):
|
2020-09-24 14:55:09 +00:00
|
|
|
return repr(("SetPredicate", self.i, self.attr, self.value, self.predicate))
|
2019-02-20 20:30:39 +00:00
|
|
|
|
2019-01-21 12:23:15 +00:00
|
|
|
|
2020-07-12 12:03:23 +00:00
|
|
|
class _ComparisonPredicate:
|
2019-03-08 10:42:26 +00:00
|
|
|
operators = ("==", "!=", ">=", "<=", ">", "<")
|
|
|
|
|
2023-01-10 09:36:17 +00:00
|
|
|
def __init__(self, i, attr, value, predicate, is_extension=False, vocab=None,
|
|
|
|
regex=False, fuzzy=None, fuzzy_compare=None):
|
2019-01-21 12:23:15 +00:00
|
|
|
self.i = i
|
|
|
|
self.attr = attr
|
|
|
|
self.value = value
|
|
|
|
self.predicate = predicate
|
|
|
|
self.is_extension = is_extension
|
2023-02-27 07:35:08 +00:00
|
|
|
self.key = _predicate_cache_key(self.attr, self.predicate, value)
|
2019-03-08 10:42:26 +00:00
|
|
|
if self.predicate not in self.operators:
|
|
|
|
raise ValueError(Errors.E126.format(good=self.operators, bad=self.predicate))
|
2019-01-21 12:23:15 +00:00
|
|
|
|
|
|
|
def __call__(self, Token token):
|
|
|
|
if self.is_extension:
|
|
|
|
value = token._.get(self.attr)
|
|
|
|
else:
|
2020-04-29 10:57:30 +00:00
|
|
|
value = get_token_attr_for_matcher(token.c, self.attr)
|
2019-03-08 10:42:26 +00:00
|
|
|
if self.predicate == "==":
|
2019-01-21 12:23:15 +00:00
|
|
|
return value == self.value
|
2019-03-08 10:42:26 +00:00
|
|
|
if self.predicate == "!=":
|
2019-01-21 12:23:15 +00:00
|
|
|
return value != self.value
|
2019-03-08 10:42:26 +00:00
|
|
|
elif self.predicate == ">=":
|
2019-01-21 12:23:15 +00:00
|
|
|
return value >= self.value
|
2019-03-08 10:42:26 +00:00
|
|
|
elif self.predicate == "<=":
|
2019-01-21 12:23:15 +00:00
|
|
|
return value <= self.value
|
2019-03-08 10:42:26 +00:00
|
|
|
elif self.predicate == ">":
|
2019-01-21 12:23:15 +00:00
|
|
|
return value > self.value
|
2019-03-08 10:42:26 +00:00
|
|
|
elif self.predicate == "<":
|
2019-01-21 12:23:15 +00:00
|
|
|
return value < self.value
|
|
|
|
|
|
|
|
|
2023-01-10 09:36:17 +00:00
|
|
|
def _get_extra_predicates(spec, extra_predicates, vocab, fuzzy_compare):
|
2019-01-21 12:23:15 +00:00
|
|
|
predicate_types = {
|
2019-03-08 10:42:26 +00:00
|
|
|
"REGEX": _RegexPredicate,
|
2020-09-24 14:55:09 +00:00
|
|
|
"IN": _SetPredicate,
|
|
|
|
"NOT_IN": _SetPredicate,
|
|
|
|
"IS_SUBSET": _SetPredicate,
|
|
|
|
"IS_SUPERSET": _SetPredicate,
|
2021-08-02 17:39:26 +00:00
|
|
|
"INTERSECTS": _SetPredicate,
|
2019-03-08 10:42:26 +00:00
|
|
|
"==": _ComparisonPredicate,
|
2020-04-14 17:14:15 +00:00
|
|
|
"!=": _ComparisonPredicate,
|
2019-03-08 10:42:26 +00:00
|
|
|
">=": _ComparisonPredicate,
|
|
|
|
"<=": _ComparisonPredicate,
|
|
|
|
">": _ComparisonPredicate,
|
|
|
|
"<": _ComparisonPredicate,
|
2023-01-10 09:36:17 +00:00
|
|
|
"FUZZY": _FuzzyPredicate,
|
|
|
|
"FUZZY1": _FuzzyPredicate,
|
|
|
|
"FUZZY2": _FuzzyPredicate,
|
|
|
|
"FUZZY3": _FuzzyPredicate,
|
|
|
|
"FUZZY4": _FuzzyPredicate,
|
|
|
|
"FUZZY5": _FuzzyPredicate,
|
|
|
|
"FUZZY6": _FuzzyPredicate,
|
|
|
|
"FUZZY7": _FuzzyPredicate,
|
|
|
|
"FUZZY8": _FuzzyPredicate,
|
|
|
|
"FUZZY9": _FuzzyPredicate,
|
2019-01-21 12:23:15 +00:00
|
|
|
}
|
2019-02-20 20:30:39 +00:00
|
|
|
seen_predicates = {pred.key: pred.i for pred in extra_predicates}
|
2019-01-21 12:23:15 +00:00
|
|
|
output = []
|
|
|
|
for attr, value in spec.items():
|
2021-09-13 15:02:17 +00:00
|
|
|
if isinstance(attr, str):
|
2019-03-08 10:42:26 +00:00
|
|
|
if attr == "_":
|
2019-01-21 12:23:15 +00:00
|
|
|
output.extend(
|
|
|
|
_get_extension_extra_predicates(
|
|
|
|
value, extra_predicates, predicate_types,
|
|
|
|
seen_predicates))
|
|
|
|
continue
|
2019-03-08 10:42:26 +00:00
|
|
|
elif attr.upper() == "OP":
|
2019-01-21 12:23:15 +00:00
|
|
|
continue
|
2019-03-08 10:42:26 +00:00
|
|
|
if attr.upper() == "TEXT":
|
|
|
|
attr = "ORTH"
|
2019-01-21 12:23:15 +00:00
|
|
|
attr = IDS.get(attr.upper())
|
|
|
|
if isinstance(value, dict):
|
2023-01-10 09:36:17 +00:00
|
|
|
output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types,
|
|
|
|
extra_predicates, seen_predicates, fuzzy_compare=fuzzy_compare))
|
|
|
|
return output
|
|
|
|
|
|
|
|
|
|
|
|
def _get_extra_predicates_dict(attr, value_dict, vocab, predicate_types,
|
|
|
|
extra_predicates, seen_predicates, regex=False, fuzzy=None, fuzzy_compare=None):
|
|
|
|
output = []
|
|
|
|
for type_, value in value_dict.items():
|
|
|
|
type_ = type_.upper()
|
|
|
|
cls = predicate_types.get(type_)
|
|
|
|
if cls is None:
|
|
|
|
warnings.warn(Warnings.W035.format(pattern=value_dict))
|
|
|
|
# ignore unrecognized predicate type
|
|
|
|
continue
|
|
|
|
elif cls == _RegexPredicate:
|
|
|
|
if isinstance(value, dict):
|
|
|
|
# add predicates inside regex operator
|
|
|
|
output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types,
|
|
|
|
extra_predicates, seen_predicates,
|
|
|
|
regex=True))
|
|
|
|
continue
|
|
|
|
elif cls == _FuzzyPredicate:
|
|
|
|
if isinstance(value, dict):
|
|
|
|
# add predicates inside fuzzy operator
|
2023-07-19 10:03:31 +00:00
|
|
|
fuzz = type_[len("FUZZY"):] # number after prefix
|
2023-01-10 09:36:17 +00:00
|
|
|
fuzzy_val = int(fuzz) if fuzz else -1
|
|
|
|
output.extend(_get_extra_predicates_dict(attr, value, vocab, predicate_types,
|
|
|
|
extra_predicates, seen_predicates,
|
|
|
|
fuzzy=fuzzy_val, fuzzy_compare=fuzzy_compare))
|
|
|
|
continue
|
|
|
|
predicate = cls(len(extra_predicates), attr, value, type_, vocab=vocab,
|
|
|
|
regex=regex, fuzzy=fuzzy, fuzzy_compare=fuzzy_compare)
|
|
|
|
# Don't create redundant predicates.
|
|
|
|
# This helps with efficiency, as we're caching the results.
|
|
|
|
if predicate.key in seen_predicates:
|
|
|
|
output.append(seen_predicates[predicate.key])
|
|
|
|
else:
|
|
|
|
extra_predicates.append(predicate)
|
|
|
|
output.append(predicate.i)
|
|
|
|
seen_predicates[predicate.key] = predicate.i
|
2019-01-21 12:23:15 +00:00
|
|
|
return output
|
|
|
|
|
|
|
|
|
2023-07-19 10:03:31 +00:00
|
|
|
def _get_extension_extra_predicates(
|
|
|
|
spec, extra_predicates, predicate_types, seen_predicates
|
|
|
|
):
|
2019-01-21 12:23:15 +00:00
|
|
|
output = []
|
|
|
|
for attr, value in spec.items():
|
|
|
|
if isinstance(value, dict):
|
|
|
|
for type_, cls in predicate_types.items():
|
|
|
|
if type_ in value:
|
2023-02-27 07:35:08 +00:00
|
|
|
key = _predicate_cache_key(attr, type_, value[type_])
|
2019-01-21 12:23:15 +00:00
|
|
|
if key in seen_predicates:
|
|
|
|
output.append(seen_predicates[key])
|
|
|
|
else:
|
|
|
|
predicate = cls(len(extra_predicates), attr, value[type_], type_,
|
|
|
|
is_extension=True)
|
|
|
|
extra_predicates.append(predicate)
|
|
|
|
output.append(predicate.i)
|
|
|
|
seen_predicates[key] = predicate.i
|
|
|
|
return output
|
|
|
|
|
|
|
|
|
|
|
|
def _get_operators(spec):
|
|
|
|
# Support 'syntactic sugar' operator '+', as combination of ONE, ZERO_PLUS
|
2019-03-08 10:42:26 +00:00
|
|
|
lookup = {"*": (ZERO_PLUS,), "+": (ONE, ZERO_PLUS),
|
|
|
|
"?": (ZERO_ONE,), "1": (ONE,), "!": (ZERO,)}
|
2019-01-21 12:23:15 +00:00
|
|
|
# Fix casing
|
|
|
|
spec = {key.upper(): values for key, values in spec.items()
|
2021-09-13 15:02:17 +00:00
|
|
|
if isinstance(key, str)}
|
2019-03-08 10:42:26 +00:00
|
|
|
if "OP" not in spec:
|
2019-01-21 12:23:15 +00:00
|
|
|
return (ONE,)
|
2019-03-08 10:42:26 +00:00
|
|
|
elif spec["OP"] in lookup:
|
|
|
|
return lookup[spec["OP"]]
|
2023-07-19 10:03:31 +00:00
|
|
|
# Min_max {n,m}
|
2022-06-30 09:01:58 +00:00
|
|
|
elif spec["OP"].startswith("{") and spec["OP"].endswith("}"):
|
|
|
|
# {n} --> {n,n} exactly n ONE,(n)
|
|
|
|
# {n,m}--> {n,m} min of n, max of m ONE,(n),ZERO_ONE,(m)
|
|
|
|
# {,m} --> {0,m} min of zero, max of m ZERO_ONE,(m)
|
|
|
|
# {n,} --> {n,∞} min of n, max of inf ONE,(n),ZERO_PLUS
|
|
|
|
|
|
|
|
min_max = spec["OP"][1:-1]
|
|
|
|
min_max = min_max if "," in min_max else f"{min_max},{min_max}"
|
|
|
|
n, m = min_max.split(",")
|
|
|
|
|
2023-07-19 10:03:31 +00:00
|
|
|
# 1. Either n or m is a blank string and the other is numeric -->isdigit
|
|
|
|
# 2. Both are numeric and n <= m
|
2022-06-30 09:01:58 +00:00
|
|
|
if (not n.isdecimal() and not m.isdecimal()) or (n.isdecimal() and m.isdecimal() and int(n) > int(m)):
|
|
|
|
keys = ", ".join(lookup.keys()) + ", {n}, {n,m}, {n,}, {,m} where n and m are integers and n <= m "
|
|
|
|
raise ValueError(Errors.E011.format(op=spec["OP"], opts=keys))
|
|
|
|
|
|
|
|
# if n is empty string, zero would be used
|
|
|
|
head = tuple(ONE for __ in range(int(n or 0)))
|
|
|
|
tail = tuple(ZERO_ONE for __ in range(int(m) - int(n or 0))) if m else (ZERO_PLUS,)
|
|
|
|
return head + tail
|
2019-01-21 12:23:15 +00:00
|
|
|
else:
|
2022-06-30 09:01:58 +00:00
|
|
|
keys = ", ".join(lookup.keys()) + ", {n}, {n,m}, {n,}, {,m} where n and m are integers and n <= m "
|
2019-08-21 12:00:37 +00:00
|
|
|
raise ValueError(Errors.E011.format(op=spec["OP"], opts=keys))
|
2019-01-21 12:23:15 +00:00
|
|
|
|
|
|
|
|
|
|
|
def _get_extensions(spec, string_store, name2index):
|
|
|
|
attr_values = []
|
2019-08-21 12:00:37 +00:00
|
|
|
if not isinstance(spec.get("_", {}), dict):
|
|
|
|
raise ValueError(Errors.E154.format())
|
2019-03-08 10:42:26 +00:00
|
|
|
for name, value in spec.get("_", {}).items():
|
2019-01-21 12:23:15 +00:00
|
|
|
if isinstance(value, dict):
|
|
|
|
# Handle predicates (e.g. "IN", in the extra_predicates, not here.
|
|
|
|
continue
|
2021-09-13 15:02:17 +00:00
|
|
|
if isinstance(value, str):
|
2019-01-21 12:23:15 +00:00
|
|
|
value = string_store.add(value)
|
|
|
|
if name not in name2index:
|
|
|
|
name2index[name] = len(name2index)
|
|
|
|
attr_values.append((name2index[name], value))
|
|
|
|
return attr_values
|