spaCy/spacy/lang/es/lemmatizer.py

from typing import List, Optional, Tuple
import re

from ...pipeline import Lemmatizer
from ...tokens import Token


class SpanishLemmatizer(Lemmatizer):
    """
    Spanish rule-based lemmatizer with morph-based rule selection.
    """

    @classmethod
    def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]:
        if mode == "rule":
            required = ["lemma_rules", "lemma_rules_groups", "lemma_index", "lemma_exc"]
            return (required, [])
        else:
            return super().get_lookups_config(mode)

    def rule_lemmatize(self, token: Token) -> List[str]:
        cache_key = (token.orth, token.pos, str(token.morph))
        if cache_key in self.cache:
            return self.cache[cache_key]
        string = token.text
        pos = token.pos_.lower()
        features = set(token.morph)
        if pos in ("", "eol", "space"):
            return [string.lower()]
        if pos in (
            "adp",
            "cconj",
            "intj",
            "part",
            "propn",
            "punct",
            "sconj",
            "sym",
            "x",
        ):
            if token.is_sent_start and pos != "propn":
                return [string.lower()]
            else:
                return [string]

        string = string.lower()
        exc = self.lookups.get_table("lemma_exc").get(pos, {}).get(string)
        if exc is not None:
            lemmas = list(exc)
        else:
            if pos == "aux":
                rule_pos = "verb"
            else:
                rule_pos = pos
            rule = self.select_rule(rule_pos, list(features))
            index = self.lookups.get_table("lemma_index").get(rule_pos, [])
            lemmas = getattr(self, "lemmatize_" + rule_pos)(
                string, features, rule, index
            )
            # Remove duplicates but preserve the ordering
            lemmas = list(dict.fromkeys(lemmas))

        self.cache[cache_key] = lemmas
        return lemmas

    def select_rule(self, pos: str, features: List[str]) -> Optional[str]:
        groups = self.lookups.get_table("lemma_rules_groups")
        if pos in groups:
            for group in groups[pos]:
                if set(group[1]).issubset(features):
                    return group[0]
        return None

    def lemmatize_adj(
        self, word: str, features: List[str], rule: str, index: List[str]
    ) -> List[str]:
        """
        Lemmatize an adjective.

        word (str): The word to lemmatize.
        features (List[str]): The morphological features as a list of Feat=Val
            pairs.
        index (List[str]): The POS-specific lookup list.

        RETURNS (List[str]): The list of lemmas.
        """

        # Initialize empty lists for the generated lemmas
        possible_lemmas = []
        selected_lemmas = []

        # Apply lemmatization rules
        for old, new in self.lookups.get_table("lemma_rules").get(rule, []):
            possible_lemma = re.sub(old + "$", new, word)
            if possible_lemma != word:
                possible_lemmas.append(possible_lemma)

        # Additional rule for plurals that go from esdrújula to grave and end in
        # 'n' or 's', e.g., jóvenes -> joven
        additional_lemmas = []
        if "Number=Plur" in features:
            for possible_lemma in possible_lemmas:
                if possible_lemma.endswith("n") or possible_lemma.endswith("s"):
                    for old, new in self.lookups.get_table("lemma_rules").get(
                        "accents", []
                    ):
                        additional_lemmas.append(re.sub(old, new, possible_lemma))
        possible_lemmas.extend(additional_lemmas)

        for lemma in possible_lemmas:
            if lemma in index:
                selected_lemmas.append(lemma)
        # If one or more of the created possible lemmas are in the lookup list,
        # return all of them
        if len(selected_lemmas) > 0:
            return selected_lemmas
        elif len(possible_lemmas) > 0:
            return possible_lemmas
        else:
            return [word]

    def lemmatize_adv(
        self, word: str, features: List[str], rule: str, index: List[str]
    ) -> List[str]:
        """
        Lemmatize an adverb.

        word (str): The word to lemmatize.
        features (List[str]): The morphological features as a list of Feat=Val
            pairs.
        index (List[str]): The POS-specific lookup list.

        RETURNS (List[str]): The list of lemmas.
        """

        # Apply lemmatization rules
        for old, new in self.lookups.get_table("lemma_rules").get("adverbs", []):
            if word == old:
                return [new]

        # If none of the rules applies, return the original word
        return [word]

    def lemmatize_det(
        self, word: str, features: List[str], rule: str, index: List[str]
    ) -> List[str]:
        """
        Lemmatize a determiner.

        word (str): The word to lemmatize.
        features (List[str]): The morphological features as a list of Feat=Val
            pairs.
        index (List[str]): The POS-specific lookup list.

        RETURNS (List[str]): The list of lemmas.
        """

        # Initialize empty lists for the generated lemmas
        possible_lemmas = []
        selected_lemmas = []

        # First, search in rules specific to determiners
        for old, new in self.lookups.get_table("lemma_rules").get("det", []):
            if word == old:
                return [new]
        # If none of the specfic rules apply, search in the common rules for
        # determiners and pronouns that follow a unique pattern for
        # lemmatization. If the word is in the list, return the corresponding
        # lemma.
        for old, new in self.lookups.get_table("lemma_rules").get(
            "det_and_pron_fixed", []
        ):
            if word == old:
                return [new]
        # If the word is not in the list of unique determiners and pronouns,
        # apply general rules of lemmatization. Include the original word in the        # list of possible lemmas.
        for old, new in self.lookups.get_table("lemma_rules").get(
            "det_and_pron_general", []
        ):
            possible_lemma = re.sub(old + "$", new, word)
            possible_lemmas.append(possible_lemma)
        possible_lemmas.append(word)

        if len(possible_lemmas) == 1:
            return possible_lemmas
        elif len(possible_lemmas) > 1:
            for lemma in possible_lemmas:
                if lemma in index:
                    selected_lemmas.append(lemma)
            if len(selected_lemmas) >= 1:
                return selected_lemmas
            else:
                return possible_lemmas
        else:
            return []

    def lemmatize_noun(
        self, word: str, features: List[str], rule: str, index: List[str]
    ) -> List[str]:
        """
        Lemmatize a noun.

        word (str): The word to lemmatize.
        features (List[str]): The morphological features as a list of Feat=Val
            pairs.
        index (List[str]): The POS-specific lookup list.

        RETURNS (List[str]): The list of lemmas.
        """

        # Initialize empty lists for the generated lemmas
        possible_lemmas = []
        selected_lemmas = []

        # Apply lemmatization rules
        for old, new in self.lookups.get_table("lemma_rules").get(rule, []):
            possible_lemma = re.sub(old + "$", new, word)
            if possible_lemma != word:
                possible_lemmas.append(possible_lemma)

        # Additional rule for plurals that go from esdrújula to grave and end in
        # 'n' or 's', e.g., órdenes -> orden, exámenes -> examen
        additional_lemmas = []
        if "Number=Plur" in features:
            for possible_lemma in possible_lemmas:
                if possible_lemma.endswith("n") or possible_lemma.endswith("s"):
                    for old, new in self.lookups.get_table("lemma_rules").get(
                        "accents", []
                    ):
                        additional_lemmas.append(re.sub(old, new, possible_lemma))
        possible_lemmas.extend(additional_lemmas)

        for lemma in possible_lemmas:
            if lemma in index:
                selected_lemmas.append(lemma)
        # If one or more of the created possible lemmas are in the lookup list,
        # return all of them
        if len(selected_lemmas) > 0:
            return selected_lemmas
        elif len(possible_lemmas) > 0:
            return possible_lemmas
        else:
            return [word]

    def lemmatize_num(
        self, word: str, features: List[str], rule: str, index: List[str]
    ) -> List[str]:
        """
        Lemmatize a numeral.

        word (str): The word to lemmatize.
        features (List[str]): The morphological features as a list of Feat=Val
            pairs.
        index (List[str]): The POS-specific lookup list.

        RETURNS (List[str]): The list of lemmas.
        """

        # If the word is in the list of rules for numerals, return the
        # corresponding lemma
        for old, new in self.lookups.get_table("lemma_rules").get("num", []):
            if word == old:
                return [new]

        # Normalize punctuation
        splitted_word = word.split(",")
        if re.search(r"(\.)([0-9]{3})$", splitted_word[0]):
            word = re.sub(r"\.", r"", word)
        word = re.sub(r",", r".", word)
        return [word]

    def lemmatize_pron(
        self, word: str, features: List[str], rule: Optional[str], index: List[str]
    ) -> List[str]:
        """
        Lemmatize a pronoun.

        word (str): The word to lemmatize.
        features (List[str]): The morphological features as a list of Feat=Val
            pairs.
        index (List[str]): The POS-specific lookup list.

        RETURNS (List[str]): The list of lemmas.
        """

        # Initialize empty lists for the generated lemmas
        possible_lemmas = []
        selected_lemmas = []

        # First, search in rules specific to pronouns
        for old, new in self.lookups.get_table("lemma_rules").get("pron", []):
            if word == old:
                return [new]
        # If none of the specfic rules apply, search in the common rules for
        # determiners and pronouns that follow a unique pattern for
        # lemmatization. If the word is in the list, return the corresponding
        # lemma.
        for old, new in self.lookups.get_table("lemma_rules").get(
            "det_and_pron_fixed", []
        ):
            if word == old:
                return [new]
        # If the word is not in the list of unique determiners and pronouns,
        # apply general rules of lemmatization. Include the original word in the
        # list of possible lemmas.
        for old, new in self.lookups.get_table("lemma_rules").get(
            "det_and_pron_general", []
        ):
            possible_lemma = re.sub(old + "$", new, word)
            if possible_lemma != word:
                possible_lemmas.append(possible_lemma)
        possible_lemmas.append(word)

        if len(possible_lemmas) == 1:
            return possible_lemmas
        elif len(possible_lemmas) > 1:
            for lemma in possible_lemmas:
                if lemma in index:
                    selected_lemmas.append(lemma)
            if len(selected_lemmas) >= 1:
                return selected_lemmas
            else:
                return possible_lemmas
        else:
            return []

    def lemmatize_verb(
        self, word: str, features: List[str], rule: Optional[str], index: List[str]
    ) -> List[str]:
        """
        Lemmatize a verb.

        word (str): The word to lemmatize.
        features (List[str]): The morphological features as a list of Feat=Val
            pairs.
        index (List[str]): The POS-specific lookup list.

        RETURNS (List[str]): The list of lemmas.
        """
        # Exceptions for verb+pronoun(s)
        if "PronType=Prs" in features:
            return self.lemmatize_verb_pron(word, features, rule, index)

        # Initialize empty lists for the generated lemmas
        possible_lemmas = []
        selected_lemmas = []

        # Apply lemmatization rules
        rule = str(rule or "")
        for old, new in self.lookups.get_table("lemma_rules").get(rule, []):
            possible_lemma = re.sub(old + "$", new, word)
            if possible_lemma != word:
                possible_lemmas.append(possible_lemma)

        for lemma in possible_lemmas:
            if lemma in index:
                selected_lemmas.append(lemma)
        if len(selected_lemmas) == 0:
            # If none of the possible lemmas are in the lookup list,
            # apply vocalic alternation rules and search in the lookup list
            # again
            for lemma in possible_lemmas:
                for old, new in self.lookups.get_table("lemma_rules").get(
                    "voc_alt_1", []
                ):
                    if old in lemma:
                        for i, char in enumerate(lemma):
                            if char == old:
                                voc_alt_lemma = lemma[:i] + new + lemma[i + 1 :]
                                if voc_alt_lemma in index:
                                    selected_lemmas.append(voc_alt_lemma)
                for old, new in self.lookups.get_table("lemma_rules").get(
                    "voc_alt_2", []
                ):
                    if old in lemma:
                        voc_alt_lemma = lemma.replace(old, new, 1)
                        if voc_alt_lemma in index:
                            selected_lemmas.append(voc_alt_lemma)
        # Additional rule for verbs that lose the accent mark when lemmatized,
        # e.g., amplían -> ampliar
        additional_lemmas = []
        for possible_lemma in possible_lemmas:
            for old, new in self.lookups.get_table("lemma_rules").get("accents", []):
                additional_lemmas.append(re.sub(old, new, possible_lemma))
        possible_lemmas.extend(additional_lemmas)

        # If one or more of the created possible lemmas are in the lookup list,
        # return all of them
        if len(selected_lemmas) > 0:
            return selected_lemmas
        elif len(possible_lemmas) > 0:
            return possible_lemmas
        else:
            return [word]

    def lemmatize_verb_pron(
        self, word: str, features: List[str], rule: Optional[str], index: List[str]
    ) -> List[str]:
        # Strip and collect pronouns
        pron_patt = "^(.*?)([mts]e|l[aeo]s?|n?os)$"
        prons: List[str] = []
        verb = word
        m = re.search(pron_patt, verb)
        while m is not None and len(prons) <= 3:
            verb = re.sub(m.group(2) + "$", "", verb)
            prons = [m.group(2)] + prons
            m = re.search(pron_patt, verb)
        # Strip accents from verb form
        for old, new in self.lookups.get_table("lemma_rules").get("accents", []):
            verb = re.sub(old, new, verb)
        # Lemmatize the verb and pronouns, checking for exceptions
        exc = self.lookups.get_table("lemma_exc").get("verb", {}).get(verb)
        if exc is not None:
            verb_lemma = exc[0]
        else:
            rule = self.select_rule("verb", features)
            verb_lemma = self.lemmatize_verb(
                verb, features - {"PronType=Prs"}, rule, index  # type: ignore[operator]
            )[0]
        pron_lemmas = []
        for pron in prons:
            exc = self.lookups.get_table("lemma_exc").get("pron", {}).get(pron)
            if exc is not None:
                pron_lemmas.append(exc[0])
            else:
                rule = self.select_rule("pron", features)
                pron_lemmas.append(self.lemmatize_pron(pron, features, rule, index)[0])
        return [verb_lemma + " " + " ".join(pron_lemmas)]