From 0ec9a696e60933807c189c7be22623a81a840289 Mon Sep 17 00:00:00 2001 From: Madeesh Kannan Date: Mon, 12 Sep 2022 14:55:41 +0200 Subject: [PATCH] Fix config validation failures caused by NVTX pipeline wrappers (#11460) * Enable Cython<->Python bindings for `Pipe` and `TrainablePipe` methods * `pipes_with_nvtx_range`: Skip hooking methods whose signature cannot be ascertained When loading pipelines from a config file, the arguments passed to individual pipeline components is validated by `pydantic` during init. For this, the validation model attempts to parse the function signature of the component's c'tor/entry point so that it can check if all mandatory parameters are present in the config file. When using the `models_and_pipes_with_nvtx_range` as a `after_pipeline_creation` callback, the methods of all pipeline components get replaced by a NVTX range wrapper **before** the above-mentioned validation takes place. This can be problematic for components that are implemented as Cython extension types - if the extension type is not compiled with Python bindings for its methods, they will have no signatures at runtime. This resulted in `pydantic` matching the *wrapper's* parameters with the those in the config and raising errors. To avoid this, we now skip applying the wrapper to any (Cython) methods that do not have signatures. --- spacy/ml/callbacks.py | 7 +++++-- spacy/pipeline/pipe.pyx | 2 +- spacy/pipeline/trainable_pipe.pyx | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/spacy/ml/callbacks.py b/spacy/ml/callbacks.py index 18290b947..3b60ec2ab 100644 --- a/spacy/ml/callbacks.py +++ b/spacy/ml/callbacks.py @@ -89,11 +89,14 @@ def pipes_with_nvtx_range( types.MethodType(nvtx_range_wrapper_for_pipe_method, pipe), func ) - # Try to preserve the original function signature. + # We need to preserve the original function signature so that + # the original parameters are passed to pydantic for validation downstream. try: wrapped_func.__signature__ = inspect.signature(func) # type: ignore except: - pass + # Can fail for Cython methods that do not have bindings. + warnings.warn(Warnings.W122.format(method=name, pipe=pipe.name)) + continue try: setattr( diff --git a/spacy/pipeline/pipe.pyx b/spacy/pipeline/pipe.pyx index 4e3ae1cf0..8407acc45 100644 --- a/spacy/pipeline/pipe.pyx +++ b/spacy/pipeline/pipe.pyx @@ -1,4 +1,4 @@ -# cython: infer_types=True, profile=True +# cython: infer_types=True, profile=True, binding=True from typing import Optional, Tuple, Iterable, Iterator, Callable, Union, Dict import srsly import warnings diff --git a/spacy/pipeline/trainable_pipe.pyx b/spacy/pipeline/trainable_pipe.pyx index 76b0733cf..3f0507d4b 100644 --- a/spacy/pipeline/trainable_pipe.pyx +++ b/spacy/pipeline/trainable_pipe.pyx @@ -1,4 +1,4 @@ -# cython: infer_types=True, profile=True +# cython: infer_types=True, profile=True, binding=True from typing import Iterable, Iterator, Optional, Dict, Tuple, Callable import srsly from thinc.api import set_dropout_rate, Model, Optimizer