2020-10-18 12:50:41 +00:00
|
|
|
from typing import Optional, List
|
|
|
|
|
|
|
|
from thinc.types import Floats2d
|
2020-07-22 11:42:59 +00:00
|
|
|
from thinc.api import Model, reduce_mean, Linear, list2ragged, Logistic
|
|
|
|
from thinc.api import chain, concatenate, clone, Dropout, ParametricAttention
|
|
|
|
from thinc.api import SparseLinear, Softmax, softmax_activation, Maxout, reduce_sum
|
2020-07-31 15:02:54 +00:00
|
|
|
from thinc.api import HashEmbed, with_array, with_cpu, uniqued
|
2020-10-01 14:22:48 +00:00
|
|
|
from thinc.api import Relu, residual, expand_window
|
2021-01-06 11:44:04 +00:00
|
|
|
from thinc.layers.chain import init as init_chain
|
2020-02-27 17:42:27 +00:00
|
|
|
|
2020-06-20 12:15:04 +00:00
|
|
|
from ...attrs import ID, ORTH, PREFIX, SUFFIX, SHAPE, LOWER
|
2020-02-28 10:57:41 +00:00
|
|
|
from ...util import registry
|
|
|
|
from ..extract_ngrams import extract_ngrams
|
2020-07-29 12:35:36 +00:00
|
|
|
from ..staticvectors import StaticVectors
|
2020-10-01 14:22:48 +00:00
|
|
|
from ..featureextractor import FeatureExtractor
|
2020-10-18 12:50:41 +00:00
|
|
|
from ...tokens import Doc
|
2021-01-06 11:44:04 +00:00
|
|
|
from .tok2vec import get_tok2vec_width
|
2020-02-27 17:42:27 +00:00
|
|
|
|
|
|
|
|
|
|
|
@registry.architectures.register("spacy.TextCatCNN.v1")
|
2020-07-22 11:42:59 +00:00
|
|
|
def build_simple_cnn_text_classifier(
|
|
|
|
tok2vec: Model, exclusive_classes: bool, nO: Optional[int] = None
|
2020-10-18 12:50:41 +00:00
|
|
|
) -> Model[List[Doc], Floats2d]:
|
2020-02-27 17:42:27 +00:00
|
|
|
"""
|
|
|
|
Build a simple CNN text classifier, given a token-to-vector model as inputs.
|
|
|
|
If exclusive_classes=True, a softmax non-linearity is applied, so that the
|
|
|
|
outputs sum to 1. If exclusive_classes=False, a logistic non-linearity
|
|
|
|
is applied instead, so that outputs are in the range [0, 1].
|
|
|
|
"""
|
|
|
|
with Model.define_operators({">>": chain}):
|
2020-10-18 12:50:41 +00:00
|
|
|
cnn = tok2vec >> list2ragged() >> reduce_mean()
|
2020-02-27 17:42:27 +00:00
|
|
|
if exclusive_classes:
|
2020-10-09 13:40:58 +00:00
|
|
|
output_layer = Softmax(nO=nO, nI=tok2vec.maybe_get_dim("nO"))
|
2020-10-18 12:50:41 +00:00
|
|
|
model = cnn >> output_layer
|
2020-02-27 17:42:27 +00:00
|
|
|
model.set_ref("output_layer", output_layer)
|
|
|
|
else:
|
2020-10-09 13:40:58 +00:00
|
|
|
linear_layer = Linear(nO=nO, nI=tok2vec.maybe_get_dim("nO"))
|
2020-10-18 12:50:41 +00:00
|
|
|
model = cnn >> linear_layer >> Logistic()
|
2020-02-27 17:42:27 +00:00
|
|
|
model.set_ref("output_layer", linear_layer)
|
|
|
|
model.set_ref("tok2vec", tok2vec)
|
|
|
|
model.set_dim("nO", nO)
|
2020-06-12 00:02:07 +00:00
|
|
|
model.attrs["multi_label"] = not exclusive_classes
|
2020-02-27 17:42:27 +00:00
|
|
|
return model
|
|
|
|
|
|
|
|
|
|
|
|
@registry.architectures.register("spacy.TextCatBOW.v1")
|
2020-07-31 15:02:54 +00:00
|
|
|
def build_bow_text_classifier(
|
|
|
|
exclusive_classes: bool,
|
|
|
|
ngram_size: int,
|
|
|
|
no_output_layer: bool,
|
|
|
|
nO: Optional[int] = None,
|
2020-10-18 12:50:41 +00:00
|
|
|
) -> Model[List[Doc], Floats2d]:
|
2020-02-27 17:42:27 +00:00
|
|
|
with Model.define_operators({">>": chain}):
|
2020-03-29 17:40:36 +00:00
|
|
|
sparse_linear = SparseLinear(nO)
|
|
|
|
model = extract_ngrams(ngram_size, attr=ORTH) >> sparse_linear
|
|
|
|
model = with_cpu(model, model.ops)
|
2020-02-27 17:42:27 +00:00
|
|
|
if not no_output_layer:
|
2020-03-29 17:40:36 +00:00
|
|
|
output_layer = softmax_activation() if exclusive_classes else Logistic()
|
|
|
|
model = model >> with_cpu(output_layer, output_layer.ops)
|
|
|
|
model.set_ref("output_layer", sparse_linear)
|
2020-06-12 00:02:07 +00:00
|
|
|
model.attrs["multi_label"] = not exclusive_classes
|
2020-03-29 17:40:36 +00:00
|
|
|
return model
|
|
|
|
|
|
|
|
|
2020-10-18 12:50:41 +00:00
|
|
|
@registry.architectures.register("spacy.TextCatEnsemble.v2")
|
2020-11-10 12:14:47 +00:00
|
|
|
def build_text_classifier_v2(
|
2020-10-18 12:50:41 +00:00
|
|
|
tok2vec: Model[List[Doc], List[Floats2d]],
|
|
|
|
linear_model: Model[List[Doc], Floats2d],
|
|
|
|
nO: Optional[int] = None,
|
|
|
|
) -> Model[List[Doc], Floats2d]:
|
|
|
|
exclusive_classes = not linear_model.attrs["multi_label"]
|
|
|
|
with Model.define_operators({">>": chain, "|": concatenate}):
|
2020-11-10 12:14:47 +00:00
|
|
|
width = tok2vec.maybe_get_dim("nO")
|
2021-01-06 11:44:04 +00:00
|
|
|
attention_layer = ParametricAttention(width) # TODO: benchmark performance difference of this layer
|
|
|
|
maxout_layer = Maxout(nO=width, nI=width)
|
|
|
|
linear_layer = Linear(nO=nO, nI=width)
|
2020-10-18 12:50:41 +00:00
|
|
|
cnn_model = (
|
2021-01-06 11:44:04 +00:00
|
|
|
tok2vec
|
|
|
|
>> list2ragged()
|
|
|
|
>> attention_layer
|
|
|
|
>> reduce_sum()
|
|
|
|
>> residual(maxout_layer)
|
|
|
|
>> linear_layer
|
|
|
|
>> Dropout(0.0)
|
2020-10-18 12:50:41 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
nO_double = nO * 2 if nO else None
|
|
|
|
if exclusive_classes:
|
|
|
|
output_layer = Softmax(nO=nO, nI=nO_double)
|
|
|
|
else:
|
|
|
|
output_layer = Linear(nO=nO, nI=nO_double) >> Dropout(0.0) >> Logistic()
|
|
|
|
model = (linear_model | cnn_model) >> output_layer
|
|
|
|
model.set_ref("tok2vec", tok2vec)
|
|
|
|
if model.has_dim("nO") is not False:
|
|
|
|
model.set_dim("nO", nO)
|
|
|
|
model.set_ref("output_layer", linear_model.get_ref("output_layer"))
|
2021-01-06 11:44:04 +00:00
|
|
|
model.set_ref("attention_layer", attention_layer)
|
|
|
|
model.set_ref("maxout_layer", maxout_layer)
|
|
|
|
model.set_ref("linear_layer", linear_layer)
|
2020-10-18 12:50:41 +00:00
|
|
|
model.attrs["multi_label"] = not exclusive_classes
|
2021-01-06 11:44:04 +00:00
|
|
|
|
|
|
|
model.init = init_ensemble_textcat
|
|
|
|
return model
|
|
|
|
|
|
|
|
|
|
|
|
def init_ensemble_textcat(model, X, Y) -> Model:
|
|
|
|
tok2vec_width = get_tok2vec_width(model)
|
|
|
|
model.get_ref("attention_layer").set_dim("nO", tok2vec_width)
|
|
|
|
model.get_ref("maxout_layer").set_dim("nO", tok2vec_width)
|
|
|
|
model.get_ref("maxout_layer").set_dim("nI", tok2vec_width)
|
|
|
|
model.get_ref("linear_layer").set_dim("nI", tok2vec_width)
|
|
|
|
init_chain(model, X, Y)
|
2020-10-18 12:50:41 +00:00
|
|
|
return model
|
|
|
|
|
2021-01-05 02:41:53 +00:00
|
|
|
|
2020-10-18 12:50:41 +00:00
|
|
|
# TODO: move to legacy
|
2020-07-26 13:11:43 +00:00
|
|
|
@registry.architectures.register("spacy.TextCatEnsemble.v1")
|
2020-11-10 12:14:47 +00:00
|
|
|
def build_text_classifier_v1(
|
2020-07-31 15:02:54 +00:00
|
|
|
width: int,
|
|
|
|
embed_size: int,
|
|
|
|
pretrained_vectors: Optional[bool],
|
|
|
|
exclusive_classes: bool,
|
|
|
|
ngram_size: int,
|
|
|
|
window_size: int,
|
|
|
|
conv_depth: int,
|
|
|
|
dropout: Optional[float],
|
|
|
|
nO: Optional[int] = None,
|
|
|
|
) -> Model:
|
2020-08-07 14:17:34 +00:00
|
|
|
# Don't document this yet, I'm not sure it's right.
|
2020-03-29 17:40:36 +00:00
|
|
|
cols = [ORTH, LOWER, PREFIX, SUFFIX, SHAPE, ID]
|
|
|
|
with Model.define_operators({">>": chain, "|": concatenate, "**": clone}):
|
2020-06-20 12:15:04 +00:00
|
|
|
lower = HashEmbed(
|
2020-07-09 17:39:31 +00:00
|
|
|
nO=width, nV=embed_size, column=cols.index(LOWER), dropout=dropout, seed=10
|
2020-06-20 12:15:04 +00:00
|
|
|
)
|
|
|
|
prefix = HashEmbed(
|
2020-07-22 11:42:59 +00:00
|
|
|
nO=width // 2,
|
|
|
|
nV=embed_size,
|
|
|
|
column=cols.index(PREFIX),
|
|
|
|
dropout=dropout,
|
|
|
|
seed=11,
|
2020-06-20 12:15:04 +00:00
|
|
|
)
|
|
|
|
suffix = HashEmbed(
|
2020-07-22 11:42:59 +00:00
|
|
|
nO=width // 2,
|
|
|
|
nV=embed_size,
|
|
|
|
column=cols.index(SUFFIX),
|
|
|
|
dropout=dropout,
|
|
|
|
seed=12,
|
2020-06-20 12:15:04 +00:00
|
|
|
)
|
|
|
|
shape = HashEmbed(
|
2020-07-22 11:42:59 +00:00
|
|
|
nO=width // 2,
|
|
|
|
nV=embed_size,
|
|
|
|
column=cols.index(SHAPE),
|
|
|
|
dropout=dropout,
|
|
|
|
seed=13,
|
2020-06-20 12:15:04 +00:00
|
|
|
)
|
2020-03-29 17:40:36 +00:00
|
|
|
width_nI = sum(layer.get_dim("nO") for layer in [lower, prefix, suffix, shape])
|
|
|
|
trained_vectors = FeatureExtractor(cols) >> with_array(
|
|
|
|
uniqued(
|
|
|
|
(lower | prefix | suffix | shape)
|
|
|
|
>> Maxout(nO=width, nI=width_nI, normalize=True),
|
|
|
|
column=cols.index(ORTH),
|
|
|
|
)
|
|
|
|
)
|
|
|
|
if pretrained_vectors:
|
2020-07-29 12:35:36 +00:00
|
|
|
static_vectors = StaticVectors(width)
|
2020-03-29 17:40:36 +00:00
|
|
|
vector_layer = trained_vectors | static_vectors
|
|
|
|
vectors_width = width * 2
|
|
|
|
else:
|
|
|
|
vector_layer = trained_vectors
|
|
|
|
vectors_width = width
|
|
|
|
tok2vec = vector_layer >> with_array(
|
|
|
|
Maxout(width, vectors_width, normalize=True)
|
2020-06-20 12:15:04 +00:00
|
|
|
>> residual(
|
|
|
|
(
|
|
|
|
expand_window(window_size=window_size)
|
|
|
|
>> Maxout(
|
|
|
|
nO=width, nI=width * ((window_size * 2) + 1), normalize=True
|
|
|
|
)
|
|
|
|
)
|
|
|
|
)
|
|
|
|
** conv_depth,
|
2020-03-29 17:40:36 +00:00
|
|
|
pad=conv_depth,
|
|
|
|
)
|
|
|
|
cnn_model = (
|
2020-06-20 12:15:04 +00:00
|
|
|
tok2vec
|
|
|
|
>> list2ragged()
|
|
|
|
>> ParametricAttention(width)
|
|
|
|
>> reduce_sum()
|
|
|
|
>> residual(Maxout(nO=width, nI=width))
|
|
|
|
>> Linear(nO=nO, nI=width)
|
|
|
|
>> Dropout(0.0)
|
2020-03-29 17:40:36 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
linear_model = build_bow_text_classifier(
|
2020-06-20 12:15:04 +00:00
|
|
|
nO=nO,
|
|
|
|
ngram_size=ngram_size,
|
|
|
|
exclusive_classes=exclusive_classes,
|
|
|
|
no_output_layer=False,
|
2020-03-29 17:40:36 +00:00
|
|
|
)
|
2020-06-20 12:15:04 +00:00
|
|
|
nO_double = nO * 2 if nO else None
|
2020-03-29 17:40:36 +00:00
|
|
|
if exclusive_classes:
|
|
|
|
output_layer = Softmax(nO=nO, nI=nO_double)
|
|
|
|
else:
|
2020-06-20 12:15:04 +00:00
|
|
|
output_layer = Linear(nO=nO, nI=nO_double) >> Dropout(0.0) >> Logistic()
|
2020-03-29 17:40:36 +00:00
|
|
|
model = (linear_model | cnn_model) >> output_layer
|
|
|
|
model.set_ref("tok2vec", tok2vec)
|
|
|
|
if model.has_dim("nO") is not False:
|
|
|
|
model.set_dim("nO", nO)
|
|
|
|
model.set_ref("output_layer", linear_model.get_ref("output_layer"))
|
2020-06-12 00:02:07 +00:00
|
|
|
model.attrs["multi_label"] = not exclusive_classes
|
2020-03-29 17:40:36 +00:00
|
|
|
return model
|
|
|
|
|
|
|
|
|
|
|
|
@registry.architectures.register("spacy.TextCatLowData.v1")
|
2020-07-31 15:02:54 +00:00
|
|
|
def build_text_classifier_lowdata(
|
2020-10-18 12:50:41 +00:00
|
|
|
width: int, dropout: Optional[float], nO: Optional[int] = None
|
|
|
|
) -> Model[List[Doc], Floats2d]:
|
2020-08-07 14:17:34 +00:00
|
|
|
# Don't document this yet, I'm not sure it's right.
|
2020-03-29 17:40:36 +00:00
|
|
|
# Note, before v.3, this was the default if setting "low_data" and "pretrained_dims"
|
|
|
|
with Model.define_operators({">>": chain, "**": clone}):
|
|
|
|
model = (
|
2020-07-29 12:35:36 +00:00
|
|
|
StaticVectors(width)
|
2020-03-29 17:40:36 +00:00
|
|
|
>> list2ragged()
|
|
|
|
>> ParametricAttention(width)
|
|
|
|
>> reduce_sum()
|
|
|
|
>> residual(Relu(width, width)) ** 2
|
|
|
|
>> Linear(nO, width)
|
|
|
|
)
|
2020-06-03 09:50:16 +00:00
|
|
|
if dropout:
|
|
|
|
model = model >> Dropout(dropout)
|
|
|
|
model = model >> Logistic()
|
2020-02-27 17:42:27 +00:00
|
|
|
return model
|