mirror of
https://github.com/explosion/spaCy.git
synced 2025-07-22 22:20:08 +03:00
31 lines
959 B
Python
31 lines
959 B
Python
from typing import Callable, List, Tuple, Union
|
|
|
|
from thinc.api import Model, registry
|
|
from thinc.types import Ints2d
|
|
|
|
from ..tokens import Doc
|
|
|
|
|
|
def FeatureExtractor(
|
|
columns: Union[List[str], List[int], List[Union[int, str]]]
|
|
) -> Model[List[Doc], List[Ints2d]]:
|
|
return Model("extract_features", forward, attrs={"columns": columns})
|
|
|
|
|
|
def forward(
|
|
model: Model[List[Doc], List[Ints2d]], docs, is_train: bool
|
|
) -> Tuple[List[Ints2d], Callable]:
|
|
columns = model.attrs["columns"]
|
|
features: List[Ints2d] = []
|
|
for doc in docs:
|
|
if hasattr(doc, "to_array"):
|
|
attrs = doc.to_array(columns)
|
|
else:
|
|
attrs = doc.doc.to_array(columns)[doc.start : doc.end]
|
|
if attrs.ndim == 1:
|
|
attrs = attrs.reshape((attrs.shape[0], 1))
|
|
features.append(model.ops.asarray2i(attrs, dtype="uint64"))
|
|
|
|
backprop: Callable[[List[Ints2d]], List] = lambda d_features: []
|
|
return features, backprop
|