mirror of
https://github.com/explosion/spaCy.git
synced 2024-11-10 19:57:17 +03:00
268 lines
8.4 KiB
Python
268 lines
8.4 KiB
Python
"""
|
|
This example shows how to use an LSTM sentiment classification model trained
|
|
using Keras in spaCy. spaCy splits the document into sentences, and each
|
|
sentence is classified using the LSTM. The scores for the sentences are then
|
|
aggregated to give the document score. This kind of hierarchical model is quite
|
|
difficult in "pure" Keras or Tensorflow, but it's very effective. The Keras
|
|
example on this dataset performs quite poorly, because it cuts off the documents
|
|
so that they're a fixed size. This hurts review accuracy a lot, because people
|
|
often summarise their rating in the final sentence
|
|
|
|
Prerequisites:
|
|
spacy download en_vectors_web_lg
|
|
pip install keras==2.0.9
|
|
|
|
Compatible with: spaCy v2.0.0+
|
|
"""
|
|
|
|
import plac
|
|
import random
|
|
import pathlib
|
|
import cytoolz
|
|
import numpy
|
|
from keras.models import Sequential, model_from_json
|
|
from keras.layers import LSTM, Dense, Embedding, Bidirectional
|
|
from keras.layers import TimeDistributed
|
|
from keras.optimizers import Adam
|
|
import thinc.extra.datasets
|
|
from spacy.compat import pickle
|
|
import spacy
|
|
|
|
|
|
class SentimentAnalyser(object):
|
|
@classmethod
|
|
def load(cls, path, nlp, max_length=100):
|
|
with (path / "config.json").open() as file_:
|
|
model = model_from_json(file_.read())
|
|
with (path / "model").open("rb") as file_:
|
|
lstm_weights = pickle.load(file_)
|
|
embeddings = get_embeddings(nlp.vocab)
|
|
model.set_weights([embeddings] + lstm_weights)
|
|
return cls(model, max_length=max_length)
|
|
|
|
def __init__(self, model, max_length=100):
|
|
self._model = model
|
|
self.max_length = max_length
|
|
|
|
def __call__(self, doc):
|
|
X = get_features([doc], self.max_length)
|
|
y = self._model.predict(X)
|
|
self.set_sentiment(doc, y)
|
|
|
|
def pipe(self, docs, batch_size=1000):
|
|
for minibatch in cytoolz.partition_all(batch_size, docs):
|
|
minibatch = list(minibatch)
|
|
sentences = []
|
|
for doc in minibatch:
|
|
sentences.extend(doc.sents)
|
|
Xs = get_features(sentences, self.max_length)
|
|
ys = self._model.predict(Xs)
|
|
for sent, label in zip(sentences, ys):
|
|
sent.doc.sentiment += label - 0.5
|
|
for doc in minibatch:
|
|
yield doc
|
|
|
|
def set_sentiment(self, doc, y):
|
|
doc.sentiment = float(y[0])
|
|
# Sentiment has a native slot for a single float.
|
|
# For arbitrary data storage, there's:
|
|
# doc.user_data['my_data'] = y
|
|
|
|
|
|
def get_labelled_sentences(docs, doc_labels):
|
|
labels = []
|
|
sentences = []
|
|
for doc, y in zip(docs, doc_labels):
|
|
for sent in doc.sents:
|
|
sentences.append(sent)
|
|
labels.append(y)
|
|
return sentences, numpy.asarray(labels, dtype="int32")
|
|
|
|
|
|
def get_features(docs, max_length):
|
|
docs = list(docs)
|
|
Xs = numpy.zeros((len(docs), max_length), dtype="int32")
|
|
for i, doc in enumerate(docs):
|
|
j = 0
|
|
for token in doc:
|
|
vector_id = token.vocab.vectors.find(key=token.orth)
|
|
if vector_id >= 0:
|
|
Xs[i, j] = vector_id
|
|
else:
|
|
Xs[i, j] = 0
|
|
j += 1
|
|
if j >= max_length:
|
|
break
|
|
return Xs
|
|
|
|
|
|
def train(
|
|
train_texts,
|
|
train_labels,
|
|
dev_texts,
|
|
dev_labels,
|
|
lstm_shape,
|
|
lstm_settings,
|
|
lstm_optimizer,
|
|
batch_size=100,
|
|
nb_epoch=5,
|
|
by_sentence=True,
|
|
):
|
|
|
|
print("Loading spaCy")
|
|
nlp = spacy.load("en_vectors_web_lg")
|
|
nlp.add_pipe(nlp.create_pipe("sentencizer"))
|
|
embeddings = get_embeddings(nlp.vocab)
|
|
model = compile_lstm(embeddings, lstm_shape, lstm_settings)
|
|
|
|
print("Parsing texts...")
|
|
train_docs = list(nlp.pipe(train_texts))
|
|
dev_docs = list(nlp.pipe(dev_texts))
|
|
if by_sentence:
|
|
train_docs, train_labels = get_labelled_sentences(train_docs, train_labels)
|
|
dev_docs, dev_labels = get_labelled_sentences(dev_docs, dev_labels)
|
|
|
|
train_X = get_features(train_docs, lstm_shape["max_length"])
|
|
dev_X = get_features(dev_docs, lstm_shape["max_length"])
|
|
model.fit(
|
|
train_X,
|
|
train_labels,
|
|
validation_data=(dev_X, dev_labels),
|
|
epochs=nb_epoch,
|
|
batch_size=batch_size,
|
|
)
|
|
return model
|
|
|
|
|
|
def compile_lstm(embeddings, shape, settings):
|
|
model = Sequential()
|
|
model.add(
|
|
Embedding(
|
|
embeddings.shape[0],
|
|
embeddings.shape[1],
|
|
input_length=shape["max_length"],
|
|
trainable=False,
|
|
weights=[embeddings],
|
|
mask_zero=True,
|
|
)
|
|
)
|
|
model.add(TimeDistributed(Dense(shape["nr_hidden"], use_bias=False)))
|
|
model.add(
|
|
Bidirectional(
|
|
LSTM(
|
|
shape["nr_hidden"],
|
|
recurrent_dropout=settings["dropout"],
|
|
dropout=settings["dropout"],
|
|
)
|
|
)
|
|
)
|
|
model.add(Dense(shape["nr_class"], activation="sigmoid"))
|
|
model.compile(
|
|
optimizer=Adam(lr=settings["lr"]),
|
|
loss="binary_crossentropy",
|
|
metrics=["accuracy"],
|
|
)
|
|
return model
|
|
|
|
|
|
def get_embeddings(vocab):
|
|
return vocab.vectors.data
|
|
|
|
|
|
def evaluate(model_dir, texts, labels, max_length=100):
|
|
nlp = spacy.load("en_vectors_web_lg")
|
|
nlp.add_pipe(nlp.create_pipe("sentencizer"))
|
|
nlp.add_pipe(SentimentAnalyser.load(model_dir, nlp, max_length=max_length))
|
|
|
|
correct = 0
|
|
i = 0
|
|
for doc in nlp.pipe(texts, batch_size=1000):
|
|
correct += bool(doc.sentiment >= 0.5) == bool(labels[i])
|
|
i += 1
|
|
return float(correct) / i
|
|
|
|
|
|
def read_data(data_dir, limit=0):
|
|
examples = []
|
|
for subdir, label in (("pos", 1), ("neg", 0)):
|
|
for filename in (data_dir / subdir).iterdir():
|
|
with filename.open() as file_:
|
|
text = file_.read()
|
|
examples.append((text, label))
|
|
random.shuffle(examples)
|
|
if limit >= 1:
|
|
examples = examples[:limit]
|
|
return zip(*examples) # Unzips into two lists
|
|
|
|
|
|
@plac.annotations(
|
|
train_dir=("Location of training file or directory"),
|
|
dev_dir=("Location of development file or directory"),
|
|
model_dir=("Location of output model directory",),
|
|
is_runtime=("Demonstrate run-time usage", "flag", "r", bool),
|
|
nr_hidden=("Number of hidden units", "option", "H", int),
|
|
max_length=("Maximum sentence length", "option", "L", int),
|
|
dropout=("Dropout", "option", "d", float),
|
|
learn_rate=("Learn rate", "option", "e", float),
|
|
nb_epoch=("Number of training epochs", "option", "i", int),
|
|
batch_size=("Size of minibatches for training LSTM", "option", "b", int),
|
|
nr_examples=("Limit to N examples", "option", "n", int),
|
|
)
|
|
def main(
|
|
model_dir=None,
|
|
train_dir=None,
|
|
dev_dir=None,
|
|
is_runtime=False,
|
|
nr_hidden=64,
|
|
max_length=100, # Shape
|
|
dropout=0.5,
|
|
learn_rate=0.001, # General NN config
|
|
nb_epoch=5,
|
|
batch_size=256,
|
|
nr_examples=-1,
|
|
): # Training params
|
|
if model_dir is not None:
|
|
model_dir = pathlib.Path(model_dir)
|
|
if train_dir is None or dev_dir is None:
|
|
imdb_data = thinc.extra.datasets.imdb()
|
|
if is_runtime:
|
|
if dev_dir is None:
|
|
dev_texts, dev_labels = zip(*imdb_data[1])
|
|
else:
|
|
dev_texts, dev_labels = read_data(dev_dir)
|
|
acc = evaluate(model_dir, dev_texts, dev_labels, max_length=max_length)
|
|
print(acc)
|
|
else:
|
|
if train_dir is None:
|
|
train_texts, train_labels = zip(*imdb_data[0])
|
|
else:
|
|
print("Read data")
|
|
train_texts, train_labels = read_data(train_dir, limit=nr_examples)
|
|
if dev_dir is None:
|
|
dev_texts, dev_labels = zip(*imdb_data[1])
|
|
else:
|
|
dev_texts, dev_labels = read_data(dev_dir, imdb_data, limit=nr_examples)
|
|
train_labels = numpy.asarray(train_labels, dtype="int32")
|
|
dev_labels = numpy.asarray(dev_labels, dtype="int32")
|
|
lstm = train(
|
|
train_texts,
|
|
train_labels,
|
|
dev_texts,
|
|
dev_labels,
|
|
{"nr_hidden": nr_hidden, "max_length": max_length, "nr_class": 1},
|
|
{"dropout": dropout, "lr": learn_rate},
|
|
{},
|
|
nb_epoch=nb_epoch,
|
|
batch_size=batch_size,
|
|
)
|
|
weights = lstm.get_weights()
|
|
if model_dir is not None:
|
|
with (model_dir / "model").open("wb") as file_:
|
|
pickle.dump(weights[1:], file_)
|
|
with (model_dir / "config.json").open("w") as file_:
|
|
file_.write(lstm.to_json())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
plac.call(main)
|