'''WIP --- Doesn't work well yet''' import plac import random import pathlib import cPickle as pickle from itertools import izip import spacy import cytoolz import numpy as np import chainer.links as L import chainer.functions as F from chainer import Chain, Variable, report import chainer.training import chainer.optimizers from chainer.training import extensions from chainer.iterators import SerialIterator from chainer.datasets.tuple_dataset import TupleDataset class SentimentAnalyser(object): @classmethod def load(cls, path, nlp, max_length=100): raise NotImplementedError #with (path / 'config.json').open() as file_: # model = model_from_json(file_.read()) #with (path / 'model').open('rb') as file_: # lstm_weights = pickle.load(file_) #embeddings = get_embeddings(nlp.vocab) #model.set_weights([embeddings] + lstm_weights) #return cls(model, max_length=max_length) def __init__(self, model, max_length=100): self._model = model self.max_length = max_length def __call__(self, doc): X = get_features([doc], self.max_length) y = self._model.predict(X) self.set_sentiment(doc, y) def pipe(self, docs, batch_size=1000, n_threads=2): for minibatch in cytoolz.partition_all(batch_size, docs): minibatch = list(minibatch) sentences = [] for doc in minibatch: sentences.extend(doc.sents) Xs = get_features(sentences, self.max_length) ys = self._model.predict(Xs) for sent, label in zip(sentences, ys): sent.doc.sentiment += label - 0.5 for doc in minibatch: yield doc def set_sentiment(self, doc, y): doc.sentiment = float(y[0]) # Sentiment has a native slot for a single float. # For arbitrary data storage, there's: # doc.user_data['my_data'] = y class Classifier(Chain): def __init__(self, predictor): super(Classifier, self).__init__(predictor=predictor) def __call__(self, x, t): y = self.predictor(x) loss = F.softmax_cross_entropy(y, t) accuracy = F.accuracy(y, t) report({'loss': loss, 'accuracy': accuracy}, self) return loss class SentimentModel(Chain): def __init__(self, shape, **settings): Chain.__init__(self, embed=_Embed(shape['nr_vector'], shape['nr_dim'], shape['nr_hidden']), encode=_Encode(shape['nr_hidden'], shape['nr_hidden']), attend=_Attend(shape['nr_hidden'], shape['nr_hidden']), predict=_Predict(shape['nr_hidden'], shape['nr_class'])) def __call__(self, sentence): return self.predict( self.attend( self.encode( self.embed(sentence)))) class _Embed(Chain): def __init__(self, nr_vector, nr_dim, nr_out): Chain.__init__(self, embed=L.EmbedID(nr_vector, nr_dim), project=L.Linear(None, nr_out, nobias=True)) #self.embed.unchain_backward() def __call__(self, sentence): return [self.project(self.embed(ts)) for ts in F.transpose(sentence)] class _Encode(Chain): def __init__(self, nr_in, nr_out): Chain.__init__(self, fwd=L.LSTM(nr_in, nr_out), bwd=L.LSTM(nr_in, nr_out), mix=L.Bilinear(nr_out, nr_out, nr_out)) def __call__(self, sentence): self.fwd.reset_state() fwds = map(self.fwd, sentence) self.bwd.reset_state() bwds = reversed(map(self.bwd, reversed(sentence))) return [F.elu(self.mix(f, b)) for f, b in zip(fwds, bwds)] class _Attend(Chain): def __init__(self, nr_in, nr_out): Chain.__init__(self) def __call__(self, sentence): sent = sum(sentence) return sent class _Predict(Chain): def __init__(self, nr_in, nr_out): Chain.__init__(self, l1=L.Linear(nr_in, nr_in), l2=L.Linear(nr_in, nr_out)) def __call__(self, vector): vector = self.l1(vector) vector = F.elu(vector) vector = self.l2(vector) return vector class SentenceDataset(TupleDataset): def __init__(self, nlp, texts, labels, max_length): self.max_length = max_length sents, labels = self._get_labelled_sentences( nlp.pipe(texts, batch_size=5000, n_threads=3), labels) TupleDataset.__init__(self, get_features(sents, max_length), labels) def _get_labelled_sentences(self, docs, doc_labels): labels = [] sentences = [] for doc, y in izip(docs, doc_labels): for sent in doc.sents: sentences.append(sent) labels.append(y) return sentences, labels class DocDataset(TupleDataset): def __init__(self, nlp, texts, labels): self.max_length = max_length TupleDataset.__init__(self, get_features( nlp.pipe(texts, batch_size=5000, n_threads=3), self.max_length), labels) def read_data(data_dir, limit=0): examples = [] for subdir, label in (('pos', 1), ('neg', 0)): for filename in (data_dir / subdir).iterdir(): with filename.open() as file_: text = file_.read() examples.append((text, label)) random.shuffle(examples) if limit >= 1: examples = examples[:limit] return zip(*examples) # Unzips into two lists def get_features(docs, max_length): docs = list(docs) Xs = np.zeros((len(docs), max_length), dtype='int32') for i, doc in enumerate(docs): j = 0 for token in doc: if token.has_vector and not token.is_punct and not token.is_space: Xs[i, j] = token.norm j += 1 if j >= max_length: break return Xs def get_embeddings(vocab, max_rank=1000): if max_rank is None: max_rank = max(lex.rank+1 for lex in vocab if lex.has_vector) vectors = np.ndarray((max_rank+1, vocab.vectors_length), dtype='float32') for lex in vocab: if lex.has_vector and lex.rank < max_rank: lex.norm = lex.rank+1 vectors[lex.rank + 1] = lex.vector else: lex.norm = 0 return vectors def train(train_texts, train_labels, dev_texts, dev_labels, lstm_shape, lstm_settings, lstm_optimizer, batch_size=100, nb_epoch=5, by_sentence=True): print("Loading spaCy") nlp = spacy.load('en', entity=False) for lex in nlp.vocab: if lex.rank >= (lstm_shape['nr_vector'] - 1): lex.norm = 0 else: lex.norm = lex.rank+1 #print("Get embeddings") #embeddings = get_embeddings(nlp.vocab) print("Make model") model = Classifier(SentimentModel(lstm_shape, **lstm_settings)) print("Parsing texts...") if by_sentence: train_data = SentenceDataset(nlp, train_texts, train_labels, lstm_shape['max_length']) dev_data = SentenceDataset(nlp, dev_texts, dev_labels, lstm_shape['max_length']) else: train_data = DocDataset(nlp, train_texts, train_labels) dev_data = DocDataset(nlp, dev_texts, dev_labels) train_iter = SerialIterator(train_data, batch_size=batch_size, shuffle=True, repeat=True) dev_iter = SerialIterator(dev_data, batch_size=batch_size, shuffle=False, repeat=False) optimizer = chainer.optimizers.Adam() optimizer.setup(model) updater = chainer.training.StandardUpdater(train_iter, optimizer) trainer = chainer.training.Trainer(updater, (20, 'epoch'), out='result') trainer.extend(extensions.Evaluator(dev_iter, model)) trainer.extend(extensions.LogReport()) trainer.extend(extensions.PrintReport([ 'epoch', 'main/accuracy', 'validation/main/accuracy'])) trainer.extend(extensions.ProgressBar()) trainer.run() def evaluate(model_dir, texts, labels, max_length=100): def create_pipeline(nlp): ''' This could be a lambda, but named functions are easier to read in Python. ''' return [nlp.tagger, nlp.parser, SentimentAnalyser.load(model_dir, nlp, max_length=max_length)] nlp = spacy.load('en') nlp.pipeline = create_pipeline(nlp) correct = 0 i = 0 for doc in nlp.pipe(texts, batch_size=1000, n_threads=4): correct += bool(doc.sentiment >= 0.5) == bool(labels[i]) i += 1 return float(correct) / i @plac.annotations( train_dir=("Location of training file or directory"), dev_dir=("Location of development file or directory"), model_dir=("Location of output model directory",), is_runtime=("Demonstrate run-time usage", "flag", "r", bool), nr_hidden=("Number of hidden units", "option", "H", int), max_length=("Maximum sentence length", "option", "L", int), dropout=("Dropout", "option", "d", float), learn_rate=("Learn rate", "option", "e", float), nb_epoch=("Number of training epochs", "option", "i", int), batch_size=("Size of minibatches for training LSTM", "option", "b", int), nr_examples=("Limit to N examples", "option", "n", int) ) def main(model_dir, train_dir, dev_dir, is_runtime=False, nr_hidden=64, max_length=100, # Shape dropout=0.5, learn_rate=0.001, # General NN config nb_epoch=5, batch_size=32, nr_examples=-1): # Training params model_dir = pathlib.Path(model_dir) train_dir = pathlib.Path(train_dir) dev_dir = pathlib.Path(dev_dir) if is_runtime: dev_texts, dev_labels = read_data(dev_dir) acc = evaluate(model_dir, dev_texts, dev_labels, max_length=max_length) print(acc) else: print("Read data") train_texts, train_labels = read_data(train_dir, limit=nr_examples) dev_texts, dev_labels = read_data(dev_dir, limit=nr_examples) train_labels = np.asarray(train_labels, dtype='int32') dev_labels = np.asarray(dev_labels, dtype='int32') lstm = train(train_texts, train_labels, dev_texts, dev_labels, {'nr_hidden': nr_hidden, 'max_length': max_length, 'nr_class': 2, 'nr_vector': 2000, 'nr_dim': 32}, {'dropout': 0.5, 'lr': learn_rate}, {}, nb_epoch=nb_epoch, batch_size=batch_size) if __name__ == '__main__': plac.call(main)