Новое в моделях TensorFlow и RNN.
Я тестирую модель RNN (LSTM) на основе анализа тональности ряда обзоров ImdB.
Код работает до тех пор, пока данные не будут вычислены для обеспечения точности и потеря:
RuntimeError: индекс вне допустимого диапазона
Код, как показано ниже:
import numpy as np
from collections import Counter
from string import punctuation
from sklearn.model_selection import train_test_split
import torch.nn as nn
#read text from .txt files
with open('reviews.txt', 'r') as f:
reviews = f.read()
with open('labels.txt', 'r') as f:
labels = f.read()
# REMOVE PUNCTUATION
from string import punctuation
reviews = reviews.lower()
all_text = ''.join([c for c in reviews if c not in punctuation])
# SPLIT BY NEW LINES AND SPACES
reviews_split = all_text.split('\n')
all_text = ' '.join(reviews_split)
#CREATE A LIST OF WORDS
words= all_text.split()
counts = Counter(words)
vocab = sorted(counts, key=counts.get, reverse=True)
vocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}
# USE THE NEW DICTIONARY TO TOKENIZE EACH REVIEW IN reviews_split
# STORE THE 'TOKENIZED' REVIEWS IN reviews_int
reviews_int =[]
for review in reviews_split:
reviews_int.append([vocab_to_int[word] for word in review.split()])
# stats about vocabulary
vocab_size = len(vocab_to_int)
# 1 = positive, 0 = negative {label conversion}
labels_split = labels.split('\n')
encoded_labels = np.array([1 if label == 'positive' else 0 for label in labels_split])
# OUTLIER REVIEW STATS
review_lens = Counter([len(x) for x in reviews_int])
## remove any reviews/labels with zero length from the reviews_int list.
# get indices of any reviews with length 0
non_zero_idx = [ii for ii, review in enumerate(reviews_int) if len(review) != 0]
# remove 0-length reviews and their labels
reviews_int = [reviews_int[ii] for ii in non_zero_idx]
encoded_labels = np.array([encoded_labels[ii] for ii in non_zero_idx])
def pad_features(reviews_ints, seq_length):
''' Return features of review_ints, where each review is padded with 0's
or truncated to the input seq_length.'''
# getting the correct rows x cols shape
features = np.zeros((len(reviews_ints), seq_length), dtype=int)
# for each review, I grab that review and
for i, row in enumerate(reviews_ints):
features[i, -len(row):] = np.array(row)[:seq_length]
return features
# Test your implementation!
seq_length = 200
features = pad_features(reviews_int, seq_length=seq_length)
## test statements - do not change - ##
assert len(features)==len(reviews_int), "Your features should have as many rows as reviews."
assert len(features[0])==seq_length, "Each feature row should contain seq_length values."
# Split train/test/dev
X_train, X_remainder, Y_train, Y_remainder = train_test_split(features, encoded_labels, test_size=0.2, random_state=7)
X_test, X_valid, Y_test, Y_valid = train_test_split(X_remainder, Y_remainder, test_size=0.5, random_state=7)
from torch.utils.data import TensorDataset, DataLoader
import torch
#CREATE TENSOR DATASETS
train_data = TensorDataset(torch.from_numpy(X_train), torch.from_numpy(Y_train))
valid_data = TensorDataset(torch.from_numpy(X_valid), torch.from_numpy(Y_valid))
test_data = TensorDataset(torch.from_numpy(X_test), torch.from_numpy(Y_test))
# DATALOADERS
batch_size = 50
# SHUFFLE DATA
train_loader = DataLoader(train_data, shuffle=True,batch_size=batch_size)
valid_loader = DataLoader(valid_data, shuffle=True,batch_size=batch_size)
test_loader = DataLoader(test_data, shuffle=True,batch_size=batch_size)
# OBTAIN BATCH OF TRAINING DATA
detailer = iter(train_loader)
sample_x, sample_y = detailer.next()
#RNN MODEL
class SentimentRNN(nn.Module):
"""
The RNN model that will be used to perform Sentiment analysis.
"""
def __init__(self, vocab_size, output_size, embedding_dim, hidden_dim, n_layers, drop_prob=0.5):
"""
Initialize the model by setting up the layers.
"""
super(SentimentRNN, self).__init__()
self.output_size = output_size
self.n_layers = n_layers
self.hidden_dim = hidden_dim
# embedding and LSTM layers
self.embedding = nn.Embedding(vocab_size, embedding_dim)
self.lstm = nn.LSTM(embedding_dim, hidden_dim, n_layers, dropout=drop_prob, batch_first=True)
# dropout layer
self.dropout = nn.Dropout(0.3)
# linear and sigmoid layers
self.fc = nn.Linear(hidden_dim, output_size)
self.sig = nn.Sigmoid()
def forward(self, x, hidden):
"""
Perform a forward pass of our model on some input and hidden state.
"""
batch_size = x.size(0)
# embeddings and lstm_out
x = x.long()
embeds = self.embedding(x)
lstm_out, hidden = self.lstm(embeds, hidden)
# stack up lstm outputs
lstm_out = lstm_out.contiguous().view(-1, self.hidden_dim)
# dropout and fully-connected layer
out = self.dropout(lstm_out)
out = self.fc(out)
# sigmoid function
sig_out = self.sig(out)
# reshape to be batch_size first
sig_out = sig_out.view(batch_size, -1)
sig_out = sig_out[:, -1] # get last batch of labels
# return last sigmoid output and hidden state
return sig_out, hidden
def init_hidden(self, batch_size):
''' Initializes hidden state '''
# Create two new tensors with sizes n_layers x batch_size x hidden_dim,
# initialized to zero, for hidden state and cell state of LSTM
weight = next(self.parameters()).data
if (train_on_gpu):
hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda(),
weight.new(self.n_layers, batch_size, self.hidden_dim).zero_().cuda())
else:
hidden = (weight.new(self.n_layers, batch_size, self.hidden_dim).zero_(),
weight.new(self.n_layers, batch_size, self.hidden_dim).zero_())
return hidden
net = SentimentRNN(vocab_size, 2, 200, 256, 3)
# loss and optimization functions
lr=0.001
criterion = nn.BCELoss()
optimizer = torch.optim.Adam(net.parameters(), lr=lr)
# training params
epochs = 4 # 3-4 is approx where I noticed the validation loss stop decreasing
counter = 0
print_every = 100
clip=5 # gradient clipping
# move model to GPU, if available
if(train_on_gpu):
net.cuda()
net.train()
# train for some number of epochs
for e in range(epochs):
# initialize hidden state
h = net.init_hidden(batch_size)
# batch loop
for inputs, labels in train_loader:
counter += 1
if(train_on_gpu):
inputs, labels = inputs.cuda(), labels.cuda()
# Creating new variables for the hidden state, otherwise
# we'd backprop through the entire training history
h = tuple([each.data for each in h])
# zero accumulated gradients
net.zero_grad()
# get the output from the model
output, h = net(inputs, h)
# calculate the loss and perform backprop
loss = criterion(output.squeeze(), labels.float())
loss.backward()
# `clip_grad_norm` helps prevent the exploding gradient problem in RNNs / LSTMs.
nn.utils.clip_grad_norm_(net.parameters(), clip)
optimizer.step()
# loss stats
if counter % print_every == 0:
# Get validation loss
val_h = net.init_hidden(batch_size)
val_losses = []
net.eval()
for inputs, labels in valid_loader:
# Creating new variables for the hidden state, otherwise
# we'd backprop through the entire training history
val_h = tuple([each.data for each in val_h])
if(train_on_gpu):
inputs, labels = inputs.cuda(), labels.cuda()
output, val_h = net(inputs, val_h)
val_loss = criterion(output.squeeze(), labels.float())
val_losses.append(val_loss.item())
net.train()
print("Epoch: {}/{}...".format(e+1, epochs),
"Step: {}...".format(counter),
"Loss: {:.6f}...".format(loss.item()),
"Val Loss: {:.6f}".format(np.mean(val_losses)))
НРАВИТСЯ ЧАРМ ДО ЗДЕСЬ: введите описание изображения здесь
Выполнение приведенного ниже кода для проверки точности и потери данных, ожидаемый результат:
"Потеря теста: 0,537 .... Точность теста: 0,811 ..."
получение:
RuntimeError: index out of range: Пытался получить доступ к индексу 74072 вне таблицы с 74071 строками
# Get test data loss and accuracy
test_losses = [] # track loss
num_correct = 0
# init hidden state
h = net.init_hidden(batch_size)
net.eval()
# iterate over test data
for inputs, labels in test_loader:
# Creating new variables for the hidden state, otherwise
# we'd backprop through the entire training history
h = tuple([each.data for each in h])
# get predicted outputs
output, h = net(inputs, h)
# calculate loss
test_loss = criterion(output.squeeze(), labels.float())
test_losses.append(test_loss.item())
# convert output probabilities to predicted class (0 or 1)
pred = torch.round(output.squeeze()) # rounds to the nearest integer
# compare predictions to true label
correct_tensor = pred.eq(labels.float().view_as(pred))
correct = np.squeeze(correct_tensor.numpy()) if not train_on_gpu else np.squeeze(correct_tensor.cpu().numpy())
num_correct += np.sum(correct)
# -- stats! -- ##
# avg test loss
print("Test loss: {:.3f}".format(np.mean(test_losses)))
# accuracy over all test data
test_acc = num_correct/len(test_loader.dataset)
print("Test accuracy: {:.3f}".format(test_acc))
Это просто проблема с индексированием во время работы, но я исчерпал свои творческие возможности.
Любая помощь будет оценена.