Я следую этому уроку PyTorch Джошуа Л. Митчелла. Главным финалом урока является следующий учебный скрипт PyTorch. Один элемент, размер пакета, я настроил в первой строке скрипта, который я запускаю во вновь запущенной записной книжке Jupyter. Ключевым параметром, о котором идет речь, является BIGGER_BATCH, изначально установленный в 4:
BIGGER_BATCH=4
import numpy as np
import torch # Tensor Package (for use on GPU)
import torch.nn as nn ## Neural Network package
import torch.optim as optim # Optimization package
import torchvision # for dealing with vision data
import torchvision.transforms as transforms # for modifying vision data to run it through models
from torch.autograd import Variable # for computational graph
import torch.nn.functional as F # Non-linearities package
import matplotlib.pyplot as plt # for plotting
def imshow(img):
img = img / 2 + 0.5 # unnormalize
npimg = img.numpy()
plt.imshow(np.transpose(npimg, (1, 2, 0)))
transform = transforms.Compose( # we're going to use this to transform our data to make each sample more uniform
[
transforms.ToTensor(), # converts each sample from a (0-255, 0-255, 0-255) PIL Image format to a (0-1, 0-1, 0-1) FloatTensor format
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) # for each of the 3 channels of the image, subtract mean 0.5 and divide by stdev 0.5
]) # the normalization makes each SGD iteration more stable and overall makes convergence easier
trainset = torchvision.datasets.CIFAR10(root='./data', train=True,
download=True, transform=transform) # this is all we need to get/wrangle the dataset!
testset = torchvision.datasets.CIFAR10(root='./data', train=False,
download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=BIGGER_BATCH,
shuffle=False)
testloader = torch.utils.data.DataLoader(testset, batch_size=BIGGER_BATCH,
shuffle=False)
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') # each image can have 1 of 10 labels
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.conv1 = nn.Conv2d(3, 10, 5) # Let's add more feature maps - that might help
self.pool = nn.MaxPool2d(2, 2)
self.conv2 = nn.Conv2d(10, 20, 5) # And another conv layer with even more feature maps
self.fc1 = nn.Linear(20 * 5 * 5, 120) # and finally, adjusting our first linear layer's input to our previous output
self.fc2 = nn.Linear(120, 10)
def forward(self, x):
x = self.conv1(x)
x = F.relu(x) # we're changing our nonlinearity / activation function from sigmoid to ReLU for a slight speedup
x = self.pool(x)
x = self.conv2(x)
x = F.relu(x)
x = self.pool(x) # after this pooling layer, we're down to a torch.Size([4, 20, 5, 5]) tensor.
x = x.view(-1, 20 * 5 * 5) # so let's adjust our tensor again.
x = self.fc1(x)
x = F.relu(x)
x = self.fc2(x)
x = F.relu(x)
return x
net = Net().cuda()
NUMBER_OF_EPOCHS = 25
LEARNING_RATE = 1e-2
loss_function = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=LEARNING_RATE)
for epoch in range(NUMBER_OF_EPOCHS):
train_loader_iter = iter(trainloader)
for batch_idx, (inputs, labels) in enumerate(train_loader_iter):
net.zero_grad()
inputs, labels = Variable(inputs.float().cuda()), Variable(labels.cuda())
output = net(inputs)
loss = loss_function(output, labels)
loss.backward()
optimizer.step()
if epoch % 5 is 0:
print("Iteration: " + str(epoch + 1))
dataiter = iter(testloader)
images, labels = dataiter.next()
imshow(torchvision.utils.make_grid(images[0:4]))
outputs = net(Variable(images.cuda()))
_, predicted = torch.max(outputs.data, 1)
print('Predicted: ', ' '.join('%5s' % classes[predicted[j]]
for j in range(4)))
print('GroundTruth: ', ' '.join('%5s' % classes[labels[j]] for j in range(4)))
correct = 0
total = 0
for data in testloader:
images, labels = data
labels = labels.cuda()
outputs = net(Variable(images.cuda()))
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum()
print('Accuracy of the network on the 10000 test images: %d %%' % (
100 * correct / total))
Это дает хороший, ожидаемый результат с точностью 58%:
Predicted: cat ship ship plane
GroundTruth: cat ship ship plane
Accuracy of the network on the 10000 test images: 58 %
Теперь, если я изменю первую строку вышеприведенного скрипта на
BIGGER_BATCH=4096
и я перезагружаю ядро и запускаю скрипт, получаю 19% точности, последовательно:
Predicted: car ship ship ship
GroundTruth: cat ship ship plane
Accuracy of the network on the 10000 test images: 19 %
Обратите внимание, что я не тасую вводы, поэтому я не могу связать это изменение с порядком входов в обучении:
trainloader = torch.utils.data.DataLoader(trainset, batch_size=BIGGER_BATCH,
shuffle=False)
testloader = torch.utils.data.DataLoader(testset, batch_size=BIGGER_BATCH,
shuffle=False)
Чем объясняется огромное снижение точности при увеличении размера партии? Что-то не так в скрипте, или что-то не так в PyTorch, или что-то еще, о чем я не думаю?