У нас есть пользователь, чей скрипт всегда истощает узел.
Обратите внимание на эту ошибку: "gres/gpu: count changed for node node002 from 0 to 1" Может ли это вводить в заблуждение? Что может привести к истощению узла? Вот содержимое файла SBATCH пользователя. Может ли трубопровод иметь эффект здесь? Еще одна вещь, которую я заметил при наборе текста, это, возможно, попытка использовать сочетание версий библиотеки. Таким образом, у него есть module load cuda10.0, но затем module load pytorch-py36-cuda10.1-gcc/1.3.1 module load ml-pythondeps-py36-cuda10.1-gcc/3.0.0 python3.6
#!/bin/sh
#SBATCH -N 1
#SBATCH -n 1
#SBATCH --mail-type=ALL
#SBATCH --gres=gpu:1
#SBATCH --job-name=$1sequentialBlur_squeezenet_training_imagewoof_crossval
module purge
module load gcc5 cuda10.0
module load openmpi/cuda/64
module load pytorch-py36-cuda10.1-gcc/1.3.1
module load ml-pythondeps-py36-cuda10.1-gcc/3.0.0
python3.6 SequentialBlur_untrained.py squeezenet 100 imagewoof $1 | tee squeeze_100_imwoof_seq_longtrain_cv_$1.txt
/u/run_seq_blur2.py
Вот содержимое скрипта:
# Banks 1978 paper:
# 1 month: 2.4 cyc/deg
# 2 month: 2.8 cyc/deg
# 3 month: 4 cyc/deg
# 224 pixels:
# 20 deg -> 11 pix in deg; 4.6 pix blur; 4 pix blur; 2.8 pix blur
# 4 deg -> 56 pix in deg; 23 pix blur (1 mo); 20 pix blur (2 mo); 14 pix blur (3 mo)
import torch
import torchvision
import torchvision.transforms as transforms
from torchvision import models
import torchvision.datasets
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.pyplot as plt
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import os
import sys
import scipy
from torch.utils.data.sampler import SubsetRandomSampler
import h5py
args = sys.argv
modelType = args[1] # 'alexnet', 'squeezenet', 'vgg16'
numEpochs = args[2] # int
image_set = str(args[3]) # 'imagewoof', 'imagenette'
block_call = args[4] # int {0:4}
# Example call:
# python3 alexnet 100 imagenette 1
def get_train_valid_loader(data_dir,block,augment=0,random_seed=69420,valid_size=0.2,shuffle=False,
show_sample=False,num_workers=4, pin_memory=False, batch_size=128):
# valid_size gotta be in [0,1]
# block must be an int between 0:(1/valid_size) (0:4 for valid_size==0.2)
transform = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]
)])
train_dataset = torchvision.datasets.ImageFolder(root=data_dir,transform=transform)
valid_dataset = torchvision.datasets.ImageFolder(root=data_dir,transform=transform)
num_train = len(train_dataset)
indices = list(range(num_train))
split = int(np.floor(valid_size * num_train))
split1 = int(np.floor(block*split))
split2 = int(np.floor((block+1)*split))
# if shuffle:
np.random.seed(100)
np.random.shuffle(indices)
valid_idx = indices[split1:split2]
train_idx = np.append(indices[:split1],indices[split2:])
train_idx = train_idx.astype('int32')
if block != 0:
for b in range(block):
indices = [indices[(i + split) % len(indices)] for i, x in enumerate(indices)]
# train_idx, valid_idx = indices[split:], indices[:split]
train_sampler = SubsetRandomSampler(train_idx)
# train_sampler = torch.utils.data.Subset(dataset, indices)
valid_sampler = SubsetRandomSampler(valid_idx)
train_loader = torch.utils.data.DataLoader(
train_dataset, sampler=train_sampler, batch_size=batch_size,
num_workers=num_workers, pin_memory=pin_memory,
)
valid_loader = torch.utils.data.DataLoader(
valid_dataset, sampler=valid_sampler, batch_size=batch_size,
num_workers=num_workers, pin_memory=pin_memory,
)
return (train_loader, valid_loader)
transform = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]
)])
blurTypes = ['gaussian']
data_dir = "/path/to/dir/" + image_set + "-320_blur/"
classes = []
for directory, subdirectories, files in os.walk(data_dir):
for file in files:
if directory.split("\\")[-1] not in classes:
classes.append(directory.split("\\")[-1])
criterion = nn.CrossEntropyLoss()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
def train():
for epoch in range(int(numEpochs)):
prev_loss = 100000.0
running_loss = 0.0
for i, data in enumerate(trainloader, 0):
# get the inputs; data is a list of [inputs, labels]
inputs, labels = data
inputs = inputs.to(device)
labels = labels.to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
outputs = net(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
running_loss += loss.item()
if epoch % 10 == 9:
print('[%d, %5d] loss: %.3f' %
(epoch + 1, i + 1, running_loss / 100))
allAccs = []
for blurType in blurTypes: # multiple types of blur
print(blurType)
print('-' * 10)
# for block in range(5):
block = int(block_call)
print("\nFOLD " + str(block+1) + ":")
for i in range(5):
if i == 0:
blurLevels = [23, 11, 5, 3, 1]
elif i == 1:
blurLevels = [11, 5, 3, 1]
elif i == 2:
blurLevels = [5, 3, 1]
elif i == 3:
blurLevels = [3, 1]
elif i == 4:
blurLevels = [1]
if modelType == 'vgg16':
net = torchvision.models.vgg16(pretrained=False)
num_ftrs = net.classifier[6].in_features
net.classifier[6] = nn.Linear(num_ftrs, len(classes))
elif modelType == 'alexnet':
net = torchvision.models.alexnet(pretrained=False)
num_ftrs = net.classifier[6].in_features
net.classifier[6] = nn.Linear(num_ftrs, len(classes))
else:
net = torchvision.models.squeezenet1_1(pretrained=False)
net.classifier[1] = nn.Conv2d(512, len(classes), kernel_size=(1, 1), stride=(1, 1))
net.num_classes = len(classes)
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
net = net.to(device)
for i in range(len(blurLevels)): #5 levels of blur: 1, 3, 5, 11, 23
mult = blurLevels[i]
trainloader, validloader = get_train_valid_loader(data_dir=data_dir + blurType + '/' + image_set +
'-320_' + str(mult) + '/train',
block=block,shuffle=False,num_workers=0,batch_size=128)
print('Start training on blur window of ' + str(mult))
train()
print('Finished Training on ' + blurType + ' with blur window of ' + str(mult))
accs = []
permBlurLevels = [23, 11, 5, 3, 1]
for j in range(len(permBlurLevels)):
tempMult = permBlurLevels[j]
correct = 0
total = 0
# newTestSet = torchvision.datasets.ImageFolder(root=data_dir + blurType + '/' + image_set + '-320_' +
# str(tempMult) + '/val',
# transform=transform)
# newTestLoader = torch.utils.data.DataLoader(newTestSet, batch_size=128,
# shuffle=True, num_workers=0)
t2, validloader2 = get_train_valid_loader(data_dir=data_dir + blurType + '/' + image_set +
'-320_' + str(mult) + '/train',
block=block,shuffle=False,num_workers=0,batch_size=128)
with torch.no_grad():
for data in validloader2:
images, labels = data
images = images.to(device)
labels = labels.to(device)
outputs = net(images)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
acc = 100 * correct / total
print('Accuracy: %f %%' % (acc))
accs.append(acc)
allAccs.append(accs)
И вот ошибки, которые мы видим каждый раз, когда он запускает это:
[2020-03-13T08:54:02.269] gres/gpu: count changed for node node002 from 0
to 1
[2020-03-13T08:54:02.269] error: Setting node node002 state to DRAIN
[2020-03-13T08:54:02.269] drain_nodes: node node002 state set to DRAIN
[2020-03-13T08:54:02.269] error: _slurm_rpc_node_registration node=node002:
Invalid argument
Я мог найти только одну ссылку на ошибку SlurMD 2015 года, которая не была ошибкой , которая даже упоминает об этом gres/gpu: count changed for node ...