Ошибка NaN при тестировании сети, в чем могут быть причины? - PullRequest
0 голосов
/ 13 мая 2019

Содержит реализацию распознавания сложных рукописных аннотаций с полностью сверточными сетями.

Во время обучения возникает ошибка, потеря NaN во время обучения.В чем могут быть причины?

Я пытался изменить функцию потерь на tanh и изменить гиперпараметры, но проблема все еще сохраняется

import torch

from PIL import Image
from matplotlib import pyplot as plt

import os
import numpy as np
from tqdm import trange, tqdm

import torch.optim as optim
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F

from torchvision import transforms, datasets
from torch.utils.data import Dataset, DataLoader

import random
import time

import math
import numbers

import torchfcn

import subprocess

def make_cv_folds(img_root, gt_root, num_folds, random_state):
    samples = []
    folds = [list() for i in range(num_folds)]
    lengths = [0]*num_folds
    img_root = os.path.expanduser(img_root)
    gt_root = os.path.expanduser(gt_root)
    for f in sorted(os.listdir(img_root)):
        if not os.path.isfile(os.path.join(img_root, f)) or not os.path.isfile(os.path.join(gt_root, f.rsplit(".", 1)[0] + ".png")):
            raise Exception("GT fehlt")
        samples.append((os.path.join(img_root, f), os.path.join(gt_root, f.rsplit(".", 1)[0] + ".png")))

    np.random.seed(random_state)
    np.random.shuffle(samples)
    np.random.seed()

    for s in samples:
        idx = np.argmin(lengths)
        folds[idx].append(s)
        lengths[idx] += 1
    return folds

def load_sample(img_path, gt_path):
    img = Image.open(img_path)
    gt = Image.open(gt_path)
    gt = np.array(gt)[:,:,2]

    #binary format
    gt[gt == 0] = 2
    gt[gt == 255] = 1

    #hisdb format
    gt[gt == 1] = 1
    gt[(gt%8) == 0] = 1
    gt[(gt%4) == 0] = 1
    gt[(gt%2) == 0] = 0
    gt = Image.fromarray(gt)
    return img, gt


class Annotations(Dataset):
    class_names = np.array(['other', 'annotation'])

    def __init__(self, img_root, gt_root, loader=load_sample, num_folds=5, preprocess=None, random_state=None):
        self.folds = make_cv_folds(img_root, gt_root, num_folds=num_folds, random_state=random_state)
        self.img_root = img_root
        self.num_folds = num_folds
        self.preprocess = preprocess
        self.loader = loader
        self.is_training = True
        self.load_split(num=0)

    def train(self, val):
        if val:
            self.is_training = True
            self.samples = self.train_samples
        else:
            self.is_training = False
            self.samples = self.test_samples

    def load_split(self, num=0):
        if len(self.folds) == 1:
            self.train_samples = self.folds[0]
            self.test_samples = self.folds[0]
        else:
            num = num%len(self.folds)
            train_folds = list(range(0,num)) + list(range(num+1, len(self.folds)))
            test_fold = num
            self.train_samples = []
            for i in train_folds:
                self.train_samples.extend(self.folds[i])
            self.test_samples = self.folds[num]
        if self.is_training:
            self.samples = self.train_samples
        else:
            self.samples = self.test_samples

    def untransform(self, img, gt):
        img = img.numpy()
        img = img.transpose(1, 2, 0)
        img = img.astype(np.uint8)
        img = img[:, :, ::-1]
        gt = gt.numpy()
        return img, gt


    def __getitem__(self, index):
        img_path, gt_path = self.samples[index]
        img, gt = self.loader(img_path, gt_path)
        if self.preprocess is not None:
            state = time.time()
            img = self.preprocess(img, random_state=state)
            gt = self.preprocess(gt, random_state=state)

        img = np.array(img, dtype=np.uint8)
        img = img[:, :, ::-1]  # RGB -> BGR
        img = img.astype(np.float64)
        img = img.transpose(2, 0, 1)
        img = torch.from_numpy(img).float()
        gt = np.array(gt, dtype=np.int32)
        gt = torch.from_numpy(gt).long()
        return img, gt

    def __len__(self):
        return len(self.samples)

    def __repr__(self):
        fmt_str = 'Dataset ' + self.__class__.__name__ + '\n'
        fmt_str += '    Number of datapoints: {}\n'.format(self.__len__())
        fmt_str += '    Number of training samples: {}\n'.format(len(self.train_samples))
        fmt_str += '    Number of testing samples: {}\n'.format(len(self.test_samples))
        return fmt_str

class CenterCrop(object):
    """Crops the given PIL Image at the center.
    Args:
        size (sequence or int): Desired output size of the crop. If size is an
            int instead of sequence like (h, w), a square crop (size, size) is
            made.
    """

    def __init__(self, size):
        if isinstance(size, numbers.Number):
            self.size = (int(size), int(size))
        else:
            self.size = size

    def __call__(self, img, random_state=None):
        """
        Args:
            img (PIL Image): Image to be cropped.
        Returns:
            PIL Image: Cropped image.
        """
        return transforms.functional.center_crop(img, self.size)

    def __repr__(self):
        return self.__class__.__name__ + '(size={0})'.format(self.size)


class RandomResizedCrop(object):
    """Crop the given PIL Image to random size and aspect ratio.
    A crop of random size (default: of 0.08 to 1.0) of the original size and a random
    aspect ratio (default: of 3/4 to 4/3) of the original aspect ratio is made. This crop
    is finally resized to given size.
    This is popularly used to train the Inception networks.
    Args:
        size: expected output size of each edge
        scale: range of size of the origin size cropped
        ratio: range of aspect ratio of the origin aspect ratio cropped
        interpolation: Default: PIL.Image.BILINEAR
    """

    def __init__(self, size, scale=(0.08, 1.0), ratio=(3. / 4., 4. / 3.), interpolation=Image.NEAREST):
        self.size = (size, size)
        self.interpolation = interpolation
        self.scale = scale
        self.ratio = ratio

    @staticmethod
    def get_params(img, scale, ratio, random_state=None):
        """Get parameters for ``crop`` for a random sized crop.
        Args:
            img (PIL Image): Image to be cropped.
            scale (tuple): range of size of the origin size cropped
            ratio (tuple): range of aspect ratio of the origin aspect ratio cropped
        Returns:
            tuple: params (i, j, h, w) to be passed to ``crop`` for a random
                sized crop.
        """


        random.seed(random_state)
        for attempt in range(10):
            area = img.size[0] * img.size[1]
            target_area = random.uniform(*scale) * area
            aspect_ratio = random.uniform(*ratio)

            w = int(round(math.sqrt(target_area * aspect_ratio)))
            h = int(round(math.sqrt(target_area / aspect_ratio)))

            if random.random() < 0.5:
                w, h = h, w

            if w <= img.size[0] and h <= img.size[1]:
                i = random.randint(0, img.size[1] - h)
                j = random.randint(0, img.size[0] - w)
                return i, j, h, w

        # Fallback
        w = min(img.size[0], img.size[1])
        i = (img.size[1] - w) // 2
        j = (img.size[0] - w) // 2
        return i, j, w, w

    def __call__(self, img, random_state=None):
        """
        Args:
            img (PIL Image): Image to be cropped and resized.
        Returns:
            PIL Image: Randomly cropped and resized image.
        """
        i, j, h, w = self.get_params(img, self.scale, self.ratio, random_state=random_state)
        return transforms.functional.resized_crop(img, i, j, h, w, self.size, self.interpolation)

    def __repr__(self):
        interpolate_str = _pil_interpolation_to_str[self.interpolation]
        format_string = self.__class__.__name__ + '(size={0}'.format(self.size)
        format_string += ', scale={0}'.format(round(self.scale, 4))
        format_string += ', ratio={0}'.format(round(self.ratio, 4))
        format_string += ', interpolation={0})'.format(interpolate_str)
        return format_string


class RandomCrop(object):
    """Crop the given PIL Image at a random location.
    Args:
        size (sequence or int): Desired output size of the crop. If size is an
            int instead of sequence like (h, w), a square crop (size, size) is
            made.
        padding (int or sequence, optional): Optional padding on each border
            of the image. Default is 0, i.e no padding. If a sequence of length
            4 is provided, it is used to pad left, top, right, bottom borders
            respectively.
    """

    def __init__(self, size, padding=0):
        if isinstance(size, numbers.Number):
            self.size = (int(size), int(size))
        else:
            self.size = size
        self.padding = padding

    @staticmethod
    def get_params(img, output_size, random_state=None):
        """Get parameters for ``crop`` for a random crop.
        Args:
            img (PIL Image): Image to be cropped.
            output_size (tuple): Expected output size of the crop.
        Returns:
            tuple: params (i, j, h, w) to be passed to ``crop`` for random crop.
        """
        random.seed(random_state)

        w, h = img.size
        th, tw = output_size
        if w == tw and h == th:
            return 0, 0, h, w

        i = random.randint(0, h - th)
        j = random.randint(0, w - tw)
        return i, j, th, tw

    def __call__(self, img, random_state=None):
        """
        Args:
            img (PIL Image): Image to be cropped.
        Returns:
            PIL Image: Cropped image.
        """
        if self.padding > 0:
            img = F.pad(img, self.padding)

        i, j, h, w = self.get_params(img, self.size, random_state=random_state)

        return transforms.functional.crop(img, i, j, h, w)

    def __repr__(self):
        return self.__class__.__name__ + '(size={0}, padding={1})'.format(self.size, self.padding)

preprocess_train = RandomResizedCrop(size=1024)
preprocess_test = RandomResizedCrop(size=1024)


trainset = Annotations(img_root='public_shared/input/',
                      gt_root='public_shared/new_bin',
                      preprocess=preprocess_train,
                      num_folds=1)

testset = Annotations(img_root='test/input/',
                      gt_root='test/new_bin/',
                      preprocess=preprocess_test,
                      num_folds=1)

testset.train(False)

train_loader = DataLoader(trainset, batch_size=1, shuffle=True, num_workers=8, drop_last=True)
test_loader = DataLoader(testset, batch_size=1, shuffle=False, num_workers=8, drop_last=True)


dat = trainset[0]
img = dat[0].numpy().transpose(1,2,0)
gt = dat[1].numpy()

print(img.shape)
print(gt.shape)

plt.imshow(img.squeeze())
plt.show()
plt.imshow(gt.squeeze())
plt.show()

log_dir = "log-mg3/"


def get_parameters(model, bias=False):
    import torch.nn as nn
    modules_skipped = (
        nn.ReLU,
        nn.MaxPool2d,
        nn.Dropout2d,
        nn.Sequential,
        torchfcn.models.FCN8s,
    )
    for m in model.modules():
        if isinstance(m, nn.Conv2d):
            if bias:
                yield m.bias
            else:
                yield m.weight
        elif isinstance(m, nn.ConvTranspose2d):
            # weight is frozen because it is just a bilinear upsampling
            if bias:
                assert m.bias is None
        elif isinstance(m, modules_skipped):
            continue
        else:
            raise ValueError('Unexpected module: %s' % str(m))



configurations = {
    # same configuration as original work
    # https://github.com/shelhamer/fcn.berkeleyvision.org
    1: dict(
        max_iteration=200000,
        lr=1.0e-10,
        momentum=0.99,
        weight_decay=0.0005,
        interval_validate=4000,
    )
}

cfg = configurations[1]
out = log_dir

cuda = torch.cuda.is_available()

torch.manual_seed(1337)
if cuda:
    torch.cuda.manual_seed(1337)

resume = ""

model = torchfcn.models.FCN8sAtOnce(n_class=2)
start_epoch = 0
start_iteration = 0
if resume:
    checkpoint = torch.load(resume)
    model.load_state_dict(checkpoint['model_state_dict'])
    start_epoch = checkpoint['epoch']
    start_iteration = checkpoint['iteration']
else:
    vgg16 = torchfcn.models.VGG16(pretrained=True)
    model.copy_params_from_vgg16(vgg16)
if cuda:
    model = model.cuda()

optimizer = torch.optim.SGD(
    [
        {'params': get_parameters(model, bias=False)},
        {'params': get_parameters(model, bias=True),
         'lr': cfg['lr'] * 2, 'weight_decay': 0},
    ],
    lr=cfg['lr'],
    momentum=cfg['momentum'],
    weight_decay=cfg['weight_decay'])
if resume:
    optimizer.load_state_dict(checkpoint['optim_state_dict'])

trainer = torchfcn.Trainer(
    cuda=cuda,
    model=model,
    optimizer=optimizer,
    train_loader=train_loader,
    val_loader=test_loader,
    out=out,
    max_iter=cfg['max_iteration'],
    interval_validate=cfg.get('interval_validate', len(train_loader)),
)
trainer.epoch = start_epoch
trainer.iteration = start_iteration
# trainer.train()

def evaluate_model(model, data_loader):
    model.eval()

    processes = []
    mius = []

    for index in tqdm(range(len(data_loader.dataset))):
        _, gt_path = data_loader.dataset.samples[index]
        image, _ = data_loader.dataset[index]
        image = image.numpy()
        image.shape = (1, image.shape[0], image.shape[1], image.shape[2])
        prediction = np.zeros((image.shape[2], image.shape[3], 3), dtype=np.uint8)
        div_arr = np.zeros((image.shape[2], image.shape[3]), dtype=np.uint8)

        offsets_vertical = list(range(0, image.shape[2], 256))
        offsets_horizontal = list(range(0, image.shape[3], 256))

        for v in offsets_vertical:
            for h in offsets_horizontal:
                data = image[:, :, v:v+1024, h:h+1024]
                data = torch.from_numpy(data)
                data = data.cuda()
                data = Variable(data, volatile=True)
                score = model(data)
                lbl_pred = score.data.max(1)[1].cpu().numpy()[:, :, :]
                lbl_pred[lbl_pred == 0] = 2
                prediction[v:v+1024, h:h+1024, 2] += lbl_pred.astype(np.uint8).squeeze()
                div_arr[v:v+1024, h:h+1024] += 1

        prediction[:,:,2] = np.round(prediction[:,:,2]/div_arr)

        im = Image.fromarray(prediction)
        prediction_path = os.path.join(log_dir, "prediction-private")
        if not os.path.isdir(prediction_path):
            os.makedirs(prediction_path)
        prediction_filename = os.path.join(prediction_path, os.path.basename(gt_path))
        im.save(prediction_filename)

        processes.append(subprocess.Popen(["java", "-jar", "DIVA_Layout_Analysis_Evaluator/out/artifacts/LayoutAnalysisEvaluator.jar", "-p", prediction_filename, "-gt", gt_path], stdout=subprocess.PIPE))

    for p in processes:
        miu = float(p.communicate()[0].splitlines()[0].split()[-1])
        mius.append(miu)

    print(mius)
    print("average:", np.mean(mius))
    return np.mean(mius)


testset = Annotations(img_root='test/input/',
                      gt_root='test/new_gt/',
                      preprocess=None,
                      num_folds=1)

testset.train(False)

test_loader = DataLoader(testset, batch_size=1, shuffle=False, num_workers=8, drop_last=True)

evaluate_model(model, test_loader)

Ошибка:

0%|          | 0/10 [00:00<?, ?it/s]/home/harsh/anaconda3/envs/vdlproject/lib/python3.6/site-packages/ipykernel_launcher.py:23: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.
100%|██████████| 10/10 [00:13<00:00,  1.28s/it]
[]
average: nan

/home/harsh/anaconda3/envs/vdlproject/lib/python3.6/site-packages/numpy/core/fromnumeric.py:2920: RuntimeWarning: Mean of empty slice.
  out=out, **kwargs)
/home/harsh/anaconda3/envs/vdlproject/lib/python3.6/site-packages/numpy/core/_methods.py:85: RuntimeWarning: invalid value encountered in double_scalars
  ret = ret.dtype.type(ret / rcount)

Вывод: средний: nan

Кто-нибудь может объяснить причину ошибки?

...