Потеря восприятия для последовательностей изображений в Керасе - PullRequest
Новая
       89

Потеря восприятия для последовательностей изображений в Керасе

1 голос
/ 30 января 2020

Я хочу реализовать потерю восприятия для последовательных данных изображения формы [batch_size, sequence_length, высота, ширина, каналы]

Предсказания моей модели также имеют ту же форму, что и входные данные. Моя проблема в том, что я не могу правильно передать свои прогнозы VGG16, чтобы рассчитать потери.

Во-первых, я строю свой vgg следующим образом: 

def build_vgg_model(self, weights="imagenet"):

    # Input image to extract features from
    img = Input(shape=(self.img_rows, self.img_cols, 3))

    # Mean center and rescale by variance as in PyTorch
    processed = Lambda(lambda x: (x - self.mean) / self.std)(img)

    # If inference only, just return empty model
    if self.inference_only:
        model = Model(inputs=img, outputs=[img for _ in range(len(self.vgg_layers))])
        model.trainable = False
        model.compile(loss='mse', optimizer='adam')
        return model

    # Get the vgg network from Keras applications
    if weights in ['imagenet', None]:
        vgg = VGG16(weights=weights, include_top=False)
        # model = Model(inputs=base_model.input, outputs=base_model.get_layer('block4_pool').output)
    else:
        vgg = VGG16(weights=None, include_top=False)
        vgg.load_weights(weights, by_name=True)

    # Output the first three pooling layers
    vgg.outputs = [vgg.layers[i].output for i in self.vgg_layers]

    # Create model and compile
    model = Model(inputs=img, outputs=vgg(processed))
    model.trainable = False

    return model

Затем я определяю свою функцию потерь: 

def total_loss(self, mask):
    def loss(y_true, y_pred):
        # Compute predicted image with non-hole pixels set to ground truth
        y_comp = mask * y_true + (1 - mask) * y_pred

        # Compute the vgg features.
        if self.vgg_device:
            with tf.device(self.vgg_device):
                vgg_out = self.vgg(y_pred)
                vgg_gt = self.vgg(y_true)
                vgg_comp = self.vgg(y_comp)
        else:
            vgg_out = self.vgg(y_pred)
            vgg_gt = self.vgg(y_true)
            vgg_comp = self.vgg(y_comp)

        # Compute loss components
        l1 = self.loss_valid(mask, y_true, y_pred)
        l2 = self.loss_hole(mask, y_true, y_pred)
        l3 = self.loss_perceptual(vgg_out, vgg_gt, vgg_comp)

        # Return loss function
        return l1 + 6 * l2 + 0.05 * l3
    return loss

def loss_perceptual(self, vgg_out, vgg_gt, vgg_comp):
    loss = 0
    for o, c, g in zip(vgg_out, vgg_comp, vgg_gt):
        loss += self.l1(o, g) + self.l1(c, g)
    return loss

Если я сейчас запустил свой код, я получаю следующую ошибку: 

Traceback (most recent call last):
  File "convlstm_main.py", line 51, in <module>
    model = ConvLSTM(64, 64, 3, 5)
  File "/mnt/workspace/lm78463/RecurrentDiFoRem/src/model/conv_lstm.py", line 40, in __init__
    self.model = self.compile()
  File "/mnt/workspace/lm78463/RecurrentDiFoRem/src/model/conv_lstm.py", line 116, in compile
    self.model.compile(loss=self.loss_total, optimizer='adadelta')
  File "/usr/local/lib/python3.6/dist-packages/keras/engine/training.py", line 229, in compile
    self.total_loss = self._prepare_total_loss(masks)
  File "/usr/local/lib/python3.6/dist-packages/keras/engine/training.py", line 692, in _prepare_total_loss
    y_true, y_pred, sample_weight=sample_weight)
  File "/usr/local/lib/python3.6/dist-packages/keras/losses.py", line 71, in __call__
    losses = self.call(y_true, y_pred)
  File "/usr/local/lib/python3.6/dist-packages/keras/losses.py", line 132, in call
    return self.fn(y_true, y_pred, **self._fn_kwargs)
  File "/mnt/workspace/lm78463/RecurrentDiFoRem/src/model/conv_lstm.py", line 180, in loss_total
    vgg_gts = self.vgg(y_trues)
  File "/usr/local/lib/python3.6/dist-packages/keras/engine/base_layer.py", line 489, in __call__
    output = self.call(inputs, **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/keras/engine/network.py", line 583, in call
    output_tensors, _, _ = self.run_internal_graph(inputs, masks)
  File "/usr/local/lib/python3.6/dist-packages/keras/engine/network.py", line 740, in run_internal_graph
    layer.call(computed_tensor, **kwargs))
  File "/usr/local/lib/python3.6/dist-packages/keras/engine/network.py", line 583, in call
    output_tensors, _, _ = self.run_internal_graph(inputs, masks)
  File "/usr/local/lib/python3.6/dist-packages/keras/engine/network.py", line 740, in run_internal_graph
    layer.call(computed_tensor, **kwargs))
  File "/usr/local/lib/python3.6/dist-packages/keras/layers/convolutional.py", line 171, in call
    dilation_rate=self.dilation_rate)
  File "/usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py", line 3717, in conv2d
    **kwargs)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/nn_ops.py", line 917, in convolution
    name=name)
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/nn_ops.py", line 979, in convolution_internal
    strides = _get_sequence(strides, n, channel_index, "strides")
  File "/usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/nn_ops.py", line 74, in _get_sequence
    name, n, n + 2, current_n))
ValueError: strides should be of length 1, 3 or 5 but was 2
...