У меня очень большая модель с IMAGE_SIZE = [2, 5928, 4728, 1]
Структуру модели можно найти ниже. Мы используем Tesla K80 для обучения, поэтому у нас есть 12 ГБ видеопамяти. Мы используем CUDA 9.0, Tensorflow 1.6.0 в качестве бэкэнда для Keras 2.1.5.
Там всегда есть некоторые проблемы с памятью. Я согласился с batch_size=1, потому что иначе не получилось. Сначала я даже не мог выполнить fit() более чем на пару шагов. Сбой с ошибкой памяти. Затем я ограничил использование памяти до 0,9, и некоторое время он работал нормально. Прямо сейчас я пытаюсь загрузить веса из предыдущей тренировки и снова получаю ошибку памяти.
Есть ли способ как-то отладить это? Почему происходит сбой при загрузке весов? Я хотел бы получить представление о том, что делать в этой ситуации.
Заранее спасибо.
def make_model():
img_1_inp = Input(shape=IMAGE_SIZE)
img_2_inp = Input(shape=IMAGE_SIZE)
conv_1 = Conv2D(32, kernel_size=(3, 3),
strides=[2, 2],
activation=None)
img_1 = conv_1(img_1_inp)
img_2 = conv_1(img_2_inp)
batch_norm_1 = BatchNormalization()
img_1 = batch_norm_1(img_1)
img_2 = batch_norm_1(img_2)
activation_1 = Activation('relu')
img_1 = activation_1(img_1)
img_2 = activation_1(img_2)
pooling_1 = MaxPooling2D(pool_size=(3, 3))
img_1 = pooling_1(img_1)
img_2 = pooling_1(img_2)
conv_2_1 = Conv2D(64, kernel_size=(3, 3),
strides=[2, 2],
activation=None)
img_1 = conv_2_1(img_1)
img_2 = conv_2_1(img_2)
batch_norm_2_1 = BatchNormalization()
img_1 = batch_norm_2_1(img_1)
img_2 = batch_norm_2_1(img_2)
activation_2_1 = Activation('relu')
img_1 = activation_2_1(img_1)
img_2 = activation_2_1(img_2)
conv_2_2 = Conv2D(64, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_2_2(img_1)
img_2 = conv_2_2(img_2)
batch_norm_2_2 = BatchNormalization()
img_1 = batch_norm_2_2(img_1)
img_2 = batch_norm_2_2(img_2)
activation_2_2 = Activation('relu')
img_1 = activation_2_2(img_1)
img_2 = activation_2_2(img_2)
conv_2_3 = Conv2D(64, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_2_3(img_1)
img_2 = conv_2_3(img_2)
batch_norm_2_3 = BatchNormalization()
img_1 = batch_norm_2_3(img_1)
img_2 = batch_norm_2_3(img_2)
activation_2_3 = Activation('relu')
img_1 = activation_2_3(img_1)
img_2 = activation_2_3(img_2)
pooling_2 = MaxPooling2D(pool_size=(3, 3))
img_1 = pooling_2(img_1)
img_2 = pooling_2(img_2)
conv_3_1 = Conv2D(128, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_3_1(img_1)
img_2 = conv_3_1(img_2)
batch_norm_3_1 = BatchNormalization()
img_1 = batch_norm_3_1(img_1)
img_2 = batch_norm_3_1(img_2)
activation_3_1 = Activation('relu')
img_1 = activation_3_1(img_1)
img_2 = activation_3_1(img_2)
conv_3_2 = Conv2D(128, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_3_2(img_1)
img_2 = conv_3_2(img_2)
batch_norm_3_2 = BatchNormalization()
img_1 = batch_norm_3_2(img_1)
img_2 = batch_norm_3_2(img_2)
activation_3_2 = Activation('relu')
img_1 = activation_3_2(img_1)
img_2 = activation_3_2(img_2)
conv_3_3 = Conv2D(128, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_3_3(img_1)
img_2 = conv_3_3(img_2)
batch_norm_3_3 = BatchNormalization()
img_1 = batch_norm_3_3(img_1)
img_2 = batch_norm_3_3(img_2)
activation_3_3 = Activation('relu')
img_1 = activation_3_3(img_1)
img_2 = activation_3_3(img_2)
pooling_3 = MaxPooling2D(pool_size=(3, 3))
img_1 = pooling_3(img_1)
img_2 = pooling_3(img_2)
conv_4_1 = Conv2D(256, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_4_1(img_1)
img_2 = conv_4_1(img_2)
batch_norm_4_1 = BatchNormalization()
img_1 = batch_norm_4_1(img_1)
img_2 = batch_norm_4_1(img_2)
activation_4_1 = Activation('relu')
img_1 = activation_4_1(img_1)
img_2 = activation_4_1(img_2)
conv_4_2 = Conv2D(256, kernel_size=(3, 3),
strides=[1, 1],
activation=None)
img_1 = conv_4_2(img_1)
img_2 = conv_4_2(img_2)
batch_norm_4_2 = BatchNormalization()
img_1 = batch_norm_4_2(img_1)
img_2 = batch_norm_4_2(img_2)
activation_4_2 = Activation('relu')
img_1 = activation_4_2(img_1)
img_2 = activation_4_2(img_2)
conv_4_3 = Conv2D(256, kernel_size=(3, 3),
strides=[10, 10],
activation=None)
img_1 = conv_4_3(img_1)
img_2 = conv_4_3(img_2)
batch_norm_4_3 = BatchNormalization()
img_1 = batch_norm_4_3(img_1)
img_2 = batch_norm_4_3(img_2)
activation_4_3 = Activation('relu')
img_1 = activation_4_3(img_1)
img_2 = activation_4_3(img_2)
g_pooling = GlobalAveragePooling2D(data_format='channels_last')
img_1 = g_pooling(img_1)
img_2 = g_pooling(img_2)
# flat_1 = Flatten()
# flatten_1 = flat_1(img_1)
# flatten_2 = flat_1(img_2)
#
# merged = keras.layers.concatenate([flatten_1, flatten_2])
merged = keras.layers.concatenate([img_1, img_2])
dropout_1 = Dropout(0.2)
merged = dropout_1(merged)
dense_1 = (Dense(1024, activation='relu'))(merged)
dropout_2 = Dropout(0.2)
dense_1 = dropout_2(dense_1)
dense_2 = (Dense(num_classes, activation='softmax'))(dense_1)
model = Model(inputs=[img_1_inp, img_2_inp], output=dense_2)
return model