Мульти входная модель с потоком из каталога

Question

Я пытаюсь научиться использовать мультимодельный ввод с помощью flow_from_directory, но есть кое-что, что я не могу понять. спасибо за вашу помощь.

Насколько я понимаю, мы поставляем fit_generator с two_image_generator, а метод and и fit выводит метки .... чего мне не хватает ...

def two_image_generator(generator, 
                        directory, 
                        batch_size,
                        shuffle = False,
                        img_size1 = (224,224), 
                        img_size2 = (299,299)):

    gen1 = generator.flow_from_directory(
        # This is the target directory
        directory,
        # All images will be resized to target height and width.
        target_size=img_size1,
        batch_size=batch_size,
        # Since we use categorical_crossentropy loss, we need categorical labels
        class_mode='categorical',
        shuffle = shuffle,
        seed = 1)

    gen2 = generator.flow_from_directory(
        # This is the target directory
        directory,
        # All images will be resized to target height and width.
        target_size=img_size2,
        batch_size=batch_size,
        # Since we use categorical_crossentropy loss, we need categorical labels
        class_mode='categorical',
        shuffle = shuffle,
        seed = 1)    

    while True:
        X1i = gen1.next()
        X2i = gen2.next()
        if y_col:
            yield [X1i[0], X2i[0]], X1i[1]  #X1i[1] is the label
        else:
            yield [X1i, X2i]

#add data_augmentation
train_aug_datagen = ImageDataGenerator(
    rotation_range = 20,
    shear_range = 0.1,
    zoom_range = 0.2,
    width_shift_range = 0.1,
    height_shift_range = 0.1,
    horizontal_flip = True
)
train_generator = two_image_generator(train_aug_datagen, 
                                      train_dir,
                                      batch_size = batch_size,  
                                      shuffle = True)

validation_datagen = ImageDataGenerator()

validation_generator = two_image_generator(validation_datagen, 
                                      validation_dir,
                                      batch_size = batch_size,  
                                      shuffle = True)
def create_base_model(MODEL, img_size, lambda_fun = None):
    inp = Input(shape = (img_size[0], img_size[1], 3))
    x = inp
    if lambda_fun:
        x = Lambda(lambda_fun)(x)

    base_model = MODEL(input_tensor = x, weights = 'imagenet', 
                       include_top = False, pooling = 'avg')

    model = Model(inp, base_model.output)
    return model

#define vgg + resnet50 + densenet
model1 = create_base_model(vgg16.VGG16, (224, 224), vgg16.preprocess_input)
model2 = create_base_model(resnet50.ResNet50, (224, 224), resnet50.preprocess_input)
model3 = create_base_model(inception_v3.InceptionV3, (299, 299), inception_v3.preprocess_input)

model1.trainable = False
model2.trainable = False
model3.trainable = False

inpA = Input(shape = (224, 224, 3))
inpB = Input(shape = (299, 299, 3))

out1 = model1(inpA)
out2 = model2(inpA)
out3 = model3(inpB)

x = Concatenate()([out1, out2, out3])                
x = Dropout(0.2)(x)
x = Dense(2, activation='softmax')(x)

model = Model([inpA, inpB], x)

############################################################################
trained_models_path = './models/VggFace_best_model'
model_names = trained_models_path + '_epoch_{epoch:02d}_val_acc_{val_accuracy:.4f}.hdf5'
checkpoint = ModelCheckpoint(model_names, 'val_accuracy', verbose=1, save_best_only=True)
############################################################################

early = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1, mode='auto')

callbacks = [checkpoint,early]

history = model.fit_generator(train_generator,
                              steps_per_epoch= NUM_TRAIN //batch_size,
                              epochs=100,
                              validation_data=validation_generator,
                              validation_steps= NUM_TEST //batch_size,
                              verbose=1,
                              use_multiprocessing=True,
                              workers=14,
                              callbacks=callbacks ) 

NameError: имя 'y_col' не определено

Answer 1

#add data_augmentation
train_aug_datagen = ImageDataGenerator(
    rescale = 1./255,
    rotation_range = 20,
    shear_range = 0.1,
    zoom_range = 0.2,
    width_shift_range = 0.1,
    height_shift_range = 0.1,
    horizontal_flip = True
)

validation_datagen = ImageDataGenerator(rescale = 1./255)

def two_image_generator(generator, 
                        directory, 
                        batch_size,
                        shuffle = False,
                        img_size1 = (224,224), 
                        img_size2 = (299,299)):

    gen1 = generator.flow_from_directory(
        # This is the target directory
        directory,
        # All images will be resized to target height and width.
        target_size=img_size1,
        batch_size=batch_size,
        # Since we use categorical_crossentropy loss, we need categorical labels
        class_mode='categorical',
        shuffle = shuffle,
        seed = 7)

    gen2 = generator.flow_from_directory(
        # This is the target directory
        directory,
        # All images will be resized to target height and width.
        target_size=img_size2,
        batch_size=batch_size,
        # Since we use categorical_crossentropy loss, we need categorical labels
        class_mode='categorical',
        shuffle = shuffle,
        seed = 7)    

    while True:
        X1i = gen1.next()
        X2i = gen2.next()
        yield [X1i[0], X2i[0]], X2i[1]  #Yield both images and their mutual label    

train_generator = two_image_generator(train_aug_datagen, 
                                      train_dir,
                                      batch_size = batch_size,  
                                      shuffle = True)

validation_generator = two_image_generator(validation_datagen, 
                                      validation_dir,
                                      batch_size = batch_size,  
                                      shuffle = True)
############################################################################
trained_models_path = './models/VggFace_best_model'
model_names = trained_models_path + '_epoch_{epoch:02d}_val_acc_{val_accuracy:.4f}.hdf5'
checkpoint = ModelCheckpoint(model_names, 'val_accuracy', verbose=1, save_best_only=True)
############################################################################

early = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=1, mode='auto')

callbacks = [checkpoint,early]

history = model.fit_generator(train_generator,
                              steps_per_epoch= NUM_TRAIN //batch_size,
                              epochs=100,
                              validation_data=validation_generator,
                              validation_steps= NUM_TEST //batch_size,
                              verbose=1,
                              use_multiprocessing=True,
#                               workers=14,
                              callbacks=callbacks ) 
Epoch 1/100
Found 5000 images belonging to 2 classes.
Found 5000 images belonging to 2 classes.
Found 52700 images belonging to 2 classes.
Found 52700 images belonging to 2 classes.
340/625 [===============>..............] - ETA: 4:37 - loss: 7.7634 - acc: 0.4926
import pandas as pd