Невозможно использовать пользовательские генераторы в непоследовательной модели, Keras R

Question

Следующая непоследовательная модель формирует два входа: 1. категорическое происхождение 2. указанная текстовая строка пункта назначения в категориальное место назначения для образца рейсов судна.

Когда я запускаю модель без использования каких-либо генераторов, она запускаетсяпросто отлично, но когда я пытаюсь использовать пользовательские генераторы, я получаю сообщение об ошибке: «Ошибка в py_call_impl (callable, многоточие $ args, многоточие $ ключевые слова): ValueError: не удалось преобразовать входной массив из формы (64,55) в форму (64)«

Вот воспроизводимая версия модели, которая не использует генераторы и работает нормально:

library(keras)

#predict where a ship is going based on orgin and stated destination

maxlen <- 15

#get data
data <- read.delim("https://www.dropbox.com/s/py2n1464ou1a2of/example%20data.csv?dl=1",sep=",",header=T,colClasses=c("character","character","character"))

#turn countires into categories
country_cat <- data.frame(country=unique(c(data$begin_country,data$end_country)))
country_cat$cat <- 1:nrow(country_cat)

#format begin and end country to categorical
data$begin_country <- country_cat$cat[match(data$begin_country,country_cat$country)]
data$end_country <- country_cat$cat[match(data$end_country,country_cat$country)]

x.begin <- to_categorical(data$begin_country,nrow(country_cat)+1)
y <- to_categorical(data$end_country,nrow(country_cat)+1)

#format dest input data
    #tokenize by character
input <- as.array(data$dest)
tokenizer <- text_tokenizer(char_level = T)
tokenizer %>% fit_text_tokenizer(input)
embedded <- tokenizer$texts_to_sequences(input)
max_features <- max(unlist(embedded))
    #pad
x.dest <- pad_sequences(embedded,maxlen=maxlen)

#split into training and validation
tv <- rep(1,nrow(data))
tv[sample(nrow(data),nrow(data)*.3)] <- 0

x.begin.train <- x.begin[which(tv==1),]
x.begin.valid <- x.begin[which(tv==0),]

x.dest.train <- x.dest[which(tv==1),]
x.dest.valid <- x.dest[which(tv==0),]

y.train <- y[which(tv==1),]
y.valid <- y[which(tv==0),]

#build model
batch_size = 64

    #stated_dest
stated_dest_input <- layer_input(shape=maxlen,name="stated_dest_input")
stated_dest <- stated_dest_input %>%
    layer_embedding(input_dim = max_features, output_dim = 128) %>%
    layer_lstm(units = 128, dropout = 0.2, recurrent_dropout = 0.2) %>% 
    layer_dense(units = 64, activation = 'relu') %>%
    layer_dense(units = 32, activation = 'relu')

    #begin layer
begin_input <- layer_input(shape=c(nrow(country_cat)+1),name="begin_input")
begin <- begin_input %>%
    layer_dense(units = 32, activation = 'relu') %>%
    layer_dense(units = 32, activation = 'relu')

    #merging layer
merging <- layer_concatenate(c(begin,stated_dest)) %>%
     layer_dense(units = 128, activation = 'relu') %>%
     layer_dropout(rate = 0.2) %>%
     layer_dense(units = 128, activation = 'relu') %>%
     layer_dropout(rate = 0.2) %>%
     layer_dense(units = 64, activation = 'relu') %>%
     layer_dropout(rate = 0.2) %>%
     layer_dense(units = 64, activation = 'relu') %>%
     layer_dense(units= nrow(country_cat)+1, activation = "softmax")


model <- keras_model(list(stated_dest_input,begin_input),merging)

model %>% compile(
    optimizer = "rmsprop",
    loss = "categorical_crossentropy",
    metrics = c("accuracy"))

model %>% fit(
    list(x.dest.train,x.begin.train),
    y.train,
    epochs = 10,
    batch_size = batch_size,
    validation_data = list(list(x.dest.valid,x.begin.valid),y.valid)
)

Вот воспроизводимая версия модели, которая использует собственные генераторы и неработа:

library(keras)

#predict where a ship is going based on orgin and stated destination

maxlen <- 15

#get data
data <- read.delim("https://www.dropbox.com/s/py2n1464ou1a2of/example%20data.csv?dl=1",sep=",",header=T,colClasses=c("character","character","character"))

#turn countires into categories
country_cat <- data.frame(country=unique(c(data$begin_country,data$end_country)))
country_cat$cat <- 1:nrow(country_cat)

#format begin and end country to categorical
data$begin_country <- country_cat$cat[match(data$begin_country,country_cat$country)]
data$end_country <- country_cat$cat[match(data$end_country,country_cat$country)]

#split into training and validation
data$tv <- rep(1,nrow(data))
data$tv[sample(nrow(data),nrow(data)*.3)] <- 0

data.train <- data[which(data$tv==1),]
data.valid <- data[which(data$tv==0),]

#generators
valid_gen <- function(batch_size, data.valid){
    function(){
        #random sample
        data.sample <- data.valid[sample(nrow(data.valid),batch_size),]

        #x.begin
        x.begin <- to_categorical(data.sample$begin_country,nrow(country_cat)+1)

        #x.dest
        input <- as.array(data.sample$dest)
        tokenizer <- text_tokenizer(char_level = T)
        tokenizer %>% fit_text_tokenizer(input)
        embedded <- tokenizer$texts_to_sequences(input)
        max_features <- max(unlist(embedded))
            #pad
        x.dest <- pad_sequences(embedded,maxlen=maxlen)

        #y
        y <- to_categorical(data.sample$end_country,nrow(country_cat)+1)

        list(list(x.begin,x.dest),y)
    }
}

train_gen <- function(batch_size, data.train){
    function(){
        #random sample
        data.sample <- data.train[sample(nrow(data.train),batch_size),]

        #x.begin
        x.begin <- to_categorical(data.sample$begin_country,nrow(country_cat)+1)

        #x.dest
        input <- as.array(data.sample$dest)
        tokenizer <- text_tokenizer(char_level = T)
        tokenizer %>% fit_text_tokenizer(input)
        embedded <- tokenizer$texts_to_sequences(input)
        max_features <- max(unlist(embedded))
            #pad
        x.dest <- pad_sequences(embedded,maxlen=maxlen)

        #y
        y <- to_categorical(data.sample$end_country,nrow(country_cat)+1)

        list(list(x.begin,x.dest),y)
    }
}

#build model
batch_size = 64

    #stated_dest
stated_dest_input <- layer_input(shape=maxlen,name="stated_dest_input")
stated_dest <- stated_dest_input %>%
    layer_embedding(input_dim = max_features, output_dim = 128) %>%
    layer_lstm(units = 128, dropout = 0.2, recurrent_dropout = 0.2) %>% 
    layer_dense(units = 64, activation = 'relu') %>%
    layer_dense(units = 32, activation = 'relu')

    #begin layer
begin_input <- layer_input(shape=c(nrow(country_cat)+1),name="begin_input")
begin <- begin_input %>%
    layer_dense(units = 32, activation = 'relu') %>%
    layer_dense(units = 32, activation = 'relu')

    #merging layer
merging <- layer_concatenate(c(begin,stated_dest)) %>%
     layer_dense(units = 128, activation = 'relu') %>%
     layer_dropout(rate = 0.2) %>%
     layer_dense(units = 128, activation = 'relu') %>%
     layer_dropout(rate = 0.2) %>%
     layer_dense(units = 64, activation = 'relu') %>%
     layer_dropout(rate = 0.2) %>%
     layer_dense(units = 64, activation = 'relu') %>%
     layer_dense(units= nrow(country_cat)+1, activation = "softmax")


model <- keras_model(list(stated_dest_input,begin_input),merging)

model %>% compile(
    optimizer = "rmsprop",
    loss = "categorical_crossentropy",
    metrics = c("accuracy"))

model %>% fit_generator(
    train_gen(batch_size,data.train), 
    steps_per_epoch = nrow(data.train)/batch_size,
    epochs = 10,
    validation_data=valid_gen(batch_size,data.valid),
    validation_steps = nrow(data.valid)/batch_size
)