Блокнот Google Colab с воспроизведением проблемы. Разверните, если какая-либо ячейка скрыта.
У меня следующая модель Keras:
Вчера я перешел на TensorFlow 2.2.0 с 2.1.0
from collections import Counter
def get_class_weights(y):
counter = Counter(y)
majority = max(counter.values())
return {cls: float(majority/count) for cls, count in counter.items()}
dict_test=get_class_weights(np.ravel(y_train, order='C'))
dict_test
"{0: 1.0, 1: 8.299129075480126}"
def create_fit_keras_model(hparams,
version_data_control,
optimizer_name,
validation_method,
callbacks,
optimizer_version = None):
sentenceLength_actors = X_train_seq_actors.shape[1] #17
vocab_size_frequent_words_actors = len(actors_tokenizer.word_index) #20000
sentenceLength_plot = X_train_seq_plot.shape[1] #20
vocab_size_frequent_words_plot = len(plot_tokenizer.word_index) #20000
sentenceLength_features = X_train_seq_features.shape[1] #43
vocab_size_frequent_words_features = len(features_tokenizer.word_index) #20000
sentenceLength_reviews = X_train_seq_reviews.shape[1] #257
vocab_size_frequent_words_reviews = len(reviews_tokenizer.word_index) #20000
model = keras.Sequential(name='{0}_{1}dim_{2}batchsize_{3}lr_{4}decaymultiplier_{5}'.format(sequential_model_name,
str(hparams[HP_EMBEDDING_DIM]),
str(hparams[HP_HIDDEN_UNITS]),
str(hparams[HP_LEARNING_RATE]),
str(hparams[HP_DECAY_STEPS_MULTIPLIER]),
version_data_control))
actors = keras.Input(shape=(sentenceLength_actors,), name='actors_input')
plot = keras.Input(shape=(sentenceLength_plot,), name='plot_input')
features = keras.Input(shape=(sentenceLength_features,), name='features_input')
reviews = keras.Input(shape=(sentenceLength_reviews,), name='reviews_input')
emb1 = layers.Embedding(input_dim = vocab_size_frequent_words_actors + 2,
output_dim = hparams[HP_EMBEDDING_DIM],
embeddings_initializer = 'uniform',
mask_zero = True,
input_length = sentenceLength_actors,
name="actors_embedding_layer")(actors)
encoded_layer1 = layers.GlobalMaxPooling1D(name="globalmaxpooling_actors_layer")(emb1)
emb2 = layers.Embedding(input_dim = vocab_size_frequent_words_plot + 2,
output_dim = hparams[HP_EMBEDDING_DIM],
embeddings_initializer = 'uniform',
mask_zero = True,
input_length = sentenceLength_plot,
name="plot_embedding_layer")(plot)
encoded_layer2 = layers.GlobalMaxPooling1D(name="globalmaxpooling_plot_summary_Layer")(emb2)
emb3 = layers.Embedding(input_dim = vocab_size_frequent_words_features + 2,
output_dim = hparams[HP_EMBEDDING_DIM],
embeddings_initializer = 'uniform',
mask_zero = True,
input_length = sentenceLength_features,
name="features_embedding_layer")(features)
encoded_layer3 = layers.GlobalMaxPooling1D(name="globalmaxpooling_movie_features_layer")(emb3)
emb4 = layers.Embedding(input_dim = vocab_size_frequent_words_reviews + 2,
output_dim = hparams[HP_EMBEDDING_DIM],
embeddings_initializer = 'uniform',
mask_zero = True,
input_length = sentenceLength_reviews,
name="reviews_embedding_layer")(reviews)
encoded_layer4 = layers.GlobalMaxPooling1D(name="globalmaxpooling_user_reviews_layer")(emb4)
merged = layers.concatenate([encoded_layer1, encoded_layer2, encoded_layer3, encoded_layer4], axis=-1)
dense_layer_1 = layers.Dense(hparams[HP_HIDDEN_UNITS],
kernel_regularizer=regularizers.l2(neural_network_parameters['l2_regularization']),
activation='relu',
name="1st_dense_hidden_layer_concatenated_inputs")(merged)
layers.Dropout(0.0)(dense_layer_1)
output_layer = layers.Dense(y_train.shape[1],
activation='sigmoid',
name='output_layer')(dense_layer_1)
model = keras.Model(inputs=[actors, plot, features, reviews], outputs=output_layer, name='{0}_{1}dim_{2}batchsize_{3}lr_{4}decaymultiplier_{5}'.format(sequential_model_name,
str(hparams[HP_EMBEDDING_DIM]),
str(hparams[HP_HIDDEN_UNITS]),
str(hparams[HP_LEARNING_RATE]),
str(hparams[HP_DECAY_STEPS_MULTIPLIER]),
version_data_control))
print(model.summary())
pruning_schedule = tfmot.sparsity.keras.PolynomialDecay(initial_sparsity=0.0,
final_sparsity=0.4,
begin_step=600,
end_step=1000)
model_for_pruning = tfmot.sparsity.keras.prune_low_magnitude(model, pruning_schedule=pruning_schedule)
if optimizer_name=="adam" and optimizer_version is None:
optimizer = optimizer_adam_v2(hparams[HP_LEARNING_RATE], hparams[HP_DECAY_STEPS_MULTIPLIER], X_train_seq_actors.shape[0], optimizer_parameters['validation_split_ratio'], hparams[HP_HIDDEN_UNITS])
elif optimizer_name=="sgd" and optimizer_version is None:
optimizer = optimizer_sgd_v1(hparams[HP_LEARNING_RATE])
elif optimizer_name=="rmsprop" and optimizer_version is None:
optimizer = optimizer_rmsprop_v1(hparams[HP_LEARNING_RATE])
model_for_pruning.compile(optimizer=optimizer,
loss=tfa.losses.sigmoid_focal_crossentropy,
metrics=[tfa.metrics.F1Score(len(y_train[0].tolist()), average="micro")]
#plot model's structure
plot_model(model, to_file=os.path.join(os.getcwd(), '{0}\\{1}_{2}batchsize_{3}lr_{4}decaymultiplier_{5}.png'.format(folder_path_model_saved,
network_structure_file_name,
str(hparams[HP_EMBEDDING_DIM]),
str(hparams[HP_HIDDEN_UNITS]),
str(hparams[HP_LEARNING_RATE]),
str(hparams[HP_DECAY_STEPS_MULTIPLIER]),
version_data_control)))
start_time = time.time()
steps_per_epoch=int(np.ceil((X_train_seq_actors.shape[0]*optimizer_parameters['validation_split_ratio'])//hparams[HP_HIDDEN_UNITS]))
print("\nSteps per epoch on current run: {0}".format(steps_per_epoch))
if validation_method=="validation_split":
fitted_model=model_for_pruning.fit([X_train_seq_actors, X_train_seq_plot, X_train_seq_features, X_train_seq_reviews],
y_train,
steps_per_epoch=int(np.ceil((X_train_seq_actors.shape[0]*optimizer_parameters['validation_split_ratio'])//hparams[HP_HIDDEN_UNITS])),
epochs=fit_parameters["epoch"],
batch_size=hparams[HP_HIDDEN_UNITS],
validation_split=fit_parameters['validation_data_ratio'],
callbacks=callbacks,
use_multiprocessing=True,
class_weight=dict_test
)
elif validation_method=="validation_data":
fitted_model=model_for_pruning.fit([X_train_seq_actors, X_train_seq_plot, X_train_seq_features, X_train_seq_reviews],
y_train,
# class_weight=class_weight_dict,
steps_per_epoch=int(np.ceil((X_train_seq_actors.shape[0]*optimizer_parameters['validation_split_ratio'])//hparams[HP_HIDDEN_UNITS])),
epochs=fit_parameters["epoch"],
verbose=fit_parameters["verbose_fit"],
batch_size=hparams[HP_HIDDEN_UNITS],
validation_data=([X_test_seq_actors, X_test_seq_plot, X_test_seq_features, X_test_seq_reviews],
y_test),
callbacks=callbacks
)
#save the model
save_model(model,
folder_path_model_saved,
"{0}_{1}dim_{2}batchsize_{3}lr_{4}decaymultiplier_{5}".format(saved_model_name,
str(hparams[HP_EMBEDDING_DIM]),
str(hparams[HP_HIDDEN_UNITS]),
str(hparams[HP_LEARNING_RATE]),
str(hparams[HP_DECAY_STEPS_MULTIPLIER]),
version_data_control))
elapsed_time = time.time() - start_time
print("\nTraining time of the multi-input keras model has finished. Duration {} secs".format(format_timespan(elapsed_time)))
_, accuracy = model_for_pruning.evaluate([X_test_seq_actors, X_test_seq_plot, X_test_seq_features, X_test_seq_reviews], y_test, batch_size=hparams[HP_HIDDEN_UNITS], verbose=2)
return accuracy, model_for_pruning, fitted_model
def run(run_dir, hparams, version_data_control, optimizer_name, validation_method, callbacks):
with tf.summary.create_file_writer(run_dir).as_default():
hp.hparams(hparams) # record the values used in this trial
accuracy, model, fitted_model = create_fit_keras_model(hparams, version_data_control, optimizer_name, validation_method, callbacks)
tf.summary.scalar(METRIC_ACCURACY, accuracy, step=1)
return model, fitted_model
"""
Model Training
"""
hp_logging_directory=os.path.join(os.getcwd(), "model_one\\logs\\hparam_tuning") #initialize the logging directory
HP_HIDDEN_UNITS = hp.HParam('batch_size', hp.Discrete([32, 64, 128]))
HP_EMBEDDING_DIM = hp.HParam('embedding_dim', hp.Discrete([100, 200, 300]))
HP_LEARNING_RATE = hp.HParam('learning_rate', hp.Discrete([0.001, 0.01, 0.1])) # Adam default: 0.001, SGD default: 0.01, RMSprop default: 0.001
HP_DECAY_STEPS_MULTIPLIER = hp.HParam('decay_steps_multiplier', hp.Discrete([100, 1000]))
METRIC_ACCURACY = "f1_score" #auc
with tf.summary.create_file_writer(hp_logging_directory).as_default():
hp.hparams_config(
hparams=[HP_HIDDEN_UNITS, HP_EMBEDDING_DIM, HP_LEARNING_RATE, HP_DECAY_STEPS_MULTIPLIER],
metrics=[hp.Metric(METRIC_ACCURACY, display_name='f1_score')], #AUC
)
try:
os.path.exists(hp_logging_directory)
print("Directory of hyper parameters logging exists!")
except Exception as e:
print(e)
print("Directory not found!")
begin_time=time.time()
print("{0}: Start execution of the cell\n".format(datetime.utcnow().strftime(date_format)))
session_num = 1
for batch_size in HP_HIDDEN_UNITS.domain.values:
for embedding_dim in HP_EMBEDDING_DIM.domain.values:
for learning_rate in HP_LEARNING_RATE.domain.values:
for decay_steps_multiplier in HP_DECAY_STEPS_MULTIPLIER.domain.values:
hparams = {
HP_HIDDEN_UNITS: batch_size,
HP_EMBEDDING_DIM: embedding_dim,
HP_LEARNING_RATE: learning_rate,
HP_DECAY_STEPS_MULTIPLIER: decay_steps_multiplier
}
run_name = "run-id {0}".format(session_num)
total_number_models=(len(HP_HIDDEN_UNITS.domain.values)*len(HP_EMBEDDING_DIM.domain.values)*len(HP_LEARNING_RATE.domain.values)*len(HP_DECAY_STEPS_MULTIPLIER.domain.values))
print('--- Starting trial: {0}/{1}\n'.format(run_name, total_number_models))
print({h.name: hparams[h] for h in hparams}, '\n')
starting_training=time.time()
model_struture, model_history=run('{0}\\'.format(hp_logging_directory) + run_name,
hparams,
version_data_control,
"adam",
"validation_split",
callback(folder_path_model_saved,
"{0}_{1}dim_{2}batchsize_{3}lr_{4}decaymultiplier_{5}".format(saved_model_name,
str(embedding_dim),
str(batch_size),
str(learning_rate),
str(decay_steps_multiplier),
version_data_control),
fit_parameters["patience_value"],
"{0}\\".format(hp_logging_directory) + datetime.now().strftime("%Y%m%d-%H%M%S"),
hparams))
print("Average time per epoch: {0}\n".format(format_timespan((time.time()-starting_training)/len(model_history.epoch))))
Как видите, я использую аргумент class_weigths, чтобы ввести веса положительного и отрицательного класса. Однако, когда я выполняю сценарий, я получаю следующий результат:
введите описание изображения здесь
Если дополнительная информация будет полезна, напишите в комментариях. Заранее спасибо.