Я все еще новичок в TF и борюсь с некоторыми проблемами при первой попытке.
Поэтому я пытаюсь использовать набор данных from_generator с model.fit
Это мой генератор train_gen ()
for file_path in file_list[:8*len(file_list)//10]:
image = nib.load("/N/u/jp109/Carbonate/train3/" + file_path)
image_data = affine_transform(image.get_fdata(), image.affine)
image_data = normalize(image_data)
number = int(re.findall(r'\d+', file_path)[0])
label = "/N/u/jp109/Carbonate/train3/label" + str(number) + ".nii.gz"
label_data, affine = load_nifti(label)
label_data = affine_transform(label_data, affine)
cnt = 0
input_batch = []
label_batch = []
for i in range(32):
for j in range(32):
for k in range(32):
input_coord = [i*6, j*6, k*6]
label_coord = [i, j, k]
input_batch.append(tf.slice(image_data, input_coord, [65, 65, 65]))
label_batch.append(tf.slice(label_data, label_coord, [8, 8, 8]))
input_batch[cnt] = tf.expand_dims(input_batch[cnt], axis=-1)
label_batch[cnt] = tf.expand_dims(label_batch[cnt], axis=-1)
cnt += 1
if cnt == 4:
inputs = tf.stack(input_batch)
inputs.set_shape([4, 65, 65, 65, 1])
labels = tf.stack(label_batch)
labels.set_shape([4, 8, 8, 8, 1])
cnt = 0
input_batch = []
label_batch = []
yield((inputs, labels))
Вот как я делаю набор данных
train_dataset = tf.data.Dataset.from_generator(
train_gen,
(tf.float16, tf.float16)
)
Это модель
inputs = tf.keras.Input(shape=(65, 65, 65, 1), name='t1_image')
x = layers.Conv3D(16, (4, 4, 4), name='cnn_1')(inputs)
x = layers.Dropout(0.3)(x)
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU()(x)
x = layers.Conv3D(24, (3, 3, 3), name='cnn_2')(x)
x = layers.Dropout(0.3)(x)
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU()(x)
x = layers.MaxPooling3D((2, 2, 2), name='max_pool_1')(x)
x = layers.Conv3D(28, (3, 3, 3), name='cnn_3')(x)
x = layers.Dropout(0.3)(x)
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU()(x)
x = layers.MaxPooling3D((2, 2, 2), name='max_pool_2')(x)
x = layers.Conv3D(34, (4, 4, 4), name='cnn_4')(x)
x = layers.Dropout(0.3)(x)
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU()(x)
x = layers.Conv3D(2, (4, 4, 4), name='cnn_5')(x)
x = layers.Dropout(0.3)(x)
x = layers.BatchNormalization()(x)
x = layers.LeakyReLU()(x)
outputs = layers.Dense(1, activation='sigmoid', name='predictions')(x)
#print(outputs.shape)
model = tf.keras.Model(inputs=inputs, outputs=outputs)
model.compile(optimizer=tf.keras.optimizers.RMSprop(lr=2e-5),
loss=tf.keras.losses.KLDivergence(), metrics=['accuracy'])
Я пытаюсь соответствовать модель, использующая этот код
callback = tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=3)
model.fit(train_dataset, epochs=30, callbacks=[callback], validation_data=val_dataset)
model.evaluate(test_dataset)
Это ошибка, которую я получаю
WARNING:tensorflow:Early stopping conditioned on metric `val_loss` which is not available. Available metrics are:
Traceback (most recent call last):
File "new_seg.py", line 194, in <module>
model.fit(train_dataset, epochs=30, callbacks=[callback], validation_data=val_dataset)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training.py", line 728, in fit
use_multiprocessing=use_multiprocessing)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 324, in fit
total_epochs=epochs)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2.py", line 123, in run_one_epoch
batch_outs = execution_function(iterator)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 86, in execution_function
distributed_function(input_fn))
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py", line 457, in __call__
result = self._call(*args, **kwds)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py", line 503, in _call
self._initialize(args, kwds, add_initializers_to=initializer_map)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py", line 408, in _initialize
*args, **kwds))
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 1848, in _get_concrete_function_internal_garbage_collected
graph_function, _, _ = self._maybe_define_function(args, kwargs)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 2150, in _maybe_define_function
graph_function = self._create_graph_function(args, kwargs)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/eager/function.py", line 2041, in _create_graph_function
capture_by_value=self._capture_by_value),
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/framework/func_graph.py", line 915, in func_graph_from_py_func
func_outputs = python_func(*func_args, **func_kwargs)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/eager/def_function.py", line 358, in wrapped_fn
return weak_wrapped_fn().__wrapped__(*args, **kwds)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 66, in distributed_function
model, input_iterator, mode)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 112, in _prepare_feed_values
inputs, targets, sample_weights = _get_input_from_iterator(inputs)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/keras/engine/training_v2_utils.py", line 149, in _get_input_from_iterator
distribution_strategy_context.get_strategy(), x, y, sample_weights)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/keras/distribute/distributed_training_utils.py", line 308, in validate_distributed_dataset_inputs
x_values_list = validate_per_replica_inputs(distribution_strategy, x)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/keras/distribute/distributed_training_utils.py", line 356, in validate_per_replica_inputs
validate_all_tensor_shapes(x, x_values)
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/keras/distribute/distributed_training_utils.py", line 373, in validate_all_tensor_shapes
x_shape = x_values[0].shape.as_list()
File "/N/soft/rhel7/deeplearning/Python-3.7.6/lib/python3.7/site-packages/tensorflow_core/python/framework/tensor_shape.py", line 1171, in as_list
raise ValueError("as_list() is not defined on an unknown TensorShape.")
ValueError: as_list() is not defined on an unknown TensorShape.
Моя функция train_gen () должна давать ввод и помечать как тензоры формы (4, 65, 65, 65, 1) и (4, 8, 8, 8, 1), поэтому я не понимаю, почему Model.fit выдает эту ошибку.
У вас, ребята, есть предложения?
============= Я нашел похожую проблему, но не решил ее, но вы можете воспроизвести мою ошибку здесь
https://colab.research.google.com/gist/gowthamkpr/ca54b9920fba141a43b2ecfa87af9232/untitled154.ipynb