Позвольте мне предисловие к этому, сказав, что я просмотрел похожие вопросы, но ничего не работает; в основном, когда я вызываю tf.data.Dataset.map и iterator.get_next (), он работает совершенно нормально (вот где у других, похоже, есть проблема), но когда я пытаюсь передать пакеты во время процесса обучения, я получаю ожидаемое изображение ошибка. Как мне это исправить, пожалуйста? Я искал решение более двух дней.
Вот полный код:
!wget --no-check-certificate \
"https://download.microsoft.com/download/3/E/1/3E1C3F21-ECDB-4869-8368-6DEBA77B919F/kagglecatsanddogs_3367a.zip" \
-O "/tmp/cats-and-dogs.zip"
local_zip = '/tmp/cats-and-dogs.zip'
zip_ref = zipfile.ZipFile(local_zip, 'r')
zip_ref.extractall('/tmp')
zip_ref.close()
try:
os.mkdir('/tmp/cats-v-dogs')
os.mkdir('/tmp/cats-v-dogs/training')
os.mkdir('/tmp/cats-v-dogs/testing')
except OSError:
pass
from shutil import copyfile
import numpy as np
def split_data(SOURCE, TRAINING, TESTING, SPLIT_SIZE):
files = []
for filename in os.listdir(SOURCE):
file = SOURCE + filename
if os.path.getsize(file) > 0:
files.append(filename)
else:
print(filename + " is zero length, so ignoring.")
training_length = int(len(files) * SPLIT_SIZE)
testing_length = int(len(files) - training_length)
shuffled_set = random.sample(files, len(files))
training_set = shuffled_set[0:training_length]
testing_set = shuffled_set[-testing_length:]
i=0
for filename in training_set:
this_file = SOURCE + filename
if 'Cat' in SOURCE:
destination = TRAINING + "Cat" + str(i) + ".jpeg"
elif 'Dog' in SOURCE:
destination = TRAINING + "Dog" + str(i) + ".jpeg"
copyfile(this_file, destination)
i=i+1
i=0
for filename in testing_set:
this_file = SOURCE + filename
if 'Cat' in SOURCE:
destination = TESTING + "Cat" + str(i) + ".jpeg"
elif 'Dog' in SOURCE:
destination = TESTING + "Dog" + str(i) + ".jpeg"
copyfile(this_file, destination)
i=i+1
CAT_SOURCE_DIR = "/tmp/PetImages/Cat/"
DOG_SOURCE_DIR = "/tmp/PetImages/Dog/"
TRAINING_DIR = "/tmp/cats-v-dogs/training/"
TESTING_DIR = "/tmp/cats-v-dogs/testing/"
split_size = .85
split_data(CAT_SOURCE_DIR, TRAINING_DIR, TESTING_DIR, split_size)
split_data(DOG_SOURCE_DIR, TRAINING_DIR, TESTING_DIR, split_size)
import glob
one_hot_cat = np.array([0.0, 1.0])
one_hot_dog = np.array([1.0, 0.0])
train_x = np.array(glob.glob("/tmp/cats-v-dogs/training/*.jpeg"), dtype=str)
test_x = np.array(glob.glob("/tmp/cats-v-dogs/testing/*.jpeg"), dtype=str)
# create placeholder for labels
train_y = np.zeros(shape=(len(train_x), 2))
test_y = np.zeros(shape=(len(test_x), 2))
for cmpt, filename in enumerate(TRAINING_DIR):
if 'Cat' in filename:
train_y[cmpt] = one_hot_cat
elif 'Dog' in filename:
train_y[cmpt] = one_hot_dog
for cmpt, filename in enumerate(TESTING_DIR):
if 'Cat' in filename:
test_y[cmpt] = one_hot_cat
elif 'Dog' in filename:
test_y[cmpt] = one_hot_dog
# split the data in train directory
from sklearn.model_selection import train_test_split
train_x, valid_x,train_y, valid_y = train_test_split(train_x, train_y, test_size=0.1)
# variables
n_epochs = 200
n_batch = 128
buffer_size = 1024
resize_shape = [128,128]
# create placeholder for input
X = tf.placeholder(tf.float32, [None, 128, 128, 3])
# create placeholder for label
y_ = tf.placeholder(tf.float32, [None, 2])
# create the image parser function# create the image parser function
def _parser_function(filename, label):
image_string = tf.io.read_file(filename)
image = tf.image.decode_jpeg(image_string,channels=3)
image_resized = tf.image.resize(image, size=[128,128])
image_resized = image_resized / 255.0
return image_resized, label
# create input pipeline using tf.data
dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y))
dataset = dataset.map(_parser_function)
dataset = dataset.shuffle(buffer_size=1024)
dataset = dataset.repeat(5)
dataset = dataset.batch(batch_size=128)
iterator = dataset.make_initializable_iterator()
# create batch iterators
input_mini_batch, label_mini_batch = iterator.get_next()
# create test pipeline using tf.data
dataset2 = tf.data.Dataset.from_tensor_slices((test_x, test_y))
dataset2 = dataset2.map(_parser_function)
dataset2 = dataset2.shuffle(buffer_size=1024)
dataset2 = dataset2.repeat()
dataset2 = dataset2.batch(batch_size=len(test_x))
iterator2 = dataset2.make_one_shot_iterator()
test_img, test_label = iterator2.get_next()
# create weights and biases with a small amount of noise
def weight_variable(shape):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial)
def bias_variable(shape):
initial = tf.constant(0.1, shape=shape)
return tf.Variable(initial)
# convolution and pooling layers
def conv2d(x, W):
return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_2x2(x):
return tf.nn.max_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
# initialize weights and biases
# convolutional layers
W_conv1 = weight_variable([5,5,3,32])
b_conv1 = bias_variable([32])
W_conv2 = weight_variable([5,5,32,32])
b_conv2 = bias_variable([32])
W_conv3 = weight_variable([5,5,32,64])
b_conv3 = bias_variable([64])
W_conv4 = weight_variable([5,5,64,64])
b_conv4 = bias_variable([64])
W_conv5 = weight_variable([5,5,64,96])
b_conv5 = bias_variable([96])
W_conv6 = weight_variable([5,5,96,96])
b_conv6 = bias_variable([96])
W_conv7 = weight_variable([5,5,96,128])
b_conv7 = bias_variable([128])
W_conv8 = weight_variable([5,5,128,128])
b_conv8 = bias_variable([128])
# fully connected layers
W_fc1 = weight_variable([8192,4096])
b_fc1 = bias_variable([4096])
W_fc2 = weight_variable([4096,4096])
b_fc2 = bias_variable([4096])
W_fc3 = weight_variable([4096,2])
b_fc3 = bias_variable([2])
# add dropout to conv layers
keep_prob_conv = tf.placeholder(tf.float32)
# add dropout to fc layers
keep_prob_fc = tf.placeholder(tf.float32)
# create tensorflow model
# conv layer 1
h_conv1 = conv2d(X, W_conv1)
h_conv1_activated = tf.nn.relu(h_conv1 + b_conv1)
# conv layer 2
h_conv2 = conv2d(h_conv1_activated, W_conv2)
h_conv2_activated = tf.nn.relu(h_conv2 + b_conv2)
h_pool2 = max_pool_2x2(h_conv2_activated)
h_pool2_dropout = tf.nn.dropout(h_pool2, keep_prob_conv)
# conv layer 3
h_conv3 = conv2d(h_pool2_dropout, W_conv3)
h_conv3_activated = tf.nn.relu(h_conv3 + b_conv3)
# conv layer 4
h_conv4 = conv2d(h_conv3_activated, W_conv4)
h_conv4_activated = tf.nn.relu(h_conv4 + b_conv4)
h_pool4 = max_pool_2x2(h_conv4_activated)
h_pool4_dropout = tf.nn.dropout(h_pool4, keep_prob_conv)
# conv layer 5
h_conv5 = conv2d(h_pool4_dropout, W_conv5)
h_conv5_activated = tf.nn.relu(h_conv5 + b_conv5)
#conv layer 6
h_conv6 = conv2d(h_conv5_activated, W_conv6)
h_conv6_activated = tf.nn.relu(h_conv6 + b_conv6)
h_pool6 = max_pool_2x2(h_conv6_activated)
h_pool6_dropout = tf.nn.dropout(h_pool6, keep_prob_conv)
# conv layer 7
h_conv7 = conv2d(h_pool6_dropout, W_conv7)
h_conv7_activated = tf.nn.relu(h_conv7 + b_conv7)
#conv layer 8
h_conv8 = conv2d(h_conv7_activated, W_conv8)
h_conv8_activated = tf.nn.relu(h_conv8 + b_conv8)
h_pool8 = max_pool_2x2(h_conv8_activated)
h_pool8_dropout = tf.nn.dropout(h_pool8, keep_prob_conv)
# flatten output from conv layers
x_fc = tf.reshape(h_pool8_dropout, shape=[-1, 8192])
# fully conected layer 1
h_fc1 = tf.matmul(x_fc, W_fc1) + b_fc1
h_fc1_activated = tf.nn.relu(h_fc1)
h_fc1_dropout = tf.nn.dropout(h_fc1_activated, keep_prob_fc)
# fully conected layer 2
h_fc2 = tf.matmul(h_fc1_dropout, W_fc2)
h_fc2_activated = tf.nn.relu(h_fc2)
# fully conected layer 3
y = tf.matmul(h_fc2_activated, W_fc3)
# define the cost function
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(logits=y, labels=y_))
# tell the tensorflow computational graph to minimize the cost using adam optimizer
train = tf.train.AdamOptimizer().minimize(cost)
# predict output using our model
# returns a boolean array
predictions = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
# cast the boolean array to float and calculate accuracy
accuracy = tf.reduce_mean(tf.cast(predictions, tf.float32))
# initialize tensorflow interactive session
sess = tf.InteractiveSession()
# initialize variables inside the graph
tf.global_variables_initializer().run()
# load graph
saver = tf.train.Saver()
# save graph
def save_graph(epoch):
saver = tf.train.Saver()
save_path = saver.save(sess, "./model_epoch_final.ckpt", global_step=epoch)
# create function to calculate train and test accuracy
def calculate_accuracy():
sess.run(iterator.initializer)
sess.run(iterator2.initializer)
train_acc = 0
train_iters = 0
for _ in range(5):
train_iters += 1
X_test, y_test = sess.run([input_mini_batch, label_mini_batch])
print(input_mini_batch.shape)
print(label_mini_batch.shape)
train_acc += sess.run(accuracy, feed_dict={X: X_test, y_: y_test, keep_prob_fc: 1.0, keep_prob_conv: 1.0})
test_acc = 0
test_iters = 0
while True:
try:
test_iters += 1
X_test, y_test = sess.run([test_img, test_label ])
test_acc += sess.run(accuracy, feed_dict={X: X_test, y_: y_test, keep_prob_fc: 1.0, keep_prob_conv: 1.0})
except tf.errors.OutOfRangeError:
break
print(f"Train accuracy: {train_acc/train_iters}")
print(f"Test accuracy: {test_acc/test_iters}")
for epoch in range(0,n_epochs):
epoch_loss = 0
sess.run(iterator.initializer)
while True:
try:
X_for_train, y_for_train = sess.run([input_mini_batch, label_mini_batch])
t, c = sess.run([train, cost], feed_dict={X: X_for_train, y_: y_for_train, keep_prob_fc: 0.5, keep_prob_conv: 0.8})
epoch_loss += c
except tf.errors.OutOfRangeError:
break
# print loss
print(f"Epoch {epoch} out of {n_epochs}, loss: {epoch_loss}")
# print train and test accuracies
calculate_accuracy()
# Save Graph
if epoch % 2 == 0 and epoch >= 2:
save_graph(epoch)
# calculate final accuracy
sess.run(iterator2.initializer)
acc = 0
iters = 0
while True:
try:
iters += 1
X_for_train, y_for_train = sess.run([test_img, test_label])
acc += sess.run(accuracy, feed_dict={X: X_for_train, y_: y_for_train, keep_prob_fc: 1.0, keep_prob_conv: 1.0})
except tf.errors.OutOfRangeError:
break
print(f"Accuracy: {acc/iters}")
sess.close()
PS: Я знаю, что не использую свои данные проверки.