Проведение состязательных атак на глубокие деревья решений, связанные с набором данных MNIST.
N_LEAF = 2 ** (DEPTH + 1) # Number of leaf node
N_LABEL = 10 # Number of classes
N_TREE = 5 # Number of trees (ensemble)
N_BATCH = 128 # Number of data points per mini-batch
Загрузка набора данных
(trX, trY), (teX, teY), min_pixel_value, max_pixel_value = load_mnist()
trX = trX.reshape(-1, 28, 28, 1)
teX = teX.reshape(-1, 28, 28, 1)
print (trX.shape[1])
Вход X, выход Y, заполнители для x и y
X = tf.placeholder("float", shape=[N_BATCH, 28, 28, 1], name='inputX')
Y = tf.placeholder("float", shape=[N_BATCH, N_LABEL], name='inputY')
def init_weights(shape):
return tf.Variable(tf.random_normal(shape, stddev=0.01))
def init_prob_weights(shape, minval=-5, maxval=5):
return tf.Variable(tf.random_uniform(shape, minval, maxval))
def model(X, w, w2, w3, w4_e, w_d_e, w_l_e, p_keep_conv, p_keep_hidden):
assert (len(w4_e) == len(w_d_e))
assert (len(w4_e) == len(w_l_e))
l1a = tf.nn.relu(tf.nn.conv2d(X, w, [1, 1, 1, 1], 'SAME'))
l1 = tf.nn.max_pool(l1a, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
l1 = tf.nn.dropout(l1, p_keep_conv)
l2a = tf.nn.relu(tf.nn.conv2d(l1, w2, [1, 1, 1, 1], 'SAME'))
l2 = tf.nn.max_pool(l2a, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
l2 = tf.nn.dropout(l2, p_keep_conv)
l3a = tf.nn.relu(tf.nn.conv2d(l2, w3, [1, 1, 1, 1], 'SAME'))
l3 = tf.nn.max_pool(l3a, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
l3 = tf.reshape(l3, [-1, w4_e[0].get_shape().as_list()[0]])
l3 = tf.nn.dropout(l3, p_keep_conv)
decision_p_e = []
leaf_p_e = []
for w4, w_d, w_l in zip(w4_e, w_d_e, w_l_e):
l4 = tf.nn.relu(tf.matmul(l3, w4))
l4 = tf.nn.dropout(l4, p_keep_hidden)
decision_p = tf.nn.sigmoid(tf.matmul(l4, w_d)) ##check here
leaf_p = tf.nn.softmax(w_l)
decision_p_e.append(decision_p)
leaf_p_e.append(leaf_p)
return decision_p_e, leaf_p_e
class TFClassifierRF(TFClassifier):
def predict(self, x, batch_size , **kwargs):
#Apply preprocessing
x_preprocessed, _ = self._apply_preprocessing(x, y=None, fit=False)
rf_feed_dict = kwargs['rf_feed_dict']
# Run prediction with batch processing
results = np.zeros((x_preprocessed.shape[0], self.nb_classes()), dtype=np.float32)
num_batch = int(np.ceil(len(x_preprocessed) / float(batch_size)))
for m in range(num_batch):
# Batch indexes
begin, end = m * batch_size, min((m + 1) * batch_size, x_preprocessed.shape[0])
# Create feed_dict
feed_dict = {self._input_ph: x_preprocessed[begin:end]}
feed_dict.update(self._feed_dict)
feed_dict.update(rf_feed_dict)
# Run prediction
results[begin:end] = self._sess.run(self._output, feed_dict=feed_dict)
return results
def fit(self, x, y, batch_size=128 , nb_epochs=10 , **kwargs):
#Check if train and output_ph available
if self._train is None or self._labels_ph is None:
raise ValueError("Need the training objective and the output placeholder to train the model.")
# Apply preprocessing
x_preprocessed, y_preprocessed = self._apply_preprocessing(x, y, fit=True)
num_batch = int(np.ceil(len(x_preprocessed) / float(batch_size)))
ind = np.arange(len(x_preprocessed))
rf_feed_dict = kwargs['rf_feed_dict']
# Start training
for _ in range(nb_epochs):
# Shuffle the examples
random.shuffle(ind)
# Train for one epoch
for m in range(num_batch):
i_batch = x_preprocessed[ind[m * batch_size:(m + 1) * batch_size]]
o_batch = y_preprocessed[ind[m * batch_size:(m + 1) * batch_size]]
# Create feed_dict
feed_dict = {self._input_ph: i_batch, self._labels_ph: o_batch}
feed_dict.update(self._feed_dict)
feed_dict.update(rf_feed_dict)
# Run train step
self._sess.run(self._train, feed_dict=feed_dict)
def fit_generator(self, generator, nb_epochs, **kwargs):
from art.data_generators import TFDataGenerator
rf_feed_dict = kwargs['rf_feed_dict']
# Train directly in TensorFlow
if isinstance(generator, TFDataGenerator) and not (
hasattr(self, 'label_smooth') or hasattr(self, 'feature_squeeze')):
for _ in range(nb_epochs):
for _ in range(int(generator.size / generator.batch_size)):
i_batch, o_batch = generator.get_batch()
# Create feed_dict
feed_dict = {self._input_ph: i_batch, self._labels_ph: o_batch}
feed_dict.update(self._feed_dict)
feed_dict.update(rf_feed_dict)
# Run train step
self._sess.run(self._train, feed_dict=feed_dict)
super(TensorFlowClassifier, self).fit_generator(generator, nb_epochs=nb_epochs, **kwargs)
инициализировать вес модели
w = init_weights([3, 3, 1, 32])
w2 = init_weights([3, 3, 32, 64])
w3 = init_weights([3, 3, 64, 128])
w4_ensemble = []
w_d_ensemble = []
w_l_ensemble = []
for i in range(N_TREE):
w4_ensemble.append(init_weights([128 * 4 * 4, 625]))
w_d_ensemble.append(init_prob_weights([625, N_LEAF], -1, 1))
w_l_ensemble.append(init_prob_weights([N_LEAF, N_LABEL], -2, 2))
p_keep_conv = tf.placeholder("float", name="p_keep_conv")
p_keep_hidden = tf.placeholder("float", name = "p_keep_hidden")
Определить модель дерева глубинных решений
# With the probability decision_p, route a sample to the right branch
decision_p_e, leaf_p_e = model(X, w, w2, w3, w4_ensemble, w_d_ensemble,
w_l_ensemble, p_keep_conv, p_keep_hidden)
flat_decision_p_e = []
# iterate over each tree
for decision_p in decision_p_e:
# Compute the complement of d, which is 1 - d
# where d is the sigmoid of fully connected output
decision_p_comp = tf.subtract(tf.ones_like(decision_p), decision_p)
# Concatenate both d, 1-d
decision_p_pack = tf.stack([decision_p, decision_p_comp])
# Flatten/vectorize the decision probabilities for efficient indexing
flat_decision_p = tf.reshape(decision_p_pack, [-1])
flat_decision_p_e.append(flat_decision_p)
# 0 index of each data instance in a mini-batch
batch_0_indices = \
tf.tile(tf.expand_dims(tf.range(0, N_BATCH * N_LEAF, N_LEAF), 1),
[1, N_LEAF])
in_repeat = N_LEAF / 2
out_repeat = N_BATCH
# N_LEAF = float(N_LEAF)
# N_BATCH = float(N_BATCH)
# Let N_BATCH * N_LEAF be N_D. flat_decision_p[N_D] will return 1-d of the
# first root node in the first tree.
batch_complement_indices = \
np.array([[0] * int(in_repeat), [N_BATCH * N_LEAF] * int(in_repeat)]
* out_repeat).reshape(N_BATCH, N_LEAF)
Сначала определить вероятности маршрутизации d для root узлов
mu_e = []
# iterate over each tree
for i, flat_decision_p in enumerate(flat_decision_p_e):
mu = tf.gather(flat_decision_p,
tf.add(batch_0_indices, batch_complement_indices))
mu_e.append(mu)
от второго до последнего уровня, мы принимаем решения узлами
for d in range(1, DEPTH + 1):
indices = tf.range(2 ** d, 2 ** (d + 1)) - 1
tile_indices = tf.reshape(tf.tile(tf.expand_dims(indices, 1),
[1, 2 ** (DEPTH - d + 1)]), [1, -1])
batch_indices = tf.add(batch_0_indices, tf.tile(tile_indices, [N_BATCH, 1]))
in_repeat = in_repeat / 2
out_repeat = out_repeat * 2
# Again define the indices that picks d and 1-d for the node
batch_complement_indices = \
np.array([[0] * int(in_repeat), [N_BATCH * N_LEAF] * int(in_repeat)]
* out_repeat).reshape(N_BATCH, N_LEAF)
mu_e_update = []
for mu, flat_decision_p in zip(mu_e, flat_decision_p_e):
mu = tf.multiply(mu, tf.gather(flat_decision_p,
tf.add(batch_indices, batch_complement_indices)))
mu_e_update.append(mu)
mu_e = mu_e_update
Определим p (y | x)
py_x_e = []
for mu, leaf_p in zip(mu_e, leaf_p_e):
# average all the leaf p
py_x_tree = tf.reduce_mean(
tf.multiply(tf.tile(tf.expand_dims(mu, 2), [1, 1, N_LABEL]),
tf.tile(tf.expand_dims(leaf_p, 0), [N_BATCH, 1, 1])), 1)
py_x_e.append(py_x_tree)
py_x_e = tf.stack(py_x_e)
# logit
py_x = tf.reduce_mean(py_x_e, 0)
Определим стоимость и метод оптимизации
# cross entropy loss
loss = tf.reduce_mean(tf.losses.softmax_cross_entropy(logits=py_x, onehot_labels=Y))
# cost = tf.reduce_mean(tf.nn.cross_entropy_with_logits(py_x, Y))
optimizer = tf.train.AdamOptimizer(0.001, 0.9)
train = optimizer.minimize(loss)
# predict = tf.argmax(py_x, 1)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
classifier = TFClassifierRF(clip_values=(min_pixel_value, max_pixel_value,), input_ph=X, output=py_x,
labels_ph=Y, train=train, loss=loss, learning=None, sess=sess)
rf_feed_dict = {p_keep_conv: 0.8, p_keep_hidden: 0.5}
classifier.fit(trX, trY, N_BATCH, nb_epochs=10, rf_feed_dict=rf_feed_dict)
ValueError: Cannot feed value of shape (96, 28, 28, 1) for Tensor 'inputX_25:0', which has shape '(128, 28, 28, 1)'
rf_feed_dict = {p_keep_conv: 1.0, p_keep_hidden: 1.0}
predictions = classifier.predict(teX, batch_size=128, rf_feed_dict=rf_feed_dict)
accuracy = np.sum(np.argmax(predictions, axis=1) == np.argmax(teY, axis=1)) / len(teY)
print('Accuracy on benign test examples: {}%'.format(accuracy * 100))