Следующий код запускается, если я опускаю строку tf.math.square или декорацию @ tf.function.
Почему квадратная линия вызывает некоторую проблему с градиентом?
import tensorflow as tf
import tensorflow.keras as keras
import tensorflow.keras.backend as K
import numpy as np
m = 1000
n = 1
X = np.random.randn(m, n).astype(np.float32)
y = (3 + 0 * np.random.randn(m)).astype(np.float32)
def create_model():
a_input = keras.layers.Input(shape=(n,), dtype=np.float32)
a = K.expand_dims(a_input, axis=2)
q = keras.layers.Conv1D(1, 1)(a)
q = - tf.math.square(q) # this breaks things, but only when using tf.function
model = keras.models.Model(inputs=a_input, outputs=q)
return model
model = create_model()
model.predict(X)
class Trainer():
def __init__(self, epochs=10):
self.epochs = epochs
self.model = create_model()
self.optimizer = tf.optimizers.Adam()
self.step = 0
def train(self, X, y, epochs=10):
X = tf.convert_to_tensor(X, dtype=tf.float32)
y = tf.convert_to_tensor(y, dtype=tf.float32)
for epoch in range(epochs):
l = self._train_one_step(X, y)
return l
@tf.function
def _train_one_step(self, X, y):
with tf.GradientTape() as tape:
yp = self.model(X)
loss = tf.reduce_mean(tf.math.square(y - yp))
gradients = tape.gradient(loss, self.model.trainable_variables)
l = self.optimizer.apply_gradients(zip(gradients, self.model.trainable_variables))
d = dict(loss=loss)
tf.print(yp[0], loss)
self.step += 1
trainer = Trainer()
l = trainer.train(X, y, epochs=100)
Трассировка в соответствии с запросом:
2019-04-18 14:23:03.776224: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
<ipython-input-1-8a9114e8a51b> in <module>
----> 1 import example_keras_tf2 as e
~/toplevelrepo/something-train/something/train/usr/HOME/example_keras_tf2.py in <module>
44
45 trainer = Trainer()
---> 46 l = trainer.train(X, y, epochs=100)
~/toplevelrepo/something-train/something/train/usr/HOME/example_keras_tf2.py in train(self, X, y, epochs)
30 y = tf.convert_to_tensor(y, dtype=tf.float32)
31 for epoch in range(epochs):
---> 32 l = self._train_one_step(X, y)
33 return l
34 @tf.function
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in __call__(self, *args, **kwds)
424 # This is the first call of __call__, so we have to initialize.
425 initializer_map = {}
--> 426 self._initialize(args, kwds, add_initializers_to=initializer_map)
427 if self._created_variables:
428 try:
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in _initialize(self, args, kwds, add_initializers_to)
368 self._concrete_stateful_fn = (
369 self._stateful_fn._get_concrete_function_internal_garbage_collected( # pylint: disable=protected-access
--> 370 *args, **kwds))
371
372 def invalid_creator_scope(*unused_args, **unused_kwds):
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _get_concrete_function_internal_garbage_collected(self, *args, **kwargs)
1311 if self._input_signature:
1312 args, kwargs = None, None
-> 1313 graph_function, _, _ = self._maybe_define_function(args, kwargs)
1314 return graph_function
1315
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _maybe_define_function(self, args, kwargs)
1578 or call_context_key not in self._function_cache.missed):
1579 self._function_cache.missed.add(call_context_key)
-> 1580 graph_function = self._create_graph_function(args, kwargs)
1581 self._function_cache.primary[cache_key] = graph_function
1582 return graph_function, args, kwargs
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _create_graph_function(self, args, kwargs, override_flat_arg_shapes)
1510 arg_names=arg_names,
1511 override_flat_arg_shapes=override_flat_arg_shapes,
-> 1512 capture_by_value=self._capture_by_value),
1513 self._function_attributes)
1514
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes)
692 converted_func)
693
--> 694 func_outputs = python_func(*func_args, **func_kwargs)
695
696 # invariant: `func_outputs` contains only Tensors, IndexedSlices,
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in wrapped_fn(*args, **kwds)
315 # __wrapped__ allows AutoGraph to swap in a converted function. We give
316 # the function a weak reference to itself to avoid a reference cycle.
--> 317 return weak_wrapped_fn().__wrapped__(*args, **kwds)
318 weak_wrapped_fn = weakref.ref(wrapped_fn)
319
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/eager/function.py in bound_method_wrapper(*args, **kwargs)
2106 # If __wrapped__ was replaced, then it is always an unbound function
2107 # that takes self as first argument.
-> 2108 return wrapped_fn(weak_instance(), *args, **kwargs)
2109 weak_bound_method_wrapper = weakref.ref(bound_method_wrapper)
2110
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
684 optional_features=autograph_options,
685 force_conversion=True,
--> 686 ), args, kwargs)
687
688 # Wrapping around a decorator allows checks like tf_inspect.getargspec
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/autograph/impl/api.py in converted_call(f, owner, options, args, kwargs)
390 return _call_unconverted(f, args, kwargs)
391
--> 392 result = converted_f(*effective_args, **kwargs)
393
394 # The converted function's closure is simply inserted into the function's
/var/folders/8s/kcb8_98n1pqbnwrv2h0b14rm0000gp/T/tmp9aj7e32f.py in tf___train_one_step(self, X, y)
5 loss = ag__.converted_call('reduce_mean', tf, ag__.ConversionOptions(recursive=True, verbose=0, strip_decorators=(tf.function, defun_1, ag__.convert, ag__.do_not_convert, ag__.converted_call), force_conversion=False, optional_features=(), internal_convert_user_code=True), (tf.math.square(y - yp),), {})
6 gradients = ag__.converted_call('gradient', tape, ag__.ConversionOptions(recursive=True, verbose=0, strip_decorators=(tf.function, defun_2, ag__.convert, ag__.do_not_convert, ag__.converted_call), force_conversion=False, optional_features=(), internal_convert_user_code=True), (loss, self.model.trainable_variables), {})
----> 7 l = ag__.converted_call('apply_gradients', self.optimizer, ag__.ConversionOptions(recursive=True, verbose=0, strip_decorators=(tf.function, defun_3, ag__.convert, ag__.do_not_convert, ag__.converted_call), force_conversion=False, optional_features=(), internal_convert_user_code=True), (zip(gradients, self.model.trainable_variables),), {})
8 d = ag__.converted_call(dict, None, ag__.ConversionOptions(recursive=True, verbose=0, strip_decorators=(tf.function, defun_4, ag__.convert, ag__.do_not_convert, ag__.converted_call), force_conversion=False, optional_features=(), internal_convert_user_code=True), (), {'loss': loss})
9 ag__.converted_call('print', tf, ag__.ConversionOptions(recursive=True, verbose=0, strip_decorators=(tf.function, defun_5, ag__.convert, ag__.do_not_convert, ag__.converted_call), force_conversion=False, optional_features=(), internal_convert_user_code=True), (yp[0], loss), {})
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/autograph/impl/api.py in converted_call(f, owner, options, args, kwargs)
265
266 if not options.force_conversion and conversion.is_whitelisted_for_graph(f):
--> 267 return _call_unconverted(f, args, kwargs)
268
269 # internal_convert_user_code is for example turned off when issuing a dynamic
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/autograph/impl/api.py in _call_unconverted(f, args, kwargs)
186 return f.__self__.call(args, kwargs)
187
--> 188 return f(*args, **kwargs)
189
190
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py in apply_gradients(self, grads_and_vars, name)
394 ValueError: If none of the variables have gradients.
395 """
--> 396 grads_and_vars = _filter_grads(grads_and_vars)
397 var_list = [v for (_, v) in grads_and_vars]
398
~/anaconda3/envs/myenv/lib/python3.7/site-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py in _filter_grads(grads_and_vars)
922 if not filtered:
923 raise ValueError("No gradients provided for any variable: %s." %
--> 924 ([v.name for _, v in grads_and_vars],))
925 if vars_with_empty_grads:
926 logging.warning(
ValueError: No gradients provided for any variable: ['conv1d_1/kernel:0', 'conv1d_1/bias:0'].
Кроме того, я не уверен, полезно ли это какЯ пока не знаю, как это читать, но вот дамп python_function:
In [6]: print(tf.autograph.to_code(e.trainer._train_one_step.python_function))
from __future__ import print_function
def tf___train_one_step(*args, **kwargs):
try:
"""Wraps either a dummy MethodType or a converted AutoGraph function."""
with ag__.function_scope('bound_method_wrapper'):
do_return = False
retval_ = None
strong_bound_method_wrapper = ag__.converted_call(weak_bound_method_wrapper, None, ag__.ConversionOptions(recursive=True, verbose=0, strip_decorators=(ag__.convert, ag__.do_not_convert, ag__.converted_call), force_conversion=False, optional_features=ag__.Feature.ALL, internal_convert_user_code=True), (), {})
wrapped_fn = strong_bound_method_wrapper.__wrapped__
cond_1 = ag__.is_(wrapped_fn, strong_bound_method_wrapper.__original_wrapped__)
def if_true_1():
with ag__.function_scope('if_true_1'):
wrapped_fn = original_function.python_function
cond = ag__.converted_call('ismethod', tf_inspect, ag__.ConversionOptions(recursive=True, verbose=0, strip_decorators=(ag__.convert, ag__.do_not_convert, ag__.converted_call), force_conversion=False, optional_features=ag__.Feature.ALL, internal_convert_user_code=True), (wrapped_fn,), {})
def if_true():
with ag__.function_scope('if_true'):
wrapped_fn_1, = wrapped_fn,
wrapped_fn_1 = ag__.converted_call('get_unbound_function', six, ag__.ConversionOptions(recursive=True, verbose=0, strip_decorators=(ag__.convert, ag__.do_not_convert, ag__.converted_call), force_conversion=False, optional_features=ag__.Feature.ALL, internal_convert_user_code=True), (wrapped_fn_1,), {})
return wrapped_fn_1
def if_false():
with ag__.function_scope('if_false'):
return wrapped_fn
wrapped_fn = ag__.if_stmt(cond, if_true, if_false)
do_return = True
retval_ = ag__.converted_call(wrapped_fn, None, ag__.ConversionOptions(recursive=True, verbose=0, strip_decorators=(ag__.convert, ag__.do_not_convert, ag__.converted_call), force_conversion=False, optional_features=ag__.Feature.ALL, internal_convert_user_code=True), (weak_instance(),) + tuple(args), dict(kwargs, **{}))
return retval_
def if_false_1():
with ag__.function_scope('if_false_1'):
do_return = True
retval_ = ag__.converted_call(wrapped_fn, None, ag__.ConversionOptions(recursive=True, verbose=0, strip_decorators=(ag__.convert, ag__.do_not_convert, ag__.converted_call), force_conversion=False, optional_features=ag__.Feature.ALL, internal_convert_user_code=True), (weak_instance(),) + tuple(args), dict(kwargs, **{}))
return retval_
retval_ = ag__.if_stmt(cond_1, if_true_1, if_false_1)
return retval_
except:
ag__.rewrite_graph_construction_error(ag_source_map__)
tf___train_one_step.autograph_info__ = {}