• 1000 2.2.0, я получаю странные ошибки и поэтому не могу использовать последнюю версию. Используя версию 1.15.2 и выбрав даже размер партии 8, я успешно начал процесс обучения, но через некоторое время процесс обучения останавливается со следующими ошибками.
TypeError: 'numpy.float64' object cannot be interpreted as an integer
self._traceback = tf_stack.extract_stack()
Мой полный журнал обучения выглядит следующим образом:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/evaluation.py", line 272, in _evaluate_once
session.run(eval_ops, feed_dict)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 754, in run
run_metadata=run_metadata)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1259, in run
run_metadata=run_metadata)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1360, in run
raise six.reraise(*original_exc_info)
File "/usr/local/lib/python3.6/dist-packages/six.py", line 693, in reraise
raise value
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1345, in run
return self._sess.run(*args, **kwargs)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1418, in run
run_metadata=run_metadata)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1176, in run
return self._sess.run(*args, **kwargs)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py", line 956, in run
run_metadata_ptr)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py", line 1180, in _run
feed_dict_tensor, options, run_metadata)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py", line 1359, in _do_run
run_metadata)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py", line 1384, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.OutOfRangeError: 2 root error(s) found.
(0) Out of range: End of sequence
[[node IteratorGetNext (defined at tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py:1748) ]]
(1) Out of range: End of sequence
[[node IteratorGetNext (defined at tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py:1748) ]]
[[Postprocessor/BatchMultiClassNonMaxSuppression/MultiClassNonMaxSuppression/non_max_suppression_with_scores_1/NonMaxSuppressionV5/_4683]]
0 successful operations.
0 derived errors ignored.
Original stack trace for 'IteratorGetNext':
File "content/models/research/object_detection/model_main.py", line 114, in <module>
tf.app.run()
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "usr/local/lib/python3.6/dist-packages/absl/app.py", line 299, in run
_run_main(main, args)
File "usr/local/lib/python3.6/dist-packages/absl/app.py", line 250, in _run_main
sys.exit(main(argv))
File "content/models/research/object_detection/model_main.py", line 110, in main
tf.estimator.train_and_evaluate(estimator, train_spec, eval_specs[0])
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 473, in train_and_evaluate
return executor.run()
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 613, in run
return self.run_local()
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 714, in run_local
saving_listeners=saving_listeners)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 370, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1161, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1195, in _train_model_default
saving_listeners)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1494, in _train_with_estimator_spec
_, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 754, in run
run_metadata=run_metadata)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1259, in run
run_metadata=run_metadata)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1345, in run
return self._sess.run(*args, **kwargs)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1426, in run
run_metadata=run_metadata))
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/training/basic_session_run_hooks.py", line 594, in after_run
if self._save(run_context.session, global_step):
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/training/basic_session_run_hooks.py", line 619, in _save
if l.after_save(session, step):
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 519, in after_save
self._evaluate(global_step_value) # updates self.eval_result
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 539, in _evaluate
self._evaluator.evaluate_and_export())
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 920, in evaluate_and_export
hooks=self._eval_spec.hooks)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 480, in evaluate
name=name)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 522, in _actual_eval
return _evaluate()
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 504, in _evaluate
self._evaluate_build_graph(input_fn, hooks, checkpoint_path))
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1511, in _evaluate_build_graph
self._call_model_fn_eval(input_fn, self.config))
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1544, in _call_model_fn_eval
input_fn, ModeKeys.EVAL)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1025, in _get_features_and_labels_from_input_fn
self._call_input_fn(input_fn, mode))
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/util.py", line 65, in parse_input_fn_result
result = iterator.get_next()
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/data/ops/iterator_ops.py", line 426, in get_next
name=name)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/gen_dataset_ops.py", line 2518, in iterator_get_next
output_shapes=output_shapes, name=name)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/op_def_library.py", line 794, in _apply_op_helper
op_def=op_def)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/util/deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py", line 3357, in create_op
attrs, op_def, compute_device)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py", line 3426, in _create_op_internal
op_def=op_def)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py", line 1748, in __init__
self._traceback = tf_stack.extract_stack()
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py", line 1365, in _do_call
return fn(*args)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py", line 1350, in _run_fn
target_list, run_metadata)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py", line 1443, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InvalidArgumentError: 2 root error(s) found.
(0) Invalid argument: TypeError: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer.
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/numpy/core/function_base.py", line 117, in linspace
num = operator.index(num)
TypeError: 'numpy.float64' object cannot be interpreted as an integer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/script_ops.py", line 235, in __call__
ret = func(*args)
File "/content/models/research/object_detection/metrics/coco_evaluation.py", line 416, in first_value_func
self._metrics = self.evaluate()
File "/content/models/research/object_detection/metrics/coco_evaluation.py", line 247, in evaluate
coco_wrapped_groundtruth, coco_wrapped_detections, agnostic_mode=False)
File "/content/models/research/object_detection/metrics/coco_tools.py", line 178, in __init__
cocoeval.COCOeval.__init__(self, groundtruth, detections, iouType=iou_type)
File "/usr/local/lib/python3.6/dist-packages/pycocotools/cocoeval.py", line 76, in __init__
self.params = Params(iouType=iouType) # parameters
File "/usr/local/lib/python3.6/dist-packages/pycocotools/cocoeval.py", line 527, in __init__
self.setDetParams()
File "/usr/local/lib/python3.6/dist-packages/pycocotools/cocoeval.py", line 507, in setDetParams
self.iouThrs = np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True)
File "<__array_function__ internals>", line 6, in linspace
File "/usr/local/lib/python3.6/dist-packages/numpy/core/function_base.py", line 121, in linspace
.format(type(num)))
TypeError: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer.
[[{{node PyFunc_3}}]]
[[cond/Detections_Left_Groundtruth_Right/0/_4927]]
(1) Invalid argument: TypeError: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer.
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/numpy/core/function_base.py", line 117, in linspace
num = operator.index(num)
TypeError: 'numpy.float64' object cannot be interpreted as an integer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/script_ops.py", line 235, in __call__
ret = func(*args)
File "/content/models/research/object_detection/metrics/coco_evaluation.py", line 416, in first_value_func
self._metrics = self.evaluate()
File "/content/models/research/object_detection/metrics/coco_evaluation.py", line 247, in evaluate
coco_wrapped_groundtruth, coco_wrapped_detections, agnostic_mode=False)
File "/content/models/research/object_detection/metrics/coco_tools.py", line 178, in __init__
cocoeval.COCOeval.__init__(self, groundtruth, detections, iouType=iou_type)
File "/usr/local/lib/python3.6/dist-packages/pycocotools/cocoeval.py", line 76, in __init__
self.params = Params(iouType=iouType) # parameters
File "/usr/local/lib/python3.6/dist-packages/pycocotools/cocoeval.py", line 527, in __init__
self.setDetParams()
File "/usr/local/lib/python3.6/dist-packages/pycocotools/cocoeval.py", line 507, in setDetParams
self.iouThrs = np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True)
File "<__array_function__ internals>", line 6, in linspace
File "/usr/local/lib/python3.6/dist-packages/numpy/core/function_base.py", line 121, in linspace
.format(type(num)))
TypeError: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer.
[[{{node PyFunc_3}}]]
0 successful operations.
0 derived errors ignored.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/content/models/research/object_detection/model_main.py", line 114, in <module>
tf.app.run()
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "/usr/local/lib/python3.6/dist-packages/absl/app.py", line 299, in run
_run_main(main, args)
File "/usr/local/lib/python3.6/dist-packages/absl/app.py", line 250, in _run_main
sys.exit(main(argv))
File "/content/models/research/object_detection/model_main.py", line 110, in main
tf.estimator.train_and_evaluate(estimator, train_spec, eval_specs[0])
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 473, in train_and_evaluate
return executor.run()
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 613, in run
return self.run_local()
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 714, in run_local
saving_listeners=saving_listeners)
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 370, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1161, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1195, in _train_model_default
saving_listeners)
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1494, in _train_with_estimator_spec
_, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 754, in run
run_metadata=run_metadata)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1259, in run
run_metadata=run_metadata)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1360, in run
raise six.reraise(*original_exc_info)
File "/usr/local/lib/python3.6/dist-packages/six.py", line 693, in reraise
raise value
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1345, in run
return self._sess.run(*args, **kwargs)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1426, in run
run_metadata=run_metadata))
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/basic_session_run_hooks.py", line 594, in after_run
if self._save(run_context.session, global_step):
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/basic_session_run_hooks.py", line 619, in _save
if l.after_save(session, step):
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 519, in after_save
self._evaluate(global_step_value) # updates self.eval_result
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 539, in _evaluate
self._evaluator.evaluate_and_export())
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 920, in evaluate_and_export
hooks=self._eval_spec.hooks)
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 480, in evaluate
name=name)
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 522, in _actual_eval
return _evaluate()
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 511, in _evaluate
output_dir=self.eval_dir(name))
File "/tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1619, in _evaluate_run
config=self._session_config)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/evaluation.py", line 272, in _evaluate_once
session.run(eval_ops, feed_dict)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 861, in __exit__
self._close_internal(exception_type)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 894, in _close_internal
h.end(self._coordinated_creator.tf_sess)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/training/basic_session_run_hooks.py", line 951, in end
self._final_ops, feed_dict=self._final_ops_feed_dict)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py", line 956, in run
run_metadata_ptr)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py", line 1180, in _run
feed_dict_tensor, options, run_metadata)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py", line 1359, in _do_run
run_metadata)
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/client/session.py", line 1384, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InvalidArgumentError: 2 root error(s) found.
(0) Invalid argument: TypeError: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer.
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/numpy/core/function_base.py", line 117, in linspace
num = operator.index(num)
TypeError: 'numpy.float64' object cannot be interpreted as an integer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/script_ops.py", line 235, in __call__
ret = func(*args)
File "/content/models/research/object_detection/metrics/coco_evaluation.py", line 416, in first_value_func
self._metrics = self.evaluate()
File "/content/models/research/object_detection/metrics/coco_evaluation.py", line 247, in evaluate
coco_wrapped_groundtruth, coco_wrapped_detections, agnostic_mode=False)
File "/content/models/research/object_detection/metrics/coco_tools.py", line 178, in __init__
cocoeval.COCOeval.__init__(self, groundtruth, detections, iouType=iou_type)
File "/usr/local/lib/python3.6/dist-packages/pycocotools/cocoeval.py", line 76, in __init__
self.params = Params(iouType=iouType) # parameters
File "/usr/local/lib/python3.6/dist-packages/pycocotools/cocoeval.py", line 527, in __init__
self.setDetParams()
File "/usr/local/lib/python3.6/dist-packages/pycocotools/cocoeval.py", line 507, in setDetParams
self.iouThrs = np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True)
File "<__array_function__ internals>", line 6, in linspace
File "/usr/local/lib/python3.6/dist-packages/numpy/core/function_base.py", line 121, in linspace
.format(type(num)))
TypeError: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer.
[[node PyFunc_3 (defined at tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py:1748) ]]
[[cond/Detections_Left_Groundtruth_Right/0/_4927]]
(1) Invalid argument: TypeError: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer.
Traceback (most recent call last):
File "/usr/local/lib/python3.6/dist-packages/numpy/core/function_base.py", line 117, in linspace
num = operator.index(num)
TypeError: 'numpy.float64' object cannot be interpreted as an integer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/script_ops.py", line 235, in __call__
ret = func(*args)
File "/content/models/research/object_detection/metrics/coco_evaluation.py", line 416, in first_value_func
self._metrics = self.evaluate()
File "/content/models/research/object_detection/metrics/coco_evaluation.py", line 247, in evaluate
coco_wrapped_groundtruth, coco_wrapped_detections, agnostic_mode=False)
File "/content/models/research/object_detection/metrics/coco_tools.py", line 178, in __init__
cocoeval.COCOeval.__init__(self, groundtruth, detections, iouType=iou_type)
File "/usr/local/lib/python3.6/dist-packages/pycocotools/cocoeval.py", line 76, in __init__
self.params = Params(iouType=iouType) # parameters
File "/usr/local/lib/python3.6/dist-packages/pycocotools/cocoeval.py", line 527, in __init__
self.setDetParams()
File "/usr/local/lib/python3.6/dist-packages/pycocotools/cocoeval.py", line 507, in setDetParams
self.iouThrs = np.linspace(.5, 0.95, np.round((0.95 - .5) / .05) + 1, endpoint=True)
File "<__array_function__ internals>", line 6, in linspace
File "/usr/local/lib/python3.6/dist-packages/numpy/core/function_base.py", line 121, in linspace
.format(type(num)))
TypeError: object of type <class 'numpy.float64'> cannot be safely interpreted as an integer.
[[node PyFunc_3 (defined at tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py:1748) ]]
0 successful operations.
0 derived errors ignored.
Original stack trace for 'PyFunc_3':
File "content/models/research/object_detection/model_main.py", line 114, in <module>
tf.app.run()
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/platform/app.py", line 40, in run
_run(main=main, argv=argv, flags_parser=_parse_flags_tolerate_undef)
File "usr/local/lib/python3.6/dist-packages/absl/app.py", line 299, in run
_run_main(main, args)
File "usr/local/lib/python3.6/dist-packages/absl/app.py", line 250, in _run_main
sys.exit(main(argv))
File "content/models/research/object_detection/model_main.py", line 110, in main
tf.estimator.train_and_evaluate(estimator, train_spec, eval_specs[0])
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 473, in train_and_evaluate
return executor.run()
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 613, in run
return self.run_local()
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 714, in run_local
saving_listeners=saving_listeners)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 370, in train
loss = self._train_model(input_fn, hooks, saving_listeners)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1161, in _train_model
return self._train_model_default(input_fn, hooks, saving_listeners)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1195, in _train_model_default
saving_listeners)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1494, in _train_with_estimator_spec
_, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 754, in run
run_metadata=run_metadata)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1259, in run
run_metadata=run_metadata)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1345, in run
return self._sess.run(*args, **kwargs)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/training/monitored_session.py", line 1426, in run
run_metadata=run_metadata))
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/training/basic_session_run_hooks.py", line 594, in after_run
if self._save(run_context.session, global_step):
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/training/basic_session_run_hooks.py", line 619, in _save
if l.after_save(session, step):
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 519, in after_save
self._evaluate(global_step_value) # updates self.eval_result
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 539, in _evaluate
self._evaluator.evaluate_and_export())
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/training.py", line 920, in evaluate_and_export
hooks=self._eval_spec.hooks)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 480, in evaluate
name=name)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 522, in _actual_eval
return _evaluate()
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 504, in _evaluate
self._evaluate_build_graph(input_fn, hooks, checkpoint_path))
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1511, in _evaluate_build_graph
self._call_model_fn_eval(input_fn, self.config))
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1547, in _call_model_fn_eval
features, labels, ModeKeys.EVAL, config)
File "tensorflow-1.15.2/python3.6/tensorflow_estimator/python/estimator/estimator.py", line 1149, in _call_model_fn
model_fn_results = self._model_fn(features=features, **kwargs)
File "content/models/research/object_detection/model_lib.py", line 570, in model_fn
eval_config, list(category_index.values()), eval_dict)
File "content/models/research/object_detection/eval_util.py", line 1045, in get_eval_metric_ops_for_evaluators
eval_dict))
File "content/models/research/object_detection/metrics/coco_evaluation.py", line 426, in get_estimator_eval_metric_ops
first_value_op = tf.py_func(first_value_func, [], tf.float32)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/util/deprecation.py", line 324, in new_func
return func(*args, **kwargs)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/script_ops.py", line 513, in py_func
return py_func_common(func, inp, Tout, stateful, name=name)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/script_ops.py", line 495, in py_func_common
func=func, inp=inp, Tout=Tout, stateful=stateful, eager=False, name=name)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/script_ops.py", line 318, in _internal_py_func
input=inp, token=token, Tout=Tout, name=name)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/ops/gen_script_ops.py", line 170, in py_func
"PyFunc", input=input, token=token, Tout=Tout, name=name)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/op_def_library.py", line 794, in _apply_op_helper
op_def=op_def)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/util/deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py", line 3357, in create_op
attrs, op_def, compute_device)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py", line 3426, in _create_op_internal
op_def=op_def)
File "tensorflow-1.15.2/python3.6/tensorflow_core/python/framework/ops.py", line 1748, in __init__
self._traceback = tf_stack.extract_stack()
Каковы возможности решения этой проблемы, какие-либо рекомендации?