Sorry to be a bother. I am running the train_default.sh using the tfrecord file in the google drive link. However, when the training progress goes to 195 steps, an error occured.
I am confused about this error, is there anything wrong with the tfrecord file?
Traceback (most recent call last):
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1334, in _do_call
return fn(*args)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1319, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1407, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.OutOfRangeError: FIFOQueue '_3_create_inputs/target_queue_1/fifo_queue' is closed and has insufficient elements (requested 4, current size 3)
[[{{node create_inputs/target_queue_1}}]]
[[{{node sparse_softmax_cross_entropy_loss/assert_broadcastable/AssertGuard/Assert/data_1}}]]
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "train_ad_tamper_aug_vgg16_fusion_2D.py", line 884, in <module>
main()
File "train_ad_tamper_aug_vgg16_fusion_2D.py", line 867, in main
D_loss_value, _ = sess.run([D_loss, D_train_op], feed_dict=feed_dict)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 929, in run
run_metadata_ptr)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1152, in _run
feed_dict_tensor, options, run_metadata)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1328, in _do_run
run_metadata)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1348, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.OutOfRangeError: FIFOQueue '_3_create_inputs/target_queue_1/fifo_queue' is closed and has insufficient elements (requested 4, current size 3)
[[node create_inputs/target_queue_1 (defined at train_ad_tamper_aug_vgg16_fusion_2D.py:202) ]]
[[node sparse_softmax_cross_entropy_loss/assert_broadcastable/AssertGuard/Assert/data_1 (defined at train_ad_tamper_aug_vgg16_fusion_2D.py:648) ]]
Caused by op 'create_inputs/target_queue_1', defined at:
File "train_ad_tamper_aug_vgg16_fusion_2D.py", line 884, in <module>
main()
File "train_ad_tamper_aug_vgg16_fusion_2D.py", line 435, in main
target_gt_batch, target_batch,target_label_batch, target_image_ids, target_distort_batch, _ =build_input(args, is_authentic=False, queue_name='target_queue',input_queue_name='target_input_queue')
File "train_ad_tamper_aug_vgg16_fusion_2D.py", line 202, in build_input
name=queue_name)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/util/deprecation.py", line 324, in new_func
return func(*args, **kwargs)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/training/input.py", line 1186, in batch_join
name=name)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/training/input.py", line 828, in _batch_join
dequeued = queue.dequeue_many(batch_size, name=name)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/data_flow_ops.py", line 488, in dequeue_many
self._queue_ref, n=n, component_types=self._dtypes, name=name)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/gen_data_flow_ops.py", line 3645, in queue_dequeue_many_v2
timeout_ms=timeout_ms, name=name)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py", line 788, in _apply_op_helper
op_def=op_def)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/util/deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 3300, in create_op
op_def=op_def)
File "/usr/local/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 1801, in __init__
self._traceback = tf_stack.extract_stack()
OutOfRangeError (see above for traceback): FIFOQueue '_3_create_inputs/target_queue_1/fifo_queue' is closed and has insufficient elements (requested 4, current size 3)
[[node create_inputs/target_queue_1 (defined at train_ad_tamper_aug_vgg16_fusion_2D.py:202) ]]
[[node sparse_softmax_cross_entropy_loss/assert_broadcastable/AssertGuard/Assert/data_1 (defined at train_ad_tamper_aug_vgg16_fusion_2D.py:648) ]]