You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@beam.apache.org by GitBox <gi...@apache.org> on 2022/06/03 22:53:53 UTC
[GitHub] [beam] kennknowles opened a new issue, #19184: Tensor flow on Flink taking too much time to process data
kennknowles opened a new issue, #19184:
URL: https://github.com/apache/beam/issues/19184
The TFX pipeline is taking too much time to process data. The SDK is looping in the following stacktrace. Also, the CPU usage is 400% which is in alignment with 4 threads that are working.
```
--- Thread #140477558937344 name: Thread-55 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 74,
in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 437, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 125, in process_encoded self.output(decoded_value) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 1448, in process *args, **kwargs)) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 587, in add_inputs accumulator = self.add_input(accumulator, element, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/tensorflow_transform/beam/analyzer_impls.py",
line 264, in add_input summary=tf.constant(summary))) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 905, in run run_metadata_ptr) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1137, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1355, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1361, in _do_call return fn(*args) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1332, in _run_fn self._extend_graph() File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1392, in _extend_graph graph_def.SerializeToString(), status)
```
Complete stack trace
```
--- Thread #140471896581888 name: Thread-153 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140480218113792 name: Thread-12 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140471528552192 name: Thread-199 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140476451624704 name: Thread-86 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140471930152704 name: Thread-143 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140471740630784 name: Thread-183 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140479144371968 name: Thread-40 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140479152764672 name: Thread-34 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140472399947520 name: Thread-122 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140475956717312 name: Thread-88 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140479689635584 name: Thread-15 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140471804360448 name: Thread-150 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140472391554816 name: Thread-126 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140474602440448 name: Thread-214 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140480226506496 name: Thread-6 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 191, in consume_request_iterator
request = next(request_iterator) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/log_handler.py",
line 111, in _write_log_entries log_entries = [self._log_entry_queue.get()] File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140476501980928 name: Thread-98 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471731975936 name: Thread-162 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140476996888320 name: Thread-92 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471692900096 name: Thread-167 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140472383162112 name: Thread-128 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140477038851840 name: Thread-74 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140477533759232 name: Thread-120 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140477558937344 name: Thread-55 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 437, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 125, in process_encoded self.output(decoded_value) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 1448, in process *args, **kwargs)) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 587, in add_inputs accumulator = self.add_input(accumulator, element, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/tensorflow_transform/beam/analyzer_impls.py",
line 264, in add_input summary=tf.constant(summary))) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 905, in run run_metadata_ptr) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1137, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1355, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1361, in _do_call return fn(*args) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1332, in _run_fn self._extend_graph() File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1392, in _extend_graph graph_def.SerializeToString(), status) --- Thread #140477575722752 name:
Thread-48 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner()
File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py",
line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140472374769408 name: Thread-129 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478070630144 name: Thread-52 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471603033856 name: Thread-181 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478112593664 name: Thread-41 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471563171584 name: Thread-178 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478607501056 name: Thread-29 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140472366376704 name: Thread-130 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471493146368 name: Thread-220 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140474594047744 name: Thread-208 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140472357984000 name: Thread-138 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) --- Thread #140471904974592 name: Thread-146 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140472005687040 name: Thread-137 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479681242880 name: Thread-16 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471795967744 name: Thread-151 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471749023488 name: Thread-176 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479723206400 name: Thread-11 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 191, in consume_request_iterator request = next(request_iterator) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 104, in get_responses response = self._responses.get() File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471972116224 name: Thread-136 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471997294336 name: Thread-133 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140476988495616 name: Thread-76 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471783118592 name: Thread-155 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471723583232 name: Thread-168 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479186335488 name: Thread-18 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 191, in consume_request_iterator request = next(request_iterator) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 244, in _write_outputs data = [self._to_send.get()] File "/usr/local/lib/python2.7/Queue.py", line
168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140477030459136 name: Thread-68 --- File "/usr/local/lib/python2.7/threading.py", line 774,
in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140471510718208 name: Thread-191 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py", line 72,
in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py", line 168,
in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait waiter.acquire()
--- Thread #140471988901632 name: Thread-134 --- File "/usr/local/lib/python2.7/threading.py", line
774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line 801,
in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run self.__target(*self.__args,
**self.__kwargs) --- Thread #140476485195520 name: Thread-80 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 437, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 125, in process_encoded self.output(decoded_value) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 1448, in process *args, **kwargs)) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 587, in add_inputs accumulator = self.add_input(accumulator, element, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/tensorflow_transform/beam/analyzer_impls.py",
line 275, in add_input next_stamp_token=self._stamp_token)) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 905, in run run_metadata_ptr) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1137, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1355, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1361, in _do_call return fn(*args) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1332, in _run_fn self._extend_graph() File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1392, in _extend_graph graph_def.SerializeToString(), status) --- Thread #140477567330048 name:
Thread-59 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner()
File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py",
line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479698028288 name: read_state --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 390, in pull_responses for response in responses: File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 366, in next return self._next() File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 348, in _next self._state.condition.wait() File "/usr/local/lib/python2.7/threading.py", line 340,
in wait waiter.acquire() --- Thread #140478062237440 name: Thread-63 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471980508928 name: Thread-135 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140474220279552 name: Thread-114 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) --- Thread #140478104200960 name: Thread-37 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471484229376 name: Thread-209 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478599108352 name: Thread-31 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471938545408 name: Thread-142 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478641071872 name: Thread-25 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471701554944 name: Thread-170 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479135979264 name: Thread-50 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479177942784 name: read_grpc_client_inputs --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 273, in <lambda> target=lambda: self._read_inputs(elements_iterator), File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 260, in _read_inputs for elements in elements_iterator: File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 366, in next return self._next() File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 348, in _next self._state.condition.wait() File "/usr/local/lib/python2.7/threading.py", line 340,
in wait waiter.acquire() --- Thread #140474568869632 name: Thread-115 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140475948324608 name: Thread-105 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140474203494144 name: Thread-117 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140481212552960 name: MainThread --- File "/usr/local/lib/python2.7/runpy.py",
line 174, in _run_module_as_main "__main__", fname, loader, pkg_name) File "/usr/local/lib/python2.7/runpy.py",
line 72, in _run_code exec code in run_globals File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker_main.py",
line 213, in <module> main(sys.argv) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker_main.py",
line 141, in main worker_count=_get_worker_count(sdk_pipeline_options)).run() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 109, in run for work_request in control_stub.Control(get_responses()): File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 366, in next return self._next() File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 348, in _next self._state.condition.wait() File "/usr/local/lib/python2.7/threading.py", line 340,
in wait waiter.acquire() --- Thread #140471913367296 name: Thread-152 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140480209721088 name: Thread-10 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 703, in channel_spin event = state.channel.next_call_event() --- Thread #140471955330816 name:
Thread-140 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner()
File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py",
line 754, in run self.__target(*self.__args, **self.__kwargs) --- Thread #140474195101440 name: Thread-212
--- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner()
File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py",
line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478079022848 name: Thread-44 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 437, in process_bundle ].process_encoded(data.data) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 125, in process_encoded self.output(decoded_value) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 1448, in process *args, **kwargs)) File "/usr/local/lib/python2.7/site-packages/apache_beam/transforms/core.py",
line 587, in add_inputs accumulator = self.add_input(accumulator, element, *args, **kwargs) File "/usr/local/lib/python2.7/site-packages/tensorflow_transform/beam/analyzer_impls.py",
line 275, in add_input next_stamp_token=self._stamp_token)) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 905, in run run_metadata_ptr) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1137, in _run feed_dict_tensor, options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1355, in _do_run options, run_metadata) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1361, in _do_call return fn(*args) File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1332, in _run_fn self._extend_graph() File "/usr/local/lib/python2.7/site-packages/tensorflow/python/client/session.py",
line 1392, in _extend_graph graph_def.SerializeToString(), status) --- Thread #140471577855744 name:
Thread-180 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner()
File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py",
line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471946938112 name: Thread-216 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140477550544640 name: Thread-65 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 74, in _worker work_item.run() File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 62, in run result = self.fn(*self.args, **self.kwargs) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in task self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 132, in _execute response = task() File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 168, in <lambda> self._execute(lambda: worker.do_instruction(work), work) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 219, in do_instruction request.instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py",
line 234, in process_bundle bundle_processor.process_bundle(instruction_id) File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py",
line 434, in process_bundle instruction_id, expected_targets): File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py",
line 205, in input_elements data = received.get(timeout=1) File "/usr/local/lib/python2.7/Queue.py",
line 177, in get self.not_empty.wait(remaining) File "/usr/local/lib/python2.7/threading.py", line 359,
in wait _sleep(delay) --- Thread #140471545337600 name: Thread-184 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471594641152 name: Thread-173 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140474186708736 name: Thread-124 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478095808256 name: Thread-38 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140474585655040 name: Thread-206 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471774725888 name: Thread-157 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140478632679168 name: Thread-46 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140471766071040 name: Thread-158 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) --- Thread #140480260077312 name: Thread-13 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py",
line 703, in channel_spin event = state.channel.next_call_event() --- Thread #140471519373056 name:
Thread-189 --- File "/usr/local/lib/python2.7/threading.py", line 774, in __bootstrap self.__bootstrap_inner()
File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py",
line 754, in run self.__target(*self.__args, **self.__kwargs) File "/usr/local/lib/python2.7/site-packages/concurrent/futures/thread.py",
line 72, in _worker work_item = work_queue.get(block=True) File "/usr/local/lib/python2.7/Queue.py",
line 168, in get self.not_empty.wait() File "/usr/local/lib/python2.7/threading.py", line 340, in wait
waiter.acquire() --- Thread #140479169550080 name: Thread-19 --- File "/usr/local/lib/python2.7/threading.py",
line 774, in __bootstrap self.__bootstrap_inner() File "/usr/local/lib/python2.7/threading.py", line
801, in __bootstrap_inner self.run() File "/usr/local/lib/python2.7/threading.py", line 754, in run
self.__target(*self.__args,
issue truncated because of its length - to see full context, see original Jira
Imported from Jira [BEAM-6029](https://issues.apache.org/jira/browse/BEAM-6029). Original Jira may contain additional context.
Reported by: angoenka.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@beam.apache.org.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org