You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@beam.apache.org by "Ankur Goenka (JIRA)" <ji...@apache.org> on 2019/01/15 21:28:00 UTC
[jira] [Updated] (BEAM-6258) Data channel failing after some time
for 1G data input
[ https://issues.apache.org/jira/browse/BEAM-6258?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Ankur Goenka updated BEAM-6258:
-------------------------------
Attachment: d44b7eda9e4c_python_client_logs.log.bz2
> Data channel failing after some time for 1G data input
> ------------------------------------------------------
>
> Key: BEAM-6258
> URL: https://issues.apache.org/jira/browse/BEAM-6258
> Project: Beam
> Issue Type: Bug
> Components: sdk-py-harness
> Reporter: Ankur Goenka
> Assignee: Ankur Goenka
> Priority: Major
> Attachments: d44b7eda9e4c_python_client_logs.log.bz2
>
> Time Spent: 1h 50m
> Remaining Estimate: 0h
>
> Data channel and logging channel are failing after some time with 1GB input data for chicago taxi.
>
> E1218 02:44:02.837680206 72 chttp2_transport.cc:1148] Received a GOAWAY with error code ENHANCE_YOUR_CALM and debug data equal to "too_many_pings"
> Exception in thread read_grpc_client_inputs:
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/threading.py", line 801, in __bootstrap_inner
> self.run()
> File "/usr/local/lib/python2.7/threading.py", line 754, in run
> self.__target(*self.__args, **self.__kwargs)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 273, in <lambda>
> target=lambda: self._read_inputs(elements_iterator),
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 260, in _read_inputs
> for elements in elements_iterator:
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in next
> return self._next()
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in _next
> raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 145, in _execute
> response = task()
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 180, in <lambda>
> self._execute(lambda: worker.do_instruction(work), work)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 253, in do_instruction
> request.instruction_id)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 269, in process_bundle
> bundle_processor.process_bundle(instruction_id)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 481, in process_bundle
> instruction_id, expected_targets):
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 209, in input_elements
> raise_(t, v, tb)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 260, in _read_inputs
> for elements in elements_iterator:
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in next
> return self._next()
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in _next
> raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 145, in _execute
> response = task()
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 180, in <lambda>
> self._execute(lambda: worker.do_instruction(work), work)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 253, in do_instruction
> request.instruction_id)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 269, in process_bundle
> bundle_processor.process_bundle(instruction_id)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 481, in process_bundle
> instruction_id, expected_targets):
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 209, in input_elements
> raise_(t, v, tb)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 260, in _read_inputs
> for elements in elements_iterator:
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in next
> return self._next()
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in _next
> raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 145, in _execute
> response = task()
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 180, in <lambda>
> self._execute(lambda: worker.do_instruction(work), work)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 253, in do_instruction
> request.instruction_id)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 269, in process_bundle
> bundle_processor.process_bundle(instruction_id)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 481, in process_bundle
> instruction_id, expected_targets):
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 209, in input_elements
> raise_(t, v, tb)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 260, in _read_inputs
> for elements in elements_iterator:
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in next
> return self._next()
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in _next
> raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
> Traceback (most recent call last):
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 145, in _execute
> response = task()
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 180, in <lambda>
> self._execute(lambda: worker.do_instruction(work), work)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 253, in do_instruction
> request.instruction_id)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/sdk_worker.py", line 269, in process_bundle
> bundle_processor.process_bundle(instruction_id)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/bundle_processor.py", line 481, in process_bundle
> instruction_id, expected_targets):
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 209, in input_elements
> raise_(t, v, tb)
> File "/usr/local/lib/python2.7/site-packages/apache_beam/runners/worker/data_plane.py", line 260, in _read_inputs
> for elements in elements_iterator:
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 347, in next
> return self._next()
> File "/usr/local/lib/python2.7/site-packages/grpc/_channel.py", line 338, in _next
> raise self
> _Rendezvous: <_Rendezvous of RPC that terminated with (StatusCode.RESOURCE_EXHAUSTED, GOAWAY received)>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)