You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@aurora.apache.org by "Zameer Manji (JIRA)" <ji...@apache.org> on 2017/01/17 23:15:26 UTC
[jira] [Commented] (AURORA-1799) Thermos does not handle low memory
scenarios gracefully
[ https://issues.apache.org/jira/browse/AURORA-1799?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15827017#comment-15827017 ]
Zameer Manji commented on AURORA-1799:
--------------------------------------
Today [~benley] reported something similar in Slack:
{noformat}
ERROR] Failed to stop health checkers:
ERROR] Traceback (most recent call last):
File "apache/aurora/executor/aurora_executor.py", line 192, in _shutdown
propagate_deadline(self._chained_checker.stop, timeout=self.STOP_TIMEOUT)
File "apache/aurora/executor/aurora_executor.py", line 35, in propagate_deadline
return deadline(*args, daemon=True, propagate=True, **kw)
File "/root/.pex/install/twitter.common.concurrent-0.3.3-py2-none-any.whl.33d9c24da69d7478b4aa6d76f474f3773a61f6f9/twitter.common.concurrent-0.3.3-py2-none-any.whl/twitter/common/concurrent/dead
line.py", line 61, in deadline
AnonymousThread().start()
File "/usr/lib/python2.7/threading.py", line 745, in start
_start_new_thread(self.__bootstrap, ())
error: can't start new thread
ERROR] Failed to stop runner:
ERROR] Traceback (most recent call last):
File "apache/aurora/executor/aurora_executor.py", line 200, in _shutdown
propagate_deadline(self._runner.stop, timeout=self.STOP_TIMEOUT)
File "apache/aurora/executor/aurora_executor.py", line 35, in propagate_deadline
return deadline(*args, daemon=True, propagate=True, **kw)
File "/root/.pex/install/twitter.common.concurrent-0.3.3-py2-none-any.whl.33d9c24da69d7478b4aa6d76f474f3773a61f6f9/twitter.common.concurrent-0.3.3-py2-none-any.whl/twitter/common/concurrent/dead
line.py", line 61, in deadline
AnonymousThread().start()
File "/usr/lib/python2.7/threading.py", line 745, in start
_start_new_thread(self.__bootstrap, ())
error: can't start new thread
Traceback (most recent call last):
File "/root/.pex/install/twitter.common.exceptions-0.3.3-py2-none-any.whl.57572b1f0a301c36c91adf2c704d0e8dd4d48429/twitter.common.exceptions-0.3.3-py2-none-any.whl/twitter/common/exceptions/__in
it__.py", line 126, in _excepting_run
self.__real_run(*args, **kw)
File "apache/aurora/executor/status_manager.py", line 50, in run
File "apache/aurora/executor/aurora_executor.py", line 218, in _shutdown
File "/root/.pex/install/twitter.common.concurrent-0.3.3-py2-none-any.whl.33d9c24da69d7478b4aa6d76f474f3773a61f6f9/twitter.common.concurrent-0.3.3-py2-none-any.whl/twitter/common/concurrent/defe
rred.py", line 56, in defer
deferred.start()
File "/usr/lib/python2.7/threading.py", line 745, in start
_start_new_thread(self.__bootstrap, ())
thread.error: can't start new thread
Traceback (most recent call last):
File "/root/.pex/install/twitter.common.exceptions-0.3.3-py2-none-any.whl.57572b1f0a301c36c91adf2c704d0e8dd4d48429/twitter.common.exceptions-0.3.3-py2-none-any.whl/twitter/common/exceptions/__in
it__.py", line 126, in _excepting_run
self.__real_run(*args, **kw)
File "apache/thermos/monitoring/resource.py", line 239, in run
File "/root/.pex/install/twitter.common.concurrent-0.3.3-py2-none-any.whl.33d9c24da69d7478b4aa6d76f474f3773a61f6f9/twitter.common.concurrent-0.3.3-py2-none-any.whl/twitter/common/concurrent/even
t_muxer.py", line 79, in wait
thread.start()
File "/usr/lib/python2.7/threading.py", line 745, in start
_start_new_thread(self.__bootstrap, ())
thread.error: can't start new thread
E0116 20:46:46.568775 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
E0116 20:46:51.789016 34 socket.hpp:174] Shutdown failed on fd=14: Transport endpoint is not connected [107]
E0116 20:50:47.904999 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
E0116 20:50:48.097457 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
E0116 20:50:50.277053 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
E0116 20:50:51.006816 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
E0116 20:50:51.022123 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
E0116 20:50:51.244179 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
E0116 20:50:55.407006 34 socket.hpp:174] Shutdown failed on fd=14: Transport endpoint is not connected [107]
E0116 20:50:55.410759 34 socket.hpp:174] Shutdown failed on fd=15: Transport endpoint is not connected [107]
E0116 20:50:56.703348 34 socket.hpp:174] Shutdown failed on fd=14: Transport endpoint is not connected [107]
E0116 20:50:56.707471 34 socket.hpp:174] Shutdown failed on fd=15: Transport endpoint is not connected [107]
E0116 20:50:56.712406 34 socket.hpp:174] Shutdown failed on fd=16: Transport endpoint is not connected [107]
E0116 20:50:57.053045 34 socket.hpp:174] Shutdown failed on fd=14: Transport endpoint is not connected [107]
E0116 20:50:57.379636 34 socket.hpp:174] Shutdown failed on fd=14: Transport endpoint is not connected [107]
E0116 20:50:57.454205 34 socket.hpp:174] Shutdown failed on fd=14: Transport endpoint is not connected [107]
E0116 20:50:57.848105 34 socket.hpp:174] Shutdown failed on fd=14: Transport endpoint is not connected [107]
E0116 20:50:59.661581 34 socket.hpp:174] Shutdown failed on fd=15: Transport endpoint is not connected [107]
E0116 20:51:03.007069 34 socket.hpp:174] Shutdown failed on fd=14: Transport endpoint is not connected [107]
E0116 20:51:08.751930 34 socket.hpp:174] Shutdown failed on fd=14: Transport endpoint is not connected [107]
E0116 20:51:08.833519 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
E0116 20:51:09.091882 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
E0116 20:51:09.166265 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
E0116 20:51:09.275421 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
E0116 20:51:09.322955 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
E0116 20:51:09.434495 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
E0116 20:51:09.495900 34 socket.hpp:174] Shutdown failed on fd=13: Transport endpoint is not connected [107]
{noformat}
> Thermos does not handle low memory scenarios gracefully
> -------------------------------------------------------
>
> Key: AURORA-1799
> URL: https://issues.apache.org/jira/browse/AURORA-1799
> Project: Aurora
> Issue Type: Bug
> Reporter: Zameer Manji
>
> Background:
> In an environment where Aurora is used to launch Docker containers via the DockerContainerizer, it was observed that some tasks would not be killed.
> What happened is that a task was allocated with a low amount of memory but demanded a lot. This caused the linux OOM killer to be invoked. Unlike the MesosContainerizer, the agent doesn't tear down the container when the OOM killer is invoked. Instead the OOM killer just kills a process in the container and thermos and mesos are unaware (unless a process directly launched by thermos is killed).
> I observed in the scheduler logs that the scheduler was trying to kill a container every reconciliation period but it never died. The slave had the logs indicating it received the killTask RPC and forwarded it to Thermos.
> The thermos logs had several entries like every hour:
> {noformat}
> I1018 20:39:18.102894 6 executor_base.py:45] Executor [aaeac4c8-2b2f-4351-874b-a16bea1b36b0-S147]: Activating kill manager.
> I1018 20:39:18.103034 6 executor_base.py:45] Executor [aaeac4c8-2b2f-4351-874b-a16bea1b36b0-S147]: killTask returned.
> I1018 21:39:17.859935 6 executor_base.py:45] Executor [aaeac4c8-2b2f-4351-874b-a16bea1b36b0-S147]: killTask got task_id: value: "<task_id>"
> {noformat}
> However, the tasks was never killed. Looking at the stderr of thermos I saw the following entries:
> {noformat}
> Logged from file resource.py, line 155
> Traceback (most recent call last):
> File "/usr/lib/python2.7/logging/__init__.py", line 883, in emit
> self.flush()
> File "/usr/lib/python2.7/logging/__init__.py", line 843, in flush
> self.stream.flush()
> IOError: [Errno 12] Cannot allocate memory
> {noformat}
> and
> {noformat}
> Logged from file thermos_task_runner.py, line 171
> Traceback (most recent call last):
> File "/root/.pex/install/twitter.common.exceptions-0.3.3-py2-none-any.whl.2a67b833b1517d179ef1c8dc6f2dac1023d51e3c/twitter.common.exceptions-0.3.3-py2-none-any.whl/twitter/common/exceptions/__init__.py", line 126, in _excepting_run
> File "apache/aurora/executor/status_manager.py", line 47, in run
> File "apache/aurora/executor/common/status_checker.py", line 97, in status
> File "apache/aurora/executor/thermos_task_runner.py", line 358, in status
> File "apache/aurora/executor/thermos_task_runner.py", line 186, in compute_status
> File "apache/aurora/executor/thermos_task_runner.py", line 136, in task_state
> File "apache/thermos/monitoring/monitor.py", line 118, in task_state
> File "apache/thermos/monitoring/monitor.py", line 114, in get_state
> File "apache/thermos/monitoring/monitor.py", line 77, in _apply_states
> File "/root/.pex/install/twitter.common.recordio-0.3.3-py2-none-any.whl.9f1e9394eca1bc33ad7d10ae3025301866824139/twitter.common.recordio-0.3.3-py2-none-any.whl/twitter/common/recordio/recordio.py", line 182, in try_read
> class InvalidTypeException(Error): pass
> File "/root/.pex/install/twitter.common.recordio-0.3.3-py2-none-any.whl.9f1e9394eca1bc33ad7d10ae3025301866824139/twitter.common.recordio-0.3.3-py2-none-any.whl/twitter/common/recordio/recordio.py", line 168, in read
> return RecordIO.Reader.do_read(self._fp, self._codec)
> File "/root/.pex/install/twitter.common.recordio-0.3.3-py2-none-any.whl.9f1e9394eca1bc33ad7d10ae3025301866824139/twitter.common.recordio-0.3.3-py2-none-any.whl/twitter/common/recordio/recordio.py", line 135, in do_read
> header = fp.read(RecordIO.RECORD_HEADER_SIZE)
> File "/root/.pex/install/twitter.common.recordio-0.3.3-py2-none-any.whl.9f1e9394eca1bc33ad7d10ae3025301866824139/twitter.common.recordio-0.3.3-py2-none-any.whl/twitter/common/recordio/filelike.py", line 81, in read
> return self._fp.read(length)
> IOError: [Errno 12] Cannot allocate memory
> {noformat}
> It seems the regular avenues of reading checkpoints or logging data, thermos would get an IOError. Some part of twitter common installs an excepthook to log the exception, but we don't seem to do anything else.
> I think we should probably install our own exception hook to send a {{LOST_TASK}} with the exception information instead of failing to kill the task.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)