You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@mesos.apache.org by "Eugene Paniot (JIRA)" <ji...@apache.org> on 2018/06/21 15:29:00 UTC
[jira] [Created] (MESOS-9018) mesos-docker-executor killed by
SIGABRTduring httpCheck
Eugene Paniot created MESOS-9018:
------------------------------------
Summary: mesos-docker-executor killed by SIGABRTduring httpCheck
Key: MESOS-9018
URL: https://issues.apache.org/jira/browse/MESOS-9018
Project: Mesos
Issue Type: Bug
Environment: {code}
MESOS_AGENT_ENDPOINT=10.62.88.21:5051
MESOS_CHECKPOINT=1
MESOS_DIRECTORY=/var/lib/mesos/slaves/93faf747-6aa5-47fb-82a5-1e5fff90e5a2-S0/frameworks/1da204e4-59c6-4884-b6f0-db3bacfa9e67-0000/executors/***********.f9666e7b-7560-11e8-959c-0050569f9f19/runs/232a8444-d6bf-4221-b0c7-fe126f051867
MESOS_EXECUTOR_ID=***********.f9666e7b-7560-11e8-959c-0050569f9f19
MESOS_EXECUTOR_SHUTDOWN_GRACE_PERIOD=3secs
MESOS_FRAMEWORK_ID=1da204e4-59c6-4884-b6f0-db3bacfa9e67-0000
MESOS_HTTP_COMMAND_EXECUTOR=0
MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos-1.6.0.so
MESOS_NATIVE_LIBRARY=/usr/lib/libmesos-1.6.0.so
MESOS_RECOVERY_TIMEOUT=3mins
MESOS_SLAVE_ID=93faf747-6aa5-47fb-82a5-1e5fff90e5a2-S0
MESOS_SLAVE_PID="slave(1)@10.62.88.21:5051"
MESOS_SUBSCRIPTION_BACKOFF_MAX=2secs
MESOS_TASK_ID=***********.f9666e7b-7560-11e8-959c-0050569f9f19
{code}
Reporter: Eugene Paniot
mesos-docker-executor killed by SIGABRTduring httpCheck if main container PID already exited
{code}
id 6375ca914f926f1aa4b1dfd10a74da77b2f253a7
reason: mesos-docker-executor killed by SIGABRT
time: Thu 21 Jun 2018 02:40:35 PM UTC
cmdline: mesos-docker-executor --cgroups_enable_cfs=false --container=mesos-232a8444-d6bf-4221-b0c7-fe126f051867 --docker=docker --docker_socket=/var/run/docker.sock --help=false --initialize_driver_logging=true --launcher_dir=/usr/libexec/mesos --logbufsecs=0 --logging_level=INFO --mapped_directory=/mnt/mesos/sandbox --quiet=false --sandbox_directory=/var/lib/mesos/slaves/93faf747-6aa5-47fb-82a5-1e5fff90e5a2-S0/frameworks/1da204e4-59c6-4884-b6f0-db3bacfa9e67-0000/executors/***********.f9666e7b-7560-11e8-959c-0050569f9f19/runs/232a8444-d6bf-4221-b0c7-fe126f051867 --stop_timeout=0ns
package: mesos-1.6.0-2.0.4
uid: 0 (root)
count: 6
Directory: /var/log/crash/abrt/ccpp-2018-06-21-14:40:35-2086701
{code}
{code}
[root@ams02-v01-ccs05 232a8444-d6bf-4221-b0c7-fe126f051867]# ll
total 9268
-rw------- 1 root root 356823040 Jun 21 14:40 core.2086701
-rw-r--r-- 1 root root 305 Jun 21 14:39 docker.tar.gz
-rw-r--r-- 1 root root 12762 Jun 21 14:40 stderr
-rw-r--r-- 1 root root 254 Jun 21 14:40 stderr.logrotate.conf
-rw-r--r-- 1 root root 8574 Jun 21 14:40 stdout
-rw-r--r-- 1 root root 254 Jun 21 14:40 stdout.logrotate.conf
cat /var/lib/mesos/slaves/93faf747-6aa5-47fb-82a5-1e5fff90e5a2-S0/frameworks/1da204e4-59c6-4884-b6f0-db3bacfa9e67-0000/executors/***********.f9666e7b-7560-11e8-959c-0050569f9f19/runs/232a8444-d6bf-4221-b0c7-fe126f051867/stderr
...
% containerized app logs %
...
+ rc=1
+ echo Exit
+ exit 1
F0621 14:40:35.647912 2086701 checker_process.cpp:114] Failed to enter the net namespace of task (pid: 2085528): Pid 2085528 does not exist
*** Check failure stack trace: ***
@ 0x7f8507e0422d google::LogMessage::Fail()
@ 0x7f8507e0607d google::LogMessage::SendToLog()
@ 0x7f8507e03e13 google::LogMessage::Flush()
@ 0x7f8507e06a79 google::LogMessageFatal::~LogMessageFatal()
@ 0x7f8506c13fd1 mesos::internal::checks::cloneWithSetns()
@ 0x7f8507db0754 process::internal::cloneChild()
@ 0x7f8507dae603 process::subprocess()
@ 0x7f8506c1a1cf mesos::internal::checks::CheckerProcess::_httpCheck()
@ 0x7f8506c1b22f mesos::internal::checks::CheckerProcess::httpCheck()
@ 0x7f8506c22c36 mesos::internal::checks::CheckerProcess::performCheck()
@ 0x7f8507d4d9d1 process::ProcessBase::consume()
@ 0x7f8507d6d4fa process::ProcessManager::resume()
@ 0x7f8507d70f96 _ZNSt6thread5_ImplISt12_Bind_simpleIFZN7process14ProcessManager12init_threadsEvEUlvE_vEEE6_M_runEv
@ 0x7f8507f75ff0 execute_native_thread_routine
@ 0x7f8505321e25 start_thread
@ 0x7f8504b3134d __clone
{code}
{code}
/var/log/crash/abrt/ccpp-2018-06-21-14:40:35-2086701/core_backtrace
{ "signal": 6
, "executable": "/usr/libexec/mesos/mesos-docker-executor"
, "stacktrace":
[ { "crash_thread": true
, "frames":
[ { "address": 140209285423607
, "build_id": "95ff02a4bebabc573c7827a66d447f7babddaa44"
, "build_id_offset": 217591
, "function_name": "raise"
, "file_name": "/usr/lib64/libc-2.17.so"
}
, { "address": 140209285429480
, "build_id": "95ff02a4bebabc573c7827a66d447f7babddaa44"
, "build_id_offset": 223464
, "function_name": "abort"
, "file_name": "/usr/lib64/libc-2.17.so"
}
, { "address": 140209339548889
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 42820825
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209339515437
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 42787373
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209339523197
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 42795133
, "function_name": "google::LogMessage::SendToLog()"
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209339514387
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 42786323
, "function_name": "google::LogMessage::Flush()"
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209339525753
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 42797689
, "function_name": "google::LogMessageFatal::~LogMessageFatal()"
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209320706001
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 23977937
, "function_name": "mesos::internal::checks::cloneWithSetns(std::function<int ()> const&, Option<int> const&, std::vector<std::string, std::allocator<std::string> > const&)"
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209339172692
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 42444628
, "function_name": "process::internal::cloneChild(std::string const&, std::vector<std::string, std::allocator<std::string> >, Option<std::map<std::string, std::string, std::less<std::string>, std::allocator<std::pair<std::string const, std::string> > > > const&, Option<std::function<int (std::function<int ()> const&)> > const&, std::vector<process::Subprocess::ParentHook, std::allocator<process::Subprocess::ParentHook> > const&, std::vector<process::Subprocess::ChildHook, std::allocator<process::Subprocess::ChildHook> > const&, process::Subprocess::IO::InputFileDescriptors, process::Subprocess::IO::OutputFileDescriptors, process::Subprocess::IO::OutputFileDescriptors)"
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209339164163
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 42436099
, "function_name": "process::subprocess(std::string const&, std::vector<std::string, std::allocator<std::string> >, process::Subprocess::IO const&, process::Subprocess::IO const&, process::Subprocess::IO const&, flags::FlagsBase const*, Option<std::map<std::string, std::string, std::less<std::string>, std::allocator<std::pair<std::string const, std::string> > > > const&, Option<std::function<int (std::function<int ()> const&)> > const&, std::vector<process::Subprocess::ParentHook, std::allocator<process::Subprocess::ParentHook> > const&, std::vector<process::Subprocess::ChildHook, std::allocator<process::Subprocess::ChildHook> > const&)"
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209320731087
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 24003023
, "function_name": "mesos::internal::checks::CheckerProcess::_httpCheck(std::vector<std::string, std::allocator<std::string> > const&, Option<mesos::internal::checks::runtime::Plain> const&)"
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209320735279
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 24007215
, "function_name": "mesos::internal::checks::CheckerProcess::httpCheck(mesos::internal::checks::check::Http const&, Option<mesos::internal::checks::runtime::Plain> const&)"
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209320766518
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 24038454
, "function_name": "mesos::internal::checks::CheckerProcess::performCheck() [clone .part.1516] [clone .constprop.1571]"
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209338767825
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 42039761
, "function_name": "process::ProcessBase::consume(process::DispatchEvent&&)"
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209338897658
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 42169594
, "function_name": "process::ProcessManager::resume(process::ProcessBase*)"
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209338912662
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 42184598
, "function_name": "std::thread::_Impl<std::_Bind_simple<process::ProcessManager::init_threads()::{lambda()#1} ()> >::_M_run()"
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209341030384
, "build_id": "665d93a2c60a1092d0079f6aaee83d2b754b05cf"
, "build_id_offset": 44302320
, "function_name": "execute_native_thread_routine"
, "file_name": "/usr/lib/libmesos-1.6.0.so"
}
, { "address": 140209294548517
, "build_id": "a48d21b2578a8381fbd8857802eaa660504248dc"
, "build_id_offset": 32293
, "function_name": "start_thread"
, "file_name": "/usr/lib64/libpthread-2.17.so"
}
, { "address": 140209286222669
, "build_id": "95ff02a4bebabc573c7827a66d447f7babddaa44"
, "build_id_offset": 1016653
, "function_name": "__clone"
, "file_name": "/usr/lib64/libc-2.17.so"
} ]
} ]
}
{code}
{code}
# rpm -qa mesos
mesos-1.6.0-2.0.4.x86_64
{code}
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)