You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kudu.apache.org by "Matthew Jacobs (JIRA)" <ji...@apache.org> on 2016/11/30 03:39:58 UTC
[jira] [Created] (KUDU-1773) Kudu client DCHECK fails
Matthew Jacobs created KUDU-1773:
------------------------------------
Summary: Kudu client DCHECK fails
Key: KUDU-1773
URL: https://issues.apache.org/jira/browse/KUDU-1773
Project: Kudu
Issue Type: Bug
Components: client
Affects Versions: 1.1.0
Reporter: Matthew Jacobs
Priority: Critical
Impalad running stress tests w/ debug libkudu_client.so gets the following DCHECK failure:
F1129 19:32:42.075459 185566 meta_cache.cc:267] Check failed: found Tablet e0eea4076a6843749389956abfed3172: Specified server not found: 520ffc6d07ad42baaf49e59f7129b080 (ve1121.halxg.cloudera.com:7050). Replicas: 275ece6d98a14be9b7dfcee3bec8d7a8 (FOLLOWER, OK), 2d52a82cb62c4064842cf628d01083fe (FOLLOWER, OK), c101883d3e82496989a5f9f667c30e38 (FOLLOWER, OK)
stack from gdb dump:
{code}
#0 0x0000003fdd232625 in raise () from /lib64/libc.so.6
#1 0x0000003fdd233e05 in abort () from /lib64/libc.so.6
#2 0x000000000281f3b4 in ?? ()
#3 0x000000000281881d in google::LogMessage::Fail() ()
#4 0x000000000281b146 in google::LogMessage::SendToLog() ()
#5 0x000000000281833d in google::LogMessage::Flush() ()
#6 0x0000000002818619 in google::LogMessage::~LogMessage() ()
#7 0x00000000011c902f in impala::LogKuduMessage(void*, kudu::client::KuduLogSeverity, char const*, int, tm const*, char const*, unsigned long) ()
#8 0x00000000011c9423 in kudu::client::KuduLoggingFunctionCallback<void*>::Run(kudu::client::KuduLogSeverity, char const*, int, tm const*, char const*, unsigned long) ()
#9 0x00007eff1b24073e in kudu::client::LoggingAdapterCB(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/client/client.cc:163
#10 0x00007eff1b25863c in kudu::internal::RunnableAdapter<void (*)(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long)>::Run(kudu::client::KuduLoggingCallback* const&, kudu::LogSeverity const&, char const* const&, int const&, tm const* const&, char const* const&, unsigned long const&) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/gutil/bind_internal.h:584
#11 0x00007eff1b2568fe in kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (*)(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long)>, void ()(kudu::client::KuduLoggingCallback*, kudu::LogSeverity const&, char const* const&, int const&, tm const* const&, char const* const&, unsigned long const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (*)(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long)>, kudu::client::KuduLoggingCallback*, kudu::LogSeverity const&, char const* const&, int const&, tm const* const&, char const* const&, unsigned long const&) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/gutil/bind_internal.h:990
#12 0x00007eff1b2540e3 in kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (*)(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long)>, void ()(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long), void ()(kudu::internal::UnretainedWrapper<kudu::client::KuduLoggingCallback>)>, void ()(kudu::client::KuduLoggingCallback*, kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long)>::Run(kudu::internal::BindStateBase*, kudu::LogSeverity const&, char const* const&, int const&, tm const* const&, char const* const&, unsigned long const&) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/gutil/bind_internal.h:2114
#13 0x00007eff1b3d1f53 in kudu::Callback<void ()(kudu::LogSeverity, char const*, int, tm const*, char const*, unsigned long)>::Run(kudu::LogSeverity const&, char const* const&, int const&, tm const* const&, char const* const&, unsigned long const&) const ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/gutil/callback.h:683
#14 0x00007eff1b3d08bc in kudu::(anonymous namespace)::SimpleSink::send(int, char const*, char const*, int, tm const*, char const*, unsigned long) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/util/logging.cc:81
#15 0x00007eff1b4ab4a7 in google::LogMessage::SendToLog() ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/glog-0.3.4/src/logging.cc:784
#16 0x00007eff1b4a8df9 in google::LogMessage::Flush() ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/glog-0.3.4/src/logging.cc:1301
#17 0x00007eff1b4ac3ef in google::LogMessageFatal::~LogMessageFatal() ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/glog-0.3.4/src/logging.cc:2013
#18 0x00007eff1b27b3bf in kudu::client::internal::RemoteTablet::MarkTServerAsLeader(kudu::client::internal::RemoteTabletServer const*) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/client/meta_cache.cc:267
#19 0x00007eff1b27c9f9 in kudu::client::internal::MetaCacheServerPicker::PickLeader(kudu::Callback<void ()(kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&, kudu::MonoTime const&) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/client/meta_cache.cc:413
#20 0x00007eff1b237aba in kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer, kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>::SendRpc() ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/rpc/retriable_rpc.h:130
#21 0x00007eff1b2f5531 in kudu::rpc::RpcRetrier::DelayedRetryCb(kudu::rpc::Rpc*, kudu::Status const&) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/rpc/rpc.cc:86
#22 0x00007eff1b2f5d68 in boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>::operator()(kudu::rpc::RpcRetrier*, kudu::rpc::Rpc*, kudu::Status const&) const ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:280
#23 0x00007eff1b2f5c38 in void boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> >::operator()<boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::rrlist1<kudu::Status const&> >(boost::_bi::type<void>, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>&, boost::_bi::rrlist1<kudu::Status const&>&, int) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/bind.hpp:398
#24 0x00007eff1b2f59f2 in void boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> > >::operator()<kudu::Status const&>(kudu::Status const&&&) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/bind.hpp:1234
#25 0x00007eff1b2f5922 in boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> > >, void, kudu::Status const&>::invoke(boost::detail::function::function_buffer&, kudu::Status const&) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/function/function_template.hpp:159
#26 0x00007eff1b2ed176 in boost::function1<void, kudu::Status const&>::operator()(kudu::Status const&) const ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/function/function_template.hpp:771
#27 0x00007eff1b2ea18d in kudu::rpc::DelayedTask::TimerHandler(ev::timer&, int) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/rpc/reactor.cc:490
#28 0x00007eff1b2ee554 in void ev::base<ev_timer, ev::timer>::method_thunk<kudu::rpc::DelayedTask, &(kudu::rpc::DelayedTask::TimerHandler(ev::timer&, int))>(ev_loop*, ev_timer*, int) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/uninstrumented/include/ev++.h:479
#29 0x00007eff1b51a4cb in ev_invoke_pending ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/libev-4.20/ev.c:3155
#30 0x00007eff1b51dbd4 in ev_run ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/src/libev-4.20/ev.c:3555
#31 0x00007eff1b2eb2df in ev::loop_ref::run(int) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/uninstrumented/include/ev++.h:211
#32 0x00007eff1b2e8280 in kudu::rpc::ReactorThread::RunThread() ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/rpc/reactor.cc:317
#33 0x00007eff1b2f3547 in boost::_mfi::mf0<void, kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:49
#34 0x00007eff1b2f3054 in void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>&, boost::_bi::list0&, int) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/bind.hpp:259
#35 0x00007eff1b2f271b in boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >::operator()() ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/bind/bind.hpp:1222
#36 0x00007eff1b2f1c20 in boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >, void>::invoke(boost::detail::function::function_buffer&) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/function/function_template.hpp:159
#37 0x00007eff1b315f3e in boost::function0<void>::operator()() const ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/thirdparty/installed/common/include/boost/function/function_template.hpp:771
#38 0x00007eff1b413434 in kudu::Thread::SuperviseThread(void*) ()
at /data/jenkins/workspace/verify-impala-toolchain-package-build/label/ec2-package-centos-6/toolchain/source/kudu/kudu-f2aeba6c059ea61f9cf8984d3e84d6c27b64d463/src/kudu/util/thread.cc:588
#39 0x0000003fdd607aa1 in start_thread () from /lib64/libpthread.so.0
#40 0x0000003fdd2e893d in clone () from /lib64/libc.so.6
{code}
Kudu client built at git hash f2aeba
Kudu tserver/master is even more recent (1.2 snapshot), git hash 2f1a2a06d4fe719ef9f35338bc7403915517718a
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)