You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kudu.apache.org by "Adar Dembo (JIRA)" <ji...@apache.org> on 2017/08/19 07:35:00 UTC
[jira] [Updated] (KUDU-2059) Data race in DnsResolver
[ https://issues.apache.org/jira/browse/KUDU-2059?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Adar Dembo updated KUDU-2059:
-----------------------------
Attachment: raft_consensus-itest.txt
Hit this again. I've attached the entire test log, and here are the TSAN warnings:
{noformat}
==================
WARNING: ThreadSanitizer: data race (pid=1150)
Write of size 8 at 0x7b2c00002cb8 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:73 (raft_consensus-itest+0x502d61)
#1 std::__1::default_delete<kudu::ThreadPoolToken>::operator()(kudu::ThreadPoolToken*) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:2397:13 (libkudu_util.so+0x1ce983)
#2 std::__1::unique_ptr<kudu::ThreadPoolToken, std::__1::default_delete<kudu::ThreadPoolToken> >::reset(kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:2603 (libkudu_util.so+0x1ce983)
#3 std::__1::unique_ptr<kudu::ThreadPoolToken, std::__1::default_delete<kudu::ThreadPoolToken> >::~unique_ptr() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:2571 (libkudu_util.so+0x1ce983)
#4 kudu::ThreadPool::~ThreadPool() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:339 (libkudu_util.so+0x1ce983)
#5 kudu::DefaultDeleter<kudu::ThreadPool>::operator()(kudu::ThreadPool*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:145:5 (libmaster.so+0xc274e)
#6 kudu::internal::gscoped_ptr_impl<kudu::ThreadPool, kudu::DefaultDeleter<kudu::ThreadPool> >::~gscoped_ptr_impl() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:228:7 (libmaster.so+0xc2719)
#7 gscoped_ptr<kudu::ThreadPool, kudu::DefaultDeleter<kudu::ThreadPool> >::~gscoped_ptr() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:318:7 (libmaster.so+0xb3789)
#8 kudu::DnsResolver::~DnsResolver() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:45:1 (libkudu_util.so+0x19274a)
#9 kudu::DefaultDeleter<kudu::DnsResolver>::operator()(kudu::DnsResolver*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:145:5 (libkudu_client.so+0xd0dae)
#10 kudu::internal::gscoped_ptr_impl<kudu::DnsResolver, kudu::DefaultDeleter<kudu::DnsResolver> >::reset(kudu::DnsResolver*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:254:7 (libkudu_client.so+0xd0d64)
#11 gscoped_ptr<kudu::DnsResolver, kudu::DefaultDeleter<kudu::DnsResolver> >::reset(kudu::DnsResolver*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:375:46 (libkudu_client.so+0xc5f90)
#12 kudu::client::KuduClient::Data::~Data() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client-internal.cc:343:17 (libkudu_client.so+0xd6add)
#13 kudu::client::KuduClient::~KuduClient() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:334:3 (libkudu_client.so+0xbbfac)
#14 std::__1::default_delete<kudu::client::KuduClient>::operator()(kudu::client::KuduClient*) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:2397:13 (libkudu_client.so+0xd067b)
#15 std::__1::__shared_ptr_pointer<kudu::client::KuduClient*, std::__1::default_delete<kudu::client::KuduClient>, std::__1::allocator<kudu::client::KuduClient> >::__on_zero_shared() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:3795 (libkudu_client.so+0xd067b)
#16 __release_shared /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/libcxx/src/memory.cpp:67:9 (libc++.so.1+0xc095d)
#17 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/libcxx/src/memory.cpp:92 (libc++.so.1+0xc095d)
#18 std::__1::shared_ptr<kudu::client::KuduClient>::~shared_ptr() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:4626:19 (raft_consensus-itest+0x549038)
#19 kudu::TestWorkload::~TestWorkload() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/test_workload.cc:78:1 (libintegration-tests.so+0x867d3)
#20 kudu::tserver::RaftConsensusITest_TestChurnyElections_WithDuplicateKeys_Test::TestBody() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1043:1 (raft_consensus-itest+0x50f9b9)
#21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
#22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
#23 testing::Test::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2474:5 (libgmock.so+0x32b97)
#24 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11 (libgmock.so+0x34036)
#25 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
#26 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
#27 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
#28 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
#29 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
#30 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
#31 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
Previous read of size 8 at 0x7b2c00002cb8 by thread T17:
#0 scoped_refptr<kudu::Histogram>::operator kudu::Histogram* scoped_refptr<kudu::Histogram>::*() const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/ref_counted.h:269:38 (libkrpc.so+0xa34b9)
#1 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:534:7 (libkudu_util.so+0x1cd7f9)
#2 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:452:10 (libkudu_util.so+0x1cf42f)
#3 kudu::ThreadPool::SubmitFunc(boost::function<void ()>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:448:10 (libkudu_util.so+0x1cf4c9)
#4 kudu::DnsResolver::ResolveAddresses(kudu::HostPort const&, std::__1::vector<kudu::Sockaddr, std::__1::allocator<kudu::Sockaddr> >*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:57:21 (libkudu_util.so+0x1927fb)
#5 kudu::client::internal::RemoteTabletServer::InitProxy(kudu::client::KuduClient*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/meta_cache.cc:135:33 (libkudu_client.so+0xf8c53)
#6 kudu::client::internal::MetaCacheServerPicker::PickLeader(kudu::Callback<void (kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&, kudu::MonoTime const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/meta_cache.cc:443:11 (libkudu_client.so+0xfaf0f)
#7 kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer, kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>::SendRpc() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/retriable_rpc.h:144:19 (libkudu_client.so+0xb2a98)
#8 kudu::rpc::RpcRetrier::DelayedRetryCb(kudu::rpc::Rpc*, kudu::Status const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/rpc.cc:90:10 (libkrpc.so+0xd4b0c)
#9 boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>::operator()(kudu::rpc::RpcRetrier*, kudu::rpc::Rpc*, kudu::Status const&) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:280:29 (libkrpc.so+0xd54f7)
#10 void boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> >::operator()<boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::rrlist1<kudu::Status const&> >(boost::_bi::type<void>, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>&, boost::_bi::rrlist1<kudu::Status const&>&, int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:398:9 (libkrpc.so+0xd5432)
#11 void boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> > >::operator()<kudu::Status const&>(kudu::Status const&) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:1234:16 (libkrpc.so+0xd5358)
#12 boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> > >, void, kudu::Status const&>::invoke(boost::detail::function::function_buffer&, kudu::Status const&) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:159:11 (libkrpc.so+0xd50f3)
#13 boost::function1<void, kudu::Status const&>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xc3234)
#14 kudu::rpc::DelayedTask::TimerHandler(ev::timer&, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:628:5 (libkrpc.so+0xc1074)
#15 void ev::base<ev_timer, ev::timer>::method_thunk<kudu::rpc::DelayedTask, &kudu::rpc::DelayedTask::TimerHandler>(ev_loop*, ev_timer*, int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/ev++.h:479:7 (libkrpc.so+0xc7d1a)
#16 ev_invoke_pending /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/libev-4.20/ev.c:3155:11 (libev.so.4+0x9800)
#17 ev_run /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/libev-4.20/ev.c:3555:7 (libev.so.4+0xa90d)
#18 ev::loop_ref::run(int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/ev++.h:211:7 (libkrpc.so+0xc2b08)
#19 kudu::rpc::ReactorThread::RunThread() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:395:9 (libkrpc.so+0xbd46a)
#20 boost::_mfi::mf0<void, kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:49:29 (libkrpc.so+0xc5f86)
#21 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>&, boost::_bi::list0&, int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:259:9 (libkrpc.so+0xc5eda)
#22 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >::operator()() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:1222:16 (libkrpc.so+0xc5e63)
#23 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >, void>::invoke(boost::detail::function::function_buffer&) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:159:11 (libkrpc.so+0xc5c69)
#24 boost::function0<void>::operator()() const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xb7251)
#25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.cc:595:3 (libkudu_util.so+0x1c76fe)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:357 (raft_consensus-itest+0x49e1e7)
#1 boost::detail::yield(unsigned int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/smart_ptr/detail/yield_k.hpp:151:9 (libmaster.so+0xd3ab6)
#2 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.cc:549:7 (libkudu_util.so+0x1c70e1)
#3 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool), kudu::ThreadPool*, bool>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(bool), kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.h:164:12 (libkudu_util.so+0x1d1736)
#4 kudu::ThreadPool::CreateThreadUnlocked() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:694:14 (libkudu_util.so+0x1cf111)
#5 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:491:21 (libkudu_util.so+0x1cd5d1)
#6 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:452:10 (libkudu_util.so+0x1cf42f)
#7 kudu::ThreadPool::SubmitFunc(boost::function<void ()>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:448:10 (libkudu_util.so+0x1cf4c9)
#8 kudu::DnsResolver::ResolveAddresses(kudu::HostPort const&, std::__1::vector<kudu::Sockaddr, std::__1::allocator<kudu::Sockaddr> >*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:57:21 (libkudu_util.so+0x1927fb)
#9 kudu::client::internal::RemoteTabletServer::InitProxy(kudu::client::KuduClient*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/meta_cache.cc:135:33 (libkudu_client.so+0xf8c53)
#10 kudu::client::KuduClient::Data::GetTabletServer(kudu::client::KuduClient*, scoped_refptr<kudu::client::internal::RemoteTablet> const&, kudu::client::KuduClient::ReplicaSelection, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::vector<kudu::client::internal::RemoteTabletServer*, std::__1::allocator<kudu::client::internal::RemoteTabletServer*> >*, kudu::client::internal::RemoteTabletServer**) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client-internal.cc:424:8 (libkudu_client.so+0xd71b9)
#11 kudu::client::KuduScanner::Data::OpenTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::MonoTime const&, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > >*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/scanner-internal.cc:388:53 (libkudu_client.so+0x11ca83)
#12 kudu::client::KuduScanner::Data::OpenNextTablet(kudu::MonoTime const&, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > >*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/scanner-internal.cc:243:10 (libkudu_client.so+0x11c23f)
#13 kudu::client::KuduScanner::Open() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:1331:3 (libkudu_client.so+0xc283d)
#14 kudu::ClusterVerifier::DoCheckRowCount(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::ClusterVerifier::ComparisonMode, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/cluster_verifier.cc:138:3 (libintegration-tests.so+0x57ad4)
#15 kudu::ClusterVerifier::CheckRowCount(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::ClusterVerifier::ComparisonMode, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/cluster_verifier.cc:116:3 (libintegration-tests.so+0x57758)
#16 kudu::tserver::RaftConsensusITest::DoTestChurnyElections(kudu::TestWorkload*, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1001:3 (raft_consensus-itest+0x50f417)
#17 kudu::tserver::RaftConsensusITest_TestChurnyElections_WithDuplicateKeys_Test::TestBody() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1042:3 (raft_consensus-itest+0x50f9ad)
#18 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
#19 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
#20 testing::Test::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2474:5 (libgmock.so+0x32b97)
#21 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11 (libgmock.so+0x34036)
#22 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
#23 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
#24 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
#25 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
#26 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
#27 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
#28 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
Thread T17 'rpc reactor-214' (tid=2145, running) created by main thread at:
#0 pthread_create /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:897 (raft_consensus-itest+0x4a28d3)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.cc:518:15 (libkudu_util.so+0x1c6ef7)
#2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ReactorThread::*)(), kudu::rpc::ReactorThread*>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, void (kudu::rpc::ReactorThread::* const&)(), kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.h:158:12 (libkrpc.so+0xc23a5)
#3 kudu::rpc::ReactorThread::Init() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:145:10 (libkrpc.so+0xbd2c5)
#4 kudu::rpc::Reactor::Init() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:645:18 (libkrpc.so+0xc14a1)
#5 kudu::rpc::Messenger::Init() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/messenger.cc:517:5 (libkrpc.so+0xa6572)
#6 kudu::rpc::MessengerBuilder::Build(std::__1::shared_ptr<kudu::rpc::Messenger>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/messenger.cc:283:3 (libkrpc.so+0xa5b65)
#7 kudu::client::KuduClientBuilder::Build(std::__1::shared_ptr<kudu::client::KuduClient>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:295:3 (libkudu_client.so+0xbb51f)
#8 kudu::ExternalMiniCluster::CreateClient(kudu::client::KuduClientBuilder*, std::__1::shared_ptr<kudu::client::KuduClient>*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/external_mini_cluster.cc:618:19 (libintegration-tests.so+0x64171)
#9 kudu::TestWorkload::Setup() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/test_workload.cc:231:5 (libintegration-tests.so+0x881fc)
#10 kudu::tserver::RaftConsensusITest::DoTestChurnyElections(kudu::TestWorkload*, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:981:13 (raft_consensus-itest+0x50f0ee)
#11 kudu::tserver::RaftConsensusITest_TestChurnyElections_WithDuplicateKeys_Test::TestBody() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1042:3 (raft_consensus-itest+0x50f9ad)
#12 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
#13 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
#14 testing::Test::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2474:5 (libgmock.so+0x32b97)
#15 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11 (libgmock.so+0x34036)
#16 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
#17 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
#18 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
#19 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
#20 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
#21 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
#22 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:73 in operator delete(void*)
==================
==================
WARNING: ThreadSanitizer: data race (pid=1150)
Write of size 8 at 0x7b5000005cf0 by main thread:
#0 pthread_cond_destroy /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1102 (raft_consensus-itest+0x4a1b5c)
#1 kudu::ConditionVariable::~ConditionVariable() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/condition_variable.cc:57:12 (libkudu_util.so+0xf77fe)
#2 kudu::ThreadPool::~ThreadPool() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:339:1 (libkudu_util.so+0x1ce9b3)
#3 kudu::DefaultDeleter<kudu::ThreadPool>::operator()(kudu::ThreadPool*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:145:5 (libmaster.so+0xc274e)
#4 kudu::internal::gscoped_ptr_impl<kudu::ThreadPool, kudu::DefaultDeleter<kudu::ThreadPool> >::~gscoped_ptr_impl() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:228:7 (libmaster.so+0xc2719)
#5 gscoped_ptr<kudu::ThreadPool, kudu::DefaultDeleter<kudu::ThreadPool> >::~gscoped_ptr() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:318:7 (libmaster.so+0xb3789)
#6 kudu::DnsResolver::~DnsResolver() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:45:1 (libkudu_util.so+0x19274a)
#7 kudu::DefaultDeleter<kudu::DnsResolver>::operator()(kudu::DnsResolver*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:145:5 (libkudu_client.so+0xd0dae)
#8 kudu::internal::gscoped_ptr_impl<kudu::DnsResolver, kudu::DefaultDeleter<kudu::DnsResolver> >::reset(kudu::DnsResolver*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:254:7 (libkudu_client.so+0xd0d64)
#9 gscoped_ptr<kudu::DnsResolver, kudu::DefaultDeleter<kudu::DnsResolver> >::reset(kudu::DnsResolver*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:375:46 (libkudu_client.so+0xc5f90)
#10 kudu::client::KuduClient::Data::~Data() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client-internal.cc:343:17 (libkudu_client.so+0xd6add)
#11 kudu::client::KuduClient::~KuduClient() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:334:3 (libkudu_client.so+0xbbfac)
#12 std::__1::default_delete<kudu::client::KuduClient>::operator()(kudu::client::KuduClient*) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:2397:13 (libkudu_client.so+0xd067b)
#13 std::__1::__shared_ptr_pointer<kudu::client::KuduClient*, std::__1::default_delete<kudu::client::KuduClient>, std::__1::allocator<kudu::client::KuduClient> >::__on_zero_shared() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:3795 (libkudu_client.so+0xd067b)
#14 __release_shared /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/libcxx/src/memory.cpp:67:9 (libc++.so.1+0xc095d)
#15 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/libcxx/src/memory.cpp:92 (libc++.so.1+0xc095d)
#16 std::__1::shared_ptr<kudu::client::KuduClient>::~shared_ptr() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:4626:19 (raft_consensus-itest+0x549038)
#17 kudu::TestWorkload::~TestWorkload() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/test_workload.cc:78:1 (libintegration-tests.so+0x867d3)
#18 kudu::tserver::RaftConsensusITest_TestChurnyElections_WithDuplicateKeys_Test::TestBody() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1043:1 (raft_consensus-itest+0x50f9b9)
#19 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
#20 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
#21 testing::Test::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2474:5 (libgmock.so+0x32b97)
#22 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11 (libgmock.so+0x34036)
#23 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
#24 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
#25 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
#26 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
#27 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
#28 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
#29 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
Previous read of size 8 at 0x7b5000005cf0 by thread T17:
#0 pthread_cond_signal /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1088 (raft_consensus-itest+0x4a1890)
#1 kudu::ConditionVariable::Signal() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/condition_variable.cc:136:12 (libkudu_util.so+0xf7c7e)
#2 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:529:14 (libkudu_util.so+0x1cd7bb)
#3 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:452:10 (libkudu_util.so+0x1cf42f)
#4 kudu::ThreadPool::SubmitFunc(boost::function<void ()>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:448:10 (libkudu_util.so+0x1cf4c9)
#5 kudu::DnsResolver::ResolveAddresses(kudu::HostPort const&, std::__1::vector<kudu::Sockaddr, std::__1::allocator<kudu::Sockaddr> >*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:57:21 (libkudu_util.so+0x1927fb)
#6 kudu::client::internal::RemoteTabletServer::InitProxy(kudu::client::KuduClient*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/meta_cache.cc:135:33 (libkudu_client.so+0xf8c53)
#7 kudu::client::internal::MetaCacheServerPicker::PickLeader(kudu::Callback<void (kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&, kudu::MonoTime const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/meta_cache.cc:443:11 (libkudu_client.so+0xfaf0f)
#8 kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer, kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>::SendRpc() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/retriable_rpc.h:144:19 (libkudu_client.so+0xb2a98)
#9 kudu::rpc::RpcRetrier::DelayedRetryCb(kudu::rpc::Rpc*, kudu::Status const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/rpc.cc:90:10 (libkrpc.so+0xd4b0c)
#10 boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>::operator()(kudu::rpc::RpcRetrier*, kudu::rpc::Rpc*, kudu::Status const&) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:280:29 (libkrpc.so+0xd54f7)
#11 void boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> >::operator()<boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::rrlist1<kudu::Status const&> >(boost::_bi::type<void>, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>&, boost::_bi::rrlist1<kudu::Status const&>&, int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:398:9 (libkrpc.so+0xd5432)
#12 void boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> > >::operator()<kudu::Status const&>(kudu::Status const&) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:1234:16 (libkrpc.so+0xd5358)
#13 boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> > >, void, kudu::Status const&>::invoke(boost::detail::function::function_buffer&, kudu::Status const&) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:159:11 (libkrpc.so+0xd50f3)
#14 boost::function1<void, kudu::Status const&>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xc3234)
#15 kudu::rpc::DelayedTask::TimerHandler(ev::timer&, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:628:5 (libkrpc.so+0xc1074)
#16 void ev::base<ev_timer, ev::timer>::method_thunk<kudu::rpc::DelayedTask, &kudu::rpc::DelayedTask::TimerHandler>(ev_loop*, ev_timer*, int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/ev++.h:479:7 (libkrpc.so+0xc7d1a)
#17 ev_invoke_pending /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/libev-4.20/ev.c:3155:11 (libev.so.4+0x9800)
#18 ev_run /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/libev-4.20/ev.c:3555:7 (libev.so.4+0xa90d)
#19 ev::loop_ref::run(int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/ev++.h:211:7 (libkrpc.so+0xc2b08)
#20 kudu::rpc::ReactorThread::RunThread() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:395:9 (libkrpc.so+0xbd46a)
#21 boost::_mfi::mf0<void, kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:49:29 (libkrpc.so+0xc5f86)
#22 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>&, boost::_bi::list0&, int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:259:9 (libkrpc.so+0xc5eda)
#23 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >::operator()() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:1222:16 (libkrpc.so+0xc5e63)
#24 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >, void>::invoke(boost::detail::function::function_buffer&) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:159:11 (libkrpc.so+0xc5c69)
#25 boost::function0<void>::operator()() const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xb7251)
#26 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.cc:595:3 (libkudu_util.so+0x1c76fe)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:357 (raft_consensus-itest+0x49e1e7)
#1 boost::detail::yield(unsigned int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/smart_ptr/detail/yield_k.hpp:151:9 (libmaster.so+0xd3ab6)
#2 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.cc:549:7 (libkudu_util.so+0x1c70e1)
#3 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool), kudu::ThreadPool*, bool>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(bool), kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.h:164:12 (libkudu_util.so+0x1d1736)
#4 kudu::ThreadPool::CreateThreadUnlocked() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:694:14 (libkudu_util.so+0x1cf111)
#5 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:491:21 (libkudu_util.so+0x1cd5d1)
#6 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:452:10 (libkudu_util.so+0x1cf42f)
#7 kudu::ThreadPool::SubmitFunc(boost::function<void ()>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:448:10 (libkudu_util.so+0x1cf4c9)
#8 kudu::DnsResolver::ResolveAddresses(kudu::HostPort const&, std::__1::vector<kudu::Sockaddr, std::__1::allocator<kudu::Sockaddr> >*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:57:21 (libkudu_util.so+0x1927fb)
#9 kudu::client::internal::RemoteTabletServer::InitProxy(kudu::client::KuduClient*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/meta_cache.cc:135:33 (libkudu_client.so+0xf8c53)
#10 kudu::client::KuduClient::Data::GetTabletServer(kudu::client::KuduClient*, scoped_refptr<kudu::client::internal::RemoteTablet> const&, kudu::client::KuduClient::ReplicaSelection, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::vector<kudu::client::internal::RemoteTabletServer*, std::__1::allocator<kudu::client::internal::RemoteTabletServer*> >*, kudu::client::internal::RemoteTabletServer**) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client-internal.cc:424:8 (libkudu_client.so+0xd71b9)
#11 kudu::client::KuduScanner::Data::OpenTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::MonoTime const&, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > >*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/scanner-internal.cc:388:53 (libkudu_client.so+0x11ca83)
#12 kudu::client::KuduScanner::Data::OpenNextTablet(kudu::MonoTime const&, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > >*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/scanner-internal.cc:243:10 (libkudu_client.so+0x11c23f)
#13 kudu::client::KuduScanner::Open() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:1331:3 (libkudu_client.so+0xc283d)
#14 kudu::ClusterVerifier::DoCheckRowCount(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::ClusterVerifier::ComparisonMode, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/cluster_verifier.cc:138:3 (libintegration-tests.so+0x57ad4)
#15 kudu::ClusterVerifier::CheckRowCount(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::ClusterVerifier::ComparisonMode, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/cluster_verifier.cc:116:3 (libintegration-tests.so+0x57758)
#16 kudu::tserver::RaftConsensusITest::DoTestChurnyElections(kudu::TestWorkload*, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1001:3 (raft_consensus-itest+0x50f417)
#17 kudu::tserver::RaftConsensusITest_TestChurnyElections_WithDuplicateKeys_Test::TestBody() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1042:3 (raft_consensus-itest+0x50f9ad)
#18 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
#19 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
#20 testing::Test::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2474:5 (libgmock.so+0x32b97)
#21 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11 (libgmock.so+0x34036)
#22 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
#23 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
#24 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
#25 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
#26 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
#27 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
#28 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
Location is heap block of size 504 at 0x7b5000005c00 allocated by main thread:
#0 operator new(unsigned long) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:41 (raft_consensus-itest+0x502ac3)
#1 kudu::ThreadPoolBuilder::Build(gscoped_ptr<kudu::ThreadPool, kudu::DefaultDeleter<kudu::ThreadPool> >*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:124:15 (libkudu_util.so+0x1cc8da)
#2 kudu::DnsResolver::DnsResolver() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:38:3 (libkudu_util.so+0x1925c0)
#3 kudu::client::KuduClientBuilder::Build(std::__1::shared_ptr<kudu::client::KuduClient>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:315:37 (libkudu_client.so+0xbb7df)
#4 kudu::ExternalMiniCluster::CreateClient(kudu::client::KuduClientBuilder*, std::__1::shared_ptr<kudu::client::KuduClient>*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/external_mini_cluster.cc:618:19 (libintegration-tests.so+0x64171)
#5 kudu::TestWorkload::Setup() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/test_workload.cc:231:5 (libintegration-tests.so+0x881fc)
#6 kudu::tserver::RaftConsensusITest::DoTestChurnyElections(kudu::TestWorkload*, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:981:13 (raft_consensus-itest+0x50f0ee)
#7 kudu::tserver::RaftConsensusITest_TestChurnyElections_WithDuplicateKeys_Test::TestBody() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1042:3 (raft_consensus-itest+0x50f9ad)
#8 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
#9 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
#10 testing::Test::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2474:5 (libgmock.so+0x32b97)
#11 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11 (libgmock.so+0x34036)
#12 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
#13 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
#14 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
#15 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
#16 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
#17 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
#18 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
Thread T17 'rpc reactor-214' (tid=2145, running) created by main thread at:
#0 pthread_create /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:897 (raft_consensus-itest+0x4a28d3)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.cc:518:15 (libkudu_util.so+0x1c6ef7)
#2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ReactorThread::*)(), kudu::rpc::ReactorThread*>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, void (kudu::rpc::ReactorThread::* const&)(), kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.h:158:12 (libkrpc.so+0xc23a5)
#3 kudu::rpc::ReactorThread::Init() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:145:10 (libkrpc.so+0xbd2c5)
#4 kudu::rpc::Reactor::Init() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:645:18 (libkrpc.so+0xc14a1)
#5 kudu::rpc::Messenger::Init() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/messenger.cc:517:5 (libkrpc.so+0xa6572)
#6 kudu::rpc::MessengerBuilder::Build(std::__1::shared_ptr<kudu::rpc::Messenger>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/messenger.cc:283:3 (libkrpc.so+0xa5b65)
#7 kudu::client::KuduClientBuilder::Build(std::__1::shared_ptr<kudu::client::KuduClient>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:295:3 (libkudu_client.so+0xbb51f)
#8 kudu::ExternalMiniCluster::CreateClient(kudu::client::KuduClientBuilder*, std::__1::shared_ptr<kudu::client::KuduClient>*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/external_mini_cluster.cc:618:19 (libintegration-tests.so+0x64171)
#9 kudu::TestWorkload::Setup() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/test_workload.cc:231:5 (libintegration-tests.so+0x881fc)
#10 kudu::tserver::RaftConsensusITest::DoTestChurnyElections(kudu::TestWorkload*, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:981:13 (raft_consensus-itest+0x50f0ee)
#11 kudu::tserver::RaftConsensusITest_TestChurnyElections_WithDuplicateKeys_Test::TestBody() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1042:3 (raft_consensus-itest+0x50f9ad)
#12 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
#13 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
#14 testing::Test::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2474:5 (libgmock.so+0x32b97)
#15 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11 (libgmock.so+0x34036)
#16 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
#17 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
#18 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
#19 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
#20 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
#21 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
#22 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/condition_variable.cc:57:12 in kudu::ConditionVariable::~ConditionVariable()
==================
==================
WARNING: ThreadSanitizer: data race (pid=1150)
Write of size 8 at 0x7b5000005dc8 by main thread:
#0 operator delete(void*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:73 (raft_consensus-itest+0x502d61)
#1 kudu::DefaultDeleter<kudu::ThreadPool>::operator()(kudu::ThreadPool*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:145:5 (libmaster.so+0xc2756)
#2 kudu::internal::gscoped_ptr_impl<kudu::ThreadPool, kudu::DefaultDeleter<kudu::ThreadPool> >::~gscoped_ptr_impl() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:228:7 (libmaster.so+0xc2719)
#3 gscoped_ptr<kudu::ThreadPool, kudu::DefaultDeleter<kudu::ThreadPool> >::~gscoped_ptr() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:318:7 (libmaster.so+0xb3789)
#4 kudu::DnsResolver::~DnsResolver() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:45:1 (libkudu_util.so+0x19274a)
#5 kudu::DefaultDeleter<kudu::DnsResolver>::operator()(kudu::DnsResolver*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:145:5 (libkudu_client.so+0xd0dae)
#6 kudu::internal::gscoped_ptr_impl<kudu::DnsResolver, kudu::DefaultDeleter<kudu::DnsResolver> >::reset(kudu::DnsResolver*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:254:7 (libkudu_client.so+0xd0d64)
#7 gscoped_ptr<kudu::DnsResolver, kudu::DefaultDeleter<kudu::DnsResolver> >::reset(kudu::DnsResolver*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:375:46 (libkudu_client.so+0xc5f90)
#8 kudu::client::KuduClient::Data::~Data() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client-internal.cc:343:17 (libkudu_client.so+0xd6add)
#9 kudu::client::KuduClient::~KuduClient() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:334:3 (libkudu_client.so+0xbbfac)
#10 std::__1::default_delete<kudu::client::KuduClient>::operator()(kudu::client::KuduClient*) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:2397:13 (libkudu_client.so+0xd067b)
#11 std::__1::__shared_ptr_pointer<kudu::client::KuduClient*, std::__1::default_delete<kudu::client::KuduClient>, std::__1::allocator<kudu::client::KuduClient> >::__on_zero_shared() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:3795 (libkudu_client.so+0xd067b)
#12 __release_shared /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/libcxx/src/memory.cpp:67:9 (libc++.so.1+0xc095d)
#13 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/libcxx/src/memory.cpp:92 (libc++.so.1+0xc095d)
#14 std::__1::shared_ptr<kudu::client::KuduClient>::~shared_ptr() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:4626:19 (raft_consensus-itest+0x549038)
#15 kudu::TestWorkload::~TestWorkload() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/test_workload.cc:78:1 (libintegration-tests.so+0x867d3)
#16 kudu::tserver::RaftConsensusITest_TestChurnyElections_WithDuplicateKeys_Test::TestBody() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1043:1 (raft_consensus-itest+0x50f9b9)
#17 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
#18 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
#19 testing::Test::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2474:5 (libgmock.so+0x32b97)
#20 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11 (libgmock.so+0x34036)
#21 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
#22 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
#23 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
#24 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
#25 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
#26 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
#27 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
Previous read of size 8 at 0x7b5000005dc8 by thread T17:
#0 scoped_refptr<kudu::Histogram>::operator kudu::Histogram* scoped_refptr<kudu::Histogram>::*() const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/ref_counted.h:269:38 (libkrpc.so+0xa34b9)
#1 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:531:7 (libkudu_util.so+0x1cd7ca)
#2 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:452:10 (libkudu_util.so+0x1cf42f)
#3 kudu::ThreadPool::SubmitFunc(boost::function<void ()>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:448:10 (libkudu_util.so+0x1cf4c9)
#4 kudu::DnsResolver::ResolveAddresses(kudu::HostPort const&, std::__1::vector<kudu::Sockaddr, std::__1::allocator<kudu::Sockaddr> >*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:57:21 (libkudu_util.so+0x1927fb)
#5 kudu::client::internal::RemoteTabletServer::InitProxy(kudu::client::KuduClient*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/meta_cache.cc:135:33 (libkudu_client.so+0xf8c53)
#6 kudu::client::internal::MetaCacheServerPicker::PickLeader(kudu::Callback<void (kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&, kudu::MonoTime const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/meta_cache.cc:443:11 (libkudu_client.so+0xfaf0f)
#7 kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer, kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>::SendRpc() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/retriable_rpc.h:144:19 (libkudu_client.so+0xb2a98)
#8 kudu::rpc::RpcRetrier::DelayedRetryCb(kudu::rpc::Rpc*, kudu::Status const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/rpc.cc:90:10 (libkrpc.so+0xd4b0c)
#9 boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>::operator()(kudu::rpc::RpcRetrier*, kudu::rpc::Rpc*, kudu::Status const&) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:280:29 (libkrpc.so+0xd54f7)
#10 void boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> >::operator()<boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::rrlist1<kudu::Status const&> >(boost::_bi::type<void>, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>&, boost::_bi::rrlist1<kudu::Status const&>&, int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:398:9 (libkrpc.so+0xd5432)
#11 void boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> > >::operator()<kudu::Status const&>(kudu::Status const&) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:1234:16 (libkrpc.so+0xd5358)
#12 boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> > >, void, kudu::Status const&>::invoke(boost::detail::function::function_buffer&, kudu::Status const&) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:159:11 (libkrpc.so+0xd50f3)
#13 boost::function1<void, kudu::Status const&>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xc3234)
#14 kudu::rpc::DelayedTask::TimerHandler(ev::timer&, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:628:5 (libkrpc.so+0xc1074)
#15 void ev::base<ev_timer, ev::timer>::method_thunk<kudu::rpc::DelayedTask, &kudu::rpc::DelayedTask::TimerHandler>(ev_loop*, ev_timer*, int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/ev++.h:479:7 (libkrpc.so+0xc7d1a)
#16 ev_invoke_pending /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/libev-4.20/ev.c:3155:11 (libev.so.4+0x9800)
#17 ev_run /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/libev-4.20/ev.c:3555:7 (libev.so.4+0xa90d)
#18 ev::loop_ref::run(int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/ev++.h:211:7 (libkrpc.so+0xc2b08)
#19 kudu::rpc::ReactorThread::RunThread() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:395:9 (libkrpc.so+0xbd46a)
#20 boost::_mfi::mf0<void, kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:49:29 (libkrpc.so+0xc5f86)
#21 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>&, boost::_bi::list0&, int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:259:9 (libkrpc.so+0xc5eda)
#22 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >::operator()() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:1222:16 (libkrpc.so+0xc5e63)
#23 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >, void>::invoke(boost::detail::function::function_buffer&) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:159:11 (libkrpc.so+0xc5c69)
#24 boost::function0<void>::operator()() const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xb7251)
#25 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.cc:595:3 (libkudu_util.so+0x1c76fe)
As if synchronized via sleep:
#0 nanosleep /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:357 (raft_consensus-itest+0x49e1e7)
#1 boost::detail::yield(unsigned int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/smart_ptr/detail/yield_k.hpp:151:9 (libmaster.so+0xd3ab6)
#2 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.cc:549:7 (libkudu_util.so+0x1c70e1)
#3 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool), kudu::ThreadPool*, bool>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(bool), kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.h:164:12 (libkudu_util.so+0x1d1736)
#4 kudu::ThreadPool::CreateThreadUnlocked() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:694:14 (libkudu_util.so+0x1cf111)
#5 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:491:21 (libkudu_util.so+0x1cd5d1)
#6 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:452:10 (libkudu_util.so+0x1cf42f)
#7 kudu::ThreadPool::SubmitFunc(boost::function<void ()>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:448:10 (libkudu_util.so+0x1cf4c9)
#8 kudu::DnsResolver::ResolveAddresses(kudu::HostPort const&, std::__1::vector<kudu::Sockaddr, std::__1::allocator<kudu::Sockaddr> >*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:57:21 (libkudu_util.so+0x1927fb)
#9 kudu::client::internal::RemoteTabletServer::InitProxy(kudu::client::KuduClient*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/meta_cache.cc:135:33 (libkudu_client.so+0xf8c53)
#10 kudu::client::KuduClient::Data::GetTabletServer(kudu::client::KuduClient*, scoped_refptr<kudu::client::internal::RemoteTablet> const&, kudu::client::KuduClient::ReplicaSelection, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::vector<kudu::client::internal::RemoteTabletServer*, std::__1::allocator<kudu::client::internal::RemoteTabletServer*> >*, kudu::client::internal::RemoteTabletServer**) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client-internal.cc:424:8 (libkudu_client.so+0xd71b9)
#11 kudu::client::KuduScanner::Data::OpenTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::MonoTime const&, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > >*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/scanner-internal.cc:388:53 (libkudu_client.so+0x11ca83)
#12 kudu::client::KuduScanner::Data::OpenNextTablet(kudu::MonoTime const&, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > >*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/scanner-internal.cc:243:10 (libkudu_client.so+0x11c23f)
#13 kudu::client::KuduScanner::Open() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:1331:3 (libkudu_client.so+0xc283d)
#14 kudu::ClusterVerifier::DoCheckRowCount(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::ClusterVerifier::ComparisonMode, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/cluster_verifier.cc:138:3 (libintegration-tests.so+0x57ad4)
#15 kudu::ClusterVerifier::CheckRowCount(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::ClusterVerifier::ComparisonMode, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/cluster_verifier.cc:116:3 (libintegration-tests.so+0x57758)
#16 kudu::tserver::RaftConsensusITest::DoTestChurnyElections(kudu::TestWorkload*, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1001:3 (raft_consensus-itest+0x50f417)
#17 kudu::tserver::RaftConsensusITest_TestChurnyElections_WithDuplicateKeys_Test::TestBody() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1042:3 (raft_consensus-itest+0x50f9ad)
#18 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
#19 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
#20 testing::Test::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2474:5 (libgmock.so+0x32b97)
#21 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11 (libgmock.so+0x34036)
#22 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
#23 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
#24 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
#25 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
#26 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
#27 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
#28 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
Thread T17 'rpc reactor-214' (tid=2145, running) created by main thread at:
#0 pthread_create /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:897 (raft_consensus-itest+0x4a28d3)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.cc:518:15 (libkudu_util.so+0x1c6ef7)
#2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ReactorThread::*)(), kudu::rpc::ReactorThread*>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, void (kudu::rpc::ReactorThread::* const&)(), kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.h:158:12 (libkrpc.so+0xc23a5)
#3 kudu::rpc::ReactorThread::Init() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:145:10 (libkrpc.so+0xbd2c5)
#4 kudu::rpc::Reactor::Init() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:645:18 (libkrpc.so+0xc14a1)
#5 kudu::rpc::Messenger::Init() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/messenger.cc:517:5 (libkrpc.so+0xa6572)
#6 kudu::rpc::MessengerBuilder::Build(std::__1::shared_ptr<kudu::rpc::Messenger>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/messenger.cc:283:3 (libkrpc.so+0xa5b65)
#7 kudu::client::KuduClientBuilder::Build(std::__1::shared_ptr<kudu::client::KuduClient>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:295:3 (libkudu_client.so+0xbb51f)
#8 kudu::ExternalMiniCluster::CreateClient(kudu::client::KuduClientBuilder*, std::__1::shared_ptr<kudu::client::KuduClient>*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/external_mini_cluster.cc:618:19 (libintegration-tests.so+0x64171)
#9 kudu::TestWorkload::Setup() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/test_workload.cc:231:5 (libintegration-tests.so+0x881fc)
#10 kudu::tserver::RaftConsensusITest::DoTestChurnyElections(kudu::TestWorkload*, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:981:13 (raft_consensus-itest+0x50f0ee)
#11 kudu::tserver::RaftConsensusITest_TestChurnyElections_WithDuplicateKeys_Test::TestBody() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1042:3 (raft_consensus-itest+0x50f9ad)
#12 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
#13 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
#14 testing::Test::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2474:5 (libgmock.so+0x32b97)
#15 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11 (libgmock.so+0x34036)
#16 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
#17 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
#18 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
#19 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
#20 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
#21 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
#22 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:73 in operator delete(void*)
==================
{noformat}
> Data race in DnsResolver
> ------------------------
>
> Key: KUDU-2059
> URL: https://issues.apache.org/jira/browse/KUDU-2059
> Project: Kudu
> Issue Type: Bug
> Components: server
> Affects Versions: 1.4.0
> Reporter: Mike Percy
> Labels: tsan
> Attachments: raft_consensus-itest.txt, raft_consensus-itest.txt.gz
>
>
> I got a TSAN failure in a Jenkins run of RaftConsensusITest.MultiThreadedInsertWithFailovers:
> http://dist-test.cloudera.org/job?job_id=jenkins-slave.1498799877.11199
> {code}
> WARNING: ThreadSanitizer: data race (pid=14861)
> Write of size 8 at 0x7b5000006af0 by main thread:
> #0 pthread_cond_destroy /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1102 (raft_consensus-itest+0x4a198c)
> #1 kudu::ConditionVariable::~ConditionVariable() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/condition_variable.cc:57:12 (libkudu_util.so+0xf107e)
> #2 kudu::ThreadPool::~ThreadPool() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:339:1 (libkudu_util.so+0x1c1b93)
> #3 kudu::DefaultDeleter<kudu::ThreadPool>::operator()(kudu::ThreadPool*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:145:5 (libmaster.so+0xc0bbe)
> #4 kudu::internal::gscoped_ptr_impl<kudu::ThreadPool, kudu::DefaultDeleter<kudu::ThreadPool> >::~gscoped_ptr_impl() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:228:7 (libmaster.so+0xc0b89)
> #5 gscoped_ptr<kudu::ThreadPool, kudu::DefaultDeleter<kudu::ThreadPool> >::~gscoped_ptr() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:318:7 (libmaster.so+0xb1b79)
> #6 kudu::DnsResolver::~DnsResolver() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:45:1 (libkudu_util.so+0x1863fa)
> #7 kudu::DefaultDeleter<kudu::DnsResolver>::operator()(kudu::DnsResolver*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:145:5 (libkudu_client.so+0xd0dee)
> #8 kudu::internal::gscoped_ptr_impl<kudu::DnsResolver, kudu::DefaultDeleter<kudu::DnsResolver> >::reset(kudu::DnsResolver*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:254:7 (libkudu_client.so+0xd0da4)
> #9 gscoped_ptr<kudu::DnsResolver, kudu::DefaultDeleter<kudu::DnsResolver> >::reset(kudu::DnsResolver*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/gutil/gscoped_ptr.h:375:46 (libkudu_client.so+0xc5fd0)
> #10 kudu::client::KuduClient::Data::~Data() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client-internal.cc:342:17 (libkudu_client.so+0xd6b1d)
> #11 kudu::client::KuduClient::~KuduClient() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:334:3 (libkudu_client.so+0xbbfec)
> #12 std::__1::default_delete<kudu::client::KuduClient>::operator()(kudu::client::KuduClient*) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:2397:13 (libkudu_client.so+0xd06bb)
> #13 std::__1::__shared_ptr_pointer<kudu::client::KuduClient*, std::__1::default_delete<kudu::client::KuduClient>, std::__1::allocator<kudu::client::KuduClient> >::__on_zero_shared() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:3795 (libkudu_client.so+0xd06bb)
> #14 __release_shared /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/libcxx/src/memory.cpp:67:9 (libc++.so.1+0xc095d)
> #15 std::__1::__shared_weak_count::__release_shared() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/libcxx/src/memory.cpp:92 (libc++.so.1+0xc095d)
> #16 std::__1::shared_ptr<kudu::client::KuduClient>::~shared_ptr() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/c++/v1/memory:4626:19 (raft_consensus-itest+0x548818)
> #17 kudu::tserver::TabletServerIntegrationTestBase::~TabletServerIntegrationTestBase() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/ts_itest-base.h:69:7 (raft_consensus-itest+0x54890f)
> #18 kudu::tserver::RaftConsensusITest::~RaftConsensusITest() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:107:7 (raft_consensus-itest+0x547887)
> #19 kudu::tserver::RaftConsensusITest_MultiThreadedInsertWithFailovers_Test::~RaftConsensusITest_MultiThreadedInsertWithFailovers_Test() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1040:1 (raft_consensus-itest+0x547ac9)
> #20 testing::Test::DeleteSelf_() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/include/gtest/gtest.h:453:24 (libgmock.so+0x53187)
> #21 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
> #22 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
> #23 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2661:3 (libgmock.so+0x340a0)
> #24 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
> #25 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
> #26 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
> #27 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
> #28 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
> #29 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
> #30 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
> Previous read of size 8 at 0x7b5000006af0 by thread T33:
> #0 pthread_cond_signal /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1088 (raft_consensus-itest+0x4a16c0)
> #1 kudu::ConditionVariable::Signal() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/condition_variable.cc:136:12 (libkudu_util.so+0xf14fe)
> #2 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:529:14 (libkudu_util.so+0x1c099b)
> #3 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:452:10 (libkudu_util.so+0x1c260f)
> #4 kudu::ThreadPool::SubmitFunc(boost::function<void ()>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:448:10 (libkudu_util.so+0x1c26a9)
> #5 kudu::DnsResolver::ResolveAddresses(kudu::HostPort const&, std::__1::vector<kudu::Sockaddr, std::__1::allocator<kudu::Sockaddr> >*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:57:21 (libkudu_util.so+0x1864ab)
> #6 kudu::client::internal::RemoteTabletServer::InitProxy(kudu::client::KuduClient*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/meta_cache.cc:133:33 (libkudu_client.so+0xf8bf3)
> #7 kudu::client::internal::MetaCacheServerPicker::PickLeader(kudu::Callback<void (kudu::Status const&, kudu::client::internal::RemoteTabletServer*)> const&, kudu::MonoTime const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/meta_cache.cc:441:11 (libkudu_client.so+0xfaeaf)
> #8 kudu::rpc::RetriableRpc<kudu::client::internal::RemoteTabletServer, kudu::tserver::WriteRequestPB, kudu::tserver::WriteResponsePB>::SendRpc() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/retriable_rpc.h:144:19 (libkudu_client.so+0xb2ad8)
> #9 kudu::rpc::RpcRetrier::DelayedRetryCb(kudu::rpc::Rpc*, kudu::Status const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/rpc.cc:89:10 (libkrpc.so+0xcc72c)
> #10 boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>::operator()(kudu::rpc::RpcRetrier*, kudu::rpc::Rpc*, kudu::Status const&) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:280:29 (libkrpc.so+0xcd117)
> #11 void boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> >::operator()<boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::rrlist1<kudu::Status const&> >(boost::_bi::type<void>, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>&, boost::_bi::rrlist1<kudu::Status const&>&, int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:398:9 (libkrpc.so+0xcd052)
> #12 void boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> > >::operator()<kudu::Status const&>(kudu::Status const&) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:1234:16 (libkrpc.so+0xccf78)
> #13 boost::detail::function::void_function_obj_invoker1<boost::_bi::bind_t<void, boost::_mfi::mf2<void, kudu::rpc::RpcRetrier, kudu::rpc::Rpc*, kudu::Status const&>, boost::_bi::list3<boost::_bi::value<kudu::rpc::RpcRetrier*>, boost::_bi::value<kudu::rpc::Rpc*>, boost::arg<1> > >, void, kudu::Status const&>::invoke(boost::detail::function::function_buffer&, kudu::Status const&) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:159:11 (libkrpc.so+0xccd13)
> #14 boost::function1<void, kudu::Status const&>::operator()(kudu::Status const&) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xbb064)
> #15 kudu::rpc::DelayedTask::TimerHandler(ev::timer&, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:579:5 (libkrpc.so+0xb90b4)
> #16 void ev::base<ev_timer, ev::timer>::method_thunk<kudu::rpc::DelayedTask, &kudu::rpc::DelayedTask::TimerHandler>(ev_loop*, ev_timer*, int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/ev++.h:479:7 (libkrpc.so+0xbf95a)
> #17 ev_invoke_pending /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/libev-4.20/ev.c:3155:11 (libev.so.4+0x97ec)
> #18 ev_run /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/libev-4.20/ev.c:3555:7 (libev.so.4+0xa8fd)
> #19 ev::loop_ref::run(int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/ev++.h:211:7 (libkrpc.so+0xba938)
> #20 kudu::rpc::ReactorThread::RunThread() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:356:9 (libkrpc.so+0xb566a)
> #21 boost::_mfi::mf0<void, kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/mem_fn_template.hpp:49:29 (libkrpc.so+0xbdbc6)
> #22 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>&, boost::_bi::list0&, int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:259:9 (libkrpc.so+0xbdb1a)
> #23 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >::operator()() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/bind/bind.hpp:1222:16 (libkrpc.so+0xbdaa3)
> #24 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >, void>::invoke(boost::detail::function::function_buffer&) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:159:11 (libkrpc.so+0xbd8a9)
> #25 boost::function0<void>::operator()() const /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xb1391)
> #26 kudu::Thread::SuperviseThread(void*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.cc:591:3 (libkudu_util.so+0x1ba91e)
> As if synchronized via sleep:
> #0 nanosleep /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:357 (raft_consensus-itest+0x49e017)
> #1 boost::detail::yield(unsigned int) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/common/include/boost/smart_ptr/detail/yield_k.hpp:151:9 (libmaster.so+0xd1ff6)
> #2 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.cc:545:7 (libkudu_util.so+0x1ba301)
> #3 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool), kudu::ThreadPool*, bool>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, void (kudu::ThreadPool::* const&)(bool), kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.h:164:12 (libkudu_util.so+0x1c4916)
> #4 kudu::ThreadPool::CreateThreadUnlocked() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:694:14 (libkudu_util.so+0x1c22f1)
> #5 kudu::ThreadPool::DoSubmit(std::__1::shared_ptr<kudu::Runnable>, kudu::ThreadPoolToken*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:491:21 (libkudu_util.so+0x1c07b1)
> #6 kudu::ThreadPool::Submit(std::__1::shared_ptr<kudu::Runnable>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:452:10 (libkudu_util.so+0x1c260f)
> #7 kudu::ThreadPool::SubmitFunc(boost::function<void ()>) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:448:10 (libkudu_util.so+0x1c26a9)
> #8 kudu::DnsResolver::ResolveAddresses(kudu::HostPort const&, std::__1::vector<kudu::Sockaddr, std::__1::allocator<kudu::Sockaddr> >*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:57:21 (libkudu_util.so+0x1864ab)
> #9 kudu::client::internal::RemoteTabletServer::InitProxy(kudu::client::KuduClient*, kudu::Callback<void (kudu::Status const&)> const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/meta_cache.cc:133:33 (libkudu_client.so+0xf8bf3)
> #10 kudu::client::KuduClient::Data::GetTabletServer(kudu::client::KuduClient*, scoped_refptr<kudu::client::internal::RemoteTablet> const&, kudu::client::KuduClient::ReplicaSelection, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::vector<kudu::client::internal::RemoteTabletServer*, std::__1::allocator<kudu::client::internal::RemoteTabletServer*> >*, kudu::client::internal::RemoteTabletServer**) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client-internal.cc:423:8 (libkudu_client.so+0xd71f9)
> #11 kudu::client::KuduScanner::Data::OpenTablet(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::MonoTime const&, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > >*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/scanner-internal.cc:389:53 (libkudu_client.so+0x11ca33)
> #12 kudu::client::KuduScanner::Data::OpenNextTablet(kudu::MonoTime const&, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > >*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/scanner-internal.cc:244:10 (libkudu_client.so+0x11c1ef)
> #13 kudu::client::KuduScanner::Open() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:1331:3 (libkudu_client.so+0xc287d)
> #14 kudu::ClusterVerifier::DoCheckRowCount(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::ClusterVerifier::ComparisonMode, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/cluster_verifier.cc:138:3 (libintegration-tests.so+0x574b4)
> #15 kudu::ClusterVerifier::CheckRowCount(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, kudu::ClusterVerifier::ComparisonMode, int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/cluster_verifier.cc:116:3 (libintegration-tests.so+0x57138)
> #16 kudu::tserver::TabletServerIntegrationTestBase::AssertAllReplicasAgree(int) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/ts_itest-base.h:523:5 (raft_consensus-itest+0x53ca2f)
> #17 kudu::tserver::RaftConsensusITest_MultiThreadedInsertWithFailovers_Test::TestBody() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1096:3 (raft_consensus-itest+0x5100cb)
> #18 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
> #19 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
> #20 testing::Test::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2474:5 (libgmock.so+0x32b97)
> #21 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11 (libgmock.so+0x34036)
> #22 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
> #23 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
> #24 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
> #25 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
> #26 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
> #27 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
> #28 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
> Location is heap block of size 504 at 0x7b5000006a00 allocated by main thread:
> #0 operator new(unsigned long) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:41 (raft_consensus-itest+0x5028f3)
> #1 kudu::ThreadPoolBuilder::Build(gscoped_ptr<kudu::ThreadPool, kudu::DefaultDeleter<kudu::ThreadPool> >*) const /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/threadpool.cc:124:15 (libkudu_util.so+0x1bfaba)
> #2 kudu::DnsResolver::DnsResolver() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/net/dns_resolver.cc:38:3 (libkudu_util.so+0x186270)
> #3 kudu::client::KuduClientBuilder::Build(std::__1::shared_ptr<kudu::client::KuduClient>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:315:37 (libkudu_client.so+0xbb81f)
> #4 kudu::tserver::TabletServerIntegrationTestBase::CreateClient(std::__1::shared_ptr<kudu::client::KuduClient>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/ts_itest-base.h:488:5 (raft_consensus-itest+0x545f82)
> #5 kudu::tserver::TabletServerIntegrationTestBase::BuildAndStart(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/ts_itest-base.h:513:5 (raft_consensus-itest+0x53b576)
> #6 kudu::tserver::RaftConsensusITest_MultiThreadedInsertWithFailovers_Test::TestBody() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1055:3 (raft_consensus-itest+0x50f8fd)
> #7 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
> #8 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
> #9 testing::Test::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2474:5 (libgmock.so+0x32b97)
> #10 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11 (libgmock.so+0x34036)
> #11 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
> #12 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
> #13 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
> #14 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
> #15 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
> #16 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
> #17 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
> Thread T33 'rpc reactor-160' (tid=16071, running) created by main thread at:
> #0 pthread_create /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/llvm-4.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:897 (raft_consensus-itest+0x4a2703)
> #1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.cc:514:15 (libkudu_util.so+0x1ba117)
> #2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ReactorThread::*)(), kudu::rpc::ReactorThread*>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, void (kudu::rpc::ReactorThread::* const&)(), kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/thread.h:158:12 (libkrpc.so+0xba1d5)
> #3 kudu::rpc::ReactorThread::Init() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:129:10 (libkrpc.so+0xb54c5)
> #4 kudu::rpc::Reactor::Init() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/reactor.cc:594:18 (libkrpc.so+0xb9461)
> #5 kudu::rpc::Messenger::Init() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/messenger.cc:515:5 (libkrpc.so+0xa0f32)
> #6 kudu::rpc::MessengerBuilder::Build(std::__1::shared_ptr<kudu::rpc::Messenger>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/rpc/messenger.cc:286:3 (libkrpc.so+0xa0525)
> #7 kudu::client::KuduClientBuilder::Build(std::__1::shared_ptr<kudu::client::KuduClient>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/client/client.cc:295:3 (libkudu_client.so+0xbb55f)
> #8 kudu::tserver::TabletServerIntegrationTestBase::CreateClient(std::__1::shared_ptr<kudu::client::KuduClient>*) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/ts_itest-base.h:488:5 (raft_consensus-itest+0x545f82)
> #9 kudu::tserver::TabletServerIntegrationTestBase::BuildAndStart(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/ts_itest-base.h:513:5 (raft_consensus-itest+0x53b576)
> #10 kudu::tserver::RaftConsensusITest_MultiThreadedInsertWithFailovers_Test::TestBody() /home/jenkins-slave/workspace/kudu-master/3/src/kudu/integration-tests/raft_consensus-itest.cc:1055:3 (raft_consensus-itest+0x50f8fd)
> #11 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x52b39)
> #12 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x52b39)
> #13 testing::Test::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2474:5 (libgmock.so+0x32b97)
> #14 testing::TestInfo::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2656:11 (libgmock.so+0x34036)
> #15 testing::TestCase::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2774:28 (libgmock.so+0x34da6)
> #16 testing::internal::UnitTestImpl::RunAllTests() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4649:43 (libgmock.so+0x40896)
> #17 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2402:10 (libgmock.so+0x53a19)
> #18 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:2438 (libgmock.so+0x53a19)
> #19 testing::UnitTest::Run() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/src/googletest-release-1.8.0/googletest/src/gtest.cc:4257:10 (libgmock.so+0x402e1)
> #20 RUN_ALL_TESTS() /home/jenkins-slave/workspace/kudu-master/3/thirdparty/installed/tsan/include/gtest/gtest.h:2233:46 (libkudu_test_main.so+0x33db)
> #21 main /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/test_main.cc:104:13 (libkudu_test_main.so+0x2b96)
> SUMMARY: ThreadSanitizer: data race /home/jenkins-slave/workspace/kudu-master/3/src/kudu/util/condition_variable.cc:57:12 in kudu::ConditionVariable::~ConditionVariable()
> {code}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)