You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kudu.apache.org by "Hao Hao (JIRA)" <ji...@apache.org> on 2019/08/05 17:49:00 UTC
[jira] [Created] (KUDU-2916) Admin.TestDumpMemTrackers is flaky in
tsan
Hao Hao created KUDU-2916:
-----------------------------
Summary: Admin.TestDumpMemTrackers is flaky in tsan
Key: KUDU-2916
URL: https://issues.apache.org/jira/browse/KUDU-2916
Project: Kudu
Issue Type: Bug
Affects Versions: 1.10.0
Reporter: Hao Hao
I saw a tsan failure for AdminCliTest.TestDumpMemTrackers with the following log:
{noformat}
/data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/kudu-admin-test.cc:2162: Failure
Value of: s.ok()
Actual: false
Expected: true
Runtime error: /tmp/dist-test-taskUWtx7r/build/tsan/bin/kudu: process exited with non-zero status 66
stdout: {"id":"root","limit":-1,"current_consumption":481,"peak_consumption":481,"child_trackers":[{"id":"server","parent_id":"root","limit":-1,"current_consumption":313,"peak_consumption":313,"child_trackers":[{"id":"result-tracker","parent_id":"server","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"log_block_manager","parent_id":"server","limit":-1,"current_consumption":48,"peak_consumption":48},{"id":"tablet-9276b163452b4b0399ff2cae579f7251","parent_id":"server","limit":-1,"current_consumption":265,"peak_consumption":265,"child_trackers":[{"id":"DeltaMemStores","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"MemRowSet-0","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":-1,"current_consumption":265,"peak_consumption":265},{"id":"txn_tracker","parent_id":"tablet-9276b163452b4b0399ff2cae579f7251","limit":67108864,"current_consumption":0,"peak_consumption":0}]}]},{"id":"ttl-cache-sharded_fifo_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"code_cache-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"block_cache-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"lbm-sharded_lru_cache","parent_id":"root","limit":-1,"current_consumption":0,"peak_consumption":0},{"id":"log_cache","parent_id":"root","limit":1073741824,"current_consumption":168,"peak_consumption":168,"child_trackers":[{"id":"log_cache:457a3168758d4f4f8f4c59e8dd179cd3:9276b163452b4b0399ff2cae579f7251","parent_id":"log_cache","limit":10485760,"current_consumption":168,"peak_consumption":168}]}]}
stderr: W0803 14:03:00.206982 8443 flags.cc:404] Enabled unsafe flag: --never_fsync=true
W0803 14:03:00.849385 8443 thread.cc:599] rpc reactor (reactor) Time spent creating pthread: real 0.590s user 0.230s sys 0.360s
W0803 14:03:00.849658 8443 thread.cc:566] rpc reactor (reactor) Time spent starting thread: real 0.591s user 0.230s sys 0.360s
==================
WARNING: ThreadSanitizer: destroy of a locked mutex (pid=8443)
#0 pthread_rwlock_destroy /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1313 (kudu+0x4bbb24)
#1 glog_internal_namespace_::Mutex::~Mutex() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:249:30 (libglog.so.0+0x16488)
#2 cxa_at_exit_wrapper(void*) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:386 (kudu+0x48beb3)
and:
#0 operator new(unsigned long) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cc:57 (kudu+0x52ae83)
#1 __allocate /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/new:228:10 (libc++.so.1+0xd63f3)
#2 allocate /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/memory:1793 (libc++.so.1+0xd63f3)
#3 allocate /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/memory:1547 (libc++.so.1+0xd63f3)
#4 __init /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/string:1591 (libc++.so.1+0xd63f3)
#5 std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >::basic_string(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/libcxx/include/string:1653 (libc++.so.1+0xd63f3)
#6 std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >::pair(std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/utility:324:5 (libprotobuf.so.14+0x188711)
#7 void std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*> >::construct<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&>(std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >*, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1805 (libprotobuf.so.14+0x188711)
#8 void std::__1::allocator_traits<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*> > >::__construct<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&>(std::__1::integral_constant<bool, true>, std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*> >&, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >*, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1715 (libprotobuf.so.14+0x188711)
#9 void std::__1::allocator_traits<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*> > >::construct<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&>(std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*> >&, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> >*, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:1561 (libprotobuf.so.14+0x188711)
#10 std::__1::unique_ptr<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*>, std::__1::__tree_node_destructor<std::__1::allocator<std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*> > > > std::__1::__tree<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, std::__1::__map_value_compare<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, true>, std::__1::allocator<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> > > >::__construct_node<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&>(std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:2192 (libprotobuf.so.14+0x188711)
#11 std::__1::pair<std::__1::__tree_iterator<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, std::__1::__tree_node<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, void*>*, long>, bool> std::__1::__tree<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, std::__1::__map_value_compare<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, true>, std::__1::allocator<std::__1::__value_type<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int> > > >::__emplace_unique_key_args<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:2137 (libprotobuf.so.14+0x188711)
#12 bool google::protobuf::InsertIfNotPresent<std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int>, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > > > >(std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int>, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > > >*, std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int>, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > > >::value_type::first_type const&, std::__1::map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::pair<void const*, int>, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::pair<void const*, int> > > >::value_type::second_type const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__tree:1273:16 (libprotobuf.so.14+0x184c18)
#13 google::protobuf::SimpleDescriptorDatabase::DescriptorIndex<std::__1::pair<void const*, int> >::AddFile(google::protobuf::FileDescriptorProto const&, std::__1::pair<void const*, int>) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor_database.cc:57 (libprotobuf.so.14+0x184c18)
#14 google::protobuf::EncodedDescriptorDatabase::Add(void const*, int) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor_database.cc:316:19 (libprotobuf.so.14+0x182d7c)
#15 google::protobuf::DescriptorPool::InternalAddGeneratedFile(void const*, int) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/descriptor.cc:1315:3 (libprotobuf.so.14+0x13b705)
#16 google::protobuf::protobuf_google_2fprotobuf_2ftype_2eproto::(anonymous namespace)::AddDescriptorsImpl() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/type.pb.cc:240:3 (libprotobuf.so.14+0x237c10)
#17 google::protobuf::internal::FunctionClosure0::Run() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/callback.h:129:5 (libprotobuf.so.14+0xd330b)
#18 google::protobuf::GoogleOnceInitImpl(long*, google::protobuf::Closure*) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.cc:83:14 (libprotobuf.so.14+0xd5d6a)
#19 google::protobuf::GoogleOnceInit(long*, void (*)()) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.h:128:5 (libprotobuf.so.14+0x237b66)
#20 google::protobuf::protobuf_google_2fprotobuf_2ftype_2eproto::AddDescriptors() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/type.pb.cc:251 (libprotobuf.so.14+0x237b66)
#21 google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::(anonymous namespace)::AddDescriptorsImpl() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:179:3 (libprotobuf.so.14+0x12ca1d)
#22 google::protobuf::internal::FunctionClosure0::Run() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/callback.h:129:5 (libprotobuf.so.14+0xd330b)
#23 google::protobuf::GoogleOnceInitImpl(long*, google::protobuf::Closure*) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.cc:83:14 (libprotobuf.so.14+0xd5d6a)
#24 google::protobuf::GoogleOnceInit(long*, void (*)()) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/stubs/once.h:128:5 (libprotobuf.so.14+0xcf0a6)
#25 google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::AddDescriptors() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:185 (libprotobuf.so.14+0xcf0a6)
#26 google::protobuf::protobuf_google_2fprotobuf_2fapi_2eproto::StaticDescriptorInitializer::StaticDescriptorInitializer() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:190 (libprotobuf.so.14+0xcf0a6)
#27 __cxx_global_var_init /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc:192 (libprotobuf.so.14+0xcf0a6)
#28 _GLOBAL__sub_I_api.pb.cc /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/protobuf-3.4.1/src/google/protobuf/api.pb.cc (libprotobuf.so.14+0xcf0a6)
#29 call_init /build/eglibc-SvCtMH/eglibc-2.19/elf/dl-init.c:78 (ld-linux-x86-64.so.2+0x102d9)
Location is global 'google::vmodule_lock' of size 64 at 0x7f67b632f878 (libglog.so.0+0x00000023c878)
Mutex M134 (0x7f67b632f878) created at:
#0 pthread_rwlock_init /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1304 (kudu+0x4bbd94)
#1 glog_internal_namespace_::Mutex::Mutex() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:247:19 (libglog.so.0+0xcca6)
#2 __cxx_global_var_init.15 /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc:120 (libglog.so.0+0xcca6)
#3 _GLOBAL__sub_I_vlog_is_on.cc /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc (libglog.so.0+0xcca6)
#4 call_init /build/eglibc-SvCtMH/eglibc-2.19/elf/dl-init.c:78 (ld-linux-x86-64.so.2+0x102d9)
SUMMARY: ThreadSanitizer: destroy of a locked mutex /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:249:30 in glog_internal_namespace_::Mutex::~Mutex()
==================
==================
WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=8443)
Cycle in lock order graph: M1860 (0x7b14000167b8) => M37857730294380664 (0x000000000000) => M1860
Mutex M37857730294380664 acquired here while holding mutex M1860 in main thread:
#0 pthread_rwlock_wrlock /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:1352 (kudu+0x4bb154)
#1 glog_internal_namespace_::Mutex::Lock() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:250:30 (libglog.so.0+0x1abb7)
#2 glog_internal_namespace_::MutexLock::MutexLock(glog_internal_namespace_::Mutex*) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:290 (libglog.so.0+0x1abb7)
#3 google::InitVLOG3__(int**, int*, char const*, int) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/vlog_is_on.cc:199 (libglog.so.0+0x1abb7)
#4 kudu::rpc::Messenger::ShutdownInternal(kudu::rpc::Messenger::ShutdownMode) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:283:5 (libkrpc.so+0xa9471)
#5 kudu::rpc::Messenger::AllExternalReferencesDropped() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:249:3 (libkrpc.so+0xa9227)
#6 std::__1::mem_fun_t<void, kudu::rpc::Messenger>::operator()(kudu::rpc::Messenger*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1120:17 (libkrpc.so+0xad755)
#7 std::__1::__shared_ptr_pointer<kudu::rpc::Messenger*, std::__1::mem_fun_t<void, kudu::rpc::Messenger>, std::__1::allocator<kudu::rpc::Messenger> >::__on_zero_shared() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3586 (libkrpc.so+0xad755)
#8 std::__1::__shared_count::__release_shared() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3490:9 (kudu+0x576f2e)
#9 std::__1::__shared_weak_count::__release_shared() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3532 (kudu+0x576f2e)
#10 std::__1::shared_ptr<kudu::rpc::Messenger>::~shared_ptr() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4468 (kudu+0x576f2e)
#11 kudu::rpc::Proxy::~Proxy() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/proxy.cc:72:1 (libkrpc.so+0xbd0f3)
#12 kudu::server::GenericServiceProxy::~GenericServiceProxy() /data/somelongdirectorytoavoidrpathissues/src/kudu/build/tsan/src/kudu/server/server_base.proxy.cc:26:1 (libserver_base_proto.so+0x4bd79)
#13 std::__1::default_delete<kudu::server::GenericServiceProxy>::operator()(kudu::server::GenericServiceProxy*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2285:5 (libkudu_tools_util.so+0x74acf)
#14 std::__1::unique_ptr<kudu::server::GenericServiceProxy, std::__1::default_delete<kudu::server::GenericServiceProxy> >::reset(kudu::server::GenericServiceProxy*) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2598 (libkudu_tools_util.so+0x74acf)
#15 std::__1::unique_ptr<kudu::server::GenericServiceProxy, std::__1::default_delete<kudu::server::GenericServiceProxy> >::~unique_ptr() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2552 (libkudu_tools_util.so+0x74acf)
#16 kudu::tools::DumpMemTrackers(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned short) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:629 (libkudu_tools_util.so+0x74acf)
#17 kudu::tools::(anonymous namespace)::TserverDumpMemTrackers(kudu::tools::RunnerContext const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_tserver.cc:161:10 (kudu+0x5b8599)
#18 _ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_ /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:4482:1 (kudu+0x53956b)
#19 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:318 (kudu+0x53956b)
#20 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1562:12 (kudu+0x53949d)
#21 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1916:12 (libkudu_tools_util.so+0x6b2a4)
#22 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action.cc:258:10 (libkudu_tools_util.so+0x69974)
#23 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:132:15 (kudu+0x5bbcf6)
#24 kudu::tools::RunTool(int, char**, bool) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:204:16 (kudu+0x5bcc51)
#25 main /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:265:10 (kudu+0x5bcfbe)
Hint: use TSAN_OPTIONS=second_deadlock_stack=1 to get more informative warning message
Mutex M1860 acquired here while holding mutex M37857730294380664 in thread T7:
#0 AnnotateRWLockAcquired /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_ann.cc:271 (kudu+0x4dbd0c)
#1 kudu::rw_spinlock::lock() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/locks.h:112:5 (libkudu_client.so+0x16d842)
#2 kudu::percpu_rwlock::lock() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/locks.h:222:22 (libkudu_client.so+0x16d7d2)
#3 std::__1::lock_guard<kudu::percpu_rwlock>::lock_guard(kudu::percpu_rwlock&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__mutex_base:104:27 (libkrpc.so+0xaad39)
#4 kudu::rpc::Messenger::~Messenger() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:430 (libkrpc.so+0xaad39)
#5 std::__1::default_delete<kudu::rpc::Messenger>::operator()(kudu::rpc::Messenger*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2285:5 (libkrpc.so+0xb081b)
#6 std::__1::__shared_ptr_pointer<kudu::rpc::Messenger*, std::__1::default_delete<kudu::rpc::Messenger>, std::__1::allocator<kudu::rpc::Messenger> >::__on_zero_shared() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3586 (libkrpc.so+0xb081b)
#7 std::__1::__shared_count::__release_shared() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3490:9 (kudu+0x576f2e)
#8 std::__1::__shared_weak_count::__release_shared() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:3532 (kudu+0x576f2e)
#9 std::__1::shared_ptr<kudu::rpc::Messenger>::~shared_ptr() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4468 (kudu+0x576f2e)
#10 std::__1::shared_ptr<kudu::rpc::Messenger>::reset() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/memory:4603:5 (libkrpc.so+0xbf151)
#11 kudu::rpc::ReactorThread::RunThread() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:499 (libkrpc.so+0xbf151)
#12 boost::_mfi::mf0<void, kudu::rpc::ReactorThread>::operator()(kudu::rpc::ReactorThread*) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/mem_fn_template.hpp:49:29 (libkrpc.so+0xc9539)
#13 void boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> >::operator()<boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>&, boost::_bi::list0&, int) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:259:9 (libkrpc.so+0xc948a)
#14 boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >::operator()() /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/bind/bind.hpp:1222:16 (libkrpc.so+0xc9413)
#15 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf0<void, kudu::rpc::ReactorThread>, boost::_bi::list1<boost::_bi::value<kudu::rpc::ReactorThread*> > >, void>::invoke(boost::detail::function::function_buffer&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:159:11 (libkrpc.so+0xc9209)
#16 boost::function0<void>::operator()() const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/boost/function/function_template.hpp:770:14 (libkrpc.so+0xb8a11)
#17 kudu::Thread::SuperviseThread(void*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:657:3 (libkudu_util.so+0x1e1824)
Thread T7 'rpc reactor-845' (tid=8451, running) created by main thread at:
#0 pthread_create /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/llvm-6.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:992 (kudu+0x4bd8eb)
#1 kudu::Thread::StartThread(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, boost::function<void ()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:601:15 (libkudu_util.so+0x1e100b)
#2 kudu::Status kudu::Thread::Create<void (kudu::rpc::ReactorThread::*)(), kudu::rpc::ReactorThread*>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, void (kudu::rpc::ReactorThread::* const&)(), kudu::rpc::ReactorThread* const&, scoped_refptr<kudu::Thread>*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.h:164:12 (libkrpc.so+0xc48e5)
#3 kudu::rpc::ReactorThread::Init() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:185:10 (libkrpc.so+0xbec4e)
#4 kudu::rpc::Reactor::Init() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:759:18 (libkrpc.so+0xc32f1)
#5 kudu::rpc::Messenger::Init() /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:446:5 (libkrpc.so+0xa90e2)
#6 kudu::rpc::MessengerBuilder::Build(std::__1::shared_ptr<kudu::rpc::Messenger>*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/messenger.cc:205:3 (libkrpc.so+0xa8b3d)
#7 kudu::Status kudu::tools::BuildProxy<kudu::server::GenericServiceProxy>(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned short, std::__1::unique_ptr<kudu::server::GenericServiceProxy, std::__1::default_delete<kudu::server::GenericServiceProxy> >*) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:328:3 (libkudu_tools_util.so+0x7c55e)
#8 kudu::tools::DumpMemTrackers(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned short) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:591:3 (libkudu_tools_util.so+0x747e3)
#9 kudu::tools::(anonymous namespace)::TserverDumpMemTrackers(kudu::tools::RunnerContext const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_tserver.cc:161:10 (kudu+0x5b8599)
#10 _ZNSt3__18__invokeIRPFN4kudu6StatusERKNS1_5tools13RunnerContextEEJS6_EEEDTclclsr3std3__1E7forwardIT_Efp_Espclsr3std3__1E7forwardIT0_Efp0_EEEOSA_DpOSB_ /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:4482:1 (kudu+0x53956b)
#11 kudu::Status std::__1::__invoke_void_return_wrapper<kudu::Status>::__call<kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&>(kudu::Status (*&)(kudu::tools::RunnerContext const&), kudu::tools::RunnerContext const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:318 (kudu+0x53956b)
#12 std::__1::__function::__func<kudu::Status (*)(kudu::tools::RunnerContext const&), std::__1::allocator<kudu::Status (*)(kudu::tools::RunnerContext const&)>, kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1562:12 (kudu+0x53949d)
#13 std::__1::function<kudu::Status (kudu::tools::RunnerContext const&)>::operator()(kudu::tools::RunnerContext const&) const /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1916:12 (libkudu_tools_util.so+0x6b2a4)
#14 kudu::tools::Action::Run(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, std::__1::unordered_map<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::hash<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::equal_to<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::pair<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) const /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action.cc:258:10 (libkudu_tools_util.so+0x69974)
#15 kudu::tools::DispatchCommand(std::__1::vector<kudu::tools::Mode*, std::__1::allocator<kudu::tools::Mode*> > const&, kudu::tools::Action*, std::__1::deque<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:132:15 (kudu+0x5bbcf6)
#16 kudu::tools::RunTool(int, char**, bool) /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:204:16 (kudu+0x5bcc51)
#17 main /data/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_main.cc:265:10 (kudu+0x5bcfbe)
SUMMARY: ThreadSanitizer: lock-order-inversion (potential deadlock) /data/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/glog-0.3.5/src/base/mutex.h:250:30 in glog_internal_namespace_::Mutex::Lock()
==================
ThreadSanitizer: reported 2 warnings
{noformat}
--
This message was sent by Atlassian JIRA
(v7.6.14#76016)