You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kudu.apache.org by "Todd Lipcon (JIRA)" <ji...@apache.org> on 2016/06/30 21:24:10 UTC
[jira] [Commented] (KUDU-1509) tsan: Data race in CBTree in
disk_reservation-itest
[ https://issues.apache.org/jira/browse/KUDU-1509?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15357881#comment-15357881 ]
Todd Lipcon commented on KUDU-1509:
-----------------------------------
You sure you had the suppressions set properly? These should be suppressed.
> tsan: Data race in CBTree in disk_reservation-itest
> ---------------------------------------------------
>
> Key: KUDU-1509
> URL: https://issues.apache.org/jira/browse/KUDU-1509
> Project: Kudu
> Issue Type: Bug
> Components: tablet
> Affects Versions: 0.9.0
> Reporter: Mike Percy
>
> CBTree reports a data race in disk_reservation-itest:
> {code}
> WARNING: ThreadSanitizer: data race (pid=20459)
> Write of size 1 at 0x7dd000002889 by thread T145:
> #0 memcpy /home/mpercy/src/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:677 (kudu-tserver+0x00000044cb1b)
> #1 void kudu::InlineSlice<8ul, true>::set<kudu::ThreadSafeMemoryTrackingArena>(unsigned char const*, unsigned long, kudu::ThreadSafeMemoryTrackingArena*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/inline_slice.h:116:7 (libtablet.so+0x0000002972b6)
> #2 void kudu::InlineSlice<8ul, true>::set<kudu::ThreadSafeMemoryTrackingArena>(kudu::Slice const&, kudu::ThreadSafeMemoryTrackingArena*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/inline_slice.h:95:5 (libtablet.so+0x000000297240)
> #3 void kudu::tablet::btree::InsertInSliceArray<kudu::InlineSlice<8ul, true>, kudu::ThreadSafeMemoryTrackingArena>(kudu::InlineSlice<8ul, true>*, unsigned long, kudu::Slice const&, unsigned long, kudu::ThreadSafeMemoryTrackingArena*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:347:3 (libtablet.so+0x00000028bc4a)
> #4 kudu::tablet::btree::InternalNode<kudu::tablet::MSBTreeTraits>::Insert(kudu::Slice const&, kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::ThreadSafeMemoryTrackingArena*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:558:5 (libtablet.so+0x000000298a34)
> #5 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::PropagateSplitUpward(kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1524:13 (libtablet.so+0x0000002979d1)
> #6 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::PropagateSplitUpward(kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1556:9 (libtablet.so+0x000000297dca)
> #7 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::PropagateSplitUpward(kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::tablet::btree::NodePtr<kudu::tablet::MSBTreeTraits>, kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1556:9 (libtablet.so+0x000000297dca)
> #8 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::SplitLeafAndInsertUp(kudu::tablet::btree::PreparedMutation<kudu::tablet::MSBTreeTraits>*, kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1485:5 (libtablet.so+0x000000296aeb)
> #9 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::Insert(kudu::tablet::btree::PreparedMutation<kudu::tablet::MSBTreeTraits>*, kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1312:16 (libtablet.so+0x0000002965d3)
> #10 kudu::tablet::btree::PreparedMutation<kudu::tablet::MSBTreeTraits>::Insert(kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:861:12 (libtablet.so+0x00000028cafd)
> #11 kudu::tablet::MemRowSet::Insert(kudu::Timestamp, kudu::ConstContiguousRow const&, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/memrowset.cc:166:27 (libtablet.so+0x00000028936b)
> #12 kudu::tablet::Tablet::InsertOrUpsertUnlocked(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:419:14 (libtablet.so+0x0000001dcebd)
> #13 kudu::tablet::Tablet::ApplyRowOperation(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:618:21 (libtablet.so+0x0000001dde5d)
> #14 kudu::tablet::Tablet::ApplyRowOperations(kudu::tablet::WriteTransactionState*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:598:5 (libtablet.so+0x0000001ddcaf)
> #15 kudu::tablet::WriteTransaction::Apply(gscoped_ptr<kudu::consensus::CommitMsg, kudu::DefaultDeleter<kudu::consensus::CommitMsg> >*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/write_transaction.cc:124:3 (libtablet.so+0x0000002438f1)
> #16 kudu::tablet::TransactionDriver::ApplyTask() /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:377:31 (libtablet.so+0x00000023b2f5)
> #17 kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>::Run(kudu::tablet::TransactionDriver*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:134:12 (libtablet.so+0x00000023d2c6)
> #18 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, void ()(kudu::tablet::TransactionDriver*)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, kudu::tablet::TransactionDriver*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:871:5 (libtablet.so+0x00000023d255)
> #19 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, void ()(kudu::tablet::TransactionDriver*), void ()(kudu::internal::UnretainedWrapper<kudu::tablet::TransactionDriver>)>, void ()(kudu::tablet::TransactionDriver*)>::Run(kudu::internal::BindStateBase*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1063:12 (libtablet.so+0x00000023d1ea)
> #20 kudu::Callback<void ()()>::Run() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:396:12 (libconsensus.so+0x0000000befdd)
> #21 boost::_mfi::cmf0<void, kudu::Callback<void ()()> >::operator()(kudu::Callback<void ()()> const&) const /usr/include/boost/bind/mem_fn_template.hpp:117:16 (libconsensus.so+0x0000000c5726)
> #22 void boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > >::operator()<boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246:9 (libconsensus.so+0x0000000c568e)
> #23 boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libconsensus.so+0x0000000c5643)
> #24 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libconsensus.so+0x0000000c5401)
> #25 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
> #26 kudu::FunctionRunnable::Run() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:48:5 (libkudu_util.so+0x000000226bcd)
> #27 kudu::ThreadPool::DispatchThread(bool) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:343:7 (libkudu_util.so+0x000000224fcf)
> #28 boost::_mfi::mf1<void, kudu::ThreadPool, bool>::operator()(kudu::ThreadPool*, bool) const /usr/include/boost/bind/mem_fn_template.hpp:162:16 (libkudu_util.so+0x00000022941e)
> #29 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:306:9 (libkudu_util.so+0x00000022937b)
> #30 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libkudu_util.so+0x000000229303)
> #31 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libkudu_util.so+0x000000229101)
> #32 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
> #33 kudu::Thread::SuperviseThread(void*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021b55f)
> Previous atomic read of size 8 at 0x7dd000002888 by thread T142:
> #0 __tsan_atomic64_load /home/mpercy/src/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interface_atomic.cc:538 (kudu-tserver+0x0000004661ac)
> #1 base::subtle::Acquire_Load(long const volatile*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/atomicops-internals-tsan.h:179:10 (libtserver.so+0x00000013d2ee)
> #2 kudu::InlineSlice<8ul, true>::LoadValue() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/inline_slice.h:144:27 (libtablet.so+0x0000001fbd19)
> #3 kudu::InlineSlice<8ul, true>::as_slice() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/inline_slice.h:79:33 (libtablet.so+0x0000001fb826)
> #4 unsigned long kudu::tablet::btree::FindInSliceArray<8ul>(kudu::InlineSlice<8ul, true> const*, long, kudu::Slice const&, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:330 (libtablet.so+0x0000001fb826)
> #5 kudu::tablet::btree::InternalNode<kudu::tablet::MSBTreeTraits>::Find(kudu::Slice const&, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:576:12 (libtablet.so+0x0000001fb609)
> #6 kudu::tablet::btree::InternalNode<kudu::tablet::MSBTreeTraits>::FindChild(kudu::Slice const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:585:18 (libtablet.so+0x0000001fb3e7)
> #7 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::TraverseToLeaf(kudu::Slice const&, long*) const /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1161:31 (libtablet.so+0x0000001fad7b)
> #8 kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>::PrepareMutation(kudu::tablet::btree::PreparedMutation<kudu::tablet::MSBTreeTraits>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:1269:33 (libtablet.so+0x000000295cf2)
> #9 kudu::tablet::btree::PreparedMutation<kudu::tablet::MSBTreeTraits>::Prepare(kudu::tablet::btree::CBTree<kudu::tablet::MSBTreeTraits>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/concurrent_btree.h:855:5 (libtablet.so+0x00000028c6b7)
> #10 kudu::tablet::MemRowSet::Insert(kudu::Timestamp, kudu::ConstContiguousRow const&, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/memrowset.cc:140:5 (libtablet.so+0x000000289219)
> #11 kudu::tablet::Tablet::InsertOrUpsertUnlocked(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:419:14 (libtablet.so+0x0000001dcebd)
> #12 kudu::tablet::Tablet::ApplyRowOperation(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:618:21 (libtablet.so+0x0000001dde5d)
> #13 kudu::tablet::Tablet::ApplyRowOperations(kudu::tablet::WriteTransactionState*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:598:5 (libtablet.so+0x0000001ddcaf)
> #14 kudu::tablet::WriteTransaction::Apply(gscoped_ptr<kudu::consensus::CommitMsg, kudu::DefaultDeleter<kudu::consensus::CommitMsg> >*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/write_transaction.cc:124:3 (libtablet.so+0x0000002438f1)
> #15 kudu::tablet::TransactionDriver::ApplyTask() /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:377:31 (libtablet.so+0x00000023b2f5)
> #16 kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>::Run(kudu::tablet::TransactionDriver*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:134:12 (libtablet.so+0x00000023d2c6)
> #17 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, void ()(kudu::tablet::TransactionDriver*)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, kudu::tablet::TransactionDriver*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:871:5 (libtablet.so+0x00000023d255)
> #18 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, void ()(kudu::tablet::TransactionDriver*), void ()(kudu::internal::UnretainedWrapper<kudu::tablet::TransactionDriver>)>, void ()(kudu::tablet::TransactionDriver*)>::Run(kudu::internal::BindStateBase*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1063:12 (libtablet.so+0x00000023d1ea)
> #19 kudu::Callback<void ()()>::Run() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:396:12 (libconsensus.so+0x0000000befdd)
> #20 boost::_mfi::cmf0<void, kudu::Callback<void ()()> >::operator()(kudu::Callback<void ()()> const&) const /usr/include/boost/bind/mem_fn_template.hpp:117:16 (libconsensus.so+0x0000000c5726)
> #21 void boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > >::operator()<boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246:9 (libconsensus.so+0x0000000c568e)
> #22 boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libconsensus.so+0x0000000c5643)
> #23 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libconsensus.so+0x0000000c5401)
> #24 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
> #25 kudu::FunctionRunnable::Run() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:48:5 (libkudu_util.so+0x000000226bcd)
> #26 kudu::ThreadPool::DispatchThread(bool) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:343:7 (libkudu_util.so+0x000000224fcf)
> #27 boost::_mfi::mf1<void, kudu::ThreadPool, bool>::operator()(kudu::ThreadPool*, bool) const /usr/include/boost/bind/mem_fn_template.hpp:162:16 (libkudu_util.so+0x00000022941e)
> #28 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:306:9 (libkudu_util.so+0x00000022937b)
> #29 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libkudu_util.so+0x000000229303)
> #30 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libkudu_util.so+0x000000229101)
> #31 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
> #32 kudu::Thread::SuperviseThread(void*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021b55f)
> Location is heap block of size 127488 at 0x7dd000000000 allocated by thread T134:
> #0 malloc /home/mpercy/src/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:595 (kudu-tserver+0x000000440858)
> #1 kudu::HeapBufferAllocator::Malloc(unsigned long) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/memory.cc:149:12 (libkudu_util.so+0x0000001d048f)
> #2 kudu::HeapBufferAllocator::AllocateInternal(unsigned long, unsigned long, kudu::BufferAllocator*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/memory.cc:100:50 (libkudu_util.so+0x0000001d039a)
> #3 kudu::BufferAllocator::DelegateAllocate(kudu::BufferAllocator*, unsigned long, unsigned long, kudu::BufferAllocator*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/memory.h:211:12 (libkudu_util.so+0x0000001d11a9)
> #4 kudu::MemoryTrackingBufferAllocator::AllocateInternal(unsigned long, unsigned long, kudu::BufferAllocator*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/memory.cc:308:22 (libkudu_util.so+0x0000001d0e5f)
> #5 kudu::BufferAllocator::BestEffortAllocate(unsigned long, unsigned long) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/memory.h:131:22 (libkudu_util.so+0x0000001cc3ab)
> #6 kudu::ArenaBase<true>::NewComponent(unsigned long, unsigned long) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/arena.cc:113:20 (libkudu_util.so+0x0000001cb1d4)
> #7 kudu::ArenaBase<true>::AllocateBytesFallback(unsigned long, unsigned long) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/arena.cc:93:26 (libkudu_util.so+0x0000001cba25)
> #8 kudu::ArenaBase<true>::AllocateBytesAligned(unsigned long, unsigned long) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/arena.h:442:10 (libtablet.so+0x000000295539)
> #9 kudu::ArenaBase<true>::AllocateBytes(unsigned long) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/arena.h:141:12 (libtablet.so+0x0000002964a5)
> #10 kudu::ArenaBase<true>::RelocateSlice(kudu::Slice const&, kudu::Slice*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/memory/arena.h:460:23 (libtablet.so+0x0000002963c3)
> #11 kudu::Status kudu::RelocateIndirectDataToArena<kudu::tablet::MRSRow, kudu::ThreadSafeMemoryTrackingArena>(kudu::tablet::MRSRow*, kudu::ThreadSafeMemoryTrackingArena*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/common/row.h:351:12 (libtablet.so+0x0000002962c4)
> #12 kudu::Status kudu::tablet::MRSRow::CopyRow<kudu::ThreadSafeMemoryTrackingArena>(kudu::ConstContiguousRow const&, kudu::ThreadSafeMemoryTrackingArena*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/memrowset.h:135:12 (libtablet.so+0x00000028ca72)
> #13 kudu::tablet::MemRowSet::Insert(kudu::Timestamp, kudu::ConstContiguousRow const&, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/memrowset.cc:164:31 (libtablet.so+0x00000028933f)
> #14 kudu::tablet::Tablet::InsertOrUpsertUnlocked(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:419:14 (libtablet.so+0x0000001dcebd)
> #15 kudu::tablet::Tablet::ApplyRowOperation(kudu::tablet::WriteTransactionState*, kudu::tablet::RowOp*, kudu::tablet::ProbeStats*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:618:21 (libtablet.so+0x0000001dde5d)
> #16 kudu::tablet::Tablet::ApplyRowOperations(kudu::tablet::WriteTransactionState*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/tablet.cc:598:5 (libtablet.so+0x0000001ddcaf)
> #17 kudu::tablet::WriteTransaction::Apply(gscoped_ptr<kudu::consensus::CommitMsg, kudu::DefaultDeleter<kudu::consensus::CommitMsg> >*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/write_transaction.cc:124:3 (libtablet.so+0x0000002438f1)
> #18 kudu::tablet::TransactionDriver::ApplyTask() /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:377:31 (libtablet.so+0x00000023b2f5)
> #19 kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>::Run(kudu::tablet::TransactionDriver*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:134:12 (libtablet.so+0x00000023d2c6)
> #20 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, void ()(kudu::tablet::TransactionDriver*)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, kudu::tablet::TransactionDriver*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:871:5 (libtablet.so+0x00000023d255)
> #21 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)()>, void ()(kudu::tablet::TransactionDriver*), void ()(kudu::internal::UnretainedWrapper<kudu::tablet::TransactionDriver>)>, void ()(kudu::tablet::TransactionDriver*)>::Run(kudu::internal::BindStateBase*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1063:12 (libtablet.so+0x00000023d1ea)
> #22 kudu::Callback<void ()()>::Run() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:396:12 (libconsensus.so+0x0000000befdd)
> #23 boost::_mfi::cmf0<void, kudu::Callback<void ()()> >::operator()(kudu::Callback<void ()()> const&) const /usr/include/boost/bind/mem_fn_template.hpp:117:16 (libconsensus.so+0x0000000c5726)
> #24 void boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > >::operator()<boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246:9 (libconsensus.so+0x0000000c568e)
> #25 boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libconsensus.so+0x0000000c5643)
> #26 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libconsensus.so+0x0000000c5401)
> #27 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
> #28 kudu::FunctionRunnable::Run() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:48:5 (libkudu_util.so+0x000000226bcd)
> #29 kudu::ThreadPool::DispatchThread(bool) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:343:7 (libkudu_util.so+0x000000224fcf)
> #30 boost::_mfi::mf1<void, kudu::ThreadPool, bool>::operator()(kudu::ThreadPool*, bool) const /usr/include/boost/bind/mem_fn_template.hpp:162:16 (libkudu_util.so+0x00000022941e)
> #31 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:306:9 (libkudu_util.so+0x00000022937b)
> #32 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libkudu_util.so+0x000000229303)
> #33 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libkudu_util.so+0x000000229101)
> #34 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
> #35 kudu::Thread::SuperviseThread(void*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021b55f)
> Thread T145 'apply [worker]-' (tid=20643, running) created by thread T124 at:
> #0 pthread_create /home/mpercy/src/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954 (kudu-tserver+0x00000044a123)
> #1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:509:15 (libkudu_util.so+0x00000021ade7)
> #2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool), kudu::ThreadPool*, bool>(std::string const&, std::string const&, void (kudu::ThreadPool::* const&)(bool), kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.h:162:12 (libkudu_util.so+0x0000002262ae)
> #3 kudu::ThreadPool::CreateThreadUnlocked() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:381:14 (libkudu_util.so+0x000000224304)
> #4 kudu::ThreadPool::Submit(std::shared_ptr<kudu::Runnable> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:211:21 (libkudu_util.so+0x000000224902)
> #5 kudu::ThreadPool::SubmitFunc(boost::function<void ()()> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:182:10 (libkudu_util.so+0x000000224704)
> #6 kudu::ThreadPool::SubmitClosure(kudu::Callback<void ()()> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:178:10 (libkudu_util.so+0x000000224632)
> #7 kudu::tablet::TransactionDriver::ApplyAsync() /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:358:10 (libtablet.so+0x00000023ae76)
> #8 kudu::tablet::TransactionDriver::ReplicationFinished(kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:311:31 (libtablet.so+0x00000023908d)
> #9 kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>::Run(kudu::tablet::TransactionDriver*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:190:12 (libtablet.so+0x000000234615)
> #10 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, kudu::tablet::TransactionDriver*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:887:5 (libtablet.so+0x000000234508)
> #11 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&), void ()(kudu::internal::UnretainedWrapper<kudu::tablet::TransactionDriver>)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&)>::Run(kudu::internal::BindStateBase*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1116:12 (libtablet.so+0x00000023444a)
> #12 kudu::Callback<void ()(kudu::Status const&)>::Run(kudu::Status const&) const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:436:12 (libmaster_rpc.so+0x000000011c31)
> #13 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/consensus.cc:60:3 (libconsensus.so+0x000000096280)
> #14 kudu::consensus::ReplicaState::AdvanceCommittedIndexUnlocked(kudu::consensus::OpId const&, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus_state.cc:624:5 (libconsensus.so+0x0000000f0f2d)
> #15 kudu::consensus::ReplicaState::UpdateMajorityReplicatedUnlocked(kudu::consensus::OpId const&, kudu::consensus::OpId*, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus_state.cc:520:31 (libconsensus.so+0x0000000f0017)
> #16 kudu::consensus::RaftConsensus::UpdateMajorityReplicatedUnlocked(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:592:14 (libconsensus.so+0x0000000d76b1)
> #17 kudu::consensus::RaftConsensus::UpdateMajorityReplicated(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:583:3 (libconsensus.so+0x0000000d73d0)
> #18 non-virtual thunk to kudu::consensus::RaftConsensus::UpdateMajorityReplicated(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:573:21 (libconsensus.so+0x0000000d79cd)
> #19 kudu::consensus::PeerMessageQueue::NotifyObserversOfMajorityReplOpChangeTask(kudu::consensus::OpId) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/consensus_queue.cc:807:5 (libconsensus.so+0x0000000a97f2)
> #20 kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>::Run(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:190:12 (libconsensus.so+0x0000000ba5e8)
> #21 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:887:5 (libconsensus.so+0x0000000ba558)
> #22 kudu::internal::Invoker<2, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId), void ()(kudu::internal::UnretainedWrapper<kudu::consensus::PeerMessageQueue>, kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId)>::Run(kudu::internal::BindStateBase*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1145:12 (libconsensus.so+0x0000000ba4d3)
> #23 kudu::Callback<void ()()>::Run() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:396:12 (libconsensus.so+0x0000000befdd)
> #24 boost::_mfi::cmf0<void, kudu::Callback<void ()()> >::operator()(kudu::Callback<void ()()> const&) const /usr/include/boost/bind/mem_fn_template.hpp:117:16 (libconsensus.so+0x0000000c5726)
> #25 void boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > >::operator()<boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246:9 (libconsensus.so+0x0000000c568e)
> #26 boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libconsensus.so+0x0000000c5643)
> #27 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libconsensus.so+0x0000000c5401)
> #28 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
> #29 kudu::FunctionRunnable::Run() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:48:5 (libkudu_util.so+0x000000226bcd)
> #30 kudu::ThreadPool::DispatchThread(bool) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:343:7 (libkudu_util.so+0x000000224fcf)
> #31 boost::_mfi::mf1<void, kudu::ThreadPool, bool>::operator()(kudu::ThreadPool*, bool) const /usr/include/boost/bind/mem_fn_template.hpp:162:16 (libkudu_util.so+0x00000022941e)
> #32 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:306:9 (libkudu_util.so+0x00000022937b)
> #33 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libkudu_util.so+0x000000229303)
> #34 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libkudu_util.so+0x000000229101)
> #35 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
> #36 kudu::Thread::SuperviseThread(void*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021b55f)
> Thread T142 'apply [worker]-' (tid=20640, running) created by thread T124 at:
> #0 pthread_create /home/mpercy/src/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954 (kudu-tserver+0x00000044a123)
> #1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:509:15 (libkudu_util.so+0x00000021ade7)
> #2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool), kudu::ThreadPool*, bool>(std::string const&, std::string const&, void (kudu::ThreadPool::* const&)(bool), kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.h:162:12 (libkudu_util.so+0x0000002262ae)
> #3 kudu::ThreadPool::CreateThreadUnlocked() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:381:14 (libkudu_util.so+0x000000224304)
> #4 kudu::ThreadPool::Submit(std::shared_ptr<kudu::Runnable> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:211:21 (libkudu_util.so+0x000000224902)
> #5 kudu::ThreadPool::SubmitFunc(boost::function<void ()()> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:182:10 (libkudu_util.so+0x000000224704)
> #6 kudu::ThreadPool::SubmitClosure(kudu::Callback<void ()()> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:178:10 (libkudu_util.so+0x000000224632)
> #7 kudu::tablet::TransactionDriver::ApplyAsync() /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:358:10 (libtablet.so+0x00000023ae76)
> #8 kudu::tablet::TransactionDriver::ReplicationFinished(kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:311:31 (libtablet.so+0x00000023908d)
> #9 kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>::Run(kudu::tablet::TransactionDriver*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:190:12 (libtablet.so+0x000000234615)
> #10 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, kudu::tablet::TransactionDriver*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:887:5 (libtablet.so+0x000000234508)
> #11 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&), void ()(kudu::internal::UnretainedWrapper<kudu::tablet::TransactionDriver>)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&)>::Run(kudu::internal::BindStateBase*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1116:12 (libtablet.so+0x00000023444a)
> #12 kudu::Callback<void ()(kudu::Status const&)>::Run(kudu::Status const&) const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:436:12 (libmaster_rpc.so+0x000000011c31)
> #13 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/consensus.cc:60:3 (libconsensus.so+0x000000096280)
> #14 kudu::consensus::ReplicaState::AdvanceCommittedIndexUnlocked(kudu::consensus::OpId const&, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus_state.cc:624:5 (libconsensus.so+0x0000000f0f2d)
> #15 kudu::consensus::ReplicaState::UpdateMajorityReplicatedUnlocked(kudu::consensus::OpId const&, kudu::consensus::OpId*, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus_state.cc:520:31 (libconsensus.so+0x0000000f0017)
> #16 kudu::consensus::RaftConsensus::UpdateMajorityReplicatedUnlocked(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:592:14 (libconsensus.so+0x0000000d76b1)
> #17 kudu::consensus::RaftConsensus::UpdateMajorityReplicated(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:583:3 (libconsensus.so+0x0000000d73d0)
> #18 non-virtual thunk to kudu::consensus::RaftConsensus::UpdateMajorityReplicated(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:573:21 (libconsensus.so+0x0000000d79cd)
> #19 kudu::consensus::PeerMessageQueue::NotifyObserversOfMajorityReplOpChangeTask(kudu::consensus::OpId) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/consensus_queue.cc:807:5 (libconsensus.so+0x0000000a97f2)
> #20 kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>::Run(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:190:12 (libconsensus.so+0x0000000ba5e8)
> #21 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:887:5 (libconsensus.so+0x0000000ba558)
> #22 kudu::internal::Invoker<2, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId), void ()(kudu::internal::UnretainedWrapper<kudu::consensus::PeerMessageQueue>, kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId)>::Run(kudu::internal::BindStateBase*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1145:12 (libconsensus.so+0x0000000ba4d3)
> #23 kudu::Callback<void ()()>::Run() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:396:12 (libconsensus.so+0x0000000befdd)
> #24 boost::_mfi::cmf0<void, kudu::Callback<void ()()> >::operator()(kudu::Callback<void ()()> const&) const /usr/include/boost/bind/mem_fn_template.hpp:117:16 (libconsensus.so+0x0000000c5726)
> #25 void boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > >::operator()<boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246:9 (libconsensus.so+0x0000000c568e)
> #26 boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libconsensus.so+0x0000000c5643)
> #27 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libconsensus.so+0x0000000c5401)
> #28 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
> #29 kudu::FunctionRunnable::Run() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:48:5 (libkudu_util.so+0x000000226bcd)
> #30 kudu::ThreadPool::DispatchThread(bool) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:343:7 (libkudu_util.so+0x000000224fcf)
> #31 boost::_mfi::mf1<void, kudu::ThreadPool, bool>::operator()(kudu::ThreadPool*, bool) const /usr/include/boost/bind/mem_fn_template.hpp:162:16 (libkudu_util.so+0x00000022941e)
> #32 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:306:9 (libkudu_util.so+0x00000022937b)
> #33 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libkudu_util.so+0x000000229303)
> #34 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libkudu_util.so+0x000000229101)
> #35 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
> #36 kudu::Thread::SuperviseThread(void*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021b55f)
> Thread T134 'apply [worker]-' (tid=20632, running) created by thread T124 at:
> #0 pthread_create /home/mpercy/src/kudu/thirdparty/llvm-3.8.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors.cc:954 (kudu-tserver+0x00000044a123)
> #1 kudu::Thread::StartThread(std::string const&, std::string const&, boost::function<void ()()> const&, unsigned long, scoped_refptr<kudu::Thread>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:509:15 (libkudu_util.so+0x00000021ade7)
> #2 kudu::Status kudu::Thread::Create<void (kudu::ThreadPool::*)(bool), kudu::ThreadPool*, bool>(std::string const&, std::string const&, void (kudu::ThreadPool::* const&)(bool), kudu::ThreadPool* const&, bool const&, scoped_refptr<kudu::Thread>*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.h:162:12 (libkudu_util.so+0x0000002262ae)
> #3 kudu::ThreadPool::CreateThreadUnlocked() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:381:14 (libkudu_util.so+0x000000224304)
> #4 kudu::ThreadPool::Submit(std::shared_ptr<kudu::Runnable> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:211:21 (libkudu_util.so+0x000000224902)
> #5 kudu::ThreadPool::SubmitFunc(boost::function<void ()()> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:182:10 (libkudu_util.so+0x000000224704)
> #6 kudu::ThreadPool::SubmitClosure(kudu::Callback<void ()()> const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:178:10 (libkudu_util.so+0x000000224632)
> #7 kudu::tablet::TransactionDriver::ApplyAsync() /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:358:10 (libtablet.so+0x00000023ae76)
> #8 kudu::tablet::TransactionDriver::ReplicationFinished(kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/tablet/transactions/transaction_driver.cc:311:31 (libtablet.so+0x00000023908d)
> #9 kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>::Run(kudu::tablet::TransactionDriver*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:190:12 (libtablet.so+0x000000234615)
> #10 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, kudu::tablet::TransactionDriver*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:887:5 (libtablet.so+0x000000234508)
> #11 kudu::internal::Invoker<1, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::tablet::TransactionDriver::*)(kudu::Status const&)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&), void ()(kudu::internal::UnretainedWrapper<kudu::tablet::TransactionDriver>)>, void ()(kudu::tablet::TransactionDriver*, kudu::Status const&)>::Run(kudu::internal::BindStateBase*, kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1116:12 (libtablet.so+0x00000023444a)
> #12 kudu::Callback<void ()(kudu::Status const&)>::Run(kudu::Status const&) const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:436:12 (libmaster_rpc.so+0x000000011c31)
> #13 kudu::consensus::ConsensusRound::NotifyReplicationFinished(kudu::Status const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/consensus.cc:60:3 (libconsensus.so+0x000000096280)
> #14 kudu::consensus::ReplicaState::AdvanceCommittedIndexUnlocked(kudu::consensus::OpId const&, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus_state.cc:624:5 (libconsensus.so+0x0000000f0f2d)
> #15 kudu::consensus::ReplicaState::UpdateMajorityReplicatedUnlocked(kudu::consensus::OpId const&, kudu::consensus::OpId*, bool*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus_state.cc:520:31 (libconsensus.so+0x0000000f0017)
> #16 kudu::consensus::RaftConsensus::UpdateMajorityReplicatedUnlocked(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:592:14 (libconsensus.so+0x0000000d76b1)
> #17 kudu::consensus::RaftConsensus::UpdateMajorityReplicated(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:583:3 (libconsensus.so+0x0000000d73d0)
> #18 non-virtual thunk to kudu::consensus::RaftConsensus::UpdateMajorityReplicated(kudu::consensus::OpId const&, kudu::consensus::OpId*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/raft_consensus.cc:573:21 (libconsensus.so+0x0000000d79cd)
> #19 kudu::consensus::PeerMessageQueue::NotifyObserversOfMajorityReplOpChangeTask(kudu::consensus::OpId) /home/mpercy/src/kudu/build/tsan/../../src/kudu/consensus/consensus_queue.cc:807:5 (libconsensus.so+0x0000000a97f2)
> #20 kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>::Run(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:190:12 (libconsensus.so+0x0000000ba5e8)
> #21 kudu::internal::InvokeHelper<false, void, kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&)>::MakeItSo(kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId const&) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:887:5 (libconsensus.so+0x0000000ba558)
> #22 kudu::internal::Invoker<2, kudu::internal::BindState<kudu::internal::RunnableAdapter<void (kudu::consensus::PeerMessageQueue::*)(kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId), void ()(kudu::internal::UnretainedWrapper<kudu::consensus::PeerMessageQueue>, kudu::consensus::OpId)>, void ()(kudu::consensus::PeerMessageQueue*, kudu::consensus::OpId)>::Run(kudu::internal::BindStateBase*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/bind_internal.h:1145:12 (libconsensus.so+0x0000000ba4d3)
> #23 kudu::Callback<void ()()>::Run() const /home/mpercy/src/kudu/build/tsan/../../src/kudu/gutil/callback.h:396:12 (libconsensus.so+0x0000000befdd)
> #24 boost::_mfi::cmf0<void, kudu::Callback<void ()()> >::operator()(kudu::Callback<void ()()> const&) const /usr/include/boost/bind/mem_fn_template.hpp:117:16 (libconsensus.so+0x0000000c5726)
> #25 void boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > >::operator()<boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:246:9 (libconsensus.so+0x0000000c568e)
> #26 boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libconsensus.so+0x0000000c5643)
> #27 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::cmf0<void, kudu::Callback<void ()()> >, boost::_bi::list1<boost::_bi::value<kudu::Callback<void ()()> > > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libconsensus.so+0x0000000c5401)
> #28 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
> #29 kudu::FunctionRunnable::Run() /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:48:5 (libkudu_util.so+0x000000226bcd)
> #30 kudu::ThreadPool::DispatchThread(bool) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/threadpool.cc:343:7 (libkudu_util.so+0x000000224fcf)
> #31 boost::_mfi::mf1<void, kudu::ThreadPool, bool>::operator()(kudu::ThreadPool*, bool) const /usr/include/boost/bind/mem_fn_template.hpp:162:16 (libkudu_util.so+0x00000022941e)
> #32 void boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> >::operator()<boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list0>(boost::_bi::type<void>, boost::_mfi::mf1<void, kudu::ThreadPool, bool>&, boost::_bi::list0&, int) /usr/include/boost/bind/bind.hpp:306:9 (libkudu_util.so+0x00000022937b)
> #33 boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >::operator()() /usr/include/boost/bind/bind_template.hpp:20:16 (libkudu_util.so+0x000000229303)
> #34 boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf1<void, kudu::ThreadPool, bool>, boost::_bi::list2<boost::_bi::value<kudu::ThreadPool*>, boost::_bi::value<bool> > >, void>::invoke(boost::detail::function::function_buffer&) /usr/include/boost/function/function_template.hpp:153:11 (libkudu_util.so+0x000000229101)
> #35 boost::function0<void>::operator()() const /usr/include/boost/function/function_template.hpp:1012:12 (libkrpc.so+0x0000000cce71)
> #36 kudu::Thread::SuperviseThread(void*) /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/thread.cc:586:3 (libkudu_util.so+0x00000021b55f)
> SUMMARY: ThreadSanitizer: data race /home/mpercy/src/kudu/build/tsan/../../src/kudu/util/inline_slice.h:116:7 in void kudu::InlineSlice<8ul, true>::set<kudu::ThreadSafeMemoryTrackingArena>(unsigned char const*, unsigned long, kudu::ThreadSafeMemoryTrackingArena*)
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)