You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@kudu.apache.org by "Bakai Ádám (Jira)" <ji...@apache.org> on 2024/12/03 15:01:00 UTC
[jira] [Commented] (KUDU-3631) TSAN errors in tests when using InternalMiniCluster
[ https://issues.apache.org/jira/browse/KUDU-3631?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17902678#comment-17902678 ]
Bakai Ádám commented on KUDU-3631:
----------------------------------
One example:
{code:java}
WARNING: ThreadSanitizer: destroy of a locked mutex (pid=411)
#0 pthread_mutex_destroy /root/tmp/test123/kudu/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1241 (ksck_remote-test+0x308066)
#1 kudu::Mutex::~Mutex() /root/tmp/test123/kudu/src/kudu/util/mutex.cc:76:12 (libkudu_util.so+0x3df980)
#2 kudu::CountDownLatch::~CountDownLatch() /root/tmp/test123/kudu/src/kudu/util/countdown_latch.h:33:7 (ksck_remote-test+0x372262)
#3 kudu::server::ServerBase::~ServerBase() /root/tmp/test123/kudu/src/kudu/server/server_base.cc:772:1 (libserver_process.so+0x1427ef)
#4 kudu::kserver::KuduServer::~KuduServer() /root/tmp/test123/kudu/src/kudu/kserver/kserver.h:41:7 (libmaster.so+0x400b3f)
#5 kudu::master::Master::~Master() /root/tmp/test123/kudu/src/kudu/master/master.cc:263:1 (libmaster.so+0x3f6dc2)
#6 kudu::master::Master::~Master() /root/tmp/test123/kudu/src/kudu/master/master.cc:261:19 (libmaster.so+0x3f70b9)
#7 std::__1::default_delete<kudu::master::Master>::operator()(kudu::master::Master*) const /root/tmp/test123/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2262:5 (libmaster.so+0x434907)
#8 std::__1::unique_ptr<kudu::master::Master, std::__1::default_delete<kudu::master::Master> >::reset(kudu::master::Master*) /root/tmp/test123/kudu/thirdparty/installed/tsan/include/c++/v1/memory:2517:7 (libmaster.so+0x42d9cd)
#9 kudu::master::MiniMaster::Shutdown() /root/tmp/test123/kudu/src/kudu/master/mini_master.cc:120:13 (libmaster.so+0x4c18c2)
#10 kudu::tools::RemoteKsckTest_TestClusterWithLocation_Test::TestBody() /root/tmp/test123/kudu/src/kudu/tools/ksck_remote-test.cc:612:13 (ksck_remote-test+0x35a1f8)
#11 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64dbf)
#12 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64dbf)
#13 testing::Test::Run() /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2674:5 (libgtest.so.1.12.1+0x429c1)
#14 testing::TestInfo::Run() /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2853:11 (libgtest.so.1.12.1+0x43cd8)
#15 testing::TestSuite::Run() /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44cb4)
#16 testing::internal::UnitTestImpl::RunAllTests() /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x597a4)
#17 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65c7f)
#18 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65c7f)
#19 testing::UnitTest::Run() /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58d5c)
#20 RUN_ALL_TESTS() /root/tmp/test123/kudu/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (ksck_remote-test+0x38d91b)
#21 main /root/tmp/test123/kudu/src/kudu/util/test_main.cc:109:10 (ksck_remote-test+0x38c81c) and:
#0 pthread_cond_timedwait /root/tmp/test123/kudu/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_interceptors_posix.cpp:1181 (ksck_remote-test+0x2f5550)
#1 kudu::ConditionVariable::WaitUntil(kudu::MonoTime const&) const /root/tmp/test123/kudu/src/kudu/util/condition_variable.cc:87:12 (libkudu_util.so+0x2c400a)
#2 kudu::CountDownLatch::WaitUntil(kudu::MonoTime const&) const /root/tmp/test123/kudu/src/kudu/util/countdown_latch.h:89:18 (ksck_remote-test+0x38abe8)
#3 kudu::CountDownLatch::WaitFor(kudu::MonoDelta const&) const /root/tmp/test123/kudu/src/kudu/util/countdown_latch.h:99:12 (ksck_remote-test+0x38ab5e)
#4 kudu::master::Master::ExpiredReservedTablesDeleterThread() /root/tmp/test123/kudu/src/kudu/master/master.cc:568:42 (libmaster.so+0x3f975e)
#5 kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2::operator()() const /root/tmp/test123/kudu/src/kudu/master/master.cc:561:42 (libmaster.so+0x3ff551)
#6 decltype(std::__1::forward<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2&>(fp)()) std::__1::__invoke<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2&>(kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2&) /root/tmp/test123/kudu/thirdparty/installed/tsan/include/c++/v1/type_traits:3899:1 (libmaster.so+0x3ff509)
#7 void std::__1::__invoke_void_return_wrapper<void>::__call<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2&>(kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2&) /root/tmp/test123/kudu/thirdparty/installed/tsan/include/c++/v1/__functional_base:348:9 (libmaster.so+0x3ff499)
I20241202 12:03:01.460966 7487 leader_election.cc:304] T d599a4e3872e43828fa5b8c2b551ed54 P a459e693ccca420f83a07ccac02c6efd [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a459e693ccca420f83a07ccac02c6efd; no voters: 300ad6e7ac034b28923bac9095c765cb, 87a10d0709914f1daaa00bcc952ad5e7
#8 std::__1::__function::__alloc_func<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2, std::__1::allocator<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2>, void ()>::operator()() /root/tmp/test123/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1557:16 (libmaster.so+0x3ff461)
#9 std::__1::__function::__func<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2, std::__1::allocator<kudu::master::Master::StartExpiredReservedTablesDeleterThread()::$_2>, void ()>::operator()() /root/tmp/test123/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1731:12 (libmaster.so+0x3fe75d)
#10 std::__1::__function::__value_func<void ()>::operator()() const /root/tmp/test123/kudu/thirdparty/installed/tsan/include/c++/v1/functional:1884:16 (libtserver_test_util.so+0x5ffa4)
#11 std::__1::function<void ()>::operator()() const /root/tmp/test123/kudu/thirdparty/installed/tsan/include/c++/v1/functional:2556:12 (libtserver_test_util.so+0x5fdd9)
#12 kudu::Thread::SuperviseThread(void*) /root/tmp/test123/kudu/src/kudu/util/thread.cc:693:3 (libkudu_util.so+0x44c836) Location is heap block of size 3696 at 0x7b80004e5000 allocated by main thread:
#0 operator new(unsigned long) /root/tmp/test123/kudu/thirdparty/src/llvm-11.0.0.src/projects/compiler-rt/lib/tsan/rtl/tsan_new_delete.cpp:64 (ksck_remote-test+0x352f57)
#1 kudu::master::MiniMaster::Start() /root/tmp/test123/kudu/src/kudu/master/mini_master.cc:94:29 (libmaster.so+0x4c1a70)
#2 kudu::cluster::InternalMiniCluster::StartMasters() /root/tmp/test123/kudu/src/kudu/mini-cluster/internal_mini_cluster.cc:178:5 (libmini_cluster.so+0xd6bbf)
#3 kudu::cluster::InternalMiniCluster::Start() /root/tmp/test123/kudu/src/kudu/mini-cluster/internal_mini_cluster.cc:109:3 (libmini_cluster.so+0xd63eb)
#4 kudu::tools::RemoteKsckTest::SetUp() /root/tmp/test123/kudu/src/kudu/tools/ksck_remote-test.cc:137:5 (ksck_remote-test+0x374b66)
#5 void testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x64dbf)
#6 void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x64dbf)
#7 testing::Test::Run() /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2669:3 (libgtest.so.1.12.1+0x4279b)
#8 testing::TestInfo::Run() /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2853:11 (libgtest.so.1.12.1+0x43cd8)
#9 testing::TestSuite::Run() /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012:30 (libgtest.so.1.12.1+0x44cb4)
#10 testing::internal::UnitTestImpl::RunAllTests() /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870:44 (libgtest.so.1.12.1+0x597a4)
#11 bool testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599:10 (libgtest.so.1.12.1+0x65c7f)
#12 bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635:14 (libgtest.so.1.12.1+0x65c7f)
#13 testing::UnitTest::Run() /root/tmp/test123/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444:10 (libgtest.so.1.12.1+0x58d5c)
#14 RUN_ALL_TESTS() /root/tmp/test123/kudu/thirdparty/installed/tsan/include/gtest/gtest.h:2293:73 (ksck_remote-test+0x38d91b)
#15 main /root/tmp/test123/kudu/src/kudu/util/test_main.cc:109:10 (ksck_remote-test+0x38c81c) Mutex M64312084381192440 is already destroyed.SUMMARY: ThreadSanitizer: destroy of a locked mutex /root/tmp/test123/kudu/src/kudu/util/mutex.cc:76:12 in kudu::Mutex::~Mutex() {code}
> TSAN errors in tests when using InternalMiniCluster
> ---------------------------------------------------
>
> Key: KUDU-3631
> URL: https://issues.apache.org/jira/browse/KUDU-3631
> Project: Kudu
> Issue Type: Sub-task
> Reporter: Bakai Ádám
> Priority: Major
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)