You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@impala.apache.org by "Sailesh Mukil (JIRA)" <ji...@apache.org> on 2017/11/06 18:11:00 UTC
[jira] [Resolved] (IMPALA-6144) Coordinator threads that publish
RuntimeFilters continue to run after query failure/cancellation
[ https://issues.apache.org/jira/browse/IMPALA-6144?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Sailesh Mukil resolved IMPALA-6144.
-----------------------------------
Resolution: Fixed
Fix Version/s: Impala 2.11.0
Commit in:
https://github.com/apache/incubator-impala/commit/e6c3a01b948777affaee3dd004a526f5e4dcd5a0
> Coordinator threads that publish RuntimeFilters continue to run after query failure/cancellation
> ------------------------------------------------------------------------------------------------
>
> Key: IMPALA-6144
> URL: https://issues.apache.org/jira/browse/IMPALA-6144
> Project: IMPALA
> Issue Type: Bug
> Components: Backend
> Reporter: Mostafa Mokhtar
> Assignee: Sailesh Mukil
> Labels: memory, runtime-filters
> Fix For: Impala 2.11.0
>
>
> When running highly concurrent queries with lots of joins on a large cluster I noticed that there is lots of untracked memory on the coordinator node.
> Experiments showed that the majority of the untracked memory is coming from RunTimeFilters.
> After canceling the concurrent queries the Queries tab showed 0 "Running" and 0 "Waiting to be cancelled" queries while the memz tab show that memory allocated for the cancelled/completed queries is still there.
> Same issue happens if remote fragments finish while the coordinator node is in the progress of sending the runtime filters to the remote nodes.
> {code}
> CPU Time
> 1 of 1: 100.0% (20.348s of 20.348s)
> impalad ! apache::thrift::transport::TSSLSocket::write - [unknown source file]
> impalad ! apache::thrift::transport::TTransport::write + 0xc - TTransport.h:158
> impalad ! apache::thrift::transport::TSaslTransport::write + 0x3b - TSaslTransport.cpp:233
> impalad ! apache::thrift::transport::TBufferedTransport::writeSlow + 0x58 - [unknown source file]
> impalad ! apache::thrift::transport::TTransport::write + 0xb - TTransport.h:158
> impalad ! writeString<std::basic_string<char> > + 0x39 - TBinaryProtocol.tcc:186
> impalad ! apache::thrift::protocol::TBinaryProtocolT<apache::thrift::transport::TTransport>::writeBinary - TBinaryProtocol.tcc:193
> impalad ! apache::thrift::protocol::TVirtualProtocol<apache::thrift::protocol::TBinaryProtocolT<apache::thrift::transport::TTransport>, apache::thrift::protocol::TProtocolDefaults>::writeBinary_virt + 0xb - TVirtualProtocol.h:421
> impalad ! apache::thrift::protocol::TProtocol::writeBinary + 0x10 - TProtocol.h:468
> impalad ! impala::TBloomFilter::write + 0x72 - ImpalaInternalService_types.cpp:4397
> impalad ! impala::TPublishFilterParams::write + 0xad - ImpalaInternalService_types.cpp:4689
> impalad ! impala::ImpalaInternalService_PublishFilter_pargs::write + 0x44 - ImpalaInternalService.cpp:903
> impalad ! impala::ImpalaInternalServiceClient::send_PublishFilter + 0x78 - ImpalaInternalService.cpp:1310
> impalad ! impala::ImpalaBackendClient::PublishFilter + 0x14 - backend-client.h:117
> impalad ! impala::ClientConnection<impala::ImpalaBackendClient>::DoRpc<void (impala::TPublishFilterResult&, impala::TPublishFilterParams const&, bool*) impala::ImpalaBackendClient::*, impala::TPublishFilterParams, impala::TPublishFilterResult> + 0x57 - client-cache.h:240
> impalad ! impala::Coordinator::BackendState::PublishFilter + 0x288 - coordinator-backend-state.cc:415
> impalad ! impala::Coordinator::UpdateFilter + 0x5ff - coordinator.cc:1167
> impalad ! impala::ImpalaServer::UpdateFilter + 0x32 - impala-server.cc:2089
> impalad ! impala::ImpalaInternalServiceProcessor::process_UpdateFilter + 0x1e2 - ImpalaInternalService.cpp:1613
> impalad ! impala::ImpalaInternalServiceProcessor::dispatchCall + 0xe8 - ImpalaInternalService.cpp:1370
> impalad ! apache::thrift::TDispatchProcessor::process + 0xab - TDispatchProcessor.h:121
> impalad ! apache::thrift::server::TAcceptQueueServer::Task::run + 0x15e - TAcceptQueueServer.cpp:77
> impalad ! impala::ThriftThread::RunRunnable + 0x68 - thrift-thread.cc:74
> impalad ! boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>::operator() + 0x3a - mem_fn_template.hpp:280
> impalad ! operator()<boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<long unsigned int>*>, boost::_bi::list0> + 0x1e - bind.hpp:392
> impalad ! boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable>>, boost::_bi::value<impala::Promise<unsigned long>*>>>::operator() - bind_template.hpp:20
> impalad ! boost::detail::function::void_function_obj_invoker0<boost::_bi::bind_t<void, boost::_mfi::mf2<void, impala::ThriftThread, boost::shared_ptr<apache::thrift::concurrency::Runnable>, impala::Promise<unsigned long>*>, boost::_bi::list3<boost::_bi::value<impala::ThriftThread*>, boost::_bi::value<boost::shared_ptr<apache::thrift::concurrency::Runnable>>, boost::_bi::value<impala::Promise<unsigned long>*>>>, void>::invoke + 0x9 - function_template.hpp:153
> impalad ! boost::function0<void>::operator() + 0x1a - function_template.hpp:767
> impalad ! impala::Thread::SuperviseThread + 0x1a7 - thread.cc:352
> impalad ! operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0> + 0x5a - bind.hpp:457
> impalad ! boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void (void)>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void (void)>>, boost::_bi::value<impala::Promise<long>*>>>::operator() - bind_template.hpp:20
> impalad ! boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(std::string const&, std::string const&, boost::function<void (void)>, impala::Promise<long>*), boost::_bi::list4<boost::_bi::value<std::string>, boost::_bi::value<std::string>, boost::_bi::value<boost::function<void (void)>>, boost::_bi::value<impala::Promise<long>*>>>>::run + 0x19 - thread.hpp:116
> impalad ! thread_proxy + 0xd9 - [unknown source file]
> libpthread-2.12.so ! start_thread + 0xd0 - [unknown source file]
> libc-2.12.so ! clone + 0x6c - [unknown source file]
> {code}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)