You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@impala.apache.org by "Tim Armstrong (JIRA)" <ji...@apache.org> on 2017/08/30 21:51:01 UTC
[jira] [Resolved] (IMPALA-5855) Preaggregation crashes - unable to
initialise hash table
[ https://issues.apache.org/jira/browse/IMPALA-5855?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Tim Armstrong resolved IMPALA-5855.
-----------------------------------
Resolution: Fixed
Fix Version/s: Impala 2.10.0
IMPALA-5855: reserve enough memory for preaggs
The calculation in the planner failed to account for the behaviour of
Suballocator, which needs to obtain at least one buffer to allocate any
memory.
Testing:
Added a regression test that caused a crash before the fix.
Updated planner tests.
Was able to run local stress test binary search to completion (it
previously crashed).
Change-Id: I870fbe2f1da01c6123d3716a1198376f9a454c3b
Reviewed-on: http://gerrit.cloudera.org:8080/7871
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
> Preaggregation crashes - unable to initialise hash table
> --------------------------------------------------------
>
> Key: IMPALA-5855
> URL: https://issues.apache.org/jira/browse/IMPALA-5855
> Project: IMPALA
> Issue Type: Bug
> Components: Backend
> Affects Versions: Impala 2.10.0
> Reporter: Matthew Mulder
> Assignee: Tim Armstrong
> Priority: Blocker
> Labels: resource-management
> Fix For: Impala 2.10.0
>
>
> The preaggregation can crash if it can't get enough memory for its initial hash tables. The bug is that the Suballocator must be able to allocate at least one buffer of the default size to allocate any memory, but the calculation in the planner only reserves 512kb of memory.
> {code}
> F0828 15:54:41.898521 56405 partitioned-aggregation-node.cc:1128] Check failed: !is_streaming_preagg_ Preagg reserves enough memory for hash tables
> #0 0x0000003299a32625 in raise () from /lib64/libc.so.6
> No symbol table info available.
> #1 0x0000003299a33e05 in abort () from /lib64/libc.so.6
> No symbol table info available.
> #2 0x0000000002e7fc04 in google::DumpStackTraceAndExit() ()
> incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
> #3 0x0000000002e7667d in google::LogMessage::Fail() ()
> incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
> #4 0x0000000002e77f22 in google::LogMessage::SendToLog() ()
> incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
> #5 0x0000000002e76057 in google::LogMessage::Flush() ()
> incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
> #6 0x0000000002e7961e in google::LogMessageFatal::~LogMessageFatal() ()
> incs = {1, 4, 13, 40, 121, 364, 1093, 3280, 9841, 29524, 88573, 265720, 797161, 2391484}
> #7 0x0000000001bef4a8 in impala::PartitionedAggregationNode::CreateHashPartitions (this=0x7f5b7b97e000, level=0, single_partition_idx=-1) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/exec/partitioned-aggregation-node.cc:1128
> got_memory = false
> partition = 0x7f602ce48850
> i = 0
> num_partitions_created = 16
> #8 0x0000000001be6a9e in impala::PartitionedAggregationNode::Open (this=0x7f5b7b97e000, state=0x7f4a3adf5c00) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/exec/partitioned-aggregation-node.cc:290
> __status__ = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x68eafc0}, msg_ = 0x68ea390},
> static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x7f5a1d9c02f0}
> SCOPED_TIMER1 = {sw_ = {start_ = 2679682494289161, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counter_ = 0x7f5a8d3ecc88, is_cancelled_ = 0x0}
> batch = {static AT_CAPACITY_MEM_USAGE = 8388608, static FIXED_LEN_BUFFER_LIMIT = 4194304, num_rows_ = 228113440, capacity_ = 32605, flush_ = 496763888, needs_deep_copy_ = 90, num_tuples_per_row_ = -840859936, tuple_ptrs_size_ = 32607, tuple_ptrs_ = 0x7f5d2fd96cd8, auxiliary_mem_usage_ = 140025020547552, tuple_data_pool_ = {
> static LLVM_CLASS_NAME = 0x2fe5b94 "class.impala::MemPool", static DEFAULT_ALIGNMENT = 8, static INITIAL_CHUNK_SIZE = 4096, static MAX_CHUNK_SIZE = 1048576, static zero_length_region_ = 1722447803, current_chunk_idx_ = -840859936, next_chunk_size_ = 32607, total_allocated_bytes_ = 140025020547568,
> peak_allocated_bytes_ = 27193507, total_reserved_bytes_ = 1, chunks_ = {<std::_Vector_base<impala::MemPool::ChunkInfo, std::allocator<impala::MemPool::ChunkInfo> >> = {_M_impl = {<std::allocator<impala::MemPool::ChunkInfo>> = {<__gnu_cxx::new_allocator<impala::MemPool::ChunkInfo>> = {<No data fields>}, <No data fields>},
> _M_start = 0x7f5a1d9c03f0, _M_finish = 0x7f5fcde17ee0, _M_end_of_storage = 0x7f5d2fd96cd8}}, <No data fields>}, mem_tracker_ = 0x7f5cd932ecb0}, row_desc_ = 0x8, mem_tracker_ = 0x7f5cd932ecb0,
> io_buffers_ = {<std::_Vector_base<std::unique_ptr<impala::DiskIoMgr::BufferDescriptor, std::default_delete<impala::DiskIoMgr::BufferDescriptor> >, std::allocator<std::unique_ptr<impala::DiskIoMgr::BufferDescriptor, std::default_delete<impala::DiskIoMgr::BufferDescriptor> > > >> = {
> _M_impl = {<std::allocator<std::unique_ptr<impala::DiskIoMgr::BufferDescriptor, std::default_delete<impala::DiskIoMgr::BufferDescriptor> > >> = {<__gnu_cxx::new_allocator<std::unique_ptr<impala::DiskIoMgr::BufferDescriptor, std::default_delete<impala::DiskIoMgr::BufferDescriptor> > >> = {<No data fields>}, <No data fields>}, _M_start = 0x15c86d0, _M_finish = 0x7f5fcde17ee0, _M_end_of_storage = 0x7f5a1d9c0780}}, <No data fields>}, buffers_ = {<std::_Vector_base<impala::RowBatch::BufferInfo, std::allocator<impala::RowBatch::BufferInfo> >> = {
> _M_impl = {<std::allocator<impala::RowBatch::BufferInfo>> = {<__gnu_cxx::new_allocator<impala::RowBatch::BufferInfo>> = {<No data fields>}, <No data fields>}, _M_start = 0x7f5cd932ecb0, _M_finish = 0x7f5fcde17ee8, _M_end_of_storage = 0x0}}, <No data fields>}, compression_scratch_ = {static npos = 18446744073709551615,
> _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f5cd932ecb0 "\330\376A\323Z\177"}}}
> eos = false
> #9 0x00000000017f16a3 in impala::FragmentInstanceState::Open (this=0x7f57f3f5af80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/fragment-instance-state.cc:256
> __status__ = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x68eafc0}, msg_ = 0x68ea390},
> static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x3008ab5}
> SCOPED_TIMER9 = {sw_ = {start_ = 2679682494285694, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counter_ = 0x7f5f0222fa80, is_cancelled_ = 0x0}
> SCOPED_TIMER6 = {sw_ = {start_ = 2679678789834010, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counter_ = 0x7f4a3adf5f18, is_cancelled_ = 0x0}
> SCOPED_TIMER7 = {sw_ = {start_ = 2679678789843186, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counter_ = 0x7f55634f4900, is_cancelled_ = 0x0}
> SCOPED_THREAD_COUNTER_MEASUREMENT8 = {stop_ = false, usage_base_ = {ru_utime = {tv_sec = 0, tv_usec = 102984}, ru_stime = {tv_sec = 0, tv_usec = 18997}, ru_maxrss = 171863496, ru_ixrss = 0, ru_idrss = 0, ru_isrss = 0, ru_minflt = 407, ru_majflt = 0, ru_nswap = 0, ru_inblock = 0, ru_oublock = 24, ru_msgsnd = 0, ru_msgrcv = 0,
> ru_nsignals = 0, ru_nvcsw = 159, ru_nivcsw = 145}, sw_ = {start_ = 2679678789843580, total_time_ = 0, time_ceiling_ = 0, running_ = true}, counters_ = 0x7f59c3505200}
> #10 0x00000000017ef1cf in impala::FragmentInstanceState::Exec (this=0x7f57f3f5af80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/fragment-instance-state.cc:80
> status = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x68eafc0}, msg_ = 0x68ea390},
> static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x0}
> #11 0x00000000017d9bda in impala::QueryState::ExecFInstance (this=0x7f57d1299600, fis=0x7f57f3f5af80) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/query-state.cc:351
> status = {static CANCELLED = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = {static CANCELLED = <same as static member of an already seen type>, static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x68eafc0}, msg_ = 0x68ea390},
> static DEPRECATED_RPC = <same as static member of an already seen type>, msg_ = 0x0}
> #12 0x00000000017d870a in impala::QueryState::<lambda()>::operator()(void) const (__closure=0x7f5a1d9c0bc8) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/runtime/query-state.cc:319
> fis = 0x7f57f3f5af80
> this = 0x7f57d1299600
> #13 0x00000000017da88b in boost::detail::function::void_function_obj_invoker0<impala::QueryState::StartFInstances()::<lambda()>, void>::invoke(boost::detail::function::function_buffer &) (function_obj_ptr=...)
> at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:153
> f = 0x7f5a1d9c0bc8
> #14 0x000000000170b722 in boost::function0<void>::operator() (this=0x7f5a1d9c0bc0) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/function/function_template.hpp:767
> No locals.
> #15 0x00000000019fc6cd in impala::Thread::SuperviseThread (name=..., category=..., functor=..., thread_started=0x7f61361b8be0) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/be/src/util/thread.cc:329
> system_tid = 56730
> category_copy = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x6910678 "fragment-execution"}}
> thread_mgr_ref = {<std::__shared_ptr<impala::ThreadMgr, (__gnu_cxx::_Lock_policy)2>> = {_M_ptr = 0x695e680, _M_refcount = {_M_pi = 0x696c1a0}}, <No data fields>}
> name_copy = {static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<__gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f568a417038 "exec-finstance (finst:5a42932fcf62becc:f7747b5800000038)"}}
> #16 0x0000000001a05008 in boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> >::operator()<void (*)(const std::basic_string<char>&, const std::basic_string<char>&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list0>(boost::_bi::type<void>, void (*&)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, const std::basic_string<char, std::char_traits<char>, std::allocator<char> > &, boost::function<void()>, impala::Promise<long> *), boost::_bi::list0 &, int) (this=0x7f5f67979dc0, f=@0x7f5f67979db8, a=...) at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind.hpp:457
> No locals.
> #17 0x0000000001a04f4b in boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > >::operator()(void) (this=0x7f5f67979db8)
> at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/bind/bind_template.hpp:20
> a = {<No data fields>}
> #18 0x0000000001a04f0e in boost::detail::thread_data<boost::_bi::bind_t<void, void (*)(const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, const std::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::function<void()>, impala::Promise<long int>*), boost::_bi::list4<boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<std::basic_string<char, std::char_traits<char>, std::allocator<char> > >, boost::_bi::value<boost::function<void()> >, boost::_bi::value<impala::Promise<long int>*> > > >::run(void) (this=0x7f5f67979c00)
> at /data/jenkins/workspace/impala-private-build-binaries/repos/Impala/toolchain/boost-1.57.0-p3/include/boost/thread/detail/thread.hpp:116
> No locals.
> #19 0x00000000020889aa in thread_proxy ()
> DEBUG_MODE = true
> kint32min = -2147483648
> kint16min = -32768
> kint8max = 127 '\177'
> kMaxFprint = 18446744073709551615
> PATH_SEPARATOR = 47 '/'
> kuint32max = 4294967295
> kuint8max = 255 '\377'
> kint32max = 2147483647
> kint64min = -9223372036854775808
> kIllegalFprint = 0
> kuint16max = 65535
> kint16max = 32767
> kuint64max = 18446744073709551615
> kint64max = 9223372036854775807
> kint8min = -128 '\200'
> Bits::num_bits = 0x3fbb740 ""
> #20 0x0000003299e07aa1 in start_thread () from /lib64/libpthread.so.0
> No symbol table info available.
> #21 0x0000003299ae893d in clone () from /lib64/libc.so.6
> No symbol table info available.
> {code}
> Repro:
> {code}
> set debug_action=-1:PREPARE:SET_DENY_RESERVATION_PROBABILITY@1.0;
> select distinct o_orderdate, o_custkey, o_comment
> from tpch_parquet.orders;
> {code}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)