You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by "snizovtsev (via GitHub)" <gi...@apache.org> on 2023/03/21 22:21:25 UTC
[GitHub] [arrow] snizovtsev commented on issue #34539: [C++][Python] Setting rows per group cause segmentation fault writing parquet with write_dataset
snizovtsev commented on issue #34539:
URL: https://github.com/apache/arrow/issues/34539#issuecomment-1478672864
I've faced a similar issue with C++ / Acero dataset writer. Regardless of options, pipeline gets crashed when dataset "write" node is used.
I found that crash cause is stack overflow. Depending on `ulimit -s` settings GDB shows up to ~698997 stack frames full of repeatable future continuations:
<details><summary>backtrace</summary>
```
Thread 5 "builder" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7ffbabfff6c0 (LWP 955318)]
0x00005555556b18dd in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::__shared_count<arrow::NumericArray<arrow::UInt64Type>, std::allocator<arrow::NumericArray<arrow::UInt64Type> >, std::shared_ptr<arrow::ArrayData> const&> (this=0x0, __p=<error reading variable: Cannot access memory at address 0x0>, __a=...)
at /usr/include/c++/11/bits/shared_ptr_base.h:643
643 /usr/include/c++/11/bits/shared_ptr_base.h: No such file or directory.
(gdb) up
#1 0x00005555556ae56c in std::__shared_ptr<arrow::NumericArray<arrow::UInt64Type>, (__gnu_cxx::_Lock_policy)2>::__shared_ptr<std::allocator<arrow::NumericArray<arrow::UInt64Type> >, std::shared_ptr<arrow::ArrayData> const&> (this=0x7ffba8000170, __tag=...) at /usr/include/c++/11/bits/shared_ptr_base.h:1342
1342 in /usr/include/c++/11/bits/shared_ptr_base.h
(gdb)
#2 0x00005555556ab913 in std::shared_ptr<arrow::NumericArray<arrow::UInt64Type> >::shared_ptr<std::allocator<arrow::NumericArray<arrow::UInt64Type> >, std::shared_ptr<arrow::ArrayData> const&> (this=0x7ffba8000170, __tag=...) at /usr/include/c++/11/bits/shared_ptr.h:409
409 /usr/include/c++/11/bits/shared_ptr.h: No such file or directory.
(gdb)
#3 0x00005555556a8e09 in std::allocate_shared<arrow::NumericArray<arrow::UInt64Type>, std::allocator<arrow::NumericArray<arrow::UInt64Type> >, std::shared_ptr<arrow::ArrayData> const&> (__a=...) at /usr/include/c++/11/bits/shared_ptr.h:863
863 in /usr/include/c++/11/bits/shared_ptr.h
(gdb)
#4 0x00005555556a5a66 in std::make_shared<arrow::NumericArray<arrow::UInt64Type>, std::shared_ptr<arrow::ArrayData> const&> ()
at /usr/include/c++/11/bits/shared_ptr.h:879
879 in /usr/include/c++/11/bits/shared_ptr.h
(gdb)
#5 0x000055555569cea4 in arrow::(anonymous namespace)::ArrayDataWrapper::Visit<arrow::UInt64Type> (this=0x7ffba80001f0)
at external/com_github_apache_arrow/cpp/src/arrow/array/util.cc:65
65 *out_ = std::make_shared<ArrayType>(data_);
(gdb)
#6 0x0000555555699616 in arrow::VisitTypeInline<arrow::(anonymous namespace)::ArrayDataWrapper> (type=..., visitor=0x7ffba80001f0)
at external/com_github_apache_arrow/cpp/src/arrow/visit_type_inline.h:54
54 ARROW_GENERATE_FOR_ALL_TYPES(TYPE_VISIT_INLINE);
(gdb)
#7 0x00005555556937c7 in arrow::MakeArray (data=std::shared_ptr<arrow::ArrayData> (use count 1, weak count 0) = {...})
at external/com_github_apache_arrow/cpp/src/arrow/array/util.cc:310
310 DCHECK_OK(VisitTypeInline(*data->type, &wrapper_visitor));
(gdb)
#8 0x0000555555cbb4bf in arrow::SimpleRecordBatch::column (this=0x7ffb89829d40, i=0) at external/com_github_apache_arrow/cpp/src/arrow/record_batch.cc:82
82 result = MakeArray(columns_[i]);
(gdb)
#9 0x0000555555bfde42 in arrow::Table::FromRecordBatches (schema=std::shared_ptr<arrow::Schema> (use count 17, weak count 0) = {...},
batches=std::vector of length 1, capacity 1 = {...}) at external/com_github_apache_arrow/cpp/src/arrow/table.cc:304
304 column_arrays[j] = batches[j]->column(i);
(gdb)
#10 0x0000555555bfe188 in arrow::Table::FromRecordBatches (batches=std::vector of length 1, capacity 1 = {...})
at external/com_github_apache_arrow/cpp/src/arrow/table.cc:318
318 return FromRecordBatches(batches[0]->schema(), batches);
(gdb)
#11 0x0000555555f7a9a5 in arrow::dataset::internal::(anonymous namespace)::DatasetWriterFileQueue::PopStagedBatch (this=0x7ffba003b3b0)
at external/com_github_apache_arrow/cpp/src/arrow/dataset/dataset_writer.cc:179
179 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<Table> table,
(gdb)
#12 0x0000555555f7acc3 in arrow::dataset::internal::(anonymous namespace)::DatasetWriterFileQueue::PopAndDeliverStagedBatch (this=0x7ffba003b3b0)
at external/com_github_apache_arrow/cpp/src/arrow/dataset/dataset_writer.cc:193
193 ARROW_ASSIGN_OR_RAISE(std::shared_ptr<RecordBatch> next_batch, PopStagedBatch());
(gdb)
#13 0x0000555555f84208 in arrow::dataset::internal::(anonymous namespace)::DatasetWriterFileQueue::Push (this=0x7ffba003b3b0,
batch=std::shared_ptr<arrow::RecordBatch> (empty) = {...}) at external/com_github_apache_arrow/cpp/src/arrow/dataset/dataset_writer.cc:208
208 ARROW_ASSIGN_OR_RAISE(int64_t rows_popped, PopAndDeliverStagedBatch());
(gdb)
#14 0x0000555555f84824 in arrow::dataset::internal::(anonymous namespace)::DatasetWriterDirectoryQueue::StartWrite (this=0x7ffba003af00,
batch=std::shared_ptr<arrow::RecordBatch> (use count 3, weak count 0) = {...})
at external/com_github_apache_arrow/cpp/src/arrow/dataset/dataset_writer.cc:307
307 return latest_open_file_->Push(batch);
(gdb)
#15 0x0000555555f8678c in arrow::dataset::internal::DatasetWriter::DatasetWriterImpl::DoWriteRecordBatch (this=0x555557732190,
batch=std::shared_ptr<arrow::RecordBatch> (use count 3, weak count 0) = {...}, directory="parts/", prefix="8_")
at external/com_github_apache_arrow/cpp/src/arrow/dataset/dataset_writer.cc:611
611 RETURN_NOT_OK(dir_queue->StartWrite(next_chunk));
(gdb)
#16 0x0000555555f85869 in arrow::dataset::internal::DatasetWriter::DatasetWriterImpl::WriteAndCheckBackpressure (this=0x555557732190,
batch=std::shared_ptr<arrow::RecordBatch> (empty) = {...}, directory="", prefix="8_")
at external/com_github_apache_arrow/cpp/src/arrow/dataset/dataset_writer.cc:526
526 return DoWriteRecordBatch(std::move(batch), write_options_.base_dir, prefix);
(gdb)
#17 0x0000555555f85967 in arrow::dataset::internal::DatasetWriter::DatasetWriterImpl::WriteRecordBatch(std::shared_ptr<arrow::RecordBatch>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)::{lambda()#1}::operator()() (__closure=0x7ffb89829b50) at external/com_github_apache_arrow/cpp/src/arrow/dataset/dataset_writer.cc:535
535 WriteAndCheckBackpressure(std::move(batch), directory, prefix);
(gdb) up
#18 0x0000555555f8e1b7 in arrow::util::AsyncTaskScheduler::SimpleTask<arrow::dataset::internal::DatasetWriter::DatasetWriterImpl::WriteRecordBatch(std::shared_ptr<arrow::RecordBatch>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)::{lambda()#1}>::operator()() (this=0x7ffb89829b40)
at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.h:153
153 Result<Future<>> operator()() override { return callable(); }
(gdb)
#19 0x0000555555b3027f in WrapperTask::operator() (this=0x7ffb8982f270) at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.cc:411
411 ARROW_ASSIGN_OR_RAISE(Future<> inner_fut, (*target)());
(gdb)
#20 0x0000555555b2f8ea in operator() (__closure=0x7ffb4e58e780) at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.cc:341
341 ARROW_ASSIGN_OR_RAISE(Future<> inner_fut, (*inner_task)());
(gdb)
#21 0x0000555555b37281 in arrow::util::AsyncTaskScheduler::SimpleTask<arrow::util::(anonymous namespace)::ThrottledAsyncTaskSchedulerImpl::SubmitTask(std::unique_ptr<arrow::util::AsyncTaskScheduler::Task>, int)::<lambda()> >::operator()(void) (this=0x7ffb4e58e770)
at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.h:153
153 Result<Future<>> operator()() override { return callable(); }
(gdb)
#22 0x0000555555b2ecde in arrow::util::(anonymous namespace)::AsyncTaskSchedulerImpl::DoSubmitTask (this=0x555557730310,
task=std::unique_ptr<arrow::util::AsyncTaskScheduler::Task> = {...}) at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.cc:195
195 Result<Future<>> submit_result = (*task)();
(gdb)
#23 0x0000555555b2f15e in arrow::util::(anonymous namespace)::AsyncTaskSchedulerImpl::SubmitTaskUnlocked (this=0x555557730310,
task=std::unique_ptr<arrow::util::AsyncTaskScheduler::Task> = {...}, lk=...) at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.cc:261
261 return DoSubmitTask(std::move(task));
(gdb)
#24 0x0000555555b2ea24 in arrow::util::(anonymous namespace)::AsyncTaskSchedulerImpl::AddTask (this=0x555557730310,
task=std::unique_ptr<arrow::util::AsyncTaskScheduler::Task> = {...}) at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.cc:171
171 SubmitTaskUnlocked(std::move(task), std::move(lk));
(gdb) up 80000
#80024 0x0000555555b2ea24 in arrow::util::(anonymous namespace)::AsyncTaskSchedulerImpl::AddTask (this=0x555557730310,
task=std::unique_ptr<arrow::util::AsyncTaskScheduler::Task> = {...}) at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.cc:171
171 SubmitTaskUnlocked(std::move(task), std::move(lk));
(gdb) up
#80025 0x0000555555b311bc in arrow::util::AsyncTaskScheduler::AddSimpleTask<arrow::util::(anonymous namespace)::ThrottledAsyncTaskSchedulerImpl::SubmitTask(std::unique_ptr<arrow::util::AsyncTaskScheduler::Task>, int)::<lambda()> >(struct {...}, std::string_view) (this=0x555557730310, callable=...,
name="DatasetWriter::WriteAndCheckBackpressure") at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.h:171
171 return AddTask(std::make_unique<SimpleTask<Callable>>(std::move(callable), name));
(gdb)
#80026 0x0000555555b2fb40 in arrow::util::(anonymous namespace)::ThrottledAsyncTaskSchedulerImpl::SubmitTask (this=0x5555577380f0,
task=std::unique_ptr<arrow::util::AsyncTaskScheduler::Task> = {...}, latched_cost=1)
at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.cc:338
338 return target_->AddSimpleTask(
(gdb)
#80027 0x0000555555b2fdcb in arrow::util::(anonymous namespace)::ThrottledAsyncTaskSchedulerImpl::ContinueTasks (this=0x5555577380f0)
at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.cc:374
374 if (!SubmitTask(std::move(next_task), next_cost)) {
(gdb) up 160000
#240027 0x0000555555b2fdcb in arrow::util::(anonymous namespace)::ThrottledAsyncTaskSchedulerImpl::ContinueTasks (this=0x5555577380f0)
at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.cc:374
374 if (!SubmitTask(std::move(next_task), next_cost)) {
(gdb) up 160000
#400027 0x0000555555b2fdcb in arrow::util::(anonymous namespace)::ThrottledAsyncTaskSchedulerImpl::ContinueTasks (this=0x5555577380f0)
at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.cc:374
374 if (!SubmitTask(std::move(next_task), next_cost)) {
(gdb) up 160000
#560027 0x0000555555b2fdcb in arrow::util::(anonymous namespace)::ThrottledAsyncTaskSchedulerImpl::ContinueTasks (this=0x5555577380f0)
at external/com_github_apache_arrow/cpp/src/arrow/util/async_util.cc:374
374 if (!SubmitTask(std::move(next_task), next_cost)) {
(gdb) up 160000
#698997 0x00007ffff7b20d90 in ?? () from /usr/lib/libc.so.6
(gdb) down
#698996 0x00007ffff7a9ebb5 in ?? () from /usr/lib/libc.so.6
(gdb)
#698995 0x00007ffff7cd72c3 in std::execute_native_thread_routine (__p=0x5555577328a0) at /usr/src/debug/gcc/gcc/libstdc++-v3/src/c++11/thread.cc:82
82 /usr/src/debug/gcc/gcc/libstdc++-v3/src/c++11/thread.cc: No such file or directory.
(gdb)
#698994 0x0000555555934c1e in std::thread::_State_impl<std::thread::_Invoker<std::tuple<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()> > > >::_M_run(void) (this=0x5555577328a0) at /usr/include/c++/11/bits/std_thread.h:211
211 /usr/include/c++/11/bits/std_thread.h: No such file or directory.
(gdb)
#698993 0x0000555555934c3a in std::thread::_Invoker<std::tuple<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()> > >::operator()(void) (
this=0x5555577328a8) at /usr/include/c++/11/bits/std_thread.h:260
260 in /usr/include/c++/11/bits/std_thread.h
(gdb)
#698992 0x0000555555934c66 in std::thread::_Invoker<std::tuple<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x5555577328a8) at /usr/include/c++/11/bits/std_thread.h:253
253 in /usr/include/c++/11/bits/std_thread.h
(gdb)
#698991 0x0000555555934cb9 in std::__invoke<arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()> >(struct {...} &&) (__fn=...)
at /usr/include/c++/11/bits/invoke.h:96
96 /usr/include/c++/11/bits/invoke.h: No such file or directory.
(gdb)
#698990 0x0000555555934cf6 in std::__invoke_impl<void, arrow::internal::ThreadPool::LaunchWorkersUnlocked(int)::<lambda()> >(std::__invoke_other, struct {...} &&) (__f=...) at /usr/include/c++/11/bits/invoke.h:61
61 in /usr/include/c++/11/bits/invoke.h
(gdb)
#698989 0x000055555593110d in operator() (__closure=0x5555577328a8) at external/com_github_apache_arrow/cpp/src/arrow/util/thread_pool.cc:430
430 WorkerLoop(state, it);
(gdb)
#698988 0x00005555559303ab in arrow::internal::WorkerLoop (state=std::shared_ptr<arrow::internal::ThreadPool::State> (use count 33, weak count 1) = {...},
it=...) at external/com_github_apache_arrow/cpp/src/arrow/util/thread_pool.cc:269
269 std::move(task.callable)();
(gdb)
#698987 0x0000555555935e32 in arrow::internal::FnOnce<void ()>::operator()() && (this=0x7ffbabffeda0)
at external/com_github_apache_arrow/cpp/src/arrow/util/functional.h:140
140 return bye->invoke(std::forward<A&&>(a)...);
(gdb)
#698986 0x00005555558be170 in arrow::internal::FnOnce<void ()>::FnImpl<std::_Bind<arrow::detail::ContinueFuture (arrow::Future<arrow::internal::Empty>, std::function<arrow::Status ()>)> >::invoke() (this=0x7ffbc4002930) at external/com_github_apache_arrow/cpp/src/arrow/util/functional.h:152
152 R invoke(A&&... a) override { return std::move(fn_)(std::forward<A&&>(a)...); }
(gdb)
#698985 0x00005555558be19c in std::_Bind<arrow::detail::ContinueFuture (arrow::Future<arrow::internal::Empty>, std::function<arrow::Status ()>)>::operator()<, void>() (this=0x7ffbc4002938) at /usr/include/c++/11/functional:503
503 /usr/include/c++/11/functional: No such file or directory.
(gdb)
#698984 0x00005555558be223 in std::_Bind<arrow::detail::ContinueFuture (arrow::Future<arrow::internal::Empty>, std::function<arrow::Status ()>)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) (this=0x7ffbc4002938, __args=...) at /usr/include/c++/11/functional:420
420 in /usr/include/c++/11/functional
(gdb)
#698983 0x00005555558be305 in std::__invoke<arrow::detail::ContinueFuture&, arrow::Future<arrow::internal::Empty>&, std::function<arrow::Status ()>&>(arrow::detail::ContinueFuture&, arrow::Future<arrow::internal::Empty>&, std::function<arrow::Status ()>&) (__fn=...) at /usr/include/c++/11/bits/invoke.h:96
96 /usr/include/c++/11/bits/invoke.h: No such file or directory.
(gdb)
#698982 0x00005555558be3d8 in std::__invoke_impl<void, arrow::detail::ContinueFuture&, arrow::Future<arrow::internal::Empty>&, std::function<arrow::Status ()>&>(std::__invoke_other, arrow::detail::ContinueFuture&, arrow::Future<arrow::internal::Empty>&, std::function<arrow::Status ()>&) (__f=...)
at /usr/include/c++/11/bits/invoke.h:61
61 in /usr/include/c++/11/bits/invoke.h
(gdb)
#698981 0x00005555558be485 in arrow::detail::ContinueFuture::operator()<std::function<arrow::Status ()>&, , arrow::Status, arrow::Future<arrow::internal::Empty> >(arrow::Future<arrow::internal::Empty>, std::function<arrow::Status ()>&) const (this=0x7ffbc4002938, next=..., f=...)
at external/com_github_apache_arrow/cpp/src/arrow/util/future.h:150
150 next.MarkFinished(std::forward<ContinueFunc>(f)(std::forward<Args>(a)...));
(gdb)
#698980 0x00005555558be52f in std::function<arrow::Status ()>::operator()() const (this=0x7ffbc4002940) at /usr/include/c++/11/bits/std_function.h:590
590 /usr/include/c++/11/bits/std_function.h: No such file or directory.
(gdb)
#698979 0x00005555558cf8d3 in std::_Function_handler<arrow::Status(), arrow::compute::(anonymous namespace)::SourceNode::SliceAndDeliverMorsel(const arrow::compute::ExecBatch&)::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/11/bits/std_function.h:291
291 /usr/include/c++/11/bits/std_function.h: No such file or directory.
(gdb)
#698978 0x00005555558d12f9 in std::__invoke_r<arrow::Status, arrow::compute::(anonymous namespace)::SourceNode::SliceAndDeliverMorsel(const arrow::compute::ExecBatch&)::<lambda()>&>(struct {...} &) (__fn=...) at /usr/include/c++/11/bits/invoke.h:116
116 /usr/include/c++/11/bits/invoke.h: No such file or directory.
(gdb)
#698977 0x00005555558d2674 in std::__invoke_impl<arrow::Status, arrow::compute::(anonymous namespace)::SourceNode::SliceAndDeliverMorsel(const arrow::compute::ExecBatch&)::<lambda()>&>(std::__invoke_other, struct {...} &) (__f=...) at /usr/include/c++/11/bits/invoke.h:61
61 in /usr/include/c++/11/bits/invoke.h
(gdb)
#698976 0x00005555558ca227 in operator() (__closure=0x7ffbc4002890) at external/com_github_apache_arrow/cpp/src/arrow/compute/exec/source_node.cc:100
100 ARROW_RETURN_NOT_OK(output_->InputReceived(this, std::move(batch)));
(gdb)
#698975 0x00005555558b737d in arrow::compute::MapNode::InputReceived (this=0x555557731290, input=0x5555577309d0, batch=...)
at external/com_github_apache_arrow/cpp/src/arrow/compute/exec/map_node.cc:72
72 ARROW_RETURN_NOT_OK(output_->InputReceived(this, std::move(output_batch)));
(gdb)
#698974 0x00005555558b737d in arrow::compute::MapNode::InputReceived (this=0x555557731a40, input=0x555557731290, batch=...)
at external/com_github_apache_arrow/cpp/src/arrow/compute/exec/map_node.cc:72
72 ARROW_RETURN_NOT_OK(output_->InputReceived(this, std::move(output_batch)));
(gdb)
#698973 0x00005555558c0334 in arrow::compute::(anonymous namespace)::ConsumingSinkNode::InputReceived (this=0x555557722520, input=0x555557731a40, batch=...)
at external/com_github_apache_arrow/cpp/src/arrow/compute/exec/sink_node.cc:343
343 ARROW_RETURN_NOT_OK(consumer_->Consume(std::move(batch)));
(gdb)
#698972 0x0000555555f5bde6 in arrow::dataset::(anonymous namespace)::DatasetWritingSinkNodeConsumer::Consume (this=0x555557731b90, batch=...)
at external/com_github_apache_arrow/cpp/src/arrow/dataset/file_base.cc:415
415 return WriteNextBatch(std::move(record_batch), batch.guarantee);
(gdb)
#698971 0x0000555555f5c0b7 in arrow::dataset::(anonymous namespace)::DatasetWritingSinkNodeConsumer::WriteNextBatch (this=0x555557731b90,
batch=std::shared_ptr<arrow::RecordBatch> (use count 1, weak count 0) = {...}, guarantee=...)
at external/com_github_apache_arrow/cpp/src/arrow/dataset/file_base.cc:434
434 });
(gdb)
#698970 0x0000555555f5b5b6 in arrow::dataset::(anonymous namespace)::WriteBatch(std::shared_ptr<arrow::RecordBatch>, arrow::compute::Expression, arrow::dataset::FileSystemDatasetWriteOptions, std::function<arrow::Status(std::shared_ptr<arrow::RecordBatch>, const arrow::dataset::PartitionPathFormat&)>) (
batch=std::shared_ptr<arrow::RecordBatch> (empty) = {...}, guarantee=..., write_options=..., write=...)
at external/com_github_apache_arrow/cpp/src/arrow/dataset/file_base.cc:383
383 RETURN_NOT_OK(write(next_batch, destination));
(gdb)
#698969 0x0000555555f666a6 in std::function<arrow::Status (std::shared_ptr<arrow::RecordBatch>, arrow::dataset::PartitionPathFormat const&)>::operator()(std::shared_ptr<arrow::RecordBatch>, arrow::dataset::PartitionPathFormat const&) const (this=0x7ffbabffe570,
__args#0=std::shared_ptr<arrow::RecordBatch> (empty) = {...}, __args#1=...) at /usr/include/c++/11/bits/std_function.h:590
590 /usr/include/c++/11/bits/std_function.h: No such file or directory.
(gdb)
#698968 0x0000555555f5f4c9 in std::_Function_handler<arrow::Status(std::shared_ptr<arrow::RecordBatch>, const arrow::dataset::PartitionPathFormat&), arrow::dataset::(anonymous namespace)::DatasetWritingSinkNodeConsumer::WriteNextBatch(std::shared_ptr<arrow::RecordBatch>, arrow::compute::Expression)::<lambda(std::shared_ptr<arrow::RecordBatch>, const arrow::dataset::PartitionPathFormat&)> >::_M_invoke(const std::_Any_data &, std::shared_ptr<arrow::RecordBatch> &&, const arrow::dataset::PartitionPathFormat &) (__functor=..., __args#0=..., __args#1=...) at /usr/include/c++/11/bits/std_function.h:291
291 in /usr/include/c++/11/bits/std_function.h
(gdb)
#698967 0x0000555555f603ee in std::__invoke_r<arrow::Status, arrow::dataset::(anonymous namespace)::DatasetWritingSinkNodeConsumer::WriteNextBatch(std::shared_ptr<arrow::RecordBatch>, arrow::compute::Expression)::<lambda(std::shared_ptr<arrow::RecordBatch>, const arrow::dataset::PartitionPathFormat&)>&, std::shared_ptr<arrow::RecordBatch>, const arrow::dataset::PartitionPathFormat&>(struct {...} &) (__fn=...) at /usr/include/c++/11/bits/invoke.h:116
116 /usr/include/c++/11/bits/invoke.h: No such file or directory.
(gdb)
#698966 0x0000555555f6120f in std::__invoke_impl<arrow::Status, arrow::dataset::(anonymous namespace)::DatasetWritingSinkNodeConsumer::WriteNextBatch(std::shared_ptr<arrow::RecordBatch>, arrow::compute::Expression)::<lambda(std::shared_ptr<arrow::RecordBatch>, const arrow::dataset::PartitionPathFormat&)>&, std::shared_ptr<arrow::RecordBatch>, const arrow::dataset::PartitionPathFormat&>(std::__invoke_other, struct {...} &) (__f=...)
at /usr/include/c++/11/bits/invoke.h:61
61 in /usr/include/c++/11/bits/invoke.h
```
</details>
When I set `OMP_NUM_THREADS` up to 2 crash disappears but program leaking memory up to dataset size.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org