You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2018/11/09 15:54:40 UTC
[arrow] branch master updated: ARROW-3698: [Gandiva] Segmentation
fault when using a large table in Gandiva
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 3ae4161 ARROW-3698: [Gandiva] Segmentation fault when using a large table in Gandiva
3ae4161 is described below
commit 3ae41615fb086bd021ddcc44bd2b9e00eb265493
Author: suquark <zs...@mail.ustc.edu.cn>
AuthorDate: Fri Nov 9 10:54:32 2018 -0500
ARROW-3698: [Gandiva] Segmentation fault when using a large table in Gandiva
https://issues.apache.org/jira/browse/ARROW-3698
Author: suquark <zs...@mail.ustc.edu.cn>
Closes #2902 from suquark/gandiva and squashes the following commits:
9c7ac4ac6 <suquark> typo
24904cb9f <suquark> lint
a1650aa7b <suquark> fix test
f76cb64e4 <suquark> fix types
72ef82197 <suquark> fix types
fdb777560 <suquark> lint
a30449fac <suquark> Add tests for selection vector.
cb4fc5de1 <suquark> Add test for huge filters.
1c3dd3562 <suquark> int64 selection vector support
89d2f5e9c <suquark> Add tests for huge table.
9f9928e7d <suquark> revert changes to ZSTD
0566a9cf0 <suquark> remote verbose
c51fb5c71 <suquark> fix include
7547de33d <suquark> Lint. Better error message.
44886728c <suquark> verbose
d005c2223 <suquark> fix typo
307411ec9 <suquark> retrigger ci
706ccaa29 <suquark> fix jni types
ca0338c99 <suquark> More fixes
4e4c8a574 <suquark> fix eval_batch.h & lint
207d79230 <suquark> Fix more possible integer overflows.
c4daa9945 <suquark> Fix integer overflow in Gandiva which causes segfault.
---
cpp/src/gandiva/bitmap_accumulator.cc | 16 ++--
cpp/src/gandiva/bitmap_accumulator.h | 2 +-
cpp/src/gandiva/eval_batch.h | 10 +-
cpp/src/gandiva/filter.cc | 10 +-
cpp/src/gandiva/jni/jni_common.cc | 12 ++-
cpp/src/gandiva/local_bitmaps_holder.h | 12 +--
cpp/src/gandiva/projector.cc | 12 +--
cpp/src/gandiva/projector.h | 4 +-
cpp/src/gandiva/selection_vector.cc | 55 ++++++++---
cpp/src/gandiva/selection_vector.h | 40 +++++---
cpp/src/gandiva/selection_vector_impl.h | 23 ++---
cpp/src/gandiva/selection_vector_test.cc | 68 +++++++++++++
cpp/src/gandiva/tests/huge_table_test.cc | 157 +++++++++++++++++++++++++++++++
cpp/src/gandiva/tests/timed_evaluate.h | 3 +-
14 files changed, 352 insertions(+), 72 deletions(-)
diff --git a/cpp/src/gandiva/bitmap_accumulator.cc b/cpp/src/gandiva/bitmap_accumulator.cc
index 50d3f75..4a27dae 100644
--- a/cpp/src/gandiva/bitmap_accumulator.cc
+++ b/cpp/src/gandiva/bitmap_accumulator.cc
@@ -22,7 +22,7 @@
namespace gandiva {
void BitMapAccumulator::ComputeResult(uint8_t* dst_bitmap) {
- int num_records = eval_batch_.num_records();
+ int64_t num_records = eval_batch_.num_records();
if (all_invalid_) {
// set all bits to 0.
@@ -35,11 +35,11 @@ void BitMapAccumulator::ComputeResult(uint8_t* dst_bitmap) {
/// Compute the intersection of multiple bitmaps.
void BitMapAccumulator::IntersectBitMaps(uint8_t* dst_map,
const std::vector<uint8_t*>& src_maps,
- int num_records) {
+ int64_t num_records) {
uint64_t* dst_map64 = reinterpret_cast<uint64_t*>(dst_map);
- int num_words = (num_records + 63) / 64; // aligned to 8-byte.
- int num_bytes = num_words * 8;
- int nmaps = static_cast<int>(src_maps.size());
+ int64_t num_words = (num_records + 63) / 64; // aligned to 8-byte.
+ int64_t num_bytes = num_words * 8;
+ int64_t nmaps = src_maps.size();
switch (nmaps) {
case 0: {
@@ -58,7 +58,7 @@ void BitMapAccumulator::IntersectBitMaps(uint8_t* dst_map,
// two src_maps bitmaps. do 64-bit ANDs
uint64_t* src_maps0_64 = reinterpret_cast<uint64_t*>(src_maps[0]);
uint64_t* src_maps1_64 = reinterpret_cast<uint64_t*>(src_maps[1]);
- for (int i = 0; i < num_words; ++i) {
+ for (int64_t i = 0; i < num_words; ++i) {
dst_map64[i] = src_maps0_64[i] & src_maps1_64[i];
}
break;
@@ -68,8 +68,8 @@ void BitMapAccumulator::IntersectBitMaps(uint8_t* dst_map,
/* > 2 src_maps bitmaps. do 64-bit ANDs */
uint64_t* src_maps0_64 = reinterpret_cast<uint64_t*>(src_maps[0]);
memcpy(dst_map64, src_maps0_64, num_bytes);
- for (int m = 1; m < nmaps; ++m) {
- for (int i = 0; i < num_words; ++i) {
+ for (int64_t m = 1; m < nmaps; ++m) {
+ for (int64_t i = 0; i < num_words; ++i) {
uint64_t* src_mapsm_64 = reinterpret_cast<uint64_t*>(src_maps[m]);
dst_map64[i] &= src_mapsm_64[i];
}
diff --git a/cpp/src/gandiva/bitmap_accumulator.h b/cpp/src/gandiva/bitmap_accumulator.h
index d164b16..31b6609 100644
--- a/cpp/src/gandiva/bitmap_accumulator.h
+++ b/cpp/src/gandiva/bitmap_accumulator.h
@@ -60,7 +60,7 @@ class BitMapAccumulator : public DexDefaultVisitor {
/// Compute the intersection of the accumulated bitmaps and save the result in
/// dst_bmap.
static void IntersectBitMaps(uint8_t* dst_map, const std::vector<uint8_t*>& src_maps,
- int num_records);
+ int64_t num_records);
private:
const EvalBatch& eval_batch_;
diff --git a/cpp/src/gandiva/eval_batch.h b/cpp/src/gandiva/eval_batch.h
index 9977e62..608f420 100644
--- a/cpp/src/gandiva/eval_batch.h
+++ b/cpp/src/gandiva/eval_batch.h
@@ -33,7 +33,7 @@ namespace gandiva {
/// expression evaluation.
class EvalBatch {
public:
- explicit EvalBatch(int num_records, int num_buffers, int num_local_bitmaps)
+ explicit EvalBatch(int64_t num_records, int num_buffers, int num_local_bitmaps)
: num_records_(num_records), num_buffers_(num_buffers) {
if (num_buffers > 0) {
buffers_array_.reset(new uint8_t*[num_buffers]);
@@ -42,7 +42,7 @@ class EvalBatch {
execution_context_.reset(new ExecutionContext());
}
- int num_records() const { return num_records_; }
+ int64_t num_records() const { return num_records_; }
uint8_t** GetBufferArray() const { return buffers_array_.get(); }
@@ -60,7 +60,9 @@ class EvalBatch {
int GetNumLocalBitMaps() const { return local_bitmaps_holder_->GetNumLocalBitMaps(); }
- int GetLocalBitmapSize() const { return local_bitmaps_holder_->GetLocalBitMapSize(); }
+ int64_t GetLocalBitmapSize() const {
+ return local_bitmaps_holder_->GetLocalBitMapSize();
+ }
uint8_t* GetLocalBitMap(int idx) const {
DCHECK(idx <= GetNumLocalBitMaps());
@@ -75,7 +77,7 @@ class EvalBatch {
private:
/// number of records in the current batch.
- int num_records_;
+ int64_t num_records_;
// number of buffers.
int num_buffers_;
diff --git a/cpp/src/gandiva/filter.cc b/cpp/src/gandiva/filter.cc
index 16bfdec..7a24d95 100644
--- a/cpp/src/gandiva/filter.cc
+++ b/cpp/src/gandiva/filter.cc
@@ -94,8 +94,8 @@ Status Filter::Evaluate(const arrow::RecordBatch& batch,
// Allocate three local_bitmaps (one for output, one for validity, one to compute the
// intersection).
- LocalBitMapsHolder bitmaps(static_cast<int>(batch.num_rows()), 3 /*local_bitmaps*/);
- int bitmap_size = bitmaps.GetLocalBitMapSize();
+ LocalBitMapsHolder bitmaps(batch.num_rows(), 3 /*local_bitmaps*/);
+ int64_t bitmap_size = bitmaps.GetLocalBitMapSize();
auto validity = std::make_shared<arrow::Buffer>(bitmaps.GetLocalBitMap(0), bitmap_size);
auto value = std::make_shared<arrow::Buffer>(bitmaps.GetLocalBitMap(1), bitmap_size);
@@ -109,11 +109,9 @@ Status Filter::Evaluate(const arrow::RecordBatch& batch,
// Compute the intersection of the value and validity.
auto result = bitmaps.GetLocalBitMap(2);
BitMapAccumulator::IntersectBitMaps(
- result, {bitmaps.GetLocalBitMap(0), bitmaps.GetLocalBitMap((1))},
- static_cast<int>(batch.num_rows()));
+ result, {bitmaps.GetLocalBitMap(0), bitmaps.GetLocalBitMap((1))}, batch.num_rows());
- return out_selection->PopulateFromBitMap(result, bitmap_size,
- static_cast<int>(batch.num_rows()) - 1);
+ return out_selection->PopulateFromBitMap(result, bitmap_size, batch.num_rows() - 1);
}
} // namespace gandiva
diff --git a/cpp/src/gandiva/jni/jni_common.cc b/cpp/src/gandiva/jni/jni_common.cc
index efc3c6e..639ad36 100644
--- a/cpp/src/gandiva/jni/jni_common.cc
+++ b/cpp/src/gandiva/jni/jni_common.cc
@@ -812,7 +812,17 @@ JNIEXPORT jint JNICALL Java_org_apache_arrow_gandiva_evaluator_JniWrapper_evalua
env->ThrowNew(gandiva_exception_, status.message().c_str());
return -1;
} else {
- return selection_vector->GetNumSlots();
+ int64_t num_slots = selection_vector->GetNumSlots();
+ // Check integer overflow
+ if (num_slots > INT_MAX) {
+ std::stringstream ss;
+ ss << "The selection vector has " << num_slots
+ << " slots, which is larger than the " << INT_MAX << " limit.\n";
+ const std::string message = ss.str();
+ env->ThrowNew(gandiva_exception_, message.c_str());
+ return -1;
+ }
+ return static_cast<int>(num_slots);
}
}
diff --git a/cpp/src/gandiva/local_bitmaps_holder.h b/cpp/src/gandiva/local_bitmaps_holder.h
index e3ac598..1dc8256 100644
--- a/cpp/src/gandiva/local_bitmaps_holder.h
+++ b/cpp/src/gandiva/local_bitmaps_holder.h
@@ -32,11 +32,11 @@ namespace gandiva {
/// expression evaluation.
class LocalBitMapsHolder {
public:
- LocalBitMapsHolder(int num_records, int num_local_bitmaps);
+ LocalBitMapsHolder(int64_t num_records, int num_local_bitmaps);
int GetNumLocalBitMaps() const { return static_cast<int>(local_bitmaps_vec_.size()); }
- int GetLocalBitMapSize() const { return local_bitmap_size_; }
+ int64_t GetLocalBitMapSize() const { return local_bitmap_size_; }
uint8_t** GetLocalBitMapArray() const { return local_bitmaps_array_.get(); }
@@ -47,7 +47,7 @@ class LocalBitMapsHolder {
private:
/// number of records in the current batch.
- int num_records_;
+ int64_t num_records_;
/// A container of 'local_bitmaps_', each sized to accomodate 'num_records'.
std::vector<std::unique_ptr<uint8_t>> local_bitmaps_vec_;
@@ -55,10 +55,10 @@ class LocalBitMapsHolder {
/// An array of the local bitmaps.
std::unique_ptr<uint8_t*> local_bitmaps_array_;
- int local_bitmap_size_;
+ int64_t local_bitmap_size_;
};
-inline LocalBitMapsHolder::LocalBitMapsHolder(int num_records, int num_local_bitmaps)
+inline LocalBitMapsHolder::LocalBitMapsHolder(int64_t num_records, int num_local_bitmaps)
: num_records_(num_records) {
// alloc an array for the pointers to the bitmaps.
if (num_local_bitmaps > 0) {
@@ -66,7 +66,7 @@ inline LocalBitMapsHolder::LocalBitMapsHolder(int num_records, int num_local_bit
}
// 64-bit aligned bitmaps.
- uint32_t roundUp64Multiple = (num_records_ + 63) >> 6;
+ int64_t roundUp64Multiple = (num_records_ + 63) >> 6;
local_bitmap_size_ = roundUp64Multiple * 8;
// Alloc 'num_local_bitmaps_' number of bitmaps, each of capacity 'num_records_'.
diff --git a/cpp/src/gandiva/projector.cc b/cpp/src/gandiva/projector.cc
index 51a370e..8020a45 100644
--- a/cpp/src/gandiva/projector.cc
+++ b/cpp/src/gandiva/projector.cc
@@ -112,8 +112,8 @@ Status Projector::Evaluate(const arrow::RecordBatch& batch,
return Status::Invalid(ss.str());
}
- Status status = ValidateArrayDataCapacity(*array_data, *(output_fields_[idx]),
- static_cast<int>(batch.num_rows()));
+ Status status =
+ ValidateArrayDataCapacity(*array_data, *(output_fields_[idx]), batch.num_rows());
ARROW_RETURN_NOT_OK(status);
++idx;
}
@@ -138,8 +138,7 @@ Status Projector::Evaluate(const arrow::RecordBatch& batch, arrow::MemoryPool* p
for (auto& field : output_fields_) {
ArrayDataPtr output_data;
- status = AllocArrayData(field->type(), static_cast<int>(batch.num_rows()), pool,
- &output_data);
+ status = AllocArrayData(field->type(), batch.num_rows(), pool, &output_data);
ARROW_RETURN_NOT_OK(status);
output_data_vecs.push_back(output_data);
@@ -158,7 +157,7 @@ Status Projector::Evaluate(const arrow::RecordBatch& batch, arrow::MemoryPool* p
}
// TODO : handle variable-len vectors
-Status Projector::AllocArrayData(const DataTypePtr& type, int num_records,
+Status Projector::AllocArrayData(const DataTypePtr& type, int64_t num_records,
arrow::MemoryPool* pool, ArrayDataPtr* array_data) {
if (!arrow::is_primitive(type->id())) {
return Status::Invalid("Unsupported output data type " + type->ToString());
@@ -191,7 +190,8 @@ Status Projector::ValidateEvaluateArgsCommon(const arrow::RecordBatch& batch) {
}
Status Projector::ValidateArrayDataCapacity(const arrow::ArrayData& array_data,
- const arrow::Field& field, int num_records) {
+ const arrow::Field& field,
+ int64_t num_records) {
// verify that there are atleast two buffers (validity and data).
if (array_data.buffers.size() < 2) {
std::stringstream ss;
diff --git a/cpp/src/gandiva/projector.h b/cpp/src/gandiva/projector.h
index 7ab45ff..c9d7271 100644
--- a/cpp/src/gandiva/projector.h
+++ b/cpp/src/gandiva/projector.h
@@ -82,12 +82,12 @@ class Projector {
const FieldVector& output_fields, std::shared_ptr<Configuration>);
/// Allocate an ArrowData of length 'length'.
- Status AllocArrayData(const DataTypePtr& type, int length, arrow::MemoryPool* pool,
+ Status AllocArrayData(const DataTypePtr& type, int64_t length, arrow::MemoryPool* pool,
ArrayDataPtr* array_data);
/// Validate that the ArrayData has sufficient capacity to accomodate 'num_records'.
Status ValidateArrayDataCapacity(const arrow::ArrayData& array_data,
- const arrow::Field& field, int num_records);
+ const arrow::Field& field, int64_t num_records);
/// Validate the common args for Evaluate() APIs.
Status ValidateEvaluateArgsCommon(const arrow::RecordBatch& batch);
diff --git a/cpp/src/gandiva/selection_vector.cc b/cpp/src/gandiva/selection_vector.cc
index 0fd64a4..9266ca7 100644
--- a/cpp/src/gandiva/selection_vector.cc
+++ b/cpp/src/gandiva/selection_vector.cc
@@ -26,34 +26,39 @@
namespace gandiva {
-Status SelectionVector::PopulateFromBitMap(const uint8_t* bitmap, int bitmap_size,
- int max_bitmap_index) {
+Status SelectionVector::PopulateFromBitMap(const uint8_t* bitmap, int64_t bitmap_size,
+ int64_t max_bitmap_index) {
if (bitmap_size % 8 != 0) {
std::stringstream ss;
ss << "bitmap size " << bitmap_size << " must be padded to 64-bit size";
return Status::Invalid(ss.str());
}
- if (static_cast<unsigned int>(max_bitmap_index) > GetMaxSupportedValue()) {
+ if (max_bitmap_index < 0) {
+ std::stringstream ss;
+ ss << "max bitmap index " << max_bitmap_index << " must be positive";
+ return Status::Invalid(ss.str());
+ }
+ if (static_cast<uint64_t>(max_bitmap_index) > GetMaxSupportedValue()) {
std::stringstream ss;
ss << "max_bitmap_index " << max_bitmap_index << " must be <= maxSupportedValue "
<< GetMaxSupportedValue() << " in selection vector";
return Status::Invalid(ss.str());
}
- int max_slots = GetMaxSlots();
+ int64_t max_slots = GetMaxSlots();
- // jump 8-bytes at a time, add the index corresponding to each valid bit to the
+ // jump 8-bytes at a time, add the index corresponding to each valid bit to the
// the selection vector.
- int selection_idx = 0;
+ int64_t selection_idx = 0;
const uint64_t* bitmap_64 = reinterpret_cast<const uint64_t*>(bitmap);
- for (int bitmap_idx = 0; bitmap_idx < bitmap_size / 8; ++bitmap_idx) {
+ for (int64_t bitmap_idx = 0; bitmap_idx < bitmap_size / 8; ++bitmap_idx) {
uint64_t current_word = bitmap_64[bitmap_idx];
while (current_word != 0) {
uint64_t highest_only = current_word & -current_word;
int pos_in_word = __builtin_ctzl(highest_only);
- int pos_in_bitmap = bitmap_idx * 64 + pos_in_word;
+ int64_t pos_in_bitmap = bitmap_idx * 64 + pos_in_word;
if (pos_in_bitmap > max_bitmap_index) {
// the bitmap may be slighly larger for alignment/padding.
break;
@@ -73,7 +78,8 @@ Status SelectionVector::PopulateFromBitMap(const uint8_t* bitmap, int bitmap_siz
return Status::OK();
}
-Status SelectionVector::MakeInt16(int max_slots, std::shared_ptr<arrow::Buffer> buffer,
+Status SelectionVector::MakeInt16(int64_t max_slots,
+ std::shared_ptr<arrow::Buffer> buffer,
std::shared_ptr<SelectionVector>* selection_vector) {
auto status = SelectionVectorInt16::ValidateBuffer(max_slots, buffer);
ARROW_RETURN_NOT_OK(status);
@@ -82,7 +88,7 @@ Status SelectionVector::MakeInt16(int max_slots, std::shared_ptr<arrow::Buffer>
return Status::OK();
}
-Status SelectionVector::MakeInt16(int max_slots, arrow::MemoryPool* pool,
+Status SelectionVector::MakeInt16(int64_t max_slots, arrow::MemoryPool* pool,
std::shared_ptr<SelectionVector>* selection_vector) {
std::shared_ptr<arrow::Buffer> buffer;
auto status = SelectionVectorInt16::AllocateBuffer(max_slots, pool, &buffer);
@@ -92,7 +98,8 @@ Status SelectionVector::MakeInt16(int max_slots, arrow::MemoryPool* pool,
return Status::OK();
}
-Status SelectionVector::MakeInt32(int max_slots, std::shared_ptr<arrow::Buffer> buffer,
+Status SelectionVector::MakeInt32(int64_t max_slots,
+ std::shared_ptr<arrow::Buffer> buffer,
std::shared_ptr<SelectionVector>* selection_vector) {
auto status = SelectionVectorInt32::ValidateBuffer(max_slots, buffer);
ARROW_RETURN_NOT_OK(status);
@@ -101,7 +108,7 @@ Status SelectionVector::MakeInt32(int max_slots, std::shared_ptr<arrow::Buffer>
return Status::OK();
}
-Status SelectionVector::MakeInt32(int max_slots, arrow::MemoryPool* pool,
+Status SelectionVector::MakeInt32(int64_t max_slots, arrow::MemoryPool* pool,
std::shared_ptr<SelectionVector>* selection_vector) {
std::shared_ptr<arrow::Buffer> buffer;
auto status = SelectionVectorInt32::AllocateBuffer(max_slots, pool, &buffer);
@@ -111,9 +118,29 @@ Status SelectionVector::MakeInt32(int max_slots, arrow::MemoryPool* pool,
return Status::OK();
}
+Status SelectionVector::MakeInt64(int64_t max_slots,
+ std::shared_ptr<arrow::Buffer> buffer,
+ std::shared_ptr<SelectionVector>* selection_vector) {
+ auto status = SelectionVectorInt64::ValidateBuffer(max_slots, buffer);
+ ARROW_RETURN_NOT_OK(status);
+
+ *selection_vector = std::make_shared<SelectionVectorInt64>(max_slots, buffer);
+ return Status::OK();
+}
+
+Status SelectionVector::MakeInt64(int64_t max_slots, arrow::MemoryPool* pool,
+ std::shared_ptr<SelectionVector>* selection_vector) {
+ std::shared_ptr<arrow::Buffer> buffer;
+ auto status = SelectionVectorInt64::AllocateBuffer(max_slots, pool, &buffer);
+ ARROW_RETURN_NOT_OK(status);
+
+ *selection_vector = std::make_shared<SelectionVectorInt64>(max_slots, buffer);
+ return Status::OK();
+}
+
template <typename C_TYPE, typename A_TYPE>
Status SelectionVectorImpl<C_TYPE, A_TYPE>::AllocateBuffer(
- int max_slots, arrow::MemoryPool* pool, std::shared_ptr<arrow::Buffer>* buffer) {
+ int64_t max_slots, arrow::MemoryPool* pool, std::shared_ptr<arrow::Buffer>* buffer) {
auto buffer_len = max_slots * sizeof(C_TYPE);
auto astatus = arrow::AllocateBuffer(pool, buffer_len, buffer);
ARROW_RETURN_NOT_OK(astatus);
@@ -123,7 +150,7 @@ Status SelectionVectorImpl<C_TYPE, A_TYPE>::AllocateBuffer(
template <typename C_TYPE, typename A_TYPE>
Status SelectionVectorImpl<C_TYPE, A_TYPE>::ValidateBuffer(
- int max_slots, std::shared_ptr<arrow::Buffer> buffer) {
+ int64_t max_slots, std::shared_ptr<arrow::Buffer> buffer) {
// verify buffer is mutable
if (!buffer->is_mutable()) {
return Status::Invalid("buffer for selection vector must be mutable");
diff --git a/cpp/src/gandiva/selection_vector.h b/cpp/src/gandiva/selection_vector.h
index 4dd9fe7..dcd2f6b 100644
--- a/cpp/src/gandiva/selection_vector.h
+++ b/cpp/src/gandiva/selection_vector.h
@@ -34,22 +34,22 @@ class SelectionVector {
virtual ~SelectionVector() = default;
/// Get the value at a given index.
- virtual uint GetIndex(int index) const = 0;
+ virtual uint64_t GetIndex(int64_t index) const = 0;
/// Set the value at a given index.
- virtual void SetIndex(int index, uint value) = 0;
+ virtual void SetIndex(int64_t index, uint64_t value) = 0;
// Get the max supported value in the selection vector.
- virtual uint GetMaxSupportedValue() const = 0;
+ virtual uint64_t GetMaxSupportedValue() const = 0;
/// The maximum slots (capacity) of the selection vector.
- virtual int GetMaxSlots() const = 0;
+ virtual int64_t GetMaxSlots() const = 0;
/// The number of slots (size) of the selection vector.
- virtual int GetNumSlots() const = 0;
+ virtual int64_t GetNumSlots() const = 0;
/// Set the number of slots in the selection vector.
- virtual void SetNumSlots(int num_slots) = 0;
+ virtual void SetNumSlots(int64_t num_slots) = 0;
/// Convert to arrow-array.
virtual ArrayPtr ToArray() const = 0;
@@ -60,21 +60,22 @@ class SelectionVector {
/// \param[in] bitmap_size size of the bitmap in bytes
/// \param[in] max_bitmap_index max valid index in bitmap (can be lesser than
/// capacity in the bitmap, due to alignment/padding).
- Status PopulateFromBitMap(const uint8_t* bitmap, int bitmap_size, int max_bitmap_index);
+ Status PopulateFromBitMap(const uint8_t* bitmap, int64_t bitmap_size,
+ int64_t max_bitmap_index);
/// \brief make selection vector with int16 type records.
///
/// \param[in] max_slots max number of slots
/// \param[in] buffer buffer sized to accomodate max_slots
/// \param[out] selection_vector selection vector backed by 'buffer'
- static Status MakeInt16(int max_slots, std::shared_ptr<arrow::Buffer> buffer,
+ static Status MakeInt16(int64_t max_slots, std::shared_ptr<arrow::Buffer> buffer,
std::shared_ptr<SelectionVector>* selection_vector);
/// \param[in] max_slots max number of slots
/// \param[in] pool memory pool to allocate buffer
/// \param[out] selection_vector selection vector backed by a buffer allocated from the
/// pool.
- static Status MakeInt16(int max_slots, arrow::MemoryPool* pool,
+ static Status MakeInt16(int64_t max_slots, arrow::MemoryPool* pool,
std::shared_ptr<SelectionVector>* selection_vector);
/// \brief make selection vector with int32 type records.
@@ -82,7 +83,7 @@ class SelectionVector {
/// \param[in] max_slots max number of slots
/// \param[in] buffer buffer sized to accomodate max_slots
/// \param[out] selection_vector selection vector backed by 'buffer'
- static Status MakeInt32(int max_slots, std::shared_ptr<arrow::Buffer> buffer,
+ static Status MakeInt32(int64_t max_slots, std::shared_ptr<arrow::Buffer> buffer,
std::shared_ptr<SelectionVector>* selection_vector);
/// \brief make selection vector with int32 type records.
@@ -91,7 +92,24 @@ class SelectionVector {
/// \param[in] pool memory pool to allocate buffer
/// \param[out] selection_vector selection vector backed by a buffer allocated from the
/// pool.
- static Status MakeInt32(int max_slots, arrow::MemoryPool* pool,
+ static Status MakeInt32(int64_t max_slots, arrow::MemoryPool* pool,
+ std::shared_ptr<SelectionVector>* selection_vector);
+
+ /// \brief make selection vector with int64 type records.
+ ///
+ /// \param[in] max_slots max number of slots
+ /// \param[in] buffer buffer sized to accomodate max_slots
+ /// \param[out] selection_vector selection vector backed by 'buffer'
+ static Status MakeInt64(int64_t max_slots, std::shared_ptr<arrow::Buffer> buffer,
+ std::shared_ptr<SelectionVector>* selection_vector);
+
+ /// \brief make selection vector with int64 type records.
+ ///
+ /// \param[in] max_slots max number of slots
+ /// \param[in] pool memory pool to allocate buffer
+ /// \param[out] selection_vector selection vector backed by a buffer allocated from the
+ /// pool.
+ static Status MakeInt64(int64_t max_slots, arrow::MemoryPool* pool,
std::shared_ptr<SelectionVector>* selection_vector);
};
diff --git a/cpp/src/gandiva/selection_vector_impl.h b/cpp/src/gandiva/selection_vector_impl.h
index a628bf0..3d6a7df 100644
--- a/cpp/src/gandiva/selection_vector_impl.h
+++ b/cpp/src/gandiva/selection_vector_impl.h
@@ -35,43 +35,43 @@ namespace gandiva {
template <typename C_TYPE, typename A_TYPE>
class SelectionVectorImpl : public SelectionVector {
public:
- SelectionVectorImpl(int max_slots, std::shared_ptr<arrow::Buffer> buffer)
+ SelectionVectorImpl(int64_t max_slots, std::shared_ptr<arrow::Buffer> buffer)
: max_slots_(max_slots), num_slots_(0), buffer_(buffer) {
raw_data_ = reinterpret_cast<C_TYPE*>(buffer->mutable_data());
}
- uint GetIndex(int index) const override { return raw_data_[index]; }
+ uint64_t GetIndex(int64_t index) const override { return raw_data_[index]; }
- void SetIndex(int index, uint value) override {
+ void SetIndex(int64_t index, uint64_t value) override {
raw_data_[index] = static_cast<C_TYPE>(value);
}
ArrayPtr ToArray() const override;
- int GetMaxSlots() const override { return max_slots_; }
+ int64_t GetMaxSlots() const override { return max_slots_; }
- int GetNumSlots() const override { return num_slots_; }
+ int64_t GetNumSlots() const override { return num_slots_; }
- void SetNumSlots(int num_slots) override {
+ void SetNumSlots(int64_t num_slots) override {
DCHECK_LE(num_slots, max_slots_);
num_slots_ = num_slots;
}
- uint GetMaxSupportedValue() const override {
+ uint64_t GetMaxSupportedValue() const override {
return std::numeric_limits<C_TYPE>::max();
}
- static Status AllocateBuffer(int max_slots, arrow::MemoryPool* pool,
+ static Status AllocateBuffer(int64_t max_slots, arrow::MemoryPool* pool,
std::shared_ptr<arrow::Buffer>* buffer);
- static Status ValidateBuffer(int max_slots, std::shared_ptr<arrow::Buffer> buffer);
+ static Status ValidateBuffer(int64_t max_slots, std::shared_ptr<arrow::Buffer> buffer);
protected:
/// maximum slots in the vector
- int max_slots_;
+ int64_t max_slots_;
/// number of slots in the vector
- int num_slots_;
+ int64_t num_slots_;
std::shared_ptr<arrow::Buffer> buffer_;
C_TYPE* raw_data_;
@@ -86,6 +86,7 @@ ArrayPtr SelectionVectorImpl<C_TYPE, A_TYPE>::ToArray() const {
using SelectionVectorInt16 = SelectionVectorImpl<uint16_t, arrow::UInt16Type>;
using SelectionVectorInt32 = SelectionVectorImpl<uint32_t, arrow::UInt32Type>;
+using SelectionVectorInt64 = SelectionVectorImpl<uint64_t, arrow::UInt64Type>;
} // namespace gandiva
diff --git a/cpp/src/gandiva/selection_vector_test.cc b/cpp/src/gandiva/selection_vector_test.cc
index b857224..acb0f33 100644
--- a/cpp/src/gandiva/selection_vector_test.cc
+++ b/cpp/src/gandiva/selection_vector_test.cc
@@ -207,4 +207,72 @@ TEST_F(TestSelectionVector, TestInt32MakeNegative) {
EXPECT_EQ(status.IsInvalid(), true);
}
+TEST_F(TestSelectionVector, TestInt64Set) {
+ int max_slots = 10;
+
+ std::shared_ptr<SelectionVector> selection;
+ auto status = SelectionVector::MakeInt64(max_slots, pool_, &selection);
+ EXPECT_EQ(status.ok(), true) << status.message();
+
+ selection->SetIndex(0, 100);
+ EXPECT_EQ(selection->GetIndex(0), 100);
+
+ selection->SetIndex(1, 200);
+ EXPECT_EQ(selection->GetIndex(1), 200);
+
+ selection->SetIndex(2, 100000);
+ EXPECT_EQ(selection->GetIndex(2), 100000);
+
+ selection->SetNumSlots(3);
+ EXPECT_EQ(selection->GetNumSlots(), 3);
+
+ // TopArray() should return an array with 100,200,100000
+ auto array_raw = selection->ToArray();
+ const auto& array = dynamic_cast<const arrow::UInt64Array&>(*array_raw);
+ EXPECT_EQ(array.length(), 3) << array_raw->ToString();
+ EXPECT_EQ(array.Value(0), 100) << array_raw->ToString();
+ EXPECT_EQ(array.Value(1), 200) << array_raw->ToString();
+ EXPECT_EQ(array.Value(2), 100000) << array_raw->ToString();
+}
+
+TEST_F(TestSelectionVector, TestInt64PopulateFromBitMap) {
+ int max_slots = 200;
+
+ std::shared_ptr<SelectionVector> selection;
+ auto status = SelectionVector::MakeInt64(max_slots, pool_, &selection);
+ EXPECT_EQ(status.ok(), true) << status.message();
+
+ int bitmap_size = RoundUpNumi64(max_slots) * 8;
+ std::unique_ptr<uint8_t> bitmap(new uint8_t[bitmap_size]);
+ memset(bitmap.get(), 0, bitmap_size);
+
+ arrow::BitUtil::SetBit(bitmap.get(), 0);
+ arrow::BitUtil::SetBit(bitmap.get(), 5);
+ arrow::BitUtil::SetBit(bitmap.get(), 121);
+ arrow::BitUtil::SetBit(bitmap.get(), 220);
+
+ status = selection->PopulateFromBitMap(bitmap.get(), bitmap_size, max_slots - 1);
+ EXPECT_EQ(status.ok(), true) << status.message();
+
+ EXPECT_EQ(selection->GetNumSlots(), 3);
+ EXPECT_EQ(selection->GetIndex(0), 0);
+ EXPECT_EQ(selection->GetIndex(1), 5);
+ EXPECT_EQ(selection->GetIndex(2), 121);
+}
+
+TEST_F(TestSelectionVector, TestInt64MakeNegative) {
+ int max_slots = 10;
+
+ std::shared_ptr<SelectionVector> selection;
+ std::shared_ptr<arrow::Buffer> buffer;
+ auto buffer_len = max_slots * sizeof(int64_t);
+
+ // alloc a buffer that's insufficient.
+ auto astatus = arrow::AllocateBuffer(pool_, buffer_len - 1, &buffer);
+ EXPECT_EQ(astatus.ok(), true);
+
+ auto status = SelectionVector::MakeInt64(max_slots, buffer, &selection);
+ EXPECT_EQ(status.IsInvalid(), true);
+}
+
} // namespace gandiva
diff --git a/cpp/src/gandiva/tests/huge_table_test.cc b/cpp/src/gandiva/tests/huge_table_test.cc
new file mode 100644
index 0000000..bffcb19
--- /dev/null
+++ b/cpp/src/gandiva/tests/huge_table_test.cc
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gtest/gtest.h>
+#include "arrow/memory_pool.h"
+#include "gandiva/filter.h"
+#include "gandiva/projector.h"
+#include "gandiva/tests/test_util.h"
+#include "gandiva/tree_expr_builder.h"
+
+namespace gandiva {
+
+using arrow::boolean;
+using arrow::float32;
+using arrow::int32;
+
+class DISABLED_TestHugeProjector : public ::testing::Test {
+ public:
+ void SetUp() { pool_ = arrow::default_memory_pool(); }
+
+ protected:
+ arrow::MemoryPool* pool_;
+};
+
+class DISABLED_TestHugeFilter : public ::testing::Test {
+ public:
+ void SetUp() { pool_ = arrow::default_memory_pool(); }
+
+ protected:
+ arrow::MemoryPool* pool_;
+};
+
+TEST_F(DISABLED_TestHugeProjector, SimpleTestSumHuge) {
+ auto atype = arrow::TypeTraits<arrow::Int32Type>::type_singleton();
+
+ // schema for input fields
+ auto field0 = field("f0", atype);
+ auto field1 = field("f1", atype);
+ auto schema = arrow::schema({field0, field1});
+
+ // output fields
+ auto field_sum = field("add", atype);
+
+ // Build expression
+ auto sum_expr = TreeExprBuilder::MakeExpression("add", {field0, field1}, field_sum);
+ std::shared_ptr<Projector> projector;
+ Status status = Projector::Make(schema, {sum_expr}, &projector);
+ EXPECT_TRUE(status.ok());
+
+ // Create a row-batch with some sample data
+ // Cause an overflow in int32_t
+ int64_t num_records = static_cast<int64_t>(INT32_MAX) + 3;
+ std::vector<int32_t> input0 = {2, 29, 5, 37, 11, 59, 17, 19};
+ std::vector<int32_t> input1 = {23, 3, 31, 7, 41, 47, 13};
+ std::vector<bool> validity;
+
+ std::vector<int32_t> arr1;
+ std::vector<int32_t> arr2;
+ // expected output
+ std::vector<int32_t> sum1;
+
+ for (int64_t i = 0; i < num_records; i++) {
+ arr1.push_back(input0[i % 8]);
+ arr2.push_back(input1[i % 7]);
+ sum1.push_back(input0[i % 8] + input1[i % 7]);
+ validity.push_back(true);
+ }
+
+ auto exp_sum = MakeArrowArray<arrow::Int32Type, int32_t>(sum1, validity);
+ auto array0 = MakeArrowArray<arrow::Int32Type, int32_t>(arr1, validity);
+ auto array1 = MakeArrowArray<arrow::Int32Type, int32_t>(arr2, validity);
+
+ // prepare input record batch
+ auto in_batch = arrow::RecordBatch::Make(schema, num_records, {array0, array1});
+
+ // Evaluate expression
+ arrow::ArrayVector outputs;
+ status = projector->Evaluate(*in_batch, pool_, &outputs);
+ EXPECT_TRUE(status.ok());
+
+ // Validate results
+ EXPECT_ARROW_ARRAY_EQUALS(exp_sum, outputs.at(0));
+}
+
+TEST_F(DISABLED_TestHugeFilter, TestSimpleHugeFilter) {
+ // Create a row-batch with some sample data
+ // Cause an overflow in int32_t
+ int64_t num_records = static_cast<int64_t>(INT32_MAX) + 3;
+ std::vector<int32_t> input0 = {2, 29, 5, 37, 11, 59, 17, 19};
+ std::vector<int32_t> input1 = {23, 3, 31, 7, 41, 47, 13};
+ std::vector<bool> validity;
+
+ std::vector<int32_t> arr1;
+ std::vector<int32_t> arr2;
+ // expected output
+ std::vector<uint64_t> sel;
+
+ for (int64_t i = 0; i < num_records; i++) {
+ arr1.push_back(input0[i % 8]);
+ arr2.push_back(input1[i % 7]);
+ if (input0[i % 8] + input1[i % 7] > 50) {
+ sel.push_back(i);
+ }
+ validity.push_back(true);
+ }
+
+ auto exp = MakeArrowArrayUint64(sel);
+
+ // schema for input fields
+ auto field0 = field("f0", int32());
+ auto field1 = field("f1", int32());
+ auto schema = arrow::schema({field0, field1});
+
+ // Build condition f0 + f1 < 50
+ auto node_f0 = TreeExprBuilder::MakeField(field0);
+ auto node_f1 = TreeExprBuilder::MakeField(field1);
+ auto sum_func =
+ TreeExprBuilder::MakeFunction("add", {node_f0, node_f1}, arrow::int32());
+ auto literal_50 = TreeExprBuilder::MakeLiteral((int32_t)50);
+ auto less_than_50 = TreeExprBuilder::MakeFunction("less_than", {sum_func, literal_50},
+ arrow::boolean());
+ auto condition = TreeExprBuilder::MakeCondition(less_than_50);
+
+ std::shared_ptr<Filter> filter;
+ Status status = Filter::Make(schema, condition, &filter);
+ EXPECT_TRUE(status.ok());
+
+ // prepare input record batch
+ auto in_batch = arrow::RecordBatch::Make(schema, num_records, {arr1, arr2});
+
+ std::shared_ptr<SelectionVector> selection_vector;
+ status = SelectionVector::MakeInt64(num_records, pool_, &selection_vector);
+ EXPECT_TRUE(status.ok());
+
+ // Evaluate expression
+ status = filter->Evaluate(*in_batch, selection_vector);
+ EXPECT_TRUE(status.ok());
+
+ // Validate results
+ EXPECT_ARROW_ARRAY_EQUALS(exp, selection_vector->ToArray());
+}
+
+} // namespace gandiva
diff --git a/cpp/src/gandiva/tests/timed_evaluate.h b/cpp/src/gandiva/tests/timed_evaluate.h
index c1c08be..e5ee571 100644
--- a/cpp/src/gandiva/tests/timed_evaluate.h
+++ b/cpp/src/gandiva/tests/timed_evaluate.h
@@ -68,8 +68,7 @@ class FilterEvaluator : public BaseEvaluator {
Status Evaluate(arrow::RecordBatch& batch, arrow::MemoryPool* pool) override {
if (selection_ == nullptr || selection_->GetMaxSlots() < batch.num_rows()) {
- auto status = SelectionVector::MakeInt16(static_cast<int>(batch.num_rows()), pool,
- &selection_);
+ auto status = SelectionVector::MakeInt16(batch.num_rows(), pool, &selection_);
if (!status.ok()) {
return status;
}