You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/11/27 02:02:04 UTC
[14/14] impala git commit: IMPALA-7869: break up
parquet-column-readers.cc
IMPALA-7869: break up parquet-column-readers.cc
Move parquet classes into exec/parquet.
Move CollectionColumnReader and ParquetLevelDecoder into separate files.
Remove unnecessary 'encoding_' field from ParquetLevelDecoder.
Switch BOOLEAN decoding to use composition instead of inheritance. This
lets the boolean decoding use the faster batched implementations in
ScalarColumnReader and avoids some confusing aspects of the class
hierarchy, like the ReadValueBatch() implementation on the base class
that was shared between BoolColumnReader and CollectionColumnReader.
Improve compile times by instantiating BitPacking templates in a
separate file (this looks to give a 30s+ speedup for
compiling parquet-column-readers.cc).
Testing:
Ran exhaustive tests.
Change-Id: I0efd5c50b781fe9e3c022b33c66c06cfb529c0b8
Reviewed-on: http://gerrit.cloudera.org:8080/11949
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/07fd3320
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/07fd3320
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/07fd3320
Branch: refs/heads/master
Commit: 07fd332089c262fa8813605f12b927c8602ac0d2
Parents: 5e740f4
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Fri Nov 16 14:45:50 2018 -0800
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Nov 27 02:01:12 2018 +0000
----------------------------------------------------------------------
be/CMakeLists.txt | 3 +-
be/src/benchmarks/bit-packing-benchmark.cc | 2 +-
be/src/benchmarks/bswap-benchmark.cc | 4 +-
be/src/codegen/impala-ir.cc | 2 +-
be/src/exec/CMakeLists.txt | 11 +-
be/src/exec/data-source-scan-node.cc | 6 +-
be/src/exec/hdfs-parquet-scanner-ir.cc | 67 -
be/src/exec/hdfs-parquet-scanner-test.cc | 176 --
be/src/exec/hdfs-parquet-scanner.cc | 1681 ---------------
be/src/exec/hdfs-parquet-scanner.h | 654 ------
be/src/exec/hdfs-parquet-table-writer.cc | 1321 ------------
be/src/exec/hdfs-parquet-table-writer.h | 205 --
be/src/exec/hdfs-scan-node-base.cc | 10 +-
be/src/exec/hdfs-table-sink.cc | 14 +-
be/src/exec/parquet-column-readers.cc | 1931 ------------------
be/src/exec/parquet-column-readers.h | 607 ------
be/src/exec/parquet-column-stats.cc | 193 --
be/src/exec/parquet-column-stats.h | 299 ---
be/src/exec/parquet-column-stats.inline.h | 254 ---
be/src/exec/parquet-common.cc | 132 --
be/src/exec/parquet-common.h | 535 -----
be/src/exec/parquet-metadata-utils.cc | 732 -------
be/src/exec/parquet-metadata-utils.h | 233 ---
be/src/exec/parquet-plain-test.cc | 338 ---
be/src/exec/parquet-scratch-tuple-batch.h | 168 --
be/src/exec/parquet-version-test.cc | 84 -
be/src/exec/parquet/CMakeLists.txt | 43 +
be/src/exec/parquet/hdfs-parquet-scanner-ir.cc | 67 +
.../exec/parquet/hdfs-parquet-scanner-test.cc | 177 ++
be/src/exec/parquet/hdfs-parquet-scanner.cc | 1684 +++++++++++++++
be/src/exec/parquet/hdfs-parquet-scanner.h | 647 ++++++
.../exec/parquet/hdfs-parquet-table-writer.cc | 1321 ++++++++++++
be/src/exec/parquet/hdfs-parquet-table-writer.h | 205 ++
be/src/exec/parquet/parquet-bool-decoder.cc | 68 +
be/src/exec/parquet/parquet-bool-decoder.h | 92 +
.../parquet/parquet-collection-column-reader.cc | 160 ++
.../parquet/parquet-collection-column-reader.h | 97 +
be/src/exec/parquet/parquet-column-readers.cc | 1605 +++++++++++++++
be/src/exec/parquet/parquet-column-readers.h | 476 +++++
be/src/exec/parquet/parquet-column-stats.cc | 193 ++
be/src/exec/parquet/parquet-column-stats.h | 299 +++
.../exec/parquet/parquet-column-stats.inline.h | 254 +++
be/src/exec/parquet/parquet-common.cc | 132 ++
be/src/exec/parquet/parquet-common.h | 565 +++++
be/src/exec/parquet/parquet-level-decoder.cc | 134 ++
be/src/exec/parquet/parquet-level-decoder.h | 164 ++
be/src/exec/parquet/parquet-metadata-utils.cc | 733 +++++++
be/src/exec/parquet/parquet-metadata-utils.h | 233 +++
be/src/exec/parquet/parquet-plain-test.cc | 338 +++
.../exec/parquet/parquet-scratch-tuple-batch.h | 168 ++
be/src/exec/parquet/parquet-version-test.cc | 84 +
be/src/util/CMakeLists.txt | 1 +
be/src/util/bit-packing-test.cc | 2 +-
be/src/util/bit-packing.cc | 60 +
be/src/util/bit-packing.h | 11 +-
be/src/util/bit-packing.inline.h | 12 +-
be/src/util/bit-stream-utils.inline.h | 2 +-
be/src/util/dict-encoding.h | 2 +-
be/src/util/dict-test.cc | 1 +
be/src/util/parquet-reader.cc | 2 +-
be/src/util/rle-test.cc | 3 +-
common/thrift/generate_error_codes.py | 3 +
.../parquet-num-values-def-levels-mismatch.test | 2 +-
63 files changed, 10044 insertions(+), 9658 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/07fd3320/be/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 608998f..2cec3dd 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -386,6 +386,7 @@ set (IMPALA_LINK_LIBS
Catalog
CodeGen
Common
+ Parquet
Exec
Exprs
GlobalFlags
@@ -419,6 +420,7 @@ if (BUILD_SHARED_LIBS)
BufferPool
Io
Runtime
+ Parquet
Exec
CodeGen
Exprs
@@ -603,7 +605,6 @@ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-variable")
add_subdirectory(generated-sources/gen-cpp)
link_directories(
- ${CMAKE_CURRENT_SOURCE_DIR}/build/bufferpool
${CMAKE_CURRENT_SOURCE_DIR}/build/catalog
${CMAKE_CURRENT_SOURCE_DIR}/build/common
${CMAKE_CURRENT_SOURCE_DIR}/build/exec
http://git-wip-us.apache.org/repos/asf/impala/blob/07fd3320/be/src/benchmarks/bit-packing-benchmark.cc
----------------------------------------------------------------------
diff --git a/be/src/benchmarks/bit-packing-benchmark.cc b/be/src/benchmarks/bit-packing-benchmark.cc
index 7769182..79ac68b 100644
--- a/be/src/benchmarks/bit-packing-benchmark.cc
+++ b/be/src/benchmarks/bit-packing-benchmark.cc
@@ -266,7 +266,7 @@
#include "gutil/strings/substitute.h"
#include "util/benchmark.h"
-#include "util/bit-packing.inline.h"
+#include "util/bit-packing.h"
#include "util/bit-stream-utils.inline.h"
#include "util/cpu-info.h"
http://git-wip-us.apache.org/repos/asf/impala/blob/07fd3320/be/src/benchmarks/bswap-benchmark.cc
----------------------------------------------------------------------
diff --git a/be/src/benchmarks/bswap-benchmark.cc b/be/src/benchmarks/bswap-benchmark.cc
index f62d4fc..86a8021 100644
--- a/be/src/benchmarks/bswap-benchmark.cc
+++ b/be/src/benchmarks/bswap-benchmark.cc
@@ -22,13 +22,13 @@
#include <iostream>
#include <memory>
+#include "exec/parquet/parquet-common.h"
#include "gutil/strings/substitute.h"
-#include "exec/parquet-common.h"
#include "runtime/decimal-value.h"
#include "testutil/mem-util.h"
#include "util/benchmark.h"
-#include "util/cpu-info.h"
#include "util/bit-util.h"
+#include "util/cpu-info.h"
#include "common/names.h"
http://git-wip-us.apache.org/repos/asf/impala/blob/07fd3320/be/src/codegen/impala-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/codegen/impala-ir.cc b/be/src/codegen/impala-ir.cc
index 0fa4fe9..520c9f3 100644
--- a/be/src/codegen/impala-ir.cc
+++ b/be/src/codegen/impala-ir.cc
@@ -29,9 +29,9 @@
#include "exec/grouping-aggregator-ir.cc"
#include "exec/hash-table-ir.cc"
#include "exec/hdfs-avro-scanner-ir.cc"
-#include "exec/hdfs-parquet-scanner-ir.cc"
#include "exec/hdfs-scanner-ir.cc"
#include "exec/non-grouping-aggregator-ir.cc"
+#include "exec/parquet/hdfs-parquet-scanner-ir.cc"
#include "exec/partitioned-hash-join-builder-ir.cc"
#include "exec/partitioned-hash-join-node-ir.cc"
#include "exec/select-node-ir.cc"
http://git-wip-us.apache.org/repos/asf/impala/blob/07fd3320/be/src/exec/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt
index 385aa49..8e52e16 100644
--- a/be/src/exec/CMakeLists.txt
+++ b/be/src/exec/CMakeLists.txt
@@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
+add_subdirectory(parquet)
cmake_minimum_required(VERSION 2.6)
@@ -61,9 +62,6 @@ add_library(Exec
hdfs-plugin-text-scanner.cc
hdfs-text-scanner.cc
hdfs-text-table-writer.cc
- hdfs-parquet-scanner.cc
- hdfs-parquet-scanner-ir.cc
- hdfs-parquet-table-writer.cc
hdfs-orc-scanner.cc
hbase-scan-node.cc
hbase-table-scanner.cc
@@ -72,9 +70,6 @@ add_library(Exec
nested-loop-join-node.cc
non-grouping-aggregator.cc
non-grouping-aggregator-ir.cc
- parquet-column-readers.cc
- parquet-column-stats.cc
- parquet-metadata-utils.cc
partial-sort-node.cc
partitioned-hash-join-builder.cc
partitioned-hash-join-builder-ir.cc
@@ -87,7 +82,6 @@ add_library(Exec
kudu-scan-node-mt.cc
kudu-table-sink.cc
kudu-util.cc
- parquet-common.cc
read-write-util.cc
scan-node.cc
scanner-context.cc
@@ -111,9 +105,6 @@ ADD_BE_LSAN_TEST(zigzag-test)
ADD_BE_LSAN_TEST(hash-table-test)
ADD_BE_LSAN_TEST(delimited-text-parser-test)
ADD_BE_LSAN_TEST(read-write-util-test)
-ADD_BE_LSAN_TEST(parquet-plain-test)
-ADD_BE_LSAN_TEST(parquet-version-test)
ADD_BE_LSAN_TEST(row-batch-list-test)
ADD_BE_LSAN_TEST(incr-stats-util-test)
ADD_BE_LSAN_TEST(hdfs-avro-scanner-test)
-ADD_BE_LSAN_TEST(hdfs-parquet-scanner-test)
http://git-wip-us.apache.org/repos/asf/impala/blob/07fd3320/be/src/exec/data-source-scan-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/data-source-scan-node.cc b/be/src/exec/data-source-scan-node.cc
index 1f2a003..7d62b72 100644
--- a/be/src/exec/data-source-scan-node.cc
+++ b/be/src/exec/data-source-scan-node.cc
@@ -20,21 +20,21 @@
#include <vector>
#include <gutil/strings/substitute.h>
-#include "exec/parquet-common.h"
+#include "exec/parquet/parquet-common.h"
#include "exec/read-write-util.h"
#include "exprs/scalar-expr.h"
#include "gen-cpp/parquet_types.h"
#include "runtime/mem-pool.h"
#include "runtime/mem-tracker.h"
-#include "runtime/runtime-state.h"
#include "runtime/row-batch.h"
+#include "runtime/runtime-state.h"
#include "runtime/string-value.h"
#include "runtime/timestamp-value.inline.h"
#include "runtime/tuple-row.h"
#include "util/jni-util.h"
#include "util/periodic-counter-updater.h"
-#include "util/ubsan.h"
#include "util/runtime-profile-counters.h"
+#include "util/ubsan.h"
#include "common/names.h"
http://git-wip-us.apache.org/repos/asf/impala/blob/07fd3320/be/src/exec/hdfs-parquet-scanner-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-scanner-ir.cc b/be/src/exec/hdfs-parquet-scanner-ir.cc
deleted file mode 100644
index 2ba56c4..0000000
--- a/be/src/exec/hdfs-parquet-scanner-ir.cc
+++ /dev/null
@@ -1,67 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "exec/hdfs-parquet-scanner.h"
-
-#include "exec/filter-context.h"
-#include "exec/parquet-scratch-tuple-batch.h"
-#include "exprs/scalar-expr.h"
-#include "runtime/runtime-filter.h"
-#include "runtime/runtime-filter.inline.h"
-#include "runtime/tuple-row.h"
-
-using namespace impala;
-
-int HdfsParquetScanner::ProcessScratchBatch(RowBatch* dst_batch) {
- ScalarExprEvaluator* const* conjunct_evals = conjunct_evals_->data();
- const int num_conjuncts = conjunct_evals_->size();
-
- // Start/end/current iterators over the output rows.
- Tuple** output_row_start =
- reinterpret_cast<Tuple**>(dst_batch->GetRow(dst_batch->num_rows()));
- Tuple** output_row_end =
- output_row_start + (dst_batch->capacity() - dst_batch->num_rows());
- Tuple** output_row = output_row_start;
-
- // Start/end/current iterators over the scratch tuples.
- uint8_t* scratch_tuple_start = scratch_batch_->CurrTuple();
- uint8_t* scratch_tuple_end = scratch_batch_->TupleEnd();
- uint8_t* scratch_tuple = scratch_tuple_start;
- const int tuple_size = scratch_batch_->tuple_byte_size;
-
- // Loop until the scratch batch is exhausted or the output batch is full.
- // Do not use batch_->AtCapacity() in this loop because it is not necessary
- // to perform the memory capacity check.
- while (scratch_tuple != scratch_tuple_end) {
- *output_row = reinterpret_cast<Tuple*>(scratch_tuple);
- scratch_tuple += tuple_size;
- // Evaluate runtime filters and conjuncts. Short-circuit the evaluation if
- // the filters/conjuncts are empty to avoid function calls.
- if (!EvalRuntimeFilters(reinterpret_cast<TupleRow*>(output_row))) {
- continue;
- }
- if (!ExecNode::EvalConjuncts(conjunct_evals, num_conjuncts,
- reinterpret_cast<TupleRow*>(output_row))) {
- continue;
- }
- // Row survived runtime filters and conjuncts.
- ++output_row;
- if (output_row == output_row_end) break;
- }
- scratch_batch_->tuple_idx += (scratch_tuple - scratch_tuple_start) / tuple_size;
- return output_row - output_row_start;
-}
http://git-wip-us.apache.org/repos/asf/impala/blob/07fd3320/be/src/exec/hdfs-parquet-scanner-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-scanner-test.cc b/be/src/exec/hdfs-parquet-scanner-test.cc
deleted file mode 100644
index 85fa3ef..0000000
--- a/be/src/exec/hdfs-parquet-scanner-test.cc
+++ /dev/null
@@ -1,176 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "exec/hdfs-parquet-scanner.h"
-#include "runtime/test-env.h"
-#include "service/fe-support.h"
-#include "testutil/gtest-util.h"
-
-#include "common/names.h"
-
-static const int64_t MIN_BUFFER_SIZE = 64 * 1024;
-static const int64_t MAX_BUFFER_SIZE = 8 * 1024 * 1024;
-
-DECLARE_int64(min_buffer_size);
-DECLARE_int32(read_size);
-
-namespace impala {
-
-class HdfsParquetScannerTest : public testing::Test {
- public:
- virtual void SetUp() {
- // Override min/max buffer sizes picked up by DiskIoMgr.
- FLAGS_min_buffer_size = MIN_BUFFER_SIZE;
- FLAGS_read_size = MAX_BUFFER_SIZE;
-
- test_env_.reset(new TestEnv);
- ASSERT_OK(test_env_->Init());
- }
-
- virtual void TearDown() {
- test_env_.reset();
- }
-
- protected:
- void TestComputeIdealReservation(const vector<int64_t>& col_range_lengths,
- int64_t expected_ideal_reservation);
- void TestDivideReservation(const vector<int64_t>& col_range_lengths,
- int64_t total_col_reservation, const vector<int64_t>& expected_reservations);
-
- boost::scoped_ptr<TestEnv> test_env_;
-};
-
-/// Test the ComputeIdealReservation returns 'expected_ideal_reservation' for a list
-/// of columns with 'col_range_lengths'.
-void HdfsParquetScannerTest::TestComputeIdealReservation(
- const vector<int64_t>& col_range_lengths, int64_t expected_ideal_reservation) {
- EXPECT_EQ(expected_ideal_reservation,
- HdfsParquetScanner::ComputeIdealReservation(col_range_lengths));
-}
-
-TEST_F(HdfsParquetScannerTest, ComputeIdealReservation) {
- // Should round up to nearest power-of-two buffer size if < max scan range buffer.
- TestComputeIdealReservation({0}, MIN_BUFFER_SIZE);
- TestComputeIdealReservation({1}, MIN_BUFFER_SIZE);
- TestComputeIdealReservation({MIN_BUFFER_SIZE - 1}, MIN_BUFFER_SIZE);
- TestComputeIdealReservation({MIN_BUFFER_SIZE}, MIN_BUFFER_SIZE);
- TestComputeIdealReservation({MIN_BUFFER_SIZE + 2}, 2 * MIN_BUFFER_SIZE);
- TestComputeIdealReservation({4 * MIN_BUFFER_SIZE + 1234}, 8 * MIN_BUFFER_SIZE);
- TestComputeIdealReservation({MAX_BUFFER_SIZE - 10}, MAX_BUFFER_SIZE);
- TestComputeIdealReservation({MAX_BUFFER_SIZE}, MAX_BUFFER_SIZE);
-
- // Should round to nearest max I/O buffer size if >= max scan range buffer, up to 3
- // buffers.
- TestComputeIdealReservation({MAX_BUFFER_SIZE + 1}, 2 * MAX_BUFFER_SIZE);
- TestComputeIdealReservation({MAX_BUFFER_SIZE * 2 - 1}, 2 * MAX_BUFFER_SIZE);
- TestComputeIdealReservation({MAX_BUFFER_SIZE * 2}, 2 * MAX_BUFFER_SIZE);
- TestComputeIdealReservation({MAX_BUFFER_SIZE * 2 + 1}, 3 * MAX_BUFFER_SIZE);
- TestComputeIdealReservation({MAX_BUFFER_SIZE * 3 + 1}, 3 * MAX_BUFFER_SIZE);
- TestComputeIdealReservation({MAX_BUFFER_SIZE * 100 + 27}, 3 * MAX_BUFFER_SIZE);
-
- // Ideal reservations from multiple ranges are simply added together.
- TestComputeIdealReservation({1, 2}, 2 * MIN_BUFFER_SIZE);
- TestComputeIdealReservation(
- {MAX_BUFFER_SIZE, MAX_BUFFER_SIZE - 1}, 2 * MAX_BUFFER_SIZE);
- TestComputeIdealReservation(
- {MAX_BUFFER_SIZE, MIN_BUFFER_SIZE + 1}, MAX_BUFFER_SIZE + 2 * MIN_BUFFER_SIZE);
- TestComputeIdealReservation(
- {MAX_BUFFER_SIZE, MAX_BUFFER_SIZE * 128}, 4 * MAX_BUFFER_SIZE);
- TestComputeIdealReservation(
- {MAX_BUFFER_SIZE * 7, MAX_BUFFER_SIZE * 128, MAX_BUFFER_SIZE * 1000},
- 3L * 3L * MAX_BUFFER_SIZE);
-
- // Test col size that doesn't fit in int32.
- TestComputeIdealReservation({MAX_BUFFER_SIZE * 1024L}, 3L * MAX_BUFFER_SIZE);
-
- // Test sum of reservations that doesn't fit in int32.
- vector<int64_t> col_range_lengths;
- const int64_t LARGE_NUM_RANGES = 10000;
- for (int i = 0; i < LARGE_NUM_RANGES; ++i) {
- col_range_lengths.push_back(4 * MAX_BUFFER_SIZE);
- }
- TestComputeIdealReservation(col_range_lengths, LARGE_NUM_RANGES * 3L * MAX_BUFFER_SIZE);
-}
-
-/// Test that DivideReservationBetweenColumns() returns 'expected_reservations' for
-/// inputs 'col_range_lengths' and 'total_col_reservation'.
-void HdfsParquetScannerTest::TestDivideReservation(const vector<int64_t>& col_range_lengths,
- int64_t total_col_reservation, const vector<int64_t>& expected_reservations) {
- vector<pair<int, int64_t>> reservations =
- HdfsParquetScanner::DivideReservationBetweenColumnsHelper(
- MIN_BUFFER_SIZE, MAX_BUFFER_SIZE, col_range_lengths, total_col_reservation);
- for (int i = 0; i < reservations.size(); ++i) {
- LOG(INFO) << i << " " << reservations[i].first << " " << reservations[i].second;
- }
- EXPECT_EQ(reservations.size(), expected_reservations.size());
- vector<bool> present(expected_reservations.size(), false);
- for (auto& reservation: reservations) {
- // Ensure that each appears exactly once.
- EXPECT_FALSE(present[reservation.first]);
- present[reservation.first] = true;
- EXPECT_EQ(expected_reservations[reservation.first], reservation.second)
- << reservation.first;
- }
-}
-
-TEST_F(HdfsParquetScannerTest, DivideReservation) {
- // Test a long scan ranges with lots of memory - should allocate 3 max-size
- // buffers per range.
- TestDivideReservation({100 * 1024 * 1024}, 50 * 1024 * 1024, {3 * MAX_BUFFER_SIZE});
- TestDivideReservation({100 * 1024 * 1024, 50 * 1024 * 1024}, 100 * 1024 * 1024,
- {3 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE});
-
- // Long scan ranges, not enough memory for 3 buffers each. Should only allocate
- // max-sized buffers, preferring the longer scan range.
- TestDivideReservation({50 * 1024 * 1024, 100 * 1024 * 1024}, 5 * MAX_BUFFER_SIZE,
- {2 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE});
- TestDivideReservation({50 * 1024 * 1024, 100 * 1024 * 1024},
- 5 * MAX_BUFFER_SIZE + MIN_BUFFER_SIZE,
- {2 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE});
- TestDivideReservation({50 * 1024 * 1024, 100 * 1024 * 1024}, 6 * MAX_BUFFER_SIZE - 1,
- {2 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE});
-
- // Test a short range with lots of memory - should round up buffer size.
- TestDivideReservation({100 * 1024}, 50 * 1024 * 1024, {128 * 1024});
-
- // Test a range << MIN_BUFFER_SIZE - should round up to buffer size.
- TestDivideReservation({13}, 50 * 1024 * 1024, {MIN_BUFFER_SIZE});
-
- // Test long ranges with limited memory.
- TestDivideReservation({100 * 1024 * 1024}, 100 * 1024, {MIN_BUFFER_SIZE});
- TestDivideReservation({100 * 1024 * 1024}, MIN_BUFFER_SIZE, {MIN_BUFFER_SIZE});
- TestDivideReservation({100 * 1024 * 1024}, 2 * MIN_BUFFER_SIZE, {2 * MIN_BUFFER_SIZE});
- TestDivideReservation({100 * 1024 * 1024}, MAX_BUFFER_SIZE - 1, {MAX_BUFFER_SIZE / 2});
- TestDivideReservation({100 * 1024 * 1024, 1024 * 1024, MIN_BUFFER_SIZE},
- 3 * MIN_BUFFER_SIZE, {MIN_BUFFER_SIZE, MIN_BUFFER_SIZE, MIN_BUFFER_SIZE});
-
- // Test a mix of scan range lengths larger than and smaller than the max I/O buffer
- // size. Long ranges get allocated most memory.
- TestDivideReservation(
- {15145047, 5019635, 5019263, 15145047, 15145047, 5019635, 5019263, 317304},
- 25165824,
- {8388608, 2097152, 524288, 8388608, 4194304, 1048576, 262144, 262144});
-}
-
-}
-
-int main(int argc, char** argv) {
- ::testing::InitGoogleTest(&argc, argv);
- impala::InitCommonRuntime(argc, argv, true, impala::TestInfo::BE_TEST);
- impala::InitFeSupport();
- return RUN_ALL_TESTS();
-}