You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by lv...@apache.org on 2018/03/03 22:33:29 UTC
[1/9] impala git commit: Revert IMPALA-4835 and dependent changes
Repository: impala
Updated Branches:
refs/heads/2.x 42c604e9f -> e5689fb5c
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/tpch/queries/sort-reservation-usage.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/tpch/queries/sort-reservation-usage.test b/testdata/workloads/tpch/queries/sort-reservation-usage.test
index af31ccb..92f180d 100644
--- a/testdata/workloads/tpch/queries/sort-reservation-usage.test
+++ b/testdata/workloads/tpch/queries/sort-reservation-usage.test
@@ -1,18 +1,17 @@
====
---- QUERY
# Test that in-mem sorts incrementally give up memory when emitting output.
-# This query and scratch limit are chosen so that the query fails if the first sort
-# does not give up memory to the second sort. Scans the text formatted file so that
-# the scan uses less reservation.
+# This query and the limit is calibrated to fail if the first sort does not
+# give up memory to the second sort.
set num_nodes=1;
set scratch_limit=0;
-set buffer_pool_limit=35m;
+set buffer_pool_limit=15m;
set default_spillable_buffer_size=64kb;
SELECT *
FROM (SELECT
Rank() OVER(ORDER BY l_orderkey) AS rank,
Rank() OVER(ORDER BY l_partkey) AS rank2
- FROM tpch.lineitem
+ FROM lineitem
WHERE l_shipdate < '1992-05-09') a
WHERE rank < 10
ORDER BY rank;
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/common/test_dimensions.py
----------------------------------------------------------------------
diff --git a/tests/common/test_dimensions.py b/tests/common/test_dimensions.py
index 3c1b5e7..4171e1f 100644
--- a/tests/common/test_dimensions.py
+++ b/tests/common/test_dimensions.py
@@ -131,13 +131,13 @@ SINGLE_NODE_ONLY = [1]
ALL_NODES_ONLY = [0]
ALL_DISABLE_CODEGEN_OPTIONS = [True, False]
-def create_single_exec_option_dimension(num_nodes=0, disable_codegen_rows_threshold=5000):
+def create_single_exec_option_dimension():
"""Creates an exec_option dimension that will produce a single test vector"""
- return create_exec_option_dimension(cluster_sizes=[num_nodes],
- disable_codegen_options=[False],
- # Make sure codegen kicks in for functional.alltypes.
- disable_codegen_rows_threshold_options=[disable_codegen_rows_threshold],
- batch_sizes=[0])
+ return create_exec_option_dimension(cluster_sizes=ALL_NODES_ONLY,
+ disable_codegen_options=[False],
+ # Make sure codegen kicks in for functional.alltypes.
+ disable_codegen_rows_threshold_options=[5000],
+ batch_sizes=[0])
def create_exec_option_dimension(cluster_sizes=ALL_CLUSTER_SIZES,
disable_codegen_options=ALL_DISABLE_CODEGEN_OPTIONS,
@@ -145,15 +145,13 @@ def create_exec_option_dimension(cluster_sizes=ALL_CLUSTER_SIZES,
sync_ddl=None, exec_single_node_option=[0],
# We already run with codegen on and off explicitly -
# don't need automatic toggling.
- disable_codegen_rows_threshold_options=[0],
- debug_action_options=[None]):
+ disable_codegen_rows_threshold_options=[0]):
exec_option_dimensions = {
'abort_on_error': [1],
'exec_single_node_rows_threshold': exec_single_node_option,
'batch_size': batch_sizes,
'disable_codegen': disable_codegen_options,
'disable_codegen_rows_threshold': disable_codegen_rows_threshold_options,
- 'debug_action': debug_action_options,
'num_nodes': cluster_sizes}
if sync_ddl is not None:
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/custom_cluster/test_scratch_disk.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_scratch_disk.py b/tests/custom_cluster/test_scratch_disk.py
index 65bde66..bd3c7e4 100644
--- a/tests/custom_cluster/test_scratch_disk.py
+++ b/tests/custom_cluster/test_scratch_disk.py
@@ -39,7 +39,7 @@ class TestScratchDir(CustomClusterTestSuite):
"""
# Buffer pool limit that is low enough to force Impala to spill to disk when executing
# spill_query.
- buffer_pool_limit = "45m"
+ buffer_pool_limit = "32m"
def count_nonempty_dirs(self, dirs):
count = 0
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/query_test/test_mem_usage_scaling.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_mem_usage_scaling.py b/tests/query_test/test_mem_usage_scaling.py
index 2d6c17b..25af44d 100644
--- a/tests/query_test/test_mem_usage_scaling.py
+++ b/tests/query_test/test_mem_usage_scaling.py
@@ -96,7 +96,7 @@ class TestExprMemUsage(ImpalaTestSuite):
class TestLowMemoryLimits(ImpalaTestSuite):
'''Super class for the memory limit tests with the TPC-H and TPC-DS queries'''
- def low_memory_limit_test(self, vector, tpch_query, limit):
+ def low_memory_limit_test(self, vector, tpch_query, limit, xfail_mem_limit=None):
mem = vector.get_value('mem_limit')
# Mem consumption can be +-30MBs, depending on how many scanner threads are
# running. Adding this extra mem in order to reduce false negatives in the tests.
@@ -113,11 +113,13 @@ class TestLowMemoryLimits(ImpalaTestSuite):
try:
self.run_test_case(tpch_query, new_vector)
except ImpalaBeeswaxException as e:
- if not expects_error: raise
+ if not expects_error and not xfail_mem_limit: raise
found_expected_error = False
for error_msg in MEM_LIMIT_ERROR_MSGS:
if error_msg in str(e): found_expected_error = True
assert found_expected_error, str(e)
+ if not expects_error and xfail_mem_limit:
+ pytest.xfail(xfail_mem_limit)
@SkipIfLocal.mem_usage_different
@@ -132,7 +134,7 @@ class TestTpchMemLimitError(TestLowMemoryLimits):
'Q6' : 25, 'Q7' : 200, 'Q8' : 125, 'Q9' : 200, 'Q10' : 162,\
'Q11' : 112, 'Q12' : 150, 'Q13' : 125, 'Q14' : 125, 'Q15' : 125,\
'Q16' : 137, 'Q17' : 137, 'Q18' : 196, 'Q19' : 112, 'Q20' : 162,\
- 'Q21' : 300, 'Q22' : 125}
+ 'Q21' : 187, 'Q22' : 125}
@classmethod
def get_workload(self):
@@ -175,7 +177,8 @@ class TestTpchMemLimitError(TestLowMemoryLimits):
self.low_memory_limit_test(vector, 'tpch-q8', self.MIN_MEM_FOR_TPCH['Q8'])
def test_low_mem_limit_q9(self, vector):
- self.low_memory_limit_test(vector, 'tpch-q9', self.MIN_MEM_FOR_TPCH['Q9'])
+ self.low_memory_limit_test(vector, 'tpch-q9', self.MIN_MEM_FOR_TPCH['Q9'],
+ xfail_mem_limit="IMPALA-3328: TPC-H Q9 memory limit test is flaky")
def test_low_mem_limit_q10(self, vector):
self.low_memory_limit_test(vector, 'tpch-q10', self.MIN_MEM_FOR_TPCH['Q10'])
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/query_test/test_query_mem_limit.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_query_mem_limit.py b/tests/query_test/test_query_mem_limit.py
index 17ea9f5..2fdd6eb 100644
--- a/tests/query_test/test_query_mem_limit.py
+++ b/tests/query_test/test_query_mem_limit.py
@@ -118,9 +118,7 @@ class TestCodegenMemLimit(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
super(TestCodegenMemLimit, cls).add_test_dimensions()
- # Run with num_nodes=1 to avoid races between fragments allocating memory.
- cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension(
- num_nodes=1, disable_codegen_rows_threshold=0))
+ cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
# Only run the query for parquet
cls.ImpalaTestMatrix.add_constraint(
lambda v: v.get_value('table_format').file_format == 'parquet')
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/query_test/test_scanners.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 2d0fd0c..9b252da 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -37,9 +37,7 @@ from tests.common.skip import (
SkipIfADLS,
SkipIfIsilon,
SkipIfLocal)
-from tests.common.test_dimensions import (
- create_single_exec_option_dimension,
- create_exec_option_dimension)
+from tests.common.test_dimensions import create_single_exec_option_dimension
from tests.common.test_result_verifier import (
parse_column_types,
parse_column_labels,
@@ -51,11 +49,6 @@ from tests.util.hdfs_util import NAMENODE
from tests.util.get_parquet_metadata import get_parquet_metadata
from tests.util.test_file_parser import QueryTestSectionReader
-# Test scanners with denial of reservations at varying frequency. This will affect the
-# number of scanner threads that can be spun up.
-DEBUG_ACTION_DIMS = [None,
- '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@0.5',
- '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0']
class TestScannersAllTableFormats(ImpalaTestSuite):
BATCH_SIZES = [0, 1, 16]
@@ -73,20 +66,18 @@ class TestScannersAllTableFormats(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_dimension(cls.create_table_info_dimension('pairwise'))
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('batch_size', *TestScannersAllTableFormats.BATCH_SIZES))
- cls.ImpalaTestMatrix.add_dimension(
- ImpalaTestDimension('debug_action', *DEBUG_ACTION_DIMS))
def test_scanners(self, vector):
new_vector = deepcopy(vector)
new_vector.get_value('exec_option')['batch_size'] = vector.get_value('batch_size')
- new_vector.get_value('exec_option')['debug_action'] = vector.get_value('debug_action')
self.run_test_case('QueryTest/scanners', new_vector)
def test_hdfs_scanner_profile(self, vector):
- if vector.get_value('table_format').file_format in ('kudu', 'hbase') or \
- vector.get_value('exec_option')['num_nodes'] != 0:
+ if vector.get_value('table_format').file_format in ('kudu', 'hbase'):
pytest.skip()
- self.run_test_case('QueryTest/hdfs_scanner_profile', vector)
+ new_vector = deepcopy(vector)
+ new_vector.get_value('exec_option')['num_nodes'] = 0
+ self.run_test_case('QueryTest/hdfs_scanner_profile', new_vector)
# Test all the scanners with a simple limit clause. The limit clause triggers
# cancellation in the scanner code paths.
@@ -180,8 +171,6 @@ class TestWideRow(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
super(TestWideRow, cls).add_test_dimensions()
- cls.ImpalaTestMatrix.add_dimension(
- create_exec_option_dimension(debug_action_options=DEBUG_ACTION_DIMS))
# I can't figure out how to load a huge row into hbase
cls.ImpalaTestMatrix.add_constraint(
lambda v: v.get_value('table_format').file_format != 'hbase')
@@ -213,8 +202,6 @@ class TestWideTable(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
super(TestWideTable, cls).add_test_dimensions()
- cls.ImpalaTestMatrix.add_dimension(
- create_exec_option_dimension(debug_action_options=DEBUG_ACTION_DIMS))
cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension("num_cols", *cls.NUM_COLS))
# To cut down on test execution time, only run in exhaustive.
if cls.exploration_strategy() != 'exhaustive':
@@ -257,8 +244,6 @@ class TestParquet(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
super(TestParquet, cls).add_test_dimensions()
- cls.ImpalaTestMatrix.add_dimension(
- create_exec_option_dimension(debug_action_options=DEBUG_ACTION_DIMS))
cls.ImpalaTestMatrix.add_constraint(
lambda v: v.get_value('table_format').file_format == 'parquet')
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/query_test/test_scanners_fuzz.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_scanners_fuzz.py b/tests/query_test/test_scanners_fuzz.py
index 0b42e5d..c336a17 100644
--- a/tests/query_test/test_scanners_fuzz.py
+++ b/tests/query_test/test_scanners_fuzz.py
@@ -48,12 +48,6 @@ class TestScannersFuzzing(ImpalaTestSuite):
# Test a range of batch sizes to exercise different corner cases.
BATCH_SIZES = [0, 1, 16, 10000]
- # Test with denial of reservations at varying frequency. This will affect the number
- # of scanner threads that can be spun up.
- DEBUG_ACTION_VALUES = [None,
- '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@0.5',
- '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0']
-
@classmethod
def get_workload(cls):
return 'functional-query'
@@ -65,8 +59,7 @@ class TestScannersFuzzing(ImpalaTestSuite):
create_exec_option_dimension_from_dict({
'abort_on_error' : cls.ABORT_ON_ERROR_VALUES,
'num_nodes' : cls.NUM_NODES_VALUES,
- 'mem_limit' : cls.MEM_LIMITS,
- 'debug_action' : cls.DEBUG_ACTION_VALUES}))
+ 'mem_limit' : cls.MEM_LIMITS}))
# TODO: enable for more table formats once they consistently pass the fuzz test.
cls.ImpalaTestMatrix.add_constraint(lambda v:
v.get_value('table_format').file_format in ('avro', 'parquet') or
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/query_test/test_sort.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_sort.py b/tests/query_test/test_sort.py
index ea0760f..23de9f2 100644
--- a/tests/query_test/test_sort.py
+++ b/tests/query_test/test_sort.py
@@ -68,23 +68,14 @@ class TestQueryFullSort(ImpalaTestSuite):
order by o_orderdate"""
exec_option = copy(vector.get_value('exec_option'))
table_format = vector.get_value('table_format')
- exec_option['default_spillable_buffer_size'] = '8M'
-
- # Minimum memory for different parts of the plan.
- sort_reservation_mb = 48
- if table_format.file_format == 'parquet':
- scan_reservation_mb = 24
- else:
- scan_reservation_mb = 8
- total_reservation_mb = sort_reservation_mb + scan_reservation_mb
# The below memory value assume 8M pages.
- # Test with unlimited and minimum memory for all file formats.
- buffer_pool_limit_values = ['-1', '{0}M'.format(total_reservation_mb)]
+ exec_option['default_spillable_buffer_size'] = '8M'
+ buffer_pool_limit_values = ['-1', '48M'] # Unlimited and minimum memory.
if self.exploration_strategy() == 'exhaustive' and \
table_format.file_format == 'parquet':
# Test some intermediate values for parquet on exhaustive.
- buffer_pool_limit_values += ['128M', '256M']
+ buffer_pool_limit_values += ['64M', '128M', '256M']
for buffer_pool_limit in buffer_pool_limit_values:
exec_option['buffer_pool_limit'] = buffer_pool_limit
result = transpose_results(self.execute_query(
@@ -93,6 +84,7 @@ class TestQueryFullSort(ImpalaTestSuite):
def test_sort_join(self, vector):
"""With 200m memory limit this should be a 2-phase sort"""
+
query = """select o1.o_orderdate, o2.o_custkey, o1.o_comment from orders o1 join
orders o2 on (o1.o_orderkey = o2.o_orderkey) order by o1.o_orderdate limit 100000"""
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/query_test/test_spilling.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_spilling.py b/tests/query_test/test_spilling.py
index 05ada9f..0b36429 100644
--- a/tests/query_test/test_spilling.py
+++ b/tests/query_test/test_spilling.py
@@ -95,8 +95,3 @@ class TestSpillingNoDebugActionDimensions(ImpalaTestSuite):
setting debug_action to alternative values via query options."""
self.run_test_case('QueryTest/spilling-query-options', vector)
- def test_spilling_no_debug_action(self, vector):
- """Spilling tests that will not succeed if run with an arbitrary debug action.
- These tests either run with no debug action set or set their own debug action."""
- self.run_test_case('QueryTest/spilling-no-debug-action', vector)
-
[6/9] impala git commit: Revert IMPALA-4835 and dependent changes
Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr.cc b/be/src/runtime/io/disk-io-mgr.cc
index 0d2afe2..8ff6609 100644
--- a/be/src/runtime/io/disk-io-mgr.cc
+++ b/be/src/runtime/io/disk-io-mgr.cc
@@ -15,10 +15,8 @@
// specific language governing permissions and limitations
// under the License.
-#include "runtime/io/disk-io-mgr.h"
-
#include "common/global-flags.h"
-#include "runtime/exec-env.h"
+#include "runtime/io/disk-io-mgr.h"
#include "runtime/io/disk-io-mgr-internal.h"
#include "runtime/io/handle-cache.inline.h"
@@ -54,8 +52,6 @@ DEFINE_int32(num_threads_per_disk, 0, "Number of I/O threads per disk");
static const int THREADS_PER_ROTATIONAL_DISK = 1;
static const int THREADS_PER_SOLID_STATE_DISK = 8;
-const int64_t DiskIoMgr::IDEAL_MAX_SIZED_BUFFERS_PER_SCAN_RANGE;
-
// The maximum number of the threads per rotational disk is also the max queue depth per
// rotational disk.
static const string num_io_threads_per_rotational_disk_help_msg = Substitute("Number of "
@@ -126,6 +122,13 @@ DEFINE_uint64(unused_file_handle_timeout_sec, 21600, "Maximum time, in seconds,
DEFINE_uint64(num_file_handle_cache_partitions, 16, "Number of partitions used by the "
"file handle cache.");
+// The IoMgr is able to run with a wide range of memory usage. If a query has memory
+// remaining less than this value, the IoMgr will stop all buffering regardless of the
+// current queue size.
+static const int LOW_MEMORY = 64 * 1024 * 1024;
+
+const int DiskIoMgr::SCAN_RANGE_READY_BUFFER_LIMIT;
+
AtomicInt32 DiskIoMgr::next_disk_id_;
namespace detail {
@@ -152,6 +155,34 @@ string DiskIoMgr::DebugString() {
return ss.str();
}
+BufferDescriptor::BufferDescriptor(DiskIoMgr* io_mgr,
+ RequestContext* reader, ScanRange* scan_range, uint8_t* buffer,
+ int64_t buffer_len, MemTracker* mem_tracker)
+ : io_mgr_(io_mgr),
+ reader_(reader),
+ mem_tracker_(mem_tracker),
+ scan_range_(scan_range),
+ buffer_(buffer),
+ buffer_len_(buffer_len) {
+ DCHECK(io_mgr != nullptr);
+ DCHECK(scan_range != nullptr);
+ DCHECK(buffer != nullptr);
+ DCHECK_GE(buffer_len, 0);
+ DCHECK_NE(scan_range->external_buffer_tag_ == ScanRange::ExternalBufferTag::NO_BUFFER,
+ mem_tracker == nullptr);
+}
+
+void BufferDescriptor::TransferOwnership(MemTracker* dst) {
+ DCHECK(dst != nullptr);
+ DCHECK(!is_client_buffer());
+ // Memory of cached buffers is not tracked against a tracker.
+ if (is_cached()) return;
+ DCHECK(mem_tracker_ != nullptr);
+ dst->Consume(buffer_len_);
+ mem_tracker_->Release(buffer_len_);
+ mem_tracker_ = dst;
+}
+
WriteRange::WriteRange(
const string& file, int64_t file_offset, int disk_id, WriteDoneCallback callback)
: RequestRange(RequestType::WRITE), callback_(callback) {
@@ -192,8 +223,8 @@ DiskIoMgr::DiskIoMgr() :
num_io_threads_per_solid_state_disk_(GetFirstPositiveVal(
FLAGS_num_io_threads_per_solid_state_disk, FLAGS_num_threads_per_disk,
THREADS_PER_SOLID_STATE_DISK)),
- max_buffer_size_(BitUtil::RoundUpToPowerOfTwo(FLAGS_read_size)),
- min_buffer_size_(BitUtil::RoundDownToPowerOfTwo(FLAGS_min_buffer_size)),
+ max_buffer_size_(FLAGS_read_size),
+ min_buffer_size_(FLAGS_min_buffer_size),
shut_down_(false),
total_bytes_read_counter_(TUnit::BYTES),
read_timer_(TUnit::TIME_NS),
@@ -202,6 +233,8 @@ DiskIoMgr::DiskIoMgr() :
FLAGS_num_file_handle_cache_partitions,
FLAGS_unused_file_handle_timeout_sec) {
DCHECK_LE(READ_SIZE_MIN_VALUE, FLAGS_read_size);
+ int64_t max_buffer_size_scaled = BitUtil::Ceil(max_buffer_size_, min_buffer_size_);
+ free_buffers_.resize(BitUtil::Log2Ceiling64(max_buffer_size_scaled) + 1);
int num_local_disks = DiskInfo::num_disks();
if (FLAGS_num_disks < 0 || FLAGS_num_disks > DiskInfo::num_disks()) {
LOG(WARNING) << "Number of disks specified should be between 0 and the number of "
@@ -215,11 +248,11 @@ DiskIoMgr::DiskIoMgr() :
}
DiskIoMgr::DiskIoMgr(int num_local_disks, int threads_per_rotational_disk,
- int threads_per_solid_state_disk, int64_t min_buffer_size, int64_t max_buffer_size) :
+ int threads_per_solid_state_disk, int min_buffer_size, int max_buffer_size) :
num_io_threads_per_rotational_disk_(threads_per_rotational_disk),
num_io_threads_per_solid_state_disk_(threads_per_solid_state_disk),
- max_buffer_size_(BitUtil::RoundUpToPowerOfTwo(max_buffer_size)),
- min_buffer_size_(BitUtil::RoundDownToPowerOfTwo(min_buffer_size)),
+ max_buffer_size_(max_buffer_size),
+ min_buffer_size_(min_buffer_size),
shut_down_(false),
total_bytes_read_counter_(TUnit::BYTES),
read_timer_(TUnit::TIME_NS),
@@ -227,6 +260,8 @@ DiskIoMgr::DiskIoMgr(int num_local_disks, int threads_per_rotational_disk,
FileSystemUtil::MaxNumFileHandles()),
FLAGS_num_file_handle_cache_partitions,
FLAGS_unused_file_handle_timeout_sec) {
+ int64_t max_buffer_size_scaled = BitUtil::Ceil(max_buffer_size_, min_buffer_size_);
+ free_buffers_.resize(BitUtil::Log2Ceiling64(max_buffer_size_scaled) + 1);
if (num_local_disks == 0) num_local_disks = DiskInfo::num_disks();
disk_queues_.resize(num_local_disks + REMOTE_NUM_DISKS);
CheckSseSupport();
@@ -250,22 +285,37 @@ DiskIoMgr::~DiskIoMgr() {
for (int i = 0; i < disk_queues_.size(); ++i) {
if (disk_queues_[i] == nullptr) continue;
int disk_id = disk_queues_[i]->disk_id;
- for (RequestContext* context : disk_queues_[i]->request_contexts) {
- unique_lock<mutex> context_lock(context->lock_);
- DCHECK_EQ(context->disk_states_[disk_id].num_threads_in_op(), 0);
- DCHECK(context->disk_states_[disk_id].done());
- context->DecrementDiskRefCount(context_lock);
+ for (list<RequestContext*>::iterator it = disk_queues_[i]->request_contexts.begin();
+ it != disk_queues_[i]->request_contexts.end(); ++it) {
+ DCHECK_EQ((*it)->disk_states_[disk_id].num_threads_in_op(), 0);
+ DCHECK((*it)->disk_states_[disk_id].done());
+ (*it)->DecrementDiskRefCount();
}
}
+ DCHECK_EQ(num_buffers_in_readers_.Load(), 0);
+
+ // Delete all allocated buffers
+ int num_free_buffers = 0;
+ for (int idx = 0; idx < free_buffers_.size(); ++idx) {
+ num_free_buffers += free_buffers_[idx].size();
+ }
+ DCHECK_EQ(num_allocated_buffers_.Load(), num_free_buffers);
+ GcIoBuffers();
+
for (int i = 0; i < disk_queues_.size(); ++i) {
delete disk_queues_[i];
}
+ if (free_buffer_mem_tracker_ != nullptr) free_buffer_mem_tracker_->Close();
if (cached_read_options_ != nullptr) hadoopRzOptionsFree(cached_read_options_);
}
-Status DiskIoMgr::Init() {
+Status DiskIoMgr::Init(MemTracker* process_mem_tracker) {
+ DCHECK(process_mem_tracker != nullptr);
+ free_buffer_mem_tracker_.reset(
+ new MemTracker(-1, "Free Disk IO Buffers", process_mem_tracker, false));
+
for (int i = 0; i < disk_queues_.size(); ++i) {
disk_queues_[i] = new DiskQueue(i);
int num_threads_per_disk;
@@ -311,14 +361,101 @@ Status DiskIoMgr::Init() {
return Status::OK();
}
-unique_ptr<RequestContext> DiskIoMgr::RegisterContext() {
- return unique_ptr<RequestContext>(new RequestContext(this, num_total_disks()));
+unique_ptr<RequestContext> DiskIoMgr::RegisterContext(MemTracker* mem_tracker) {
+ return unique_ptr<RequestContext>(
+ new RequestContext(this, num_total_disks(), mem_tracker));
}
void DiskIoMgr::UnregisterContext(RequestContext* reader) {
reader->CancelAndMarkInactive();
}
+// Cancellation requires coordination from multiple threads. Each thread that currently
+// has a reference to the request context must notice the cancel and remove it from its
+// tracking structures. The last thread to touch the context should deallocate (aka
+// recycle) the request context object. Potential threads are:
+// 1. Disk threads that are currently reading for this reader.
+// 2. Caller threads that are waiting in GetNext.
+//
+// The steps are:
+// 1. Cancel will immediately set the context in the Cancelled state. This prevents any
+// other thread from adding more ready buffers to the context (they all take a lock and
+// check the state before doing so), or any write ranges to the context.
+// 2. Cancel will call cancel on each ScanRange that is not yet complete, unblocking
+// any threads in GetNext(). The reader will see the cancelled Status returned. Cancel
+// also invokes the callback for the WriteRanges with the cancelled state.
+// 3. Disk threads notice the context is cancelled either when picking the next context
+// to process or when they try to enqueue a ready buffer. Upon noticing the cancelled
+// state, removes the context from the disk queue. The last thread per disk with an
+// outstanding reference to the context decrements the number of disk queues the context
+// is on.
+void DiskIoMgr::CancelContext(RequestContext* context) {
+ context->Cancel(Status::CANCELLED);
+}
+
+void DiskIoMgr::set_read_timer(RequestContext* r, RuntimeProfile::Counter* c) {
+ r->read_timer_ = c;
+}
+
+void DiskIoMgr::set_open_file_timer(RequestContext* r, RuntimeProfile::Counter* c) {
+ r->open_file_timer_ = c;
+}
+
+void DiskIoMgr::set_bytes_read_counter(RequestContext* r, RuntimeProfile::Counter* c) {
+ r->bytes_read_counter_ = c;
+}
+
+void DiskIoMgr::set_active_read_thread_counter(RequestContext* r,
+ RuntimeProfile::Counter* c) {
+ r->active_read_thread_counter_ = c;
+}
+
+void DiskIoMgr::set_disks_access_bitmap(RequestContext* r,
+ RuntimeProfile::Counter* c) {
+ r->disks_accessed_bitmap_ = c;
+}
+
+int64_t DiskIoMgr::queue_size(RequestContext* reader) const {
+ return reader->num_ready_buffers_.Load();
+}
+
+Status DiskIoMgr::context_status(RequestContext* context) const {
+ unique_lock<mutex> lock(context->lock_);
+ return context->status_;
+}
+
+int64_t DiskIoMgr::bytes_read_local(RequestContext* reader) const {
+ return reader->bytes_read_local_.Load();
+}
+
+int64_t DiskIoMgr::bytes_read_short_circuit(RequestContext* reader) const {
+ return reader->bytes_read_short_circuit_.Load();
+}
+
+int64_t DiskIoMgr::bytes_read_dn_cache(RequestContext* reader) const {
+ return reader->bytes_read_dn_cache_.Load();
+}
+
+int DiskIoMgr::num_remote_ranges(RequestContext* reader) const {
+ return reader->num_remote_ranges_.Load();
+}
+
+int64_t DiskIoMgr::unexpected_remote_bytes(RequestContext* reader) const {
+ return reader->unexpected_remote_bytes_.Load();
+}
+
+int DiskIoMgr::cached_file_handles_hit_count(RequestContext* reader) const {
+ return reader->cached_file_handles_hit_count_.Load();
+}
+
+int DiskIoMgr::cached_file_handles_miss_count(RequestContext* reader) const {
+ return reader->cached_file_handles_miss_count_.Load();
+}
+
+int64_t DiskIoMgr::GetReadThroughput() {
+ return RuntimeProfile::UnitsPerSecond(&total_bytes_read_counter_, &read_timer_);
+}
+
Status DiskIoMgr::ValidateScanRange(ScanRange* range) {
int disk_id = range->disk_id_;
if (disk_id < 0 || disk_id >= disk_queues_.size()) {
@@ -329,91 +466,84 @@ Status DiskIoMgr::ValidateScanRange(ScanRange* range) {
return Status(TErrorCode::DISK_IO_ERROR,
Substitute("Invalid scan range. Negative offset $0", range->offset_));
}
- if (range->len_ <= 0) {
+ if (range->len_ < 0) {
return Status(TErrorCode::DISK_IO_ERROR,
- Substitute("Invalid scan range. Non-positive length $0", range->len_));
+ Substitute("Invalid scan range. Negative length $0", range->len_));
}
return Status::OK();
}
-Status DiskIoMgr::AddScanRanges(
- RequestContext* reader, const vector<ScanRange*>& ranges) {
- DCHECK_GT(ranges.size(), 0);
+Status DiskIoMgr::AddScanRanges(RequestContext* reader,
+ const vector<ScanRange*>& ranges, bool schedule_immediately) {
+ if (ranges.empty()) return Status::OK();
+
// Validate and initialize all ranges
for (int i = 0; i < ranges.size(); ++i) {
RETURN_IF_ERROR(ValidateScanRange(ranges[i]));
ranges[i]->InitInternal(this, reader);
}
+ // disks that this reader needs to be scheduled on.
unique_lock<mutex> reader_lock(reader->lock_);
DCHECK(reader->Validate()) << endl << reader->DebugString();
- if (reader->state_ == RequestContext::Cancelled) return Status::CANCELLED;
+ if (reader->state_ == RequestContext::Cancelled) {
+ DCHECK(!reader->status_.ok());
+ return reader->status_;
+ }
// Add each range to the queue of the disk the range is on
- for (ScanRange* range : ranges) {
+ for (int i = 0; i < ranges.size(); ++i) {
// Don't add empty ranges.
- DCHECK_NE(range->len(), 0);
- reader->AddActiveScanRangeLocked(reader_lock, range);
+ DCHECK_NE(ranges[i]->len(), 0);
+ ScanRange* range = ranges[i];
+
if (range->try_cache_) {
- reader->cached_ranges_.Enqueue(range);
- } else {
- reader->AddRangeToDisk(reader_lock, range, ScheduleMode::UPON_GETNEXT);
+ if (schedule_immediately) {
+ bool cached_read_succeeded;
+ RETURN_IF_ERROR(range->ReadFromCache(reader_lock, &cached_read_succeeded));
+ if (cached_read_succeeded) continue;
+ // Cached read failed, fall back to AddRequestRange() below.
+ } else {
+ reader->cached_ranges_.Enqueue(range);
+ continue;
+ }
}
+ reader->AddRequestRange(range, schedule_immediately);
}
DCHECK(reader->Validate()) << endl << reader->DebugString();
+
return Status::OK();
}
-Status DiskIoMgr::StartScanRange(RequestContext* reader, ScanRange* range,
- bool* needs_buffers) {
- RETURN_IF_ERROR(ValidateScanRange(range));
- range->InitInternal(this, reader);
-
- unique_lock<mutex> reader_lock(reader->lock_);
- DCHECK(reader->Validate()) << endl << reader->DebugString();
- if (reader->state_ == RequestContext::Cancelled) return Status::CANCELLED;
-
- DCHECK_NE(range->len(), 0);
- if (range->try_cache_) {
- bool cached_read_succeeded;
- RETURN_IF_ERROR(range->ReadFromCache(reader_lock, &cached_read_succeeded));
- if (cached_read_succeeded) {
- DCHECK(reader->Validate()) << endl << reader->DebugString();
- *needs_buffers = false;
- return Status::OK();
- }
- // Cached read failed, fall back to normal read path.
- }
- // If we don't have a buffer yet, the caller must allocate buffers for the range.
- *needs_buffers = range->external_buffer_tag_ == ScanRange::ExternalBufferTag::NO_BUFFER;
- if (*needs_buffers) range->SetBlockedOnBuffer();
- reader->AddActiveScanRangeLocked(reader_lock, range);
- reader->AddRangeToDisk(reader_lock, range,
- *needs_buffers ? ScheduleMode::BY_CALLER : ScheduleMode::IMMEDIATELY);
- DCHECK(reader->Validate()) << endl << reader->DebugString();
- return Status::OK();
+Status DiskIoMgr::AddScanRange(
+ RequestContext* reader, ScanRange* range, bool schedule_immediately) {
+ return AddScanRanges(reader, vector<ScanRange*>({range}), schedule_immediately);
}
// This function returns the next scan range the reader should work on, checking
// for eos and error cases. If there isn't already a cached scan range or a scan
// range prepared by the disk threads, the caller waits on the disk threads.
-Status DiskIoMgr::GetNextUnstartedRange(RequestContext* reader, ScanRange** range,
- bool* needs_buffers) {
+Status DiskIoMgr::GetNextRange(RequestContext* reader, ScanRange** range) {
DCHECK(reader != nullptr);
DCHECK(range != nullptr);
*range = nullptr;
- *needs_buffers = false;
+ Status status = Status::OK();
unique_lock<mutex> reader_lock(reader->lock_);
DCHECK(reader->Validate()) << endl << reader->DebugString();
+
while (true) {
- if (reader->state_ == RequestContext::Cancelled) return Status::CANCELLED;
+ if (reader->state_ == RequestContext::Cancelled) {
+ DCHECK(!reader->status_.ok());
+ status = reader->status_;
+ break;
+ }
if (reader->num_unstarted_scan_ranges_.Load() == 0 &&
reader->ready_to_start_ranges_.empty() && reader->cached_ranges_.empty()) {
// All ranges are done, just return.
- return Status::OK();
+ break;
}
if (!reader->cached_ranges_.empty()) {
@@ -425,7 +555,7 @@ Status DiskIoMgr::GetNextUnstartedRange(RequestContext* reader, ScanRange** rang
if (cached_read_succeeded) return Status::OK();
// This range ended up not being cached. Loop again and pick up a new range.
- reader->AddRangeToDisk(reader_lock, *range, ScheduleMode::UPON_GETNEXT);
+ reader->AddRequestRange(*range, false);
DCHECK(reader->Validate()) << endl << reader->DebugString();
*range = nullptr;
continue;
@@ -441,72 +571,183 @@ Status DiskIoMgr::GetNextUnstartedRange(RequestContext* reader, ScanRange** rang
// Set this to nullptr, the next time this disk runs for this reader, it will
// get another range ready.
reader->disk_states_[disk_id].set_next_scan_range_to_start(nullptr);
- ScanRange::ExternalBufferTag buffer_tag = (*range)->external_buffer_tag_;
- if (buffer_tag == ScanRange::ExternalBufferTag::NO_BUFFER) {
- // We can't schedule this range until the client gives us buffers. The context
- // must be rescheduled regardless to ensure that 'next_scan_range_to_start' is
- // refilled.
- reader->disk_states_[disk_id].ScheduleContext(reader_lock, reader, disk_id);
- (*range)->SetBlockedOnBuffer();
- *needs_buffers = true;
- } else {
- reader->ScheduleScanRange(reader_lock, *range);
- }
- return Status::OK();
+ reader->ScheduleScanRange(*range);
+ break;
}
}
+ return status;
}
-Status DiskIoMgr::AllocateBuffersForRange(RequestContext* reader,
- BufferPool::ClientHandle* bp_client, ScanRange* range, int64_t max_bytes) {
- DCHECK_GE(max_bytes, min_buffer_size_);
- DCHECK(range->external_buffer_tag_ == ScanRange::ExternalBufferTag::NO_BUFFER)
- << static_cast<int>(range->external_buffer_tag_) << " invalid to allocate buffers "
- << "when already reading into an external buffer";
- BufferPool* bp = ExecEnv::GetInstance()->buffer_pool();
- Status status;
- vector<unique_ptr<BufferDescriptor>> buffers;
- for (int64_t buffer_size : ChooseBufferSizes(range->len(), max_bytes)) {
- BufferPool::BufferHandle handle;
- status = bp->AllocateBuffer(bp_client, buffer_size, &handle);
- if (!status.ok()) goto error;
- buffers.emplace_back(new BufferDescriptor(
- this, reader, range, bp_client, move(handle)));
+Status DiskIoMgr::Read(RequestContext* reader,
+ ScanRange* range, std::unique_ptr<BufferDescriptor>* buffer) {
+ DCHECK(range != nullptr);
+ DCHECK(buffer != nullptr);
+ *buffer = nullptr;
+
+ if (range->len() > max_buffer_size_
+ && range->external_buffer_tag_ != ScanRange::ExternalBufferTag::CLIENT_BUFFER) {
+ return Status(TErrorCode::DISK_IO_ERROR, Substitute("Internal error: cannot "
+ "perform sync read of '$0' bytes that is larger than the max read buffer size "
+ "'$1'.", range->len(), max_buffer_size_));
}
- range->AddUnusedBuffers(move(buffers), false);
+
+ vector<ScanRange*> ranges;
+ ranges.push_back(range);
+ RETURN_IF_ERROR(AddScanRanges(reader, ranges, true));
+ RETURN_IF_ERROR(range->GetNext(buffer));
+ DCHECK((*buffer) != nullptr);
+ DCHECK((*buffer)->eosr());
return Status::OK();
- error:
- DCHECK(!status.ok());
- range->CleanUpBuffers(move(buffers));
- return status;
}
-vector<int64_t> DiskIoMgr::ChooseBufferSizes(int64_t scan_range_len, int64_t max_bytes) {
- DCHECK_GE(max_bytes, min_buffer_size_);
- vector<int64_t> buffer_sizes;
- int64_t bytes_allocated = 0;
- while (bytes_allocated < scan_range_len) {
- int64_t bytes_remaining = scan_range_len - bytes_allocated;
- // Either allocate a max-sized buffer or a smaller buffer to fit the rest of the
- // range.
- int64_t next_buffer_size;
- if (bytes_remaining >= max_buffer_size_) {
- next_buffer_size = max_buffer_size_;
+void DiskIoMgr::ReturnBuffer(unique_ptr<BufferDescriptor> buffer_desc) {
+ DCHECK(buffer_desc != nullptr);
+ if (!buffer_desc->status_.ok()) DCHECK(buffer_desc->buffer_ == nullptr);
+
+ RequestContext* reader = buffer_desc->reader_;
+ if (buffer_desc->buffer_ != nullptr) {
+ if (!buffer_desc->is_cached() && !buffer_desc->is_client_buffer()) {
+ // Buffers the were not allocated by DiskIoMgr don't need to be freed.
+ FreeBufferMemory(buffer_desc.get());
+ }
+ buffer_desc->buffer_ = nullptr;
+ num_buffers_in_readers_.Add(-1);
+ reader->num_buffers_in_reader_.Add(-1);
+ } else {
+ // A nullptr buffer means there was an error in which case there is no buffer
+ // to return.
+ }
+
+ if (buffer_desc->eosr_ || buffer_desc->scan_range_->is_cancelled_) {
+ // Need to close the scan range if returning the last buffer or the scan range
+ // has been cancelled (and the caller might never get the last buffer).
+ // Close() is idempotent so multiple cancelled buffers is okay.
+ buffer_desc->scan_range_->Close();
+ }
+}
+
+unique_ptr<BufferDescriptor> DiskIoMgr::GetFreeBuffer(
+ RequestContext* reader, ScanRange* range, int64_t buffer_size) {
+ DCHECK_LE(buffer_size, max_buffer_size_);
+ DCHECK_GT(buffer_size, 0);
+ buffer_size = min(static_cast<int64_t>(max_buffer_size_), buffer_size);
+ int idx = free_buffers_idx(buffer_size);
+ // Quantize buffer size to nearest power of 2 greater than the specified buffer size and
+ // convert to bytes
+ buffer_size = (1LL << idx) * min_buffer_size_;
+
+ // Track memory against the reader. This is checked the next time we start
+ // a read for the next reader in DiskIoMgr::GetNextScanRange().
+ DCHECK(reader->mem_tracker_ != nullptr);
+ reader->mem_tracker_->Consume(buffer_size);
+
+ uint8_t* buffer = nullptr;
+ {
+ unique_lock<mutex> lock(free_buffers_lock_);
+ if (free_buffers_[idx].empty()) {
+ num_allocated_buffers_.Add(1);
+ if (ImpaladMetrics::IO_MGR_NUM_BUFFERS != nullptr) {
+ ImpaladMetrics::IO_MGR_NUM_BUFFERS->Increment(1L);
+ }
+ if (ImpaladMetrics::IO_MGR_TOTAL_BYTES != nullptr) {
+ ImpaladMetrics::IO_MGR_TOTAL_BYTES->Increment(buffer_size);
+ }
+ // We already tracked this memory against the reader's MemTracker.
+ buffer = new uint8_t[buffer_size];
} else {
- next_buffer_size =
- max(min_buffer_size_, BitUtil::RoundUpToPowerOfTwo(bytes_remaining));
+ if (ImpaladMetrics::IO_MGR_NUM_UNUSED_BUFFERS != nullptr) {
+ ImpaladMetrics::IO_MGR_NUM_UNUSED_BUFFERS->Increment(-1L);
+ }
+ buffer = free_buffers_[idx].front();
+ free_buffers_[idx].pop_front();
+ free_buffer_mem_tracker_->Release(buffer_size);
+ ASAN_UNPOISON_MEMORY_REGION(buffer, buffer_size);
}
- if (next_buffer_size + bytes_allocated > max_bytes) {
- // Can't allocate the desired buffer size. Make sure to allocate at least one
- // buffer.
- if (bytes_allocated > 0) break;
- next_buffer_size = BitUtil::RoundDownToPowerOfTwo(max_bytes);
+ }
+
+ // Validate more invariants.
+ DCHECK(range != nullptr);
+ DCHECK(reader != nullptr);
+ DCHECK(buffer != nullptr);
+ return unique_ptr<BufferDescriptor>(new BufferDescriptor(
+ this, reader, range, buffer, buffer_size, reader->mem_tracker_));
+}
+
+void DiskIoMgr::GcIoBuffers(int64_t bytes_to_free) {
+ unique_lock<mutex> lock(free_buffers_lock_);
+ int buffers_freed = 0;
+ int bytes_freed = 0;
+ // Free small-to-large to avoid retaining many small buffers and fragmenting memory.
+ for (int idx = 0; idx < free_buffers_.size(); ++idx) {
+ deque<uint8_t*>* free_buffers = &free_buffers_[idx];
+ while (
+ !free_buffers->empty() && (bytes_to_free == -1 || bytes_freed <= bytes_to_free)) {
+ uint8_t* buffer = free_buffers->front();
+ free_buffers->pop_front();
+ int64_t buffer_size = (1LL << idx) * min_buffer_size_;
+ ASAN_UNPOISON_MEMORY_REGION(buffer, buffer_size);
+ delete[] buffer;
+ free_buffer_mem_tracker_->Release(buffer_size);
+ num_allocated_buffers_.Add(-1);
+
+ ++buffers_freed;
+ bytes_freed += buffer_size;
+ }
+ if (bytes_to_free != -1 && bytes_freed >= bytes_to_free) break;
+ }
+
+ if (ImpaladMetrics::IO_MGR_NUM_BUFFERS != nullptr) {
+ ImpaladMetrics::IO_MGR_NUM_BUFFERS->Increment(-buffers_freed);
+ }
+ if (ImpaladMetrics::IO_MGR_TOTAL_BYTES != nullptr) {
+ ImpaladMetrics::IO_MGR_TOTAL_BYTES->Increment(-bytes_freed);
+ }
+ if (ImpaladMetrics::IO_MGR_NUM_UNUSED_BUFFERS != nullptr) {
+ ImpaladMetrics::IO_MGR_NUM_UNUSED_BUFFERS->Increment(-buffers_freed);
+ }
+}
+
+void DiskIoMgr::FreeBufferMemory(BufferDescriptor* desc) {
+ DCHECK(!desc->is_cached());
+ DCHECK(!desc->is_client_buffer());
+ uint8_t* buffer = desc->buffer_;
+ int64_t buffer_size = desc->buffer_len_;
+ int idx = free_buffers_idx(buffer_size);
+ DCHECK_EQ(BitUtil::Ceil(buffer_size, min_buffer_size_) & ~(1LL << idx), 0)
+ << "buffer_size_ / min_buffer_size_ should be power of 2, got buffer_size = "
+ << buffer_size << ", min_buffer_size_ = " << min_buffer_size_;
+
+ {
+ unique_lock<mutex> lock(free_buffers_lock_);
+ if (!FLAGS_disable_mem_pools &&
+ free_buffers_[idx].size() < FLAGS_max_free_io_buffers) {
+ // Poison buffers stored in cache.
+ ASAN_POISON_MEMORY_REGION(buffer, buffer_size);
+ free_buffers_[idx].push_back(buffer);
+ if (ImpaladMetrics::IO_MGR_NUM_UNUSED_BUFFERS != nullptr) {
+ ImpaladMetrics::IO_MGR_NUM_UNUSED_BUFFERS->Increment(1L);
+ }
+ // This consume call needs to be protected by 'free_buffers_lock_' to avoid a race
+ // with a Release() call for the same buffer that could make consumption negative.
+ // Note: we can't use TryConsume(), which can indirectly call GcIoBuffers().
+ // TODO: after IMPALA-3200 is completed, we should be able to leverage the buffer
+ // pool's free lists, and remove these free lists.
+ free_buffer_mem_tracker_->Consume(buffer_size);
+ } else {
+ num_allocated_buffers_.Add(-1);
+ delete[] buffer;
+ if (ImpaladMetrics::IO_MGR_NUM_BUFFERS != nullptr) {
+ ImpaladMetrics::IO_MGR_NUM_BUFFERS->Increment(-1L);
+ }
+ if (ImpaladMetrics::IO_MGR_TOTAL_BYTES != nullptr) {
+ ImpaladMetrics::IO_MGR_TOTAL_BYTES->Increment(-buffer_size);
+ }
}
- DCHECK(BitUtil::IsPowerOf2(next_buffer_size)) << next_buffer_size;
- buffer_sizes.push_back(next_buffer_size);
- bytes_allocated += next_buffer_size;
}
- return buffer_sizes;
+
+ // We transferred the buffer ownership from the BufferDescriptor to the DiskIoMgr.
+ desc->mem_tracker_->Release(buffer_size);
+ desc->buffer_ = nullptr;
}
// This function gets the next RequestRange to work on for this disk. It checks for
@@ -546,7 +787,7 @@ bool DiskIoMgr::GetNextRequestRange(DiskQueue* disk_queue, RequestRange** range,
disk_queue->request_contexts.pop_front();
DCHECK(*request_context != nullptr);
request_disk_state = &((*request_context)->disk_states_[disk_id]);
- request_disk_state->IncrementDiskThreadAndDequeue();
+ request_disk_state->IncrementRequestThreadAndDequeue();
}
// NOTE: no locks were taken in between. We need to be careful about what state
@@ -554,13 +795,27 @@ bool DiskIoMgr::GetNextRequestRange(DiskQueue* disk_queue, RequestRange** range,
// There are some invariants here. Only one disk thread can have the
// same reader here (the reader is removed from the queue). There can be
// other disk threads operating on this reader in other functions though.
+
+ // We just picked a reader. Before we may allocate a buffer on its behalf, check that
+ // it has not exceeded any memory limits (e.g. the query or process limit).
+ // TODO: once IMPALA-3200 is fixed, we should be able to remove the free lists and
+ // move these memory limit checks to GetFreeBuffer().
+ // Note that calling AnyLimitExceeded() can result in a call to GcIoBuffers().
+ // TODO: IMPALA-3209: we should not force a reader over its memory limit by
+ // pushing more buffers to it. Most readers can make progress and operate within
+ // a fixed memory limit.
+ if ((*request_context)->mem_tracker_ != nullptr
+ && (*request_context)->mem_tracker_->AnyLimitExceeded()) {
+ (*request_context)->Cancel(Status::MemLimitExceeded());
+ }
+
unique_lock<mutex> request_lock((*request_context)->lock_);
VLOG_FILE << "Disk (id=" << disk_id << ") reading for "
<< (*request_context)->DebugString();
// Check if reader has been cancelled
if ((*request_context)->state_ == RequestContext::Cancelled) {
- request_disk_state->DecrementDiskThread(request_lock, *request_context);
+ request_disk_state->DecrementRequestThreadAndCheckDone(*request_context);
continue;
}
@@ -571,16 +826,16 @@ bool DiskIoMgr::GetNextRequestRange(DiskQueue* disk_queue, RequestRange** range,
!request_disk_state->unstarted_scan_ranges()->empty()) {
// We don't have a range queued for this disk for what the caller should
// read next. Populate that. We want to have one range waiting to minimize
- // wait time in GetNextUnstartedRange().
+ // wait time in GetNextRange.
ScanRange* new_range = request_disk_state->unstarted_scan_ranges()->Dequeue();
(*request_context)->num_unstarted_scan_ranges_.Add(-1);
(*request_context)->ready_to_start_ranges_.Enqueue(new_range);
request_disk_state->set_next_scan_range_to_start(new_range);
if ((*request_context)->num_unstarted_scan_ranges_.Load() == 0) {
- // All the ranges have been started, notify everyone blocked on
- // GetNextUnstartedRange(). Only one of them will get work so make sure to return
- // nullptr to the other caller threads.
+ // All the ranges have been started, notify everyone blocked on GetNextRange.
+ // Only one of them will get work so make sure to return nullptr to the other
+ // caller threads.
(*request_context)->ready_to_start_ranges_cv_.NotifyAll();
} else {
(*request_context)->ready_to_start_ranges_cv_.NotifyOne();
@@ -603,7 +858,7 @@ bool DiskIoMgr::GetNextRequestRange(DiskQueue* disk_queue, RequestRange** range,
// There are no inflight ranges, nothing to do.
if (request_disk_state->in_flight_ranges()->empty()) {
- request_disk_state->DecrementDiskThread(request_lock, *request_context);
+ request_disk_state->DecrementRequestThread();
continue;
}
DCHECK_GT(request_disk_state->num_remaining_ranges(), 0);
@@ -612,7 +867,7 @@ bool DiskIoMgr::GetNextRequestRange(DiskQueue* disk_queue, RequestRange** range,
// Now that we've picked a request range, put the context back on the queue so
// another thread can pick up another request range for this context.
- request_disk_state->ScheduleContext(request_lock, *request_context, disk_id);
+ request_disk_state->ScheduleContext(*request_context, disk_id);
DCHECK((*request_context)->Validate()) << endl << (*request_context)->DebugString();
return true;
}
@@ -626,62 +881,81 @@ void DiskIoMgr::HandleWriteFinished(
// Copy disk_id before running callback: the callback may modify write_range.
int disk_id = write_range->disk_id_;
- // Execute the callback before decrementing the thread count. Otherwise
- // RequestContext::Cancel() that waits for the disk ref count to be 0 will
- // return, creating a race, e.g. see IMPALA-1890.
+ // Execute the callback before decrementing the thread count. Otherwise CancelContext()
+ // that waits for the disk ref count to be 0 will return, creating a race, e.g. see
+ // IMPALA-1890.
// The status of the write does not affect the status of the writer context.
write_range->callback_(write_status);
{
unique_lock<mutex> writer_lock(writer->lock_);
DCHECK(writer->Validate()) << endl << writer->DebugString();
RequestContext::PerDiskState& state = writer->disk_states_[disk_id];
- state.DecrementDiskThread(writer_lock, writer);
+ if (writer->state_ == RequestContext::Cancelled) {
+ state.DecrementRequestThreadAndCheckDone(writer);
+ } else {
+ state.DecrementRequestThread();
+ }
--state.num_remaining_ranges();
}
}
void DiskIoMgr::HandleReadFinished(DiskQueue* disk_queue, RequestContext* reader,
- Status read_status, unique_ptr<BufferDescriptor> buffer) {
+ unique_ptr<BufferDescriptor> buffer) {
unique_lock<mutex> reader_lock(reader->lock_);
- RequestContext::PerDiskState* disk_state = &reader->disk_states_[disk_queue->disk_id];
+ RequestContext::PerDiskState& state = reader->disk_states_[disk_queue->disk_id];
DCHECK(reader->Validate()) << endl << reader->DebugString();
- DCHECK_GT(disk_state->num_threads_in_op(), 0);
+ DCHECK_GT(state.num_threads_in_op(), 0);
DCHECK(buffer->buffer_ != nullptr);
- DCHECK(!buffer->is_cached()) << "HDFS cache reads don't go through this code path.";
- // After calling EnqueueReadyBuffer() below, it is no longer valid to read from buffer.
- // Store the state we need before calling EnqueueReadyBuffer().
- bool eosr = buffer->eosr_;
+ if (reader->state_ == RequestContext::Cancelled) {
+ state.DecrementRequestThreadAndCheckDone(reader);
+ DCHECK(reader->Validate()) << endl << reader->DebugString();
+ if (!buffer->is_client_buffer()) FreeBufferMemory(buffer.get());
+ buffer->buffer_ = nullptr;
+ ScanRange* scan_range = buffer->scan_range_;
+ scan_range->Cancel(reader->status_);
+ // Enqueue the buffer to use the scan range's buffer cleanup path.
+ scan_range->EnqueueBuffer(reader_lock, move(buffer));
+ return;
+ }
+
+ DCHECK_EQ(reader->state_, RequestContext::Active);
+ DCHECK(buffer->buffer_ != nullptr);
- // TODO: IMPALA-4249: it safe to touch 'scan_range' until DecrementDiskThread() is
- // called because all clients of DiskIoMgr keep ScanRange objects alive until they
- // unregister their RequestContext.
+ // Update the reader's scan ranges. There are a three cases here:
+ // 1. Read error
+ // 2. End of scan range
+ // 3. Middle of scan range
+ if (!buffer->status_.ok()) {
+ // Error case
+ if (!buffer->is_client_buffer()) FreeBufferMemory(buffer.get());
+ buffer->buffer_ = nullptr;
+ buffer->eosr_ = true;
+ --state.num_remaining_ranges();
+ buffer->scan_range_->Cancel(buffer->status_);
+ } else if (buffer->eosr_) {
+ --state.num_remaining_ranges();
+ }
+
+ // After calling EnqueueBuffer(), it is no longer valid to read from buffer.
+ // Store the state we need before calling EnqueueBuffer().
+ bool eosr = buffer->eosr_;
ScanRange* scan_range = buffer->scan_range_;
- bool scan_range_done = eosr;
- if (read_status.ok() && reader->state_ != RequestContext::Cancelled) {
- DCHECK_EQ(reader->state_, RequestContext::Active);
- // Read successfully - update the reader's scan ranges. There are two cases here:
- // 1. End of scan range or cancelled scan range - don't need to reschedule.
- // 2. Middle of scan range - need to schedule to read next buffer.
- bool enqueued = scan_range->EnqueueReadyBuffer(reader_lock, move(buffer));
- if (!eosr && enqueued) reader->ScheduleScanRange(reader_lock, scan_range);
+ bool is_cached = buffer->is_cached();
+ bool queue_full = scan_range->EnqueueBuffer(reader_lock, move(buffer));
+ if (eosr) {
+ // For cached buffers, we can't close the range until the cached buffer is returned.
+ // Close() is called from DiskIoMgr::ReturnBuffer().
+ if (!is_cached) scan_range->Close();
} else {
- // The scan range will be cancelled, either because we hit an error or because the
- // request context was cancelled. The buffer is not needed - we must free it.
- reader->FreeBuffer(buffer.get());
- // Propagate 'read_status' to the scan range. If we are here because the context
- // was cancelled, the scan range is already cancelled so we do not need to re-cancel
- // it.
- if (!read_status.ok()) scan_range->CancelFromReader(reader_lock, read_status);
- scan_range_done = true;
- }
- if (scan_range_done) {
- scan_range->Close();
- --disk_state->num_remaining_ranges();
+ if (queue_full) {
+ reader->blocked_ranges_.Enqueue(scan_range);
+ } else {
+ reader->ScheduleScanRange(scan_range);
+ }
}
- DCHECK(reader->Validate()) << endl << reader->DebugString();
- disk_state->DecrementDiskThread(reader_lock, reader);
+ state.DecrementRequestThread();
}
void DiskIoMgr::WorkLoop(DiskQueue* disk_queue) {
@@ -697,12 +971,14 @@ void DiskIoMgr::WorkLoop(DiskQueue* disk_queue) {
// 3. Perform the read or write as specified.
// Cancellation checking needs to happen in both steps 1 and 3.
while (true) {
- RequestContext* worker_context = nullptr;
+ RequestContext* worker_context = nullptr;;
RequestRange* range = nullptr;
+
if (!GetNextRequestRange(disk_queue, &range, &worker_context)) {
DCHECK(shut_down_);
- return;
+ break;
}
+
if (range->request_type() == RequestType::READ) {
ReadRange(disk_queue, worker_context, static_cast<ScanRange*>(range));
} else {
@@ -710,8 +986,12 @@ void DiskIoMgr::WorkLoop(DiskQueue* disk_queue) {
Write(worker_context, static_cast<WriteRange*>(range));
}
}
+
+ DCHECK(shut_down_);
}
+// This function reads the specified scan range associated with the
+// specified reader context and disk queue.
void DiskIoMgr::ReadRange(
DiskQueue* disk_queue, RequestContext* reader, ScanRange* range) {
int64_t bytes_remaining = range->len_ - range->bytes_read_;
@@ -719,40 +999,86 @@ void DiskIoMgr::ReadRange(
unique_ptr<BufferDescriptor> buffer_desc;
if (range->external_buffer_tag_ == ScanRange::ExternalBufferTag::CLIENT_BUFFER) {
buffer_desc = unique_ptr<BufferDescriptor>(new BufferDescriptor(this, reader, range,
- range->client_buffer_.data, range->client_buffer_.len));
+ range->client_buffer_.data, range->client_buffer_.len, nullptr));
} else {
- DCHECK(range->external_buffer_tag_ == ScanRange::ExternalBufferTag::NO_BUFFER)
- << "This code path does not handle other buffer types, i.e. HDFS cache"
- << static_cast<int>(range->external_buffer_tag_);
- buffer_desc = range->GetNextUnusedBufferForRange();
- if (buffer_desc == nullptr) {
- // No buffer available - the range will be rescheduled when a buffer is added.
- unique_lock<mutex> reader_lock(reader->lock_);
- reader->disk_states_[disk_queue->disk_id].DecrementDiskThread(reader_lock, reader);
- DCHECK(reader->Validate()) << endl << reader->DebugString();
- return;
- }
+ // Need to allocate a buffer to read into.
+ int64_t buffer_size = ::min(bytes_remaining, static_cast<int64_t>(max_buffer_size_));
+ buffer_desc = TryAllocateNextBufferForRange(disk_queue, reader, range, buffer_size);
+ if (buffer_desc == nullptr) return;
}
+ reader->num_used_buffers_.Add(1);
// No locks in this section. Only working on local vars. We don't want to hold a
// lock across the read call.
- Status read_status = range->Open(detail::is_file_handle_caching_enabled());
- if (read_status.ok()) {
+ buffer_desc->status_ = range->Open(detail::is_file_handle_caching_enabled());
+ if (buffer_desc->status_.ok()) {
// Update counters.
- COUNTER_ADD_IF_NOT_NULL(reader->active_read_thread_counter_, 1L);
- COUNTER_BITOR_IF_NOT_NULL(reader->disks_accessed_bitmap_, 1LL << disk_queue->disk_id);
+ if (reader->active_read_thread_counter_) {
+ reader->active_read_thread_counter_->Add(1L);
+ }
+ if (reader->disks_accessed_bitmap_) {
+ int64_t disk_bit = 1LL << disk_queue->disk_id;
+ reader->disks_accessed_bitmap_->BitOr(disk_bit);
+ }
- read_status = range->Read(buffer_desc->buffer_, buffer_desc->buffer_len_,
+ buffer_desc->status_ = range->Read(buffer_desc->buffer_, buffer_desc->buffer_len_,
&buffer_desc->len_, &buffer_desc->eosr_);
buffer_desc->scan_range_offset_ = range->bytes_read_ - buffer_desc->len_;
- COUNTER_ADD_IF_NOT_NULL(reader->bytes_read_counter_, buffer_desc->len_);
+ if (reader->bytes_read_counter_ != nullptr) {
+ COUNTER_ADD(reader->bytes_read_counter_, buffer_desc->len_);
+ }
+
COUNTER_ADD(&total_bytes_read_counter_, buffer_desc->len_);
- COUNTER_ADD_IF_NOT_NULL(reader->active_read_thread_counter_, -1L);
+ if (reader->active_read_thread_counter_) {
+ reader->active_read_thread_counter_->Add(-1L);
+ }
}
// Finished read, update reader/disk based on the results
- HandleReadFinished(disk_queue, reader, read_status, move(buffer_desc));
+ HandleReadFinished(disk_queue, reader, move(buffer_desc));
+}
+
+unique_ptr<BufferDescriptor> DiskIoMgr::TryAllocateNextBufferForRange(
+ DiskQueue* disk_queue, RequestContext* reader, ScanRange* range,
+ int64_t buffer_size) {
+ DCHECK(reader->mem_tracker_ != nullptr);
+ bool enough_memory = reader->mem_tracker_->SpareCapacity() > LOW_MEMORY;
+ if (!enough_memory) {
+ // Low memory, GC all the buffers and try again.
+ GcIoBuffers();
+ enough_memory = reader->mem_tracker_->SpareCapacity() > LOW_MEMORY;
+ }
+
+ if (!enough_memory) {
+ RequestContext::PerDiskState& state = reader->disk_states_[disk_queue->disk_id];
+ unique_lock<mutex> reader_lock(reader->lock_);
+
+ // Just grabbed the reader lock, check for cancellation.
+ if (reader->state_ == RequestContext::Cancelled) {
+ DCHECK(reader->Validate()) << endl << reader->DebugString();
+ state.DecrementRequestThreadAndCheckDone(reader);
+ range->Cancel(reader->status_);
+ DCHECK(reader->Validate()) << endl << reader->DebugString();
+ return nullptr;
+ }
+
+ if (!range->ready_buffers_.empty()) {
+ // We have memory pressure and this range doesn't need another buffer
+ // (it already has one queued). Skip this range and pick it up later.
+ range->blocked_on_queue_ = true;
+ reader->blocked_ranges_.Enqueue(range);
+ state.DecrementRequestThread();
+ return nullptr;
+ } else {
+ // We need to get a buffer anyway since there are none queued. The query
+ // is likely to fail due to mem limits but there's nothing we can do about that
+ // now.
+ }
+ }
+ unique_ptr<BufferDescriptor> buffer_desc = GetFreeBuffer(reader, range, buffer_size);
+ DCHECK(buffer_desc != nullptr);
+ return buffer_desc;
}
void DiskIoMgr::Write(RequestContext* writer_context, WriteRange* write_range) {
@@ -806,14 +1132,30 @@ Status DiskIoMgr::WriteRangeHelper(FILE* file_handle, WriteRange* write_range) {
Substitute("fwrite(buffer, 1, $0, $1) failed with errno=$2 description=$3",
write_range->len_, write_range->file_, errno, GetStrErrMsg())));
}
- ImpaladMetrics::IO_MGR_BYTES_WRITTEN->Increment(write_range->len_);
+ if (ImpaladMetrics::IO_MGR_BYTES_WRITTEN != nullptr) {
+ ImpaladMetrics::IO_MGR_BYTES_WRITTEN->Increment(write_range->len_);
+ }
+
return Status::OK();
}
+int DiskIoMgr::free_buffers_idx(int64_t buffer_size) {
+ int64_t buffer_size_scaled = BitUtil::Ceil(buffer_size, min_buffer_size_);
+ int idx = BitUtil::Log2Ceiling64(buffer_size_scaled);
+ DCHECK_GE(idx, 0);
+ DCHECK_LT(idx, free_buffers_.size());
+ return idx;
+}
+
Status DiskIoMgr::AddWriteRange(RequestContext* writer, WriteRange* write_range) {
unique_lock<mutex> writer_lock(writer->lock_);
- if (writer->state_ == RequestContext::Cancelled) return Status::CANCELLED;
- writer->AddRangeToDisk(writer_lock, write_range, ScheduleMode::IMMEDIATELY);
+
+ if (writer->state_ == RequestContext::Cancelled) {
+ DCHECK(!writer->status_.ok());
+ return writer->status_;
+ }
+
+ writer->AddRequestRange(write_range, false);
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr.h b/be/src/runtime/io/disk-io-mgr.h
index 2b6881b..e5d7eb4 100644
--- a/be/src/runtime/io/disk-io-mgr.h
+++ b/be/src/runtime/io/disk-io-mgr.h
@@ -30,7 +30,6 @@
#include "common/hdfs.h"
#include "common/object-pool.h"
#include "common/status.h"
-#include "runtime/bufferpool/buffer-pool.h"
#include "runtime/io/handle-cache.h"
#include "runtime/io/request-ranges.h"
#include "runtime/thread-resource-mgr.h"
@@ -43,19 +42,21 @@
namespace impala {
+class MemTracker;
+
namespace io {
/// Manager object that schedules IO for all queries on all disks and remote filesystems
/// (such as S3). Each query maps to one or more RequestContext objects, each of which
/// has its own queue of scan ranges and/or write ranges.
-///
+//
/// The API splits up requesting scan/write ranges (non-blocking) and reading the data
/// (blocking). The DiskIoMgr has worker threads that will read from and write to
/// disk/hdfs/remote-filesystems, allowing interleaving of IO and CPU. This allows us to
/// keep all disks and all cores as busy as possible.
-///
+//
/// All public APIs are thread-safe. It is not valid to call any of the APIs after
/// UnregisterContext() returns.
-///
+//
/// For Readers:
/// We can model this problem as a multiple producer (threads for each disk), multiple
/// consumer (scan ranges) problem. There are multiple queues that need to be
@@ -67,102 +68,84 @@ namespace io {
/// Readers map to scan nodes. The reader then contains a queue of scan ranges. The caller
/// asks the IoMgr for the next range to process. The IoMgr then selects the best range
/// to read based on disk activity and begins reading and queuing buffers for that range.
-///
+/// TODO: We should map readers to queries. A reader is the unit of scheduling and queries
+/// that have multiple scan nodes shouldn't have more 'turns'.
+//
/// For Writers:
/// Data is written via AddWriteRange(). This is non-blocking and adds a WriteRange to a
/// per-disk queue. After the write is complete, a callback in WriteRange is invoked.
/// No memory is allocated within IoMgr for writes and no copies are made. It is the
/// responsibility of the client to ensure that the data to be written is valid and that
/// the file to be written to exists until the callback is invoked.
-///
-/// There are several key methods for scanning data with the IoMgr.
-/// 1. StartScanRange(): adds range to the IoMgr to start immediately.
-/// 2. AddScanRanges(): adds ranges to the IoMgr that the reader wants to scan, but does
-/// not start them until GetNextUnstartedRange() is called.
-/// 3. GetNextUnstartedRange(): returns to the caller the next scan range it should
-/// process.
-/// 4. ScanRange::GetNext(): returns the next buffer for this range, blocking until
-/// data is available.
-///
+//
+/// The IoMgr provides three key APIs.
+/// 1. AddScanRanges: this is non-blocking and tells the IoMgr all the ranges that
+/// will eventually need to be read.
+/// 2. GetNextRange: returns to the caller the next scan range it should process.
+/// This is based on disk load. This also begins reading the data in this scan
+/// range. This is blocking.
+/// 3. ScanRange::GetNext: returns the next buffer for this range. This is blocking.
+//
/// The disk threads do not synchronize with each other. The readers and writers don't
/// synchronize with each other. There is a lock and condition variable for each request
/// context queue and each disk queue.
/// IMPORTANT: whenever both locks are needed, the lock order is to grab the context lock
/// before the disk lock.
-///
+//
/// Scheduling: If there are multiple request contexts with work for a single disk, the
/// request contexts are scheduled in round-robin order. Multiple disk threads can
/// operate on the same request context. Exactly one request range is processed by a
-/// disk thread at a time. If there are multiple scan ranges scheduled for a single
-/// context, these are processed in round-robin order.
+/// disk thread at a time. If there are multiple scan ranges scheduled via
+/// GetNextRange() for a single context, these are processed in round-robin order.
/// If there are multiple scan and write ranges for a disk, a read is always followed
/// by a write, and a write is followed by a read, i.e. reads and writes alternate.
/// If multiple write ranges are enqueued for a single disk, they will be processed
/// by the disk threads in order, but may complete in any order. No guarantees are made
/// on ordering of writes across disks.
-///
-/// Resource Management: the IoMgr is designed to share the available disk I/O capacity
-/// between many clients and to help use the available I/O capacity efficiently. The IoMgr
-/// interfaces are designed to let clients manage their own CPU and memory usage while the
-/// IoMgr manages the allocation of the I/O capacity of different I/O devices to scan
-/// ranges of different clients.
-///
-/// IoMgr clients may want to work on multiple scan ranges at a time to maximize CPU and
-/// I/O utilization. Clients can call GetNextUnstartedRange() to start as many concurrent
-/// scan ranges as required, e.g. from each parallel scanner thread. Once a scan range has
-/// been returned via GetNextUnstartedRange(), the caller must allocate any memory needed
-/// for buffering reads, after which the IoMgr wil start to fill the buffers with data
-/// while the caller concurrently consumes and processes the data. For example, the logic
-/// in a scanner thread might look like:
+//
+/// Resource Management: effective resource management in the IoMgr is key to good
+/// performance. The IoMgr helps coordinate two resources: CPU and disk. For CPU,
+/// spinning up too many threads causes thrashing.
+/// Memory usage in the IoMgr comes from queued read buffers. If we queue the minimum
+/// (i.e. 1), then the disks are idle while we are processing the buffer. If we don't
+/// limit the queue, then it possible we end up queueing the entire data set (i.e. CPU
+/// is slower than disks) and run out of memory.
+/// For both CPU and memory, we want to model the machine as having a fixed amount of
+/// resources. If a single query is running, it should saturate either CPU or Disk
+/// as well as using as little memory as possible. With multiple queries, each query
+/// should get less CPU. In that case each query will need fewer queued buffers and
+/// therefore have less memory usage.
+//
+/// The IoMgr defers CPU management to the caller. The IoMgr provides a GetNextRange
+/// API which will return the next scan range the caller should process. The caller
+/// can call this from the desired number of reading threads. Once a scan range
+/// has been returned via GetNextRange, the IoMgr will start to buffer reads for
+/// that range and it is expected the caller will pull those buffers promptly. For
+/// example, if the caller would like to have 1 scanner thread, the read loop
+/// would look like:
/// while (more_ranges)
-/// range = GetNextUnstartedRange()
+/// range = GetNextRange()
/// while (!range.eosr)
/// buffer = range.GetNext()
-///
-/// Note that the IoMgr rather than the client is responsible for choosing which scan
-/// range to process next, which allows optimizations like distributing load across disks.
-///
-/// Buffer Management:
-/// Buffers for reads are either a) allocated on behalf of the caller with
-/// AllocateBuffersForRange() ("IoMgr-allocated"), b) cached HDFS buffers if the scan
-/// range was read from the HDFS cache, or c) a client buffer, large enough to fit the
-/// whole scan range's data, that is provided by the caller when constructing the
+/// To have multiple reading threads, the caller would simply spin up the threads
+/// and each would process the loops above.
+//
+/// To control the number of IO buffers, each scan range has a limit of two queued
+/// buffers (SCAN_RANGE_READY_BUFFER_LIMIT). If the number of buffers is at capacity,
+/// the IoMgr will no longer read for that scan range until the caller has processed
+/// a buffer. Assuming the client returns each buffer before requesting the next one
+/// from the scan range, then this will consume up to 3 * 8MB = 24MB of I/O buffers per
/// scan range.
+//
+/// Buffer Management:
+/// Buffers for reads are either a) allocated by the IoMgr and transferred to the caller,
+/// b) cached HDFS buffers if the scan range uses HDFS caching, or c) provided by the
+/// caller when constructing the scan range.
///
-/// All three kinds of buffers are wrapped in BufferDescriptors before returning to the
-/// caller. The caller must always call ReturnBuffer() on the buffer descriptor to allow
-/// recycling of the buffer memory and to release any resources associated with the buffer
-/// or scan range.
-///
-/// In case a), ReturnBuffer() may re-enqueue the buffer for GetNext() to return again if
-/// needed. E.g. if 24MB of buffers were allocated to read a 64MB scan range, each buffer
-/// must be returned multiple times. Callers must be careful to call ReturnBuffer() with
-/// the previous buffer returned from the range before calling before GetNext() so that
-/// at least one buffer is available for the I/O mgr to read data into. Calling GetNext()
-/// when the scan range has no buffers to read data into causes a resource deadlock.
-/// NB: if the scan range was allocated N buffers, then it's always ok for the caller
-/// to hold onto N - 1 buffers, but currently the IoMgr doesn't give the caller a way
-/// to determine the value of N.
-///
-/// If the caller wants to maximize I/O throughput, it can give the range enough memory
-/// for 3 max-sized buffers per scan range. Having two queued buffers (plus the buffer
-/// that is currently being processed by the client) gives good performance in most
-/// scenarios:
-/// 1. If the consumer is consuming data faster than we can read from disk, then the
-/// queue will be empty most of the time because the buffer will be immediately
-/// pulled off the queue as soon as it is added. There will always be an I/O request
-/// in the disk queue to maximize I/O throughput, which is the bottleneck in this
-/// case.
-/// 2. If we can read from disk faster than the consumer is consuming data, the queue
-/// will fill up and there will always be a buffer available for the consumer to
-/// read, so the consumer will not block and we maximize consumer throughput, which
-/// is the bottleneck in this case.
-/// 3. If the consumer is consuming data at approximately the same rate as we are
-/// reading from disk, then the steady state is that the consumer is processing one
-/// buffer and one buffer is in the disk queue. The additional buffer can absorb
-/// bursts where the producer runs faster than the consumer or the consumer runs
-/// faster than the producer without blocking either the producer or consumer.
-/// See IDEAL_MAX_SIZED_BUFFERS_PER_SCAN_RANGE.
+/// As a caller reads from a scan range, these buffers are wrapped in BufferDescriptors
+/// and returned to the caller. The caller must always call ReturnBuffer() on the buffer
+/// descriptor to allow recycling of the associated buffer (if there is an
+/// IoMgr-allocated or HDFS cached buffer).
///
/// Caching support:
/// Scan ranges contain metadata on whether or not it is cached on the DN. In that
@@ -178,13 +161,13 @@ namespace io {
/// - HDFS will time us out if we hold onto the mlock for too long
/// - Holding the lock prevents uncaching this file due to a caching policy change.
/// Therefore, we only issue the cached read when the caller is ready to process the
-/// range (GetNextUnstartedRange()) instead of when the ranges are issued. This guarantees
-/// that there will be a CPU available to process the buffer and any throttling we do with
+/// range (GetNextRange()) instead of when the ranges are issued. This guarantees that
+/// there will be a CPU available to process the buffer and any throttling we do with
/// the number of scanner threads properly controls the amount of files we mlock.
/// With cached scan ranges, we cannot close the scan range until the cached buffer
/// is returned (HDFS does not allow this). We therefore need to defer the close until
/// the cached buffer is returned (ReturnBuffer()).
-///
+//
/// Remote filesystem support (e.g. S3):
/// Remote filesystems are modeled as "remote disks". That is, there is a seperate disk
/// queue for each supported remote filesystem type. In order to maximize throughput,
@@ -193,13 +176,12 @@ namespace io {
/// intensive than local disk/hdfs because of non-direct I/O and SSL processing, and can
/// be CPU bottlenecked especially if not enough I/O threads for these queues are
/// started.
-///
-/// TODO: We should implement more sophisticated resource management. Currently readers
-/// are the unit of scheduling and we attempt to distribute IOPS between them. Instead
-/// it would be better to have policies based on queries, resource pools, etc.
+//
/// TODO: IoMgr should be able to request additional scan ranges from the coordinator
/// to help deal with stragglers.
-///
+/// TODO: look into using a lock free queue
+/// TODO: simplify the common path (less locking, memory allocations).
+//
/// Structure of the Implementation:
/// - All client APIs are defined in this file, request-ranges.h and request-context.h.
/// Clients can include only the files that they need.
@@ -222,12 +204,10 @@ class DiskIoMgr : public CacheLineAligned {
/// disk. This is also the max queue depth.
/// - threads_per_solid_state_disk: number of read threads to create per solid state
/// disk. This is also the max queue depth.
- /// - min_buffer_size: minimum io buffer size (in bytes). Will be rounded down to the
- // nearest power-of-two.
- /// - max_buffer_size: maximum io buffer size (in bytes). Will be rounded up to the
- /// nearest power-of-two. Also the max read size.
+ /// - min_buffer_size: minimum io buffer size (in bytes)
+ /// - max_buffer_size: maximum io buffer size (in bytes). Also the max read size.
DiskIoMgr(int num_disks, int threads_per_rotational_disk,
- int threads_per_solid_state_disk, int64_t min_buffer_size, int64_t max_buffer_size);
+ int threads_per_solid_state_disk, int min_buffer_size, int max_buffer_size);
/// Create DiskIoMgr with default configs.
DiskIoMgr();
@@ -237,13 +217,16 @@ class DiskIoMgr : public CacheLineAligned {
~DiskIoMgr();
/// Initialize the IoMgr. Must be called once before any of the other APIs.
- Status Init() WARN_UNUSED_RESULT;
-
+ Status Init(MemTracker* process_mem_tracker) WARN_UNUSED_RESULT;
/// Allocates tracking structure for a request context.
/// Register a new request context and return it to the caller. The caller must call
/// UnregisterContext() for each context.
- std::unique_ptr<RequestContext> RegisterContext();
+ /// reader_mem_tracker: Is non-null only for readers. IO buffers
+ /// used for this reader will be tracked by this. If the limit is exceeded
+ /// the reader will be cancelled and MEM_LIMIT_EXCEEDED will be returned via
+ /// GetNext().
+ std::unique_ptr<RequestContext> RegisterContext(MemTracker* reader_mem_tracker);
/// Unregisters context from the disk IoMgr by first cancelling it then blocking until
/// all references to the context are removed from I/O manager internal data structures.
@@ -253,60 +236,50 @@ class DiskIoMgr : public CacheLineAligned {
/// up.
void UnregisterContext(RequestContext* context);
- /// Adds the scan ranges to reader's queues, but does not start scheduling it. The range
- /// can be scheduled by a thread calling GetNextUnstartedRange(). This call is
- /// non-blocking. The caller must not deallocate the scan range pointers before
- /// UnregisterContext(). 'ranges' must not be empty.
- Status AddScanRanges(
- RequestContext* reader, const std::vector<ScanRange*>& ranges) WARN_UNUSED_RESULT;
-
- /// Adds the scan range to the queues, as with AddScanRanges(), but immediately
- /// start scheduling the scan range. This can be used to do synchronous reads as well
- /// as schedule dependent ranges, e.g. for columnar formats. This call is non-blocking.
- /// The caller must not deallocate the scan range pointers before UnregisterContext().
- ///
- /// If this returns true in '*needs_buffers', the caller must then call
- /// AllocateBuffersForRange() to add buffers for the data to be read into before the
- /// range can be scheduled. Otherwise, the range is scheduled and the IoMgr will
- /// asynchronously read the data for the range and the caller can call
- /// ScanRange::GetNext() to read the data.
- Status StartScanRange(
- RequestContext* reader, ScanRange* range, bool* needs_buffers) WARN_UNUSED_RESULT;
+ /// This function cancels the context asychronously. All outstanding requests
+ /// are aborted and tracking structures cleaned up. This does not need to be
+ /// called if the context finishes normally.
+ /// This will also fail any outstanding GetNext()/Read requests.
+ void CancelContext(RequestContext* context);
+
+ /// Adds the scan ranges to the queues. This call is non-blocking. The caller must
+ /// not deallocate the scan range pointers before UnregisterContext().
+ /// If schedule_immediately, the ranges are immediately put on the read queue
+ /// (i.e. the caller should not/cannot call GetNextRange for these ranges).
+ /// This can be used to do synchronous reads as well as schedule dependent ranges,
+ /// as in the case for columnar formats.
+ Status AddScanRanges(RequestContext* reader,
+ const std::vector<ScanRange*>& ranges,
+ bool schedule_immediately = false) WARN_UNUSED_RESULT;
+ Status AddScanRange(RequestContext* reader, ScanRange* range,
+ bool schedule_immediately = false) WARN_UNUSED_RESULT;
/// Add a WriteRange for the writer. This is non-blocking and schedules the context
/// on the IoMgr disk queue. Does not create any files.
Status AddWriteRange(
RequestContext* writer, WriteRange* write_range) WARN_UNUSED_RESULT;
- /// Tries to get an unstarted scan range that was added to 'reader' with
- /// AddScanRanges(). On success, returns OK and returns the range in '*range'.
- /// If 'reader' was cancelled, returns CANCELLED. If another error is encountered,
- /// an error status is returned. Otherwise, if error or cancellation wasn't encountered
- /// and there are no unstarted ranges for 'reader', returns OK and sets '*range' to
- /// nullptr.
- ///
- /// If '*needs_buffers' is returned as true, the caller must call
- /// AllocateBuffersForRange() to add buffers for the data to be read into before the
- /// range can be scheduled. Otherwise, the range is scheduled and the IoMgr will
- /// asynchronously read the data for the range and the caller can call
- /// ScanRange::GetNext() to read the data.
- Status GetNextUnstartedRange(RequestContext* reader, ScanRange** range,
- bool* needs_buffers) WARN_UNUSED_RESULT;
-
- /// Allocates up to 'max_bytes' buffers to read the data from 'range' into and schedules
- /// the range. Called after StartScanRange() or GetNextUnstartedRange() returns
- /// *needs_buffers=true.
- ///
- /// The buffer sizes are chosen based on range->len(). 'max_bytes' must be >=
- /// min_read_buffer_size() so that at least one buffer can be allocated. The caller
- /// must ensure that 'bp_client' has at least 'max_bytes' unused reservation. Returns ok
- /// if the buffers were successfully allocated and the range was scheduled.
- ///
- /// Setting 'max_bytes' to IDEAL_MAX_SIZED_BUFFERS_PER_SCAN_RANGE * max_buffer_size()
- /// will typically maximize I/O throughput. See the "Buffer Management" section of
- /// the class comment for explanation.
- Status AllocateBuffersForRange(RequestContext* reader,
- BufferPool::ClientHandle* bp_client, ScanRange* range, int64_t max_bytes);
+ /// Returns the next unstarted scan range for this reader. When the range is returned,
+ /// the disk threads in the IoMgr will already have started reading from it. The
+ /// caller is expected to call ScanRange::GetNext on the returned range.
+ /// If there are no more unstarted ranges, nullptr is returned.
+ /// This call is blocking.
+ Status GetNextRange(RequestContext* reader, ScanRange** range) WARN_UNUSED_RESULT;
+
+ /// Reads the range and returns the result in buffer.
+ /// This behaves like the typical synchronous read() api, blocking until the data
+ /// is read. This can be called while there are outstanding ScanRanges and is
+ /// thread safe. Multiple threads can be calling Read() per reader at a time.
+ /// range *cannot* have already been added via AddScanRanges.
+ /// This can only be used if the scan range fits in a single IO buffer (i.e. is smaller
+ /// than max_read_buffer_size()) or if reading into a client-provided buffer.
+ Status Read(RequestContext* reader, ScanRange* range,
+ std::unique_ptr<BufferDescriptor>* buffer) WARN_UNUSED_RESULT;
+
+ /// Returns the buffer to the IoMgr. This must be called for every buffer
+ /// returned by GetNext()/Read() that did not return an error. This is non-blocking.
+ /// After calling this, the buffer descriptor is invalid and cannot be accessed.
+ void ReturnBuffer(std::unique_ptr<BufferDescriptor> buffer);
/// Determine which disk queue this file should be assigned to. Returns an index into
/// disk_queues_. The disk_id is the volume ID for the local disk that holds the
@@ -314,8 +287,32 @@ class DiskIoMgr : public CacheLineAligned {
/// co-located with the datanode for this file.
int AssignQueue(const char* file, int disk_id, bool expected_local);
- int64_t min_buffer_size() const { return min_buffer_size_; }
- int64_t max_buffer_size() const { return max_buffer_size_; }
+ /// TODO: The functions below can be moved to RequestContext.
+ /// Returns the current status of the context.
+ Status context_status(RequestContext* context) const WARN_UNUSED_RESULT;
+
+ void set_bytes_read_counter(RequestContext*, RuntimeProfile::Counter*);
+ void set_read_timer(RequestContext*, RuntimeProfile::Counter*);
+ void set_open_file_timer(RequestContext*, RuntimeProfile::Counter*);
+ void set_active_read_thread_counter(RequestContext*, RuntimeProfile::Counter*);
+ void set_disks_access_bitmap(RequestContext*, RuntimeProfile::Counter*);
+
+ int64_t queue_size(RequestContext* reader) const;
+ int64_t bytes_read_local(RequestContext* reader) const;
+ int64_t bytes_read_short_circuit(RequestContext* reader) const;
+ int64_t bytes_read_dn_cache(RequestContext* reader) const;
+ int num_remote_ranges(RequestContext* reader) const;
+ int64_t unexpected_remote_bytes(RequestContext* reader) const;
+ int cached_file_handles_hit_count(RequestContext* reader) const;
+ int cached_file_handles_miss_count(RequestContext* reader) const;
+
+ /// Returns the read throughput across all readers.
+ /// TODO: should this be a sliding window? This should report metrics for the
+ /// last minute, hour and since the beginning.
+ int64_t GetReadThroughput();
+
+ /// Returns the maximum read buffer size
+ int max_read_buffer_size() const { return max_buffer_size_; }
/// Returns the total number of disk queues (both local and remote).
int num_total_disks() const { return disk_queues_.size(); }
@@ -368,6 +365,29 @@ class DiskIoMgr : public CacheLineAligned {
Status ReopenCachedHdfsFileHandle(const hdfsFS& fs, std::string* fname, int64_t mtime,
RequestContext* reader, CachedHdfsFileHandle** fid);
+ /// Garbage collect unused I/O buffers up to 'bytes_to_free', or all the buffers if
+ /// 'bytes_to_free' is -1.
+ void GcIoBuffers(int64_t bytes_to_free = -1);
+
+ /// The maximum number of ready buffers that can be queued in a scan range. Having two
+ /// queued buffers (plus the buffer that is returned to the client) gives good
+ /// performance in most scenarios:
+ /// 1. If the consumer is consuming data faster than we can read from disk, then the
+ /// queue will be empty most of the time because the buffer will be immediately
+ /// pulled off the queue as soon as it is added. There will always be an I/O request
+ /// in the disk queue to maximize I/O throughput, which is the bottleneck in this
+ /// case.
+ /// 2. If we can read from disk faster than the consumer is consuming data, the queue
+ /// will fill up and there will always be a buffer available for the consumer to
+ /// read, so the consumer will not block and we maximize consumer throughput, which
+ /// is the bottleneck in this case.
+ /// 3. If the consumer is consuming data at approximately the same rate as we are
+ /// reading from disk, then the steady state is that the consumer is processing one
+ /// buffer and one buffer is in the disk queue. The additional buffer can absorb
+ /// bursts where the producer runs faster than the consumer or the consumer runs
+ /// faster than the producer without blocking either the producer or consumer.
+ static const int SCAN_RANGE_READY_BUFFER_LIMIT = 2;
+
/// "Disk" queue offsets for remote accesses. Offset 0 corresponds to
/// disk ID (i.e. disk_queue_ index) of num_local_disks().
enum {
@@ -377,10 +397,6 @@ class DiskIoMgr : public CacheLineAligned {
REMOTE_NUM_DISKS
};
- /// The ideal number of max-sized buffers per scan range to maximise throughput.
- /// See "Buffer Management" in the class comment for explanation.
- static const int64_t IDEAL_MAX_SIZED_BUFFERS_PER_SCAN_RANGE = 3;
-
private:
friend class BufferDescriptor;
friend class RequestContext;
@@ -389,9 +405,16 @@ class DiskIoMgr : public CacheLineAligned {
struct DiskQueue;
friend class DiskIoMgrTest_Buffers_Test;
- friend class DiskIoMgrTest_BufferSizeSelection_Test;
friend class DiskIoMgrTest_VerifyNumThreadsParameter_Test;
+ /// Memory tracker for unused I/O buffers owned by DiskIoMgr.
+ boost::scoped_ptr<MemTracker> free_buffer_mem_tracker_;
+
+ /// Memory tracker for I/O buffers where the RequestContext has no MemTracker.
+ /// TODO: once IMPALA-3200 is fixed, there should be no more cases where readers don't
+ /// provide a MemTracker.
+ boost::scoped_ptr<MemTracker> unowned_buffer_mem_tracker_;
+
/// Number of worker(read) threads per rotational disk. Also the max depth of queued
/// work to the disk.
const int num_io_threads_per_rotational_disk_;
@@ -401,10 +424,10 @@ class DiskIoMgr : public CacheLineAligned {
const int num_io_threads_per_solid_state_disk_;
/// Maximum read size. This is also the maximum size of each allocated buffer.
- const int64_t max_buffer_size_;
+ const int max_buffer_size_;
- /// The minimum size of each read buffer. Must be >= BufferPool::min_buffer_len().
- const int64_t min_buffer_size_;
+ /// The minimum size of each read buffer.
+ const int min_buffer_size_;
/// Thread group containing all the worker threads.
ThreadGroup disk_thread_group_;
@@ -422,6 +445,28 @@ class DiskIoMgr : public CacheLineAligned {
/// Total time spent in hdfs reading
RuntimeProfile::Counter read_timer_;
+ /// Protects free_buffers_
+ boost::mutex free_buffers_lock_;
+
+ /// Free buffers that can be handed out to clients. There is one list for each buffer
+ /// size, indexed by the Log2 of the buffer size in units of min_buffer_size_. The
+ /// maximum buffer size is max_buffer_size_, so the maximum index is
+ /// Log2(max_buffer_size_ / min_buffer_size_).
+ //
+ /// E.g. if min_buffer_size_ = 1024 bytes:
+ /// free_buffers_[0] => list of free buffers with size 1024 B
+ /// free_buffers_[1] => list of free buffers with size 2048 B
+ /// free_buffers_[10] => list of free buffers with size 1 MB
+ /// free_buffers_[13] => list of free buffers with size 8 MB
+ /// free_buffers_[n] => list of free buffers with size 2^n * 1024 B
+ std::vector<std::deque<uint8_t*>> free_buffers_;
+
+ /// Total number of allocated buffers, used for debugging.
+ AtomicInt32 num_allocated_buffers_;
+
+ /// Total number of buffers in readers
+ AtomicInt32 num_buffers_in_readers_;
+
/// Per disk queues. This is static and created once at Init() time. One queue is
/// allocated for each local disk on the system and for each remote filesystem type.
/// It is indexed by disk id.
@@ -437,6 +482,23 @@ class DiskIoMgr : public CacheLineAligned {
// handles are closed.
FileHandleCache file_handle_cache_;
+ /// Returns the index into free_buffers_ for a given buffer size
+ int free_buffers_idx(int64_t buffer_size);
+
+ /// Returns a buffer to read into with size between 'buffer_size' and
+ /// 'max_buffer_size_', If there is an appropriately-sized free buffer in the
+ /// 'free_buffers_', that is returned, otherwise a new one is allocated.
+ /// The returned *buffer_size must be between 0 and 'max_buffer_size_'.
+ /// The buffer memory is tracked against reader's mem tracker, or
+ /// 'unowned_buffer_mem_tracker_' if the reader does not have one.
+ std::unique_ptr<BufferDescriptor> GetFreeBuffer(
+ RequestContext* reader, ScanRange* range, int64_t buffer_size);
+
+ /// Disassociates the desc->buffer_ memory from 'desc' (which cannot be nullptr), either
+ /// freeing it or returning it to 'free_buffers_'. Memory tracking is updated to
+ /// reflect the transfer of ownership from desc->mem_tracker_ to the disk I/O mgr.
+ void FreeBufferMemory(BufferDescriptor* desc);
+
/// Disk worker thread loop. This function retrieves the next range to process on
/// the disk queue and invokes ReadRange() or Write() depending on the type of Range().
/// There can be multiple threads per disk running this loop.
@@ -450,12 +512,10 @@ class DiskIoMgr : public CacheLineAligned {
bool GetNextRequestRange(DiskQueue* disk_queue, RequestRange** range,
RequestContext** request_context);
- /// Updates disk queue and reader state after a read is complete. If the read
- /// was successful, 'read_status' is ok and 'buffer' contains the result of the
- /// read. If the read failed with an error, 'read_status' contains the error and
- /// 'buffer' has the buffer that was meant to hold the result of the read.
+ /// Updates disk queue and reader state after a read is complete. The read result
+ /// is captured in the buffer descriptor.
void HandleReadFinished(DiskQueue* disk_queue, RequestContext* reader,
- Status read_status, std::unique_ptr<BufferDescriptor> buffer);
+ std::unique_ptr<BufferDescriptor> buffer);
/// Invokes write_range->callback_ after the range has been written and
/// updates per-disk state and handle state. The status of the write OK/RUNTIME_ERROR
@@ -478,14 +538,15 @@ class DiskIoMgr : public CacheLineAligned {
/// Does not open or close the file that is written.
Status WriteRangeHelper(FILE* file_handle, WriteRange* write_range) WARN_UNUSED_RESULT;
- /// Reads the specified scan range and calls HandleReadFinished() when done. If no
- /// buffer is available to read the range's data into, the read cannot proceed, the
- /// range becomes blocked and this function returns without doing I/O.
+ /// Reads the specified scan range and calls HandleReadFinished when done.
void ReadRange(DiskQueue* disk_queue, RequestContext* reader, ScanRange* range);
- /// Helper for AllocateBuffersForRange() to compute the buffer sizes for a scan range
- /// with length 'scan_range_len', given that 'max_bytes' of memory should be allocated.
- std::vector<int64_t> ChooseBufferSizes(int64_t scan_range_len, int64_t max_bytes);
+ /// Try to allocate the next buffer for the scan range, returning the new buffer
+ /// if successful. If 'reader' is cancelled, cancels the range and returns nullptr.
+ /// If there is memory pressure and buffers are already queued, adds the range
+ /// to the blocked ranges and returns nullptr.
+ std::unique_ptr<BufferDescriptor> TryAllocateNextBufferForRange(DiskQueue* disk_queue,
+ RequestContext* reader, ScanRange* range, int64_t buffer_size);
};
}
}
[4/9] impala git commit: Revert IMPALA-4835 and dependent changes
Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
index 0a945bd..23f2d88 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
@@ -153,6 +153,26 @@ public class SlotRef extends Expr {
return "<slot " + Integer.toString(desc_.getId().asInt()) + ">";
}
+ /**
+ * Checks if this slotRef refers to an array "pos" pseudo-column.
+ *
+ * Note: checking whether the column is null distinguishes between top-level columns
+ * and nested types. This check more specifically looks just for a reference to the
+ * "pos" field of an array type.
+ */
+ public boolean isArrayPosRef() {
+ TupleDescriptor parent = getDesc().getParent();
+ if (parent == null) return false;
+ Type parentType = parent.getType();
+ if (parentType instanceof CollectionStructType) {
+ if (((CollectionStructType)parentType).isArrayStruct() &&
+ getDesc().getLabel().equals(Path.ARRAY_POS_FIELD_NAME)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.SLOT_REF;
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
index 87d1806..7735f98 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
@@ -59,7 +59,6 @@ import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.ImpalaRuntimeException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.NotImplementedException;
-import org.apache.impala.common.Pair;
import org.apache.impala.common.PrintUtils;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.fb.FbFileBlock;
@@ -77,7 +76,6 @@ import org.apache.impala.thrift.TScanRange;
import org.apache.impala.thrift.TScanRangeLocation;
import org.apache.impala.thrift.TScanRangeLocationList;
import org.apache.impala.thrift.TTableStats;
-import org.apache.impala.util.BitUtil;
import org.apache.impala.util.MembershipSnapshot;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -145,7 +143,7 @@ public class HdfsScanNode extends ScanNode {
// derived experimentally: running metadata-only Parquet count(*) scans on TPC-H
// lineitem and TPC-DS store_sales of different sizes resulted in memory consumption
// between 128kb and 1.1mb.
- private static final long MIN_MEMORY_ESTIMATE = 1L * 1024L * 1024L;
+ private final static long MIN_MEMORY_ESTIMATE = 1 * 1024 * 1024;
private final HdfsTable tbl_;
@@ -168,18 +166,6 @@ public class HdfsScanNode extends ScanNode {
private long totalFiles_ = 0;
private long totalBytes_ = 0;
- // File formats scanned. Set in computeScanRangeLocations().
- private Set<HdfsFileFormat> fileFormats_;
-
- // Number of bytes in the largest scan range (i.e. hdfs split). Set in
- // computeScanRangeLocations().
- private long maxScanRangeBytes_ = 0;
-
- // The ideal reservation to process a single scan range (i.e. hdfs split), >= the
- // minimum reservation. Generally provides enough memory to overlap CPU and I/O and
- // maximize throughput. Set in computeResourceProfile().
- private long idealScanRangeReservation_ = -1;
-
// Input cardinality based on the partition row counts or extrapolation. -1 if invalid.
// Both values can be valid to report them in the explain plan, but only one of them is
// used for determining the scan cardinality.
@@ -343,25 +329,25 @@ public class HdfsScanNode extends ScanNode {
computeDictionaryFilterConjuncts(analyzer);
// compute scan range locations with optional sampling
- computeScanRangeLocations(analyzer);
+ Set<HdfsFileFormat> fileFormats = computeScanRangeLocations(analyzer);
// Determine backend scan node implementation to use. The optimized MT implementation
// is currently only supported for Parquet.
if (analyzer.getQueryOptions().isSetMt_dop() &&
analyzer.getQueryOptions().mt_dop > 0 &&
- fileFormats_.size() == 1 &&
- (fileFormats_.contains(HdfsFileFormat.PARQUET)
- || fileFormats_.contains(HdfsFileFormat.TEXT))) {
+ fileFormats.size() == 1 &&
+ (fileFormats.contains(HdfsFileFormat.PARQUET)
+ || fileFormats.contains(HdfsFileFormat.TEXT))) {
useMtScanNode_ = true;
} else {
useMtScanNode_ = false;
}
- if (fileFormats_.contains(HdfsFileFormat.PARQUET)) {
+ if (fileFormats.contains(HdfsFileFormat.PARQUET)) {
computeMinMaxTupleAndConjuncts(analyzer);
}
- if (canApplyParquetCountStarOptimization(analyzer, fileFormats_)) {
+ if (canApplyParquetCountStarOptimization(analyzer, fileFormats)) {
Preconditions.checkState(desc_.getPath().destTable() != null);
Preconditions.checkState(collectionConjuncts_.isEmpty());
countStarSlot_ = applyParquetCountStartOptimization(analyzer);
@@ -474,7 +460,7 @@ public class HdfsScanNode extends ScanNode {
// This node is a table scan, so this must be a scanning slot.
Preconditions.checkState(slotRef.getDesc().isScanSlot());
// Skip the slot ref if it refers to an array's "pos" field.
- if (slotRef.getDesc().isArrayPosRef()) return;
+ if (slotRef.isArrayPosRef()) return;
Expr constExpr = binaryPred.getChild(1);
// Only constant exprs can be evaluated against parquet::Statistics. This includes
@@ -502,7 +488,7 @@ public class HdfsScanNode extends ScanNode {
// This node is a table scan, so this must be a scanning slot.
Preconditions.checkState(slotRef.getDesc().isScanSlot());
// Skip the slot ref if it refers to an array's "pos" field.
- if (slotRef.getDesc().isArrayPosRef()) return;
+ if (slotRef.isArrayPosRef()) return;
if (inPred.isNotIn()) return;
ArrayList<Expr> children = inPred.getChildren();
@@ -709,15 +695,13 @@ public class HdfsScanNode extends ScanNode {
}
/**
- * Computes scan ranges (i.e. hdfs splits) plus their storage locations, including
- * volume ids, based on the given maximum number of bytes each scan range should scan.
+ * Computes scan ranges (hdfs splits) plus their storage locations, including volume
+ * ids, based on the given maximum number of bytes each scan range should scan.
* If 'sampleParams_' is not null, generates a sample and computes the scan ranges
* based on the sample.
- *
- * Initializes members with information about files and scan ranges, e.g. totalFiles_,
- * fileFormats_, etc.
+ * Returns the set of file formats being scanned.
*/
- private void computeScanRangeLocations(Analyzer analyzer)
+ private Set<HdfsFileFormat> computeScanRangeLocations(Analyzer analyzer)
throws ImpalaRuntimeException {
Map<Long, List<FileDescriptor>> sampledFiles = null;
if (sampleParams_ != null) {
@@ -740,8 +724,7 @@ public class HdfsScanNode extends ScanNode {
numPartitions_ = (sampledFiles != null) ? sampledFiles.size() : partitions_.size();
totalFiles_ = 0;
totalBytes_ = 0;
- maxScanRangeBytes_ = 0;
- fileFormats_ = Sets.newHashSet();
+ Set<HdfsFileFormat> fileFormats = Sets.newHashSet();
for (HdfsPartition partition: partitions_) {
List<FileDescriptor> fileDescs = partition.getFileDescriptors();
if (sampledFiles != null) {
@@ -751,7 +734,7 @@ public class HdfsScanNode extends ScanNode {
}
analyzer.getDescTbl().addReferencedPartition(tbl_, partition.getId());
- fileFormats_.add(partition.getFileFormat());
+ fileFormats.add(partition.getFileFormat());
Preconditions.checkState(partition.getId() >= 0);
// Missing disk id accounting is only done for file systems that support the notion
// of disk/storage ids.
@@ -814,7 +797,6 @@ public class HdfsScanNode extends ScanNode {
scanRangeLocations.scan_range = scanRange;
scanRangeLocations.locations = locations;
scanRanges_.add(scanRangeLocations);
- maxScanRangeBytes_ = Math.max(maxScanRangeBytes_, currentLength);
remainingLength -= currentLength;
currentOffset += currentLength;
}
@@ -829,58 +811,7 @@ public class HdfsScanNode extends ScanNode {
}
if (partitionMissingDiskIds) ++numPartitionsNoDiskIds_;
}
- }
-
- /**
- * Compute the number of columns that are read from the file, as opposed to
- * materialised based on metadata. If there are nested collections, counts the
- * number of leaf scalar slots per collection. This matches Parquet's "shredded"
- * approach to nested collections, where each nested field is stored as a separate
- * column. We may need to adjust this logic for non-shredded columnar formats if added.
- */
- private int computeNumColumnsReadFromFile() {
- HdfsTable table = (HdfsTable) desc_.getTable();
- int numColumns = 0;
- boolean havePosSlot = false;
- for (SlotDescriptor slot: desc_.getSlots()) {
- if (!slot.isMaterialized() || slot == countStarSlot_) continue;
- if (slot.getColumn() == null ||
- slot.getColumn().getPosition() >= table.getNumClusteringCols()) {
- if (slot.isArrayPosRef()) {
- // Position virtual slots can be materialized by piggybacking on another slot.
- havePosSlot = true;
- } else if (slot.getType().isScalarType()) {
- ++numColumns;
- } else {
- numColumns += computeNumColumnsReadForCollection(slot);
- }
- }
- }
- // Must scan something to materialize a position slot.
- if (havePosSlot) numColumns = Math.max(numColumns, 1);
- return numColumns;
- }
-
- /**
- * Compute the number of columns read from disk for materialized scalar slots in
- * the provided tuple.
- */
- private int computeNumColumnsReadForCollection(SlotDescriptor collectionSlot) {
- Preconditions.checkState(collectionSlot.getType().isCollectionType());
- int numColumns = 0;
- for (SlotDescriptor nestedSlot: collectionSlot.getItemTupleDesc().getSlots()) {
- // Position virtual slots can be materialized by piggybacking on another slot.
- if (!nestedSlot.isMaterialized() || nestedSlot.isArrayPosRef()) continue;
- if (nestedSlot.getType().isScalarType()) {
- ++numColumns;
- } else {
- numColumns += computeNumColumnsReadForCollection(nestedSlot);
- }
- }
- // Need to scan at least one column to materialize the pos virtual slot and/or
- // determine the size of the nested array.
- numColumns = Math.max(numColumns, 1);
- return numColumns;
+ return fileFormats;
}
/**
@@ -1075,8 +1006,6 @@ public class HdfsScanNode extends ScanNode {
}
msg.hdfs_scan_node.setRandom_replica(randomReplica_);
msg.node_type = TPlanNodeType.HDFS_SCAN_NODE;
- Preconditions.checkState(idealScanRangeReservation_ >= 0, idealScanRangeReservation_);
- msg.hdfs_scan_node.setIdeal_scan_range_reservation(idealScanRangeReservation_);
if (!collectionConjuncts_.isEmpty()) {
Map<Integer, List<TExpr>> tcollectionConjuncts = Maps.newLinkedHashMap();
for (Map.Entry<TupleDescriptor, List<Expr>> entry:
@@ -1255,20 +1184,25 @@ public class HdfsScanNode extends ScanNode {
Preconditions.checkNotNull(scanRanges_, "Cost estimation requires scan ranges.");
if (scanRanges_.isEmpty()) {
nodeResourceProfile_ = ResourceProfile.noReservation(0);
- idealScanRangeReservation_ = 0;
return;
}
Preconditions.checkState(0 < numNodes_ && numNodes_ <= scanRanges_.size());
Preconditions.checkNotNull(desc_);
Preconditions.checkNotNull(desc_.getTable() instanceof HdfsTable);
HdfsTable table = (HdfsTable) desc_.getTable();
- int numColumnsReadFromFile = computeNumColumnsReadFromFile();
int perHostScanRanges;
if (table.getMajorityFormat() == HdfsFileFormat.PARQUET) {
// For the purpose of this estimation, the number of per-host scan ranges for
// Parquet files are equal to the number of columns read from the file. I.e.
// excluding partition columns and columns that are populated from file metadata.
- perHostScanRanges = numColumnsReadFromFile;
+ perHostScanRanges = 0;
+ for (SlotDescriptor slot: desc_.getSlots()) {
+ if (!slot.isMaterialized() || slot == countStarSlot_) continue;
+ if (slot.getColumn() == null ||
+ slot.getColumn().getPosition() >= table.getNumClusteringCols()) {
+ ++perHostScanRanges;
+ }
+ }
} else {
perHostScanRanges = (int) Math.ceil((
(double) scanRanges_.size() / (double) numNodes_) * SCAN_RANGE_SKEW_FACTOR);
@@ -1290,13 +1224,12 @@ public class HdfsScanNode extends ScanNode {
long avgScanRangeBytes = (long) Math.ceil(totalBytes_ / (double) scanRanges_.size());
// The +1 accounts for an extra I/O buffer to read past the scan range due to a
// trailing record spanning Hdfs blocks.
- long maxIoBufferSize =
- BitUtil.roundUpToPowerOf2(BackendConfig.INSTANCE.getReadSize());
+ long readSize = BackendConfig.INSTANCE.getReadSize();
long perThreadIoBuffers =
- Math.min((long) Math.ceil(avgScanRangeBytes / (double) maxIoBufferSize),
+ Math.min((long) Math.ceil(avgScanRangeBytes / (double) readSize),
MAX_IO_BUFFERS_PER_THREAD) + 1;
long perInstanceMemEstimate = checkedMultiply(
- checkedMultiply(maxScannerThreads, perThreadIoBuffers), maxIoBufferSize);
+ checkedMultiply(maxScannerThreads, perThreadIoBuffers), readSize);
// Sanity check: the tighter estimation should not exceed the per-host maximum.
long perHostUpperBound = getPerHostMemUpperBound();
@@ -1307,51 +1240,7 @@ public class HdfsScanNode extends ScanNode {
perInstanceMemEstimate = perHostUpperBound;
}
perInstanceMemEstimate = Math.max(perInstanceMemEstimate, MIN_MEMORY_ESTIMATE);
-
- Pair<Long, Long> reservation = computeReservation(numColumnsReadFromFile);
- nodeResourceProfile_ = new ResourceProfileBuilder()
- .setMemEstimateBytes(perInstanceMemEstimate)
- .setMinReservationBytes(reservation.first).build();
- idealScanRangeReservation_ = reservation.second;
- }
-
- /*
- * Compute the minimum and ideal memory reservation to process a single scan range
- * (i.e. hdfs split). Bound the reservation based on:
- * - One minimum-sized buffer per IoMgr scan range, which is the absolute minimum
- * required to scan the data.
- * - A maximum of either 1 or 3 max-sized I/O buffers per IoMgr scan range for
- * the minimum and ideal reservation respectively. 1 max-sized I/O buffer avoids
- * issuing small I/O unnecessarily while 3 max-sized I/O buffers guarantees higher
- * throughput by overlapping compute and I/O efficiently.
- * - A maximum reservation of the hdfs split size, to avoid reserving excessive
- * memory for small files or ranges, e.g. small dimension tables with very few
- * rows.
- */
- private Pair<Long, Long> computeReservation(int numColumnsReadFromFile) {
- Preconditions.checkState(maxScanRangeBytes_ >= 0);
- long maxIoBufferSize =
- BitUtil.roundUpToPowerOf2(BackendConfig.INSTANCE.getReadSize());
- // Scanners for columnar formats issue one IoMgr scan range for metadata, followed by
- // one IoMgr scan range per column in parallel. Scanners for row-oriented formats
- // issue only one IoMgr scan range at a time.
- int iomgrScanRangesPerSplit = fileFormats_.contains(HdfsFileFormat.PARQUET) ?
- Math.max(1, numColumnsReadFromFile) : 1;
- // Need one buffer per IoMgr scan range to execute the scan.
- long minReservationToExecute =
- iomgrScanRangesPerSplit * BackendConfig.INSTANCE.getMinBufferSize();
-
- // Quantize the max scan range (i.e. hdfs split) size to an I/O buffer size.
- long quantizedMaxScanRangeBytes = maxScanRangeBytes_ < maxIoBufferSize ?
- BitUtil.roundUpToPowerOf2(maxScanRangeBytes_) :
- BitUtil.roundUpToPowerOf2Factor(maxScanRangeBytes_, maxIoBufferSize);
- long minReservationBytes = Math.max(minReservationToExecute,
- Math.min(iomgrScanRangesPerSplit * maxIoBufferSize,
- quantizedMaxScanRangeBytes));
- long idealReservationBytes = Math.max(minReservationToExecute,
- Math.min(iomgrScanRangesPerSplit * maxIoBufferSize * 3,
- quantizedMaxScanRangeBytes));
- return Pair.create(minReservationBytes, idealReservationBytes);
+ nodeResourceProfile_ = ResourceProfile.noReservation(perInstanceMemEstimate);
}
/**
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/fe/src/main/java/org/apache/impala/util/BitUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/BitUtil.java b/fe/src/main/java/org/apache/impala/util/BitUtil.java
index 6bb670d..839dd6e 100644
--- a/fe/src/main/java/org/apache/impala/util/BitUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/BitUtil.java
@@ -29,10 +29,4 @@ public class BitUtil {
public static long roundUpToPowerOf2(long val) {
return 1L << log2Ceiling(val);
}
-
- // Round up 'val' to the nearest multiple of a power-of-two 'factor'.
- // 'val' must be > 0.
- public static long roundUpToPowerOf2Factor(long val, long factor) {
- return (val + (factor - 1)) & ~(factor - 1);
- }
}
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/fe/src/test/java/org/apache/impala/util/BitUtilTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/util/BitUtilTest.java b/fe/src/test/java/org/apache/impala/util/BitUtilTest.java
index a6da80c..a134b6a 100644
--- a/fe/src/test/java/org/apache/impala/util/BitUtilTest.java
+++ b/fe/src/test/java/org/apache/impala/util/BitUtilTest.java
@@ -46,10 +46,4 @@ public class BitUtilTest {
assertEquals(0x8000000000000000L, BitUtil.roundUpToPowerOf2(0x8000000000000000L - 1));
}
- @Test
- public void testPowerOf2Factor() {
- assertEquals(BitUtil.roundUpToPowerOf2Factor(7, 8), 8);
- assertEquals(BitUtil.roundUpToPowerOf2Factor(8, 8), 8);
- assertEquals(BitUtil.roundUpToPowerOf2Factor(9, 8), 16);
- }
}
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test b/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test
index 533ac42..f25ad0a 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test
@@ -5,7 +5,7 @@ where 5 + 5 < c_custkey and o_orderkey = (2 + 2)
and (coalesce(2, 3, 4) * 10) + l_linenumber < (0 * 1)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=264.00MB mem-reservation=24.00MB
+| Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -59,7 +59,7 @@ PLAN-ROOT SINK
parquet dictionary predicates: c_custkey > 10
parquet dictionary predicates on o: o_orderkey = 4
parquet dictionary predicates on o_lineitems: 20 + l_linenumber < 0
- mem-estimate=264.00MB mem-reservation=24.00MB
+ mem-estimate=176.00MB mem-reservation=0B
tuple-ids=0 row-size=24B cardinality=15000
====
# Test HBase scan node.
@@ -107,7 +107,7 @@ having 1024 * 1024 * count(*) % 2 = 0
and (sm between 5 and 10)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=138.00MB mem-reservation=1.97MB
+| Per-Host Resources: mem-estimate=138.00MB mem-reservation=1.94MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -125,7 +125,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=20B cardinality=7300
====
# Test hash join.
@@ -136,7 +136,7 @@ left outer join functional.alltypes b
where round(1.11 + 2.22 + 3.33 + 4.44, 1) < cast(b.double_col as decimal(3, 2))
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=257.94MB mem-reservation=2.00MB
+| Per-Host Resources: mem-estimate=257.94MB mem-reservation=1.94MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -157,7 +157,7 @@ PLAN-ROOT SINK
| columns: all
| extrapolated-rows=disabled
| parquet dictionary predicates: CAST(b.double_col AS DECIMAL(3,2)) > 11.1
-| mem-estimate=128.00MB mem-reservation=32.00KB
+| mem-estimate=128.00MB mem-reservation=0B
| tuple-ids=1 row-size=20B cardinality=730
|
00:SCAN HDFS [functional.alltypes a]
@@ -167,7 +167,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=7300
====
# Test nested-loop join. Same as above but and with a disjunction in the On clause.
@@ -179,7 +179,7 @@ left outer join functional.alltypes b
where cast(b.double_col as decimal(3, 2)) > round(1.11 + 2.22 + 3.33 + 4.44, 1)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=256.01MB mem-reservation=64.00KB
+| Per-Host Resources: mem-estimate=256.01MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -198,7 +198,7 @@ PLAN-ROOT SINK
| columns: all
| extrapolated-rows=disabled
| parquet dictionary predicates: CAST(b.double_col AS DECIMAL(3,2)) > 11.1
-| mem-estimate=128.00MB mem-reservation=32.00KB
+| mem-estimate=128.00MB mem-reservation=0B
| tuple-ids=1 row-size=20B cardinality=730
|
00:SCAN HDFS [functional.alltypes a]
@@ -208,7 +208,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=7300
====
# Test distinct aggregation with grouping.
@@ -242,7 +242,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=20B cardinality=7300
====
# Test non-grouping distinct aggregation.
@@ -251,7 +251,7 @@ from functional.alltypes
having 1024 * 1024 * count(*) % 2 = 0
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=138.00MB mem-reservation=1.97MB
+| Per-Host Resources: mem-estimate=138.00MB mem-reservation=1.94MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -274,7 +274,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=4B cardinality=7300
====
# Test analytic eval node.
@@ -284,7 +284,7 @@ select first_value(1 + 1 + int_col - (1 - 1)) over
from functional.alltypes
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.03MB
+| Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -309,7 +309,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=29B cardinality=7300
====
# Test sort node.
@@ -317,7 +317,7 @@ select int_col from functional.alltypes
order by id * abs((factorial(5) / power(2, 4)))
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=134.00MB mem-reservation=6.03MB
+| Per-Host Resources: mem-estimate=134.00MB mem-reservation=6.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -333,7 +333,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=7300
====
# Test HDFS table sink.
@@ -342,7 +342,7 @@ select id, int_col, cast(1 + 1 + 1 + year as int), cast(month - (1 - 1 - 1) as i
from functional.alltypessmall
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=8.00KB
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
WRITE TO HDFS [functional.alltypes, OVERWRITE=false, PARTITION-KEYS=(CAST(3 + year AS INT),CAST(month - -1 AS INT))]
| partitions=4
| mem-estimate=1.56KB mem-reservation=0B
@@ -354,7 +354,7 @@ WRITE TO HDFS [functional.alltypes, OVERWRITE=false, PARTITION-KEYS=(CAST(3 + ye
partitions: 4/4 rows=100
columns: all
extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=8.00KB
+ mem-estimate=32.00MB mem-reservation=0B
tuple-ids=0 row-size=16B cardinality=100
====
# Constant folding does not work across query blocks.
@@ -366,7 +366,7 @@ select sum(id + c3) from
) v3
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=138.00MB mem-reservation=32.00KB
+| Per-Host Resources: mem-estimate=138.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -383,6 +383,6 @@ PLAN-ROOT SINK
columns: all
extrapolated-rows=disabled
limit: 2
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=4B cardinality=2
====
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test b/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test
index 3d234d1..c5081a7 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test
@@ -1,7 +1,7 @@
# Rows per node is < 3000: codegen should be disabled.
select count(*) from functional.alltypes
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=148.00MB
Codegen disabled by planner
@@ -21,7 +21,7 @@ PLAN-ROOT SINK
# Rows per node is > 3000: codegen should be enabled.
select count(*) from functional.alltypesagg
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=128.00KB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=100.00MB
PLAN-ROOT SINK
@@ -40,7 +40,7 @@ PLAN-ROOT SINK
# No stats on functional_parquet: codegen should be disabled.
select count(*) from functional_parquet.alltypes
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00KB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=21.00MB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypes
@@ -61,7 +61,7 @@ PLAN-ROOT SINK
# > 3000 rows returned to coordinator: codegen should be enabled
select * from functional_parquet.alltypes
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=88.00KB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=128.00MB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypes
@@ -78,7 +78,7 @@ select count(*)
from functional.alltypes t1
join functional.alltypestiny t2 on t1.id = t2.id
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=2.98MB
+Max Per-Host Resource Reservation: Memory=2.94MB
Per-Host Resource Estimates: Memory=182.94MB
Codegen disabled by planner
@@ -108,7 +108,7 @@ PLAN-ROOT SINK
# Optimisation is disabled by cross join producing > 3000 rows
select count(*) from functional.alltypes t1, functional.alltypes t2
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=64.00KB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=276.00MB
PLAN-ROOT SINK
@@ -137,7 +137,7 @@ select count(*) from (
union all
select * from functional.alltypestiny) v
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=148.00MB
Codegen disabled by planner
@@ -166,7 +166,7 @@ select count(*) from (
union all
select * from functional.alltypes) v
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=148.00MB
PLAN-ROOT SINK
@@ -193,7 +193,7 @@ PLAN-ROOT SINK
select sum(l_discount)
from (select * from tpch.lineitem limit 1000) v
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=274.00MB
Codegen disabled by planner
@@ -214,7 +214,7 @@ PLAN-ROOT SINK
select sum(l_discount)
from (select * from tpch.lineitem where l_orderkey > 100 limit 1000) v
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=274.00MB
PLAN-ROOT SINK
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test b/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test
index 55439d6..bb12bca 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test
@@ -5,7 +5,7 @@ on ss_customer_sk = c_customer_sk
where c_salutation = 'Mrs.'
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=185.50MB mem-reservation=25.50MB runtime-filters-memory=1.00MB
+| Per-Host Resources: mem-estimate=185.50MB mem-reservation=9.50MB runtime-filters-memory=1.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -24,7 +24,7 @@ PLAN-ROOT SINK
| columns: all
| extrapolated-rows=disabled
| parquet dictionary predicates: c_salutation = 'Mrs.'
-| mem-estimate=48.00MB mem-reservation=8.00MB
+| mem-estimate=48.00MB mem-reservation=0B
| tuple-ids=1 row-size=255B cardinality=16667
|
00:SCAN HDFS [tpcds.store_sales]
@@ -35,7 +35,7 @@ PLAN-ROOT SINK
partitions: 1824/1824 rows=2880404
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=8.00MB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=100B cardinality=2880404
====
# Single-column FK/PK join detection on left outer join. The join cardinality
@@ -46,7 +46,7 @@ on ss_customer_sk = c_customer_sk
where c_salutation = 'Mrs.'
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=184.50MB mem-reservation=24.50MB
+| Per-Host Resources: mem-estimate=184.50MB mem-reservation=8.50MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -65,7 +65,7 @@ PLAN-ROOT SINK
| columns: all
| extrapolated-rows=disabled
| parquet dictionary predicates: c_salutation = 'Mrs.'
-| mem-estimate=48.00MB mem-reservation=8.00MB
+| mem-estimate=48.00MB mem-reservation=0B
| tuple-ids=1 row-size=255B cardinality=16667
|
00:SCAN HDFS [tpcds.store_sales]
@@ -75,7 +75,7 @@ PLAN-ROOT SINK
partitions: 1824/1824 rows=2880404
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=8.00MB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=100B cardinality=2880404
====
# Single-column FK/PK join detection on right outer join. The join cardinality
@@ -86,7 +86,7 @@ on ss_customer_sk = c_customer_sk
where c_salutation = 'Mrs.'
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=185.50MB mem-reservation=25.50MB runtime-filters-memory=1.00MB
+| Per-Host Resources: mem-estimate=185.50MB mem-reservation=9.50MB runtime-filters-memory=1.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -105,7 +105,7 @@ PLAN-ROOT SINK
| columns: all
| extrapolated-rows=disabled
| parquet dictionary predicates: c_salutation = 'Mrs.'
-| mem-estimate=48.00MB mem-reservation=8.00MB
+| mem-estimate=48.00MB mem-reservation=0B
| tuple-ids=1 row-size=255B cardinality=16667
|
00:SCAN HDFS [tpcds.store_sales]
@@ -116,7 +116,7 @@ PLAN-ROOT SINK
partitions: 1824/1824 rows=2880404
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=8.00MB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=100B cardinality=2880404
====
# Multi-column FK/PK join detection
@@ -126,7 +126,7 @@ on ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number
where sr_return_quantity < 10
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=214.75MB mem-reservation=22.75MB runtime-filters-memory=2.00MB
+| Per-Host Resources: mem-estimate=214.75MB mem-reservation=6.75MB runtime-filters-memory=2.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -145,7 +145,7 @@ PLAN-ROOT SINK
| columns: all
| extrapolated-rows=disabled
| parquet dictionary predicates: sr_return_quantity < 10
-| mem-estimate=80.00MB mem-reservation=8.00MB
+| mem-estimate=80.00MB mem-reservation=0B
| tuple-ids=1 row-size=88B cardinality=28751
|
00:SCAN HDFS [tpcds.store_sales]
@@ -156,7 +156,7 @@ PLAN-ROOT SINK
partitions: 1824/1824 rows=2880404
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=8.00MB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=100B cardinality=2880404
====
# Many-to-many join detection.
@@ -165,7 +165,7 @@ tpcds.store_sales inner join tpcds.web_sales
on ss_sold_time_sk = ws_sold_time_sk
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=397.67MB mem-reservation=51.00MB runtime-filters-memory=1.00MB
+| Per-Host Resources: mem-estimate=397.67MB mem-reservation=35.00MB runtime-filters-memory=1.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -182,7 +182,7 @@ PLAN-ROOT SINK
| table: rows=719384 size=140.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=160.00MB mem-reservation=8.00MB
+| mem-estimate=160.00MB mem-reservation=0B
| tuple-ids=1 row-size=144B cardinality=719384
|
00:SCAN HDFS [tpcds.store_sales]
@@ -193,7 +193,7 @@ PLAN-ROOT SINK
partitions: 1824/1824 rows=2880404
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=8.00MB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=100B cardinality=2880404
====
# PK/PK join is detected as FK/PK.
@@ -203,7 +203,7 @@ on a.d_date_sk = b.d_date_sk
where a.d_holiday = "Y"
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=114.00MB mem-reservation=34.00MB runtime-filters-memory=1.00MB
+| Per-Host Resources: mem-estimate=114.00MB mem-reservation=18.00MB runtime-filters-memory=1.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -222,7 +222,7 @@ PLAN-ROOT SINK
| columns: all
| extrapolated-rows=disabled
| parquet dictionary predicates: a.d_holiday = 'Y'
-| mem-estimate=48.00MB mem-reservation=8.00MB
+| mem-estimate=48.00MB mem-reservation=0B
| tuple-ids=0 row-size=303B cardinality=36525
|
01:SCAN HDFS [tpcds.date_dim b]
@@ -232,7 +232,7 @@ PLAN-ROOT SINK
table: rows=73049 size=9.84MB
columns: all
extrapolated-rows=disabled
- mem-estimate=48.00MB mem-reservation=8.00MB
+ mem-estimate=48.00MB mem-reservation=0B
tuple-ids=1 row-size=303B cardinality=73049
====
# Single query with various join types combined.
@@ -246,7 +246,7 @@ where ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number
and d1.d_fy_week_seq = 1000
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=362.81MB mem-reservation=50.81MB runtime-filters-memory=5.00MB
+| Per-Host Resources: mem-estimate=362.81MB mem-reservation=12.75MB runtime-filters-memory=5.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -263,7 +263,7 @@ PLAN-ROOT SINK
| table: rows=100000 size=12.60MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=48.00MB mem-reservation=8.00MB
+| mem-estimate=48.00MB mem-reservation=0B
| tuple-ids=2 row-size=4B cardinality=100000
|
07:HASH JOIN [INNER JOIN]
@@ -279,7 +279,7 @@ PLAN-ROOT SINK
| table: rows=73049 size=9.84MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=48.00MB mem-reservation=8.00MB
+| mem-estimate=48.00MB mem-reservation=0B
| tuple-ids=4 row-size=4B cardinality=73049
|
06:HASH JOIN [INNER JOIN]
@@ -304,7 +304,7 @@ PLAN-ROOT SINK
| | columns: all
| | extrapolated-rows=disabled
| | parquet dictionary predicates: d1.d_fy_week_seq = 1000
-| | mem-estimate=48.00MB mem-reservation=8.00MB
+| | mem-estimate=48.00MB mem-reservation=0B
| | tuple-ids=3 row-size=8B cardinality=7
| |
| 00:SCAN HDFS [tpcds.store_sales]
@@ -315,7 +315,7 @@ PLAN-ROOT SINK
| partitions: 1824/1824 rows=2880404
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=128.00MB mem-reservation=8.00MB
+| mem-estimate=128.00MB mem-reservation=0B
| tuple-ids=0 row-size=24B cardinality=2880404
|
01:SCAN HDFS [tpcds.store_returns]
@@ -325,7 +325,7 @@ PLAN-ROOT SINK
table: rows=287514 size=31.19MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=1 row-size=20B cardinality=287514
====
# Assumed FK/PK join becasue of non-trivial equi-join exprs.
@@ -334,7 +334,7 @@ tpcds.store_sales inner join tpcds.customer
on ss_customer_sk % 10 = c_customer_sk / 100
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=211.00MB mem-reservation=51.00MB runtime-filters-memory=1.00MB
+| Per-Host Resources: mem-estimate=211.00MB mem-reservation=35.00MB runtime-filters-memory=1.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -351,7 +351,7 @@ PLAN-ROOT SINK
| table: rows=100000 size=12.60MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=48.00MB mem-reservation=8.00MB
+| mem-estimate=48.00MB mem-reservation=0B
| tuple-ids=1 row-size=255B cardinality=100000
|
00:SCAN HDFS [tpcds.store_sales]
@@ -362,7 +362,7 @@ PLAN-ROOT SINK
partitions: 1824/1824 rows=2880404
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=8.00MB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=100B cardinality=2880404
====
# Assumed FK/PK join due to missing stats on the rhs. Join cardinality is equal to
@@ -372,7 +372,7 @@ tpcds.store_sales inner join tpcds_seq_snap.customer
on ss_customer_sk = c_customer_sk
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=2.17GB mem-reservation=51.00MB runtime-filters-memory=1.00MB
+| Per-Host Resources: mem-estimate=2.17GB mem-reservation=35.00MB runtime-filters-memory=1.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -384,12 +384,12 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=8B cardinality=2880404
|
|--01:SCAN HDFS [tpcds_seq_snap.customer]
-| partitions=1/1 files=1 size=8.58MB
+| partitions=1/1 files=1 size=8.59MB
| stored statistics:
-| table: rows=unavailable size=8.58MB
+| table: rows=unavailable size=8.59MB
| columns: unavailable
| extrapolated-rows=disabled
-| mem-estimate=48.00MB mem-reservation=8.00MB
+| mem-estimate=48.00MB mem-reservation=0B
| tuple-ids=1 row-size=4B cardinality=unavailable
|
00:SCAN HDFS [tpcds.store_sales]
@@ -400,7 +400,7 @@ PLAN-ROOT SINK
partitions: 1824/1824 rows=2880404
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=8.00MB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=4B cardinality=2880404
====
# Assumed FK/PK join due to missing stats on the lhs. Join cardinality is unknown.
@@ -409,7 +409,7 @@ tpcds_seq_snap.store_sales inner join tpcds.customer
on ss_customer_sk = c_customer_sk
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=178.94MB mem-reservation=18.94MB runtime-filters-memory=1.00MB
+| Per-Host Resources: mem-estimate=178.94MB mem-reservation=2.94MB runtime-filters-memory=1.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -426,18 +426,18 @@ PLAN-ROOT SINK
| table: rows=100000 size=12.60MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=48.00MB mem-reservation=8.00MB
+| mem-estimate=48.00MB mem-reservation=0B
| tuple-ids=1 row-size=4B cardinality=100000
|
00:SCAN HDFS [tpcds_seq_snap.store_sales]
- partitions=1824/1824 files=1824 size=207.85MB
+ partitions=1824/1824 files=1824 size=207.90MB
runtime filters: RF000[bloom] -> ss_customer_sk
stored statistics:
table: rows=unavailable size=unavailable
partitions: 0/1824 rows=unavailable
columns: unavailable
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=8.00MB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=4B cardinality=unavailable
====
# Join is detected as many-to-many even though the rhs join columns
@@ -448,7 +448,7 @@ tpcds.store_sales inner join
on ss_sold_time_sk = ws_sold_time_sk
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=299.00MB mem-reservation=18.94MB runtime-filters-memory=1.00MB
+| Per-Host Resources: mem-estimate=299.00MB mem-reservation=4.88MB runtime-filters-memory=1.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -470,7 +470,7 @@ PLAN-ROOT SINK
| table: rows=719384 size=140.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=160.00MB mem-reservation=8.00MB
+| mem-estimate=160.00MB mem-reservation=0B
| tuple-ids=1 row-size=4B cardinality=719384
|
00:SCAN HDFS [tpcds.store_sales]
@@ -481,6 +481,6 @@ PLAN-ROOT SINK
partitions: 1824/1824 rows=2880404
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=8.00MB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=100B cardinality=2880404
====
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test b/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
index 1df9270..96015e0 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
@@ -4,7 +4,7 @@ select straight_join *
from tpch_parquet.customer
inner join tpch_parquet.nation on c_nationkey = n_nationkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=33.97MB
+Max Per-Host Resource Reservation: Memory=17.94MB
Per-Host Resource Estimates: Memory=57.94MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -17,7 +17,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=355B cardinality=150000
|
F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-Per-Host Resources: mem-estimate=41.94MB mem-reservation=33.94MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=41.94MB mem-reservation=17.94MB runtime-filters-memory=1.00MB
02:HASH JOIN [INNER JOIN, BROADCAST]
| hash predicates: c_nationkey = n_nationkey
| fk/pk conjuncts: c_nationkey = n_nationkey
@@ -30,14 +30,14 @@ Per-Host Resources: mem-estimate=41.94MB mem-reservation=33.94MB runtime-filters
| | tuple-ids=1 row-size=117B cardinality=25
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
+| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.nation, RANDOM]
| partitions=1/1 files=1 size=2.74KB
| stored statistics:
| table: rows=25 size=2.74KB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=16.00MB mem-reservation=32.00KB
+| mem-estimate=16.00MB mem-reservation=0B
| tuple-ids=1 row-size=117B cardinality=25
|
00:SCAN HDFS [tpch_parquet.customer, RANDOM]
@@ -47,7 +47,7 @@ Per-Host Resources: mem-estimate=41.94MB mem-reservation=33.94MB runtime-filters
table: rows=150000 size=12.31MB
columns: all
extrapolated-rows=disabled
- mem-estimate=24.00MB mem-reservation=16.00MB
+ mem-estimate=24.00MB mem-reservation=0B
tuple-ids=0 row-size=238B cardinality=150000
====
# Join with large build side.
@@ -56,8 +56,8 @@ select straight_join *
from tpch_parquet.lineitem
left join tpch_parquet.orders on l_orderkey = o_orderkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=166.00MB
-Per-Host Resource Estimates: Memory=428.41MB
+Max Per-Host Resource Reservation: Memory=46.00MB
+Per-Host Resource Estimates: Memory=420.41MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -69,7 +69,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1N row-size=454B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=380.41MB mem-reservation=118.00MB
+Per-Host Resources: mem-estimate=380.41MB mem-reservation=46.00MB
02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
| hash predicates: l_orderkey = o_orderkey
| fk/pk conjuncts: l_orderkey = o_orderkey
@@ -81,23 +81,23 @@ Per-Host Resources: mem-estimate=380.41MB mem-reservation=118.00MB
| | tuple-ids=1 row-size=191B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-| Per-Host Resources: mem-estimate=48.00MB mem-reservation=48.00MB
+| Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.orders, RANDOM]
| partitions=1/1 files=2 size=54.07MB
| stored statistics:
| table: rows=1500000 size=54.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=48.00MB mem-reservation=48.00MB
+| mem-estimate=40.00MB mem-reservation=0B
| tuple-ids=1 row-size=191B cardinality=1500000
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.72MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.72MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
====
# Null-aware anti-join with medium build side.
@@ -105,7 +105,7 @@ Per-Host Resources: mem-estimate=380.41MB mem-reservation=118.00MB
select * from tpch_parquet.lineitem
where l_orderkey not in (select o_orderkey from tpch_parquet.orders)
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=114.00MB
+Max Per-Host Resource Reservation: Memory=34.00MB
Per-Host Resource Estimates: Memory=154.00MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -118,7 +118,7 @@ PLAN-ROOT SINK
| tuple-ids=0 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=114.00MB mem-reservation=106.00MB
+Per-Host Resources: mem-estimate=114.00MB mem-reservation=34.00MB
02:HASH JOIN [NULL AWARE LEFT ANTI JOIN, BROADCAST]
| hash predicates: l_orderkey = o_orderkey
| mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=1.00MB
@@ -129,23 +129,23 @@ Per-Host Resources: mem-estimate=114.00MB mem-reservation=106.00MB
| | tuple-ids=1 row-size=8B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-| Per-Host Resources: mem-estimate=40.00MB mem-reservation=8.00MB
+| Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.orders, RANDOM]
| partitions=1/1 files=2 size=54.07MB
| stored statistics:
| table: rows=1500000 size=54.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=40.00MB mem-reservation=8.00MB
+| mem-estimate=40.00MB mem-reservation=0B
| tuple-ids=1 row-size=8B cardinality=1500000
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.72MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.72MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
====
# Mid NDV aggregation.
@@ -156,7 +156,7 @@ from tpch_parquet.lineitem
group by 1, 2
having count(*) = 1
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=137.00MB
+Max Per-Host Resource Reservation: Memory=113.00MB
Per-Host Resource Estimates: Memory=253.12MB
F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -201,14 +201,14 @@ Per-Host Resources: mem-estimate=86.12MB mem-reservation=66.00MB runtime-filters
| | tuple-ids=1 row-size=25B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-| Per-Host Resources: mem-estimate=40.00MB mem-reservation=16.00MB
+| Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.orders, RANDOM]
| partitions=1/1 files=2 size=54.07MB
| stored statistics:
| table: rows=1500000 size=54.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=40.00MB mem-reservation=16.00MB
+| mem-estimate=40.00MB mem-reservation=0B
| tuple-ids=1 row-size=25B cardinality=1500000
|
04:EXCHANGE [HASH(l_orderkey)]
@@ -216,15 +216,15 @@ Per-Host Resources: mem-estimate=86.12MB mem-reservation=66.00MB runtime-filters
| tuple-ids=0 row-size=8B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=81.00MB mem-reservation=9.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=81.00MB mem-reservation=1.00MB runtime-filters-memory=1.00MB
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.72MB
+ partitions=1/1 files=3 size=193.73MB
runtime filters: RF000[bloom] -> l_orderkey
stored statistics:
- table: rows=6001215 size=193.72MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
====
# High NDV aggregation.
@@ -232,7 +232,7 @@ Per-Host Resources: mem-estimate=81.00MB mem-reservation=9.00MB runtime-filters-
select distinct *
from tpch_parquet.lineitem
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=152.00MB
+Max Per-Host Resource Reservation: Memory=80.00MB
Per-Host Resource Estimates: Memory=3.31GB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -256,19 +256,19 @@ Per-Host Resources: mem-estimate=1.62GB mem-reservation=46.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=1.69GB mem-reservation=106.00MB
+Per-Host Resources: mem-estimate=1.69GB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.72MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.72MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
====
# High NDV aggregation with string aggregation function.
@@ -277,7 +277,7 @@ select l_orderkey, l_partkey, group_concat(l_linestatus, ",")
from tpch_parquet.lineitem
group by 1, 2
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=106.00MB
+Max Per-Host Resource Reservation: Memory=82.00MB
Per-Host Resource Estimates: Memory=482.91MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -302,7 +302,7 @@ Per-Host Resources: mem-estimate=201.46MB mem-reservation=48.00MB
| tuple-ids=1 row-size=32B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=281.46MB mem-reservation=58.00MB
+Per-Host Resources: mem-estimate=281.46MB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| output: group_concat(l_linestatus, ',')
| group by: l_orderkey, l_partkey
@@ -310,12 +310,12 @@ Per-Host Resources: mem-estimate=281.46MB mem-reservation=58.00MB
| tuple-ids=1 row-size=32B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.72MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.72MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=24.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=33B cardinality=6001215
====
# Sort + Analytic.
@@ -323,7 +323,7 @@ Per-Host Resources: mem-estimate=281.46MB mem-reservation=58.00MB
select max(tinyint_col) over(partition by int_col)
from functional.alltypes
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=40.03MB
+Max Per-Host Resource Reservation: Memory=40.00MB
Per-Host Resource Estimates: Memory=56.00MB
Codegen disabled by planner
@@ -354,7 +354,7 @@ Per-Host Resources: mem-estimate=40.00MB mem-reservation=40.00MB
| tuple-ids=0 row-size=5B cardinality=7300
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
+Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
00:SCAN HDFS [functional.alltypes, RANDOM]
partitions=24/24 files=24 size=478.45KB
stored statistics:
@@ -362,6 +362,6 @@ Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=32.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=5B cardinality=7300
====
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test b/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test
index 8dd2593..7f6d96b 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test
@@ -99,7 +99,7 @@ select count(*) from functional_kudu.alltypes a, functional_parquet.alltypes b,
where a.int_col = b.int_col and a.int_col = c.int_col
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=2.02GB mem-reservation=36.95MB runtime-filters-memory=1.00MB
+| Per-Host Resources: mem-estimate=2.02GB mem-reservation=36.94MB runtime-filters-memory=1.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -127,14 +127,14 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=8B cardinality=7300
|
|--01:SCAN HDFS [functional_parquet.alltypes b]
-| partitions=24/24 files=24 size=174.62KB
+| partitions=24/24 files=24 size=174.39KB
| runtime filters: RF000[bloom] -> b.int_col
| stored statistics:
| table: rows=unavailable size=unavailable
| partitions: 0/24 rows=unavailable
| columns: unavailable
| extrapolated-rows=disabled
-| mem-estimate=16.00MB mem-reservation=8.00KB
+| mem-estimate=16.00MB mem-reservation=0B
| tuple-ids=1 row-size=4B cardinality=unavailable
|
00:SCAN KUDU [functional_kudu.alltypes a]
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test b/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
index 31129ff..61d646b 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
@@ -40,7 +40,7 @@ order by cnt, bigint_col
limit 10
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.02MB
+| Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -56,7 +56,7 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=16B cardinality=unavailable
|
00:SCAN HDFS [functional_parquet.alltypes]
- partitions=24/24 files=24 size=174.39KB
+ partitions=24/24 files=24 size=179.19KB
predicates: id < 10
stored statistics:
table: rows=unavailable size=unavailable
@@ -65,7 +65,7 @@ PLAN-ROOT SINK
extrapolated-rows=disabled
parquet statistics predicates: id < 10
parquet dictionary predicates: id < 10
- mem-estimate=16.00MB mem-reservation=24.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=16B cardinality=unavailable
---- PARALLELPLANS
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -97,7 +97,7 @@ Per-Host Resources: mem-estimate=384.00MB mem-reservation=102.00MB
| tuple-ids=1 row-size=16B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=9
-Per-Host Resources: mem-estimate=432.00MB mem-reservation=102.07MB
+Per-Host Resources: mem-estimate=432.00MB mem-reservation=102.00MB
01:AGGREGATE [STREAMING]
| output: count(int_col)
| group by: bigint_col
@@ -105,7 +105,7 @@ Per-Host Resources: mem-estimate=432.00MB mem-reservation=102.07MB
| tuple-ids=1 row-size=16B cardinality=unavailable
|
00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
- partitions=24/24 files=24 size=174.39KB
+ partitions=24/24 files=24 size=179.19KB
predicates: id < 10
stored statistics:
table: rows=unavailable size=unavailable
@@ -114,7 +114,7 @@ Per-Host Resources: mem-estimate=432.00MB mem-reservation=102.07MB
extrapolated-rows=disabled
parquet statistics predicates: id < 10
parquet dictionary predicates: id < 10
- mem-estimate=16.00MB mem-reservation=24.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=16B cardinality=unavailable
====
# Single-table scan/filter/analytic should work.
@@ -123,7 +123,7 @@ from functional_parquet.alltypes
where id < 10
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=26.00MB mem-reservation=10.02MB
+| Per-Host Resources: mem-estimate=26.00MB mem-reservation=10.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -141,7 +141,7 @@ PLAN-ROOT SINK
| tuple-ids=4 row-size=8B cardinality=unavailable
|
00:SCAN HDFS [functional_parquet.alltypes]
- partitions=24/24 files=24 size=174.39KB
+ partitions=24/24 files=24 size=179.19KB
predicates: id < 10
stored statistics:
table: rows=unavailable size=unavailable
@@ -150,7 +150,7 @@ PLAN-ROOT SINK
extrapolated-rows=disabled
parquet statistics predicates: id < 10
parquet dictionary predicates: id < 10
- mem-estimate=16.00MB mem-reservation=16.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=unavailable
---- PARALLELPLANS
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -182,9 +182,9 @@ Per-Host Resources: mem-estimate=30.00MB mem-reservation=30.00MB
| tuple-ids=0 row-size=8B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=9
-Per-Host Resources: mem-estimate=48.00MB mem-reservation=48.00KB
+Per-Host Resources: mem-estimate=48.00MB mem-reservation=0B
00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
- partitions=24/24 files=24 size=174.39KB
+ partitions=24/24 files=24 size=179.19KB
predicates: id < 10
stored statistics:
table: rows=unavailable size=unavailable
@@ -193,7 +193,7 @@ Per-Host Resources: mem-estimate=48.00MB mem-reservation=48.00KB
extrapolated-rows=disabled
parquet statistics predicates: id < 10
parquet dictionary predicates: id < 10
- mem-estimate=16.00MB mem-reservation=16.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=unavailable
====
# Nested-loop join in a subplan should work.
@@ -202,7 +202,7 @@ from tpch_nested_parquet.customer c, c.c_orders o, o.o_lineitems
where c_custkey < 10 and o_orderkey < 5 and l_linenumber < 3
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=88.00MB
+| Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -257,7 +257,7 @@ PLAN-ROOT SINK
parquet dictionary predicates: c_custkey < 10
parquet dictionary predicates on o: o_orderkey < 5
parquet dictionary predicates on o_lineitems: l_linenumber < 3
- mem-estimate=88.00MB mem-reservation=88.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=254B cardinality=15000
---- PARALLELPLANS
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -270,7 +270,7 @@ PLAN-ROOT SINK
| tuple-ids=2,1,0 row-size=562B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=9
-Per-Host Resources: mem-estimate=264.00MB mem-reservation=264.00MB
+Per-Host Resources: mem-estimate=264.00MB mem-reservation=0B
01:SUBPLAN
| mem-estimate=0B mem-reservation=0B
| tuple-ids=2,1,0 row-size=562B cardinality=1500000
@@ -322,7 +322,7 @@ Per-Host Resources: mem-estimate=264.00MB mem-reservation=264.00MB
parquet dictionary predicates: c_custkey < 10
parquet dictionary predicates on o: o_orderkey < 5
parquet dictionary predicates on o_lineitems: l_linenumber < 3
- mem-estimate=88.00MB mem-reservation=88.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=254B cardinality=15000
====
# Hash-join in a subplan should work.
@@ -331,7 +331,7 @@ from tpch_nested_parquet.customer c, c.c_orders o1, c.c_orders o2
where o1.o_orderkey = o2.o_orderkey + 2 and o1.o_orderkey < 5
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=89.94MB mem-reservation=81.94MB
+| Per-Host Resources: mem-estimate=89.94MB mem-reservation=1.94MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -374,7 +374,7 @@ PLAN-ROOT SINK
extrapolated-rows=disabled
parquet statistics predicates on o1: o1.o_orderkey < 5
parquet dictionary predicates on o1: o1.o_orderkey < 5
- mem-estimate=88.00MB mem-reservation=80.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=270B cardinality=150000
---- PARALLELPLANS
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -387,7 +387,7 @@ PLAN-ROOT SINK
| tuple-ids=1,0,2 row-size=286B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=9
-Per-Host Resources: mem-estimate=269.81MB mem-reservation=245.81MB
+Per-Host Resources: mem-estimate=269.81MB mem-reservation=5.81MB
01:SUBPLAN
| mem-estimate=0B mem-reservation=0B
| tuple-ids=1,0,2 row-size=286B cardinality=1500000
@@ -427,6 +427,6 @@ Per-Host Resources: mem-estimate=269.81MB mem-reservation=245.81MB
extrapolated-rows=disabled
parquet statistics predicates on o1: o1.o_orderkey < 5
parquet dictionary predicates on o1: o1.o_orderkey < 5
- mem-estimate=88.00MB mem-reservation=80.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=270B cardinality=150000
====
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test b/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
index 3e812c6..2b602c9 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
@@ -8,7 +8,7 @@ where int_col > 1 and int_col * rand() > 50 and int_col is null
and int_col > tinyint_col;
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=42.00MB mem-reservation=16.00KB
+| Per-Host Resources: mem-estimate=42.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -18,7 +18,7 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=8B cardinality=1
|
00:SCAN HDFS [functional_parquet.alltypes]
- partitions=24/24 files=24 size=174.39KB
+ partitions=24/24 files=24 size=179.19KB
predicates: int_col IS NULL, int_col > 1, int_col > tinyint_col, int_col * rand() > 50
stored statistics:
table: rows=unavailable size=unavailable
@@ -27,7 +27,7 @@ PLAN-ROOT SINK
extrapolated-rows=disabled
parquet statistics predicates: int_col > 1
parquet dictionary predicates: int_col > 1
- mem-estimate=32.00MB mem-reservation=16.00KB
+ mem-estimate=32.00MB mem-reservation=0B
tuple-ids=0 row-size=5B cardinality=unavailable
====
# Test a variety of types
@@ -40,7 +40,7 @@ and timestamp_cmp(timestamp_col, '2016-11-20 00:00:00') = 1
and year > 2000 and month < 12;
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=138.00MB mem-reservation=88.00KB
+| Per-Host Resources: mem-estimate=138.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -50,7 +50,7 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=8B cardinality=1
|
00:SCAN HDFS [functional_parquet.alltypes]
- partitions=22/24 files=22 size=159.69KB
+ partitions=22/24 files=22 size=164.09KB
predicates: bool_col, bigint_col < 5000, double_col > 100.00, float_col > 50.00, id = 1, tinyint_col < 50, string_col IN ('aaaa', 'bbbb', 'cccc'), smallint_col IN (1, 2, 3, 4, 5), mod(int_col, 2) = 1, timestamp_cmp(timestamp_col, TIMESTAMP '2016-11-20 00:00:00') = 1, date_string_col > '1993-10-01'
stored statistics:
table: rows=unavailable size=unavailable
@@ -59,7 +59,7 @@ PLAN-ROOT SINK
extrapolated-rows=disabled
parquet statistics predicates: bigint_col < 5000, double_col > 100.00, float_col > 50.00, id = 1, tinyint_col < 50, string_col IN ('aaaa', 'bbbb', 'cccc'), smallint_col IN (1, 2, 3, 4, 5), date_string_col > '1993-10-01'
parquet dictionary predicates: bool_col, bigint_col < 5000, double_col > 100.00, float_col > 50.00, id = 1, tinyint_col < 50, string_col IN ('aaaa', 'bbbb', 'cccc'), smallint_col IN (1, 2, 3, 4, 5), mod(int_col, 2) = 1, timestamp_cmp(timestamp_col, TIMESTAMP '2016-11-20 00:00:00') = 1, date_string_col > '1993-10-01'
- mem-estimate=128.00MB mem-reservation=88.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=80B cardinality=unavailable
====
# Test negative cases for IN predicate min/max filtering
@@ -73,7 +73,7 @@ and mod(int_col,50) IN (0,1)
and id IN (int_col);
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=58.00MB mem-reservation=24.00KB
+| Per-Host Resources: mem-estimate=58.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -83,7 +83,7 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=8B cardinality=1
|
00:SCAN HDFS [functional_parquet.alltypes]
- partitions=24/24 files=24 size=174.39KB
+ partitions=24/24 files=24 size=179.19KB
predicates: id IN (int_col), id NOT IN (0, 1, 2), string_col IN ('aaaa', 'bbbb', 'cccc', NULL), mod(int_col, 50) IN (0, 1)
stored statistics:
table: rows=unavailable size=unavailable
@@ -91,7 +91,7 @@ PLAN-ROOT SINK
columns: unavailable
extrapolated-rows=disabled
parquet dictionary predicates: id NOT IN (0, 1, 2), string_col IN ('aaaa', 'bbbb', 'cccc', NULL), mod(int_col, 50) IN (0, 1)
- mem-estimate=48.00MB mem-reservation=24.00KB
+ mem-estimate=48.00MB mem-reservation=0B
tuple-ids=0 row-size=24B cardinality=unavailable
====
# Test collection types where all collections on the path are required (inner
@@ -101,7 +101,7 @@ select id from functional_parquet.complextypestbl c, c.nested_struct.c.d cn, cn.
where a.item.e < -10;
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=16.00KB
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -152,7 +152,7 @@ PLAN-ROOT SINK
extrapolated-rows=disabled
parquet statistics predicates on a: a.item.e < -10
parquet dictionary predicates on a: a.item.e < -10
- mem-estimate=32.00MB mem-reservation=16.00KB
+ mem-estimate=32.00MB mem-reservation=0B
tuple-ids=0 row-size=24B cardinality=unavailable
====
# Test collection types where the lower collection in the path is optional
@@ -164,7 +164,7 @@ left outer join cn.item a
where a.item.e < -10;
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=16.00KB
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -213,7 +213,7 @@ PLAN-ROOT SINK
table: rows=unavailable size=unavailable
columns missing stats: id
extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=16.00KB
+ mem-estimate=32.00MB mem-reservation=0B
tuple-ids=0 row-size=24B cardinality=unavailable
====
# Tests collection types where the outer is optional (outer join descent)
@@ -223,7 +223,7 @@ select id from functional_parquet.complextypestbl c
left outer join c.nested_struct.c.d cn, cn.item a where a.item.e < -10;
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=16.00KB
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -270,7 +270,7 @@ PLAN-ROOT SINK
table: rows=unavailable size=unavailable
columns missing stats: id
extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=16.00KB
+ mem-estimate=32.00MB mem-reservation=0B
tuple-ids=0 row-size=24B cardinality=unavailable
====
# Test collections so that each level has a filter applied.
@@ -278,7 +278,7 @@ select c_custkey from tpch_nested_parquet.customer c, c.c_orders o,
o.o_lineitems l where c_custkey > 0 and o.o_orderkey > 0 and l.l_partkey > 0;
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=264.00MB mem-reservation=24.00MB
+| Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -333,7 +333,7 @@ PLAN-ROOT SINK
parquet dictionary predicates: c_custkey > 0
parquet dictionary predicates on o: o.o_orderkey > 0
parquet dictionary predicates on l: l.l_partkey > 0
- mem-estimate=264.00MB mem-reservation=24.00MB
+ mem-estimate=176.00MB mem-reservation=0B
tuple-ids=0 row-size=24B cardinality=15000
====
# Test collections in a way that would incorrectly apply a min-max
@@ -342,7 +342,7 @@ select count(*) from functional_parquet.complextypestbl c left outer join
(select * from c.int_array where item > 10) v;
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=26.00MB mem-reservation=8.00KB
+| Per-Host Resources: mem-estimate=26.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -376,7 +376,7 @@ PLAN-ROOT SINK
table: rows=unavailable size=unavailable
columns: unavailable
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=8.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=16B cardinality=unavailable
====
# Multiple nested collection values (at the same nesting level) where dictionary
@@ -388,7 +388,7 @@ l.l_receiptdate = '1994-08-24' and l.l_shipmode = 'RAIL' and l.l_returnflag = 'R
l.l_comment is null;
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=616.00MB mem-reservation=56.00MB
+| Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -439,6 +439,6 @@ PLAN-ROOT SINK
extrapolated-rows=disabled
parquet statistics predicates on l: l.l_shipdate = '1994-08-19', l.l_receiptdate = '1994-08-24', l.l_shipmode = 'RAIL', l.l_returnflag = 'R'
parquet dictionary predicates on l: l.l_shipdate = '1994-08-19', l.l_receiptdate = '1994-08-24', l.l_shipmode = 'RAIL', l.l_returnflag = 'R'
- mem-estimate=616.00MB mem-reservation=56.00MB
+ mem-estimate=176.00MB mem-reservation=0B
tuple-ids=0 row-size=50B cardinality=150000
====
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/partition-pruning.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/partition-pruning.test b/testdata/workloads/functional-planner/queries/PlannerTest/partition-pruning.test
index 5cbba7b..610136d 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/partition-pruning.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/partition-pruning.test
@@ -4,7 +4,7 @@ select * from functional.stringpartitionkey
where string_col=cast("2009-01-01 00:00:00" as timestamp);
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=8.00KB
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -15,6 +15,6 @@ PLAN-ROOT SINK
partitions: 1/1 rows=1
columns: all
extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=8.00KB
+ mem-estimate=32.00MB mem-reservation=0B
tuple-ids=0 row-size=20B cardinality=1
====
[5/9] impala git commit: Revert IMPALA-4835 and dependent changes
Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/request-context.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/request-context.cc b/be/src/runtime/io/request-context.cc
index dec6aa6..287f53a 100644
--- a/be/src/runtime/io/request-context.cc
+++ b/be/src/runtime/io/request-context.cc
@@ -17,122 +17,74 @@
#include "runtime/io/disk-io-mgr-internal.h"
-#include "runtime/exec-env.h"
-
#include "common/names.h"
using namespace impala;
using namespace impala::io;
-BufferDescriptor::BufferDescriptor(DiskIoMgr* io_mgr,
- RequestContext* reader, ScanRange* scan_range, uint8_t* buffer,
- int64_t buffer_len)
- : io_mgr_(io_mgr),
- reader_(reader),
- scan_range_(scan_range),
- buffer_(buffer),
- buffer_len_(buffer_len) {
- DCHECK(io_mgr != nullptr);
- DCHECK(scan_range != nullptr);
- DCHECK(buffer != nullptr);
- DCHECK_GE(buffer_len, 0);
-}
+void RequestContext::Cancel(const Status& status) {
+ DCHECK(!status.ok());
-BufferDescriptor::BufferDescriptor(DiskIoMgr* io_mgr, RequestContext* reader,
- ScanRange* scan_range, BufferPool::ClientHandle* bp_client,
- BufferPool::BufferHandle handle) :
- io_mgr_(io_mgr),
- reader_(reader),
- scan_range_(scan_range),
- buffer_(handle.data()),
- buffer_len_(handle.len()),
- bp_client_(bp_client),
- handle_(move(handle)) {
- DCHECK(io_mgr != nullptr);
- DCHECK(scan_range != nullptr);
- DCHECK(bp_client_->is_registered());
- DCHECK(handle_.is_open());
-}
-
-void RequestContext::FreeBuffer(BufferDescriptor* buffer) {
- DCHECK(buffer->buffer_ != nullptr);
- if (!buffer->is_cached() && !buffer->is_client_buffer()) {
- // Only buffers that were allocated by DiskIoMgr need to be freed.
- ExecEnv::GetInstance()->buffer_pool()->FreeBuffer(
- buffer->bp_client_, &buffer->handle_);
- }
- buffer->buffer_ = nullptr;
-}
-
-// Cancellation of a RequestContext requires coordination from multiple threads that may
-// hold references to the context:
-// 1. Disk threads that are currently processing a range for this context.
-// 2. Caller threads that are waiting in GetNext().
-//
-// Each thread that currently has a reference to the request context must notice the
-// cancel, cancel any pending operations involving the context and remove the contxt from
-// tracking structures. Once no more operations are pending on the context and no more
-// I/O mgr threads hold references to the context, the context can be marked inactive
-// (see CancelAndMarkInactive()), after which the owner of the context object can free
-// it.
-//
-// The steps are:
-// 1. Cancel() will immediately set the context in the Cancelled state. This prevents any
-// other thread from adding more ready buffers to the context (they all take a lock and
-// check the state before doing so), or any write ranges to the context.
-// 2. Cancel() will call Cancel() on each ScanRange that is not yet complete, unblocking
-// any threads in GetNext(). If there was no prior error for a scan range, any reads from
-// that scan range will return a CANCELLED Status. Cancel() also invokes callbacks for
-// all WriteRanges with a CANCELLED Status.
-// 3. Disk threads notice the context is cancelled either when picking the next context
-// to process or when they try to enqueue a ready buffer. Upon noticing the cancelled
-// state, removes the context from the disk queue. The last thread per disk then calls
-// DecrementDiskRefCount(). After the last disk thread has called DecrementDiskRefCount(),
-// cancellation is done and it is safe to unregister the context.
-void RequestContext::Cancel() {
// Callbacks are collected in this vector and invoked while no lock is held.
vector<WriteRange::WriteDoneCallback> write_callbacks;
{
- unique_lock<mutex> lock(lock_);
+ lock_guard<mutex> lock(lock_);
DCHECK(Validate()) << endl << DebugString();
// Already being cancelled
if (state_ == RequestContext::Cancelled) return;
+ DCHECK(status_.ok());
+ status_ = status;
+
// The reader will be put into a cancelled state until call cleanup is complete.
state_ = RequestContext::Cancelled;
- // Clear out all request ranges from queues for this reader. Cancel the scan
- // ranges and invoke the write range callbacks to propagate the cancellation.
- for (ScanRange* range : active_scan_ranges_) range->CancelInternal(Status::CANCELLED);
- active_scan_ranges_.clear();
- for (PerDiskState& disk_state : disk_states_) {
- RequestRange* range;
- while ((range = disk_state.in_flight_ranges()->Dequeue()) != nullptr) {
- if (range->request_type() == RequestType::WRITE) {
+ // Cancel all scan ranges for this reader. Each range could be one one of
+ // four queues.
+ for (int i = 0; i < disk_states_.size(); ++i) {
+ RequestContext::PerDiskState& state = disk_states_[i];
+ RequestRange* range = NULL;
+ while ((range = state.in_flight_ranges()->Dequeue()) != NULL) {
+ if (range->request_type() == RequestType::READ) {
+ static_cast<ScanRange*>(range)->Cancel(status);
+ } else {
+ DCHECK(range->request_type() == RequestType::WRITE);
write_callbacks.push_back(static_cast<WriteRange*>(range)->callback_);
}
}
- while (disk_state.unstarted_scan_ranges()->Dequeue() != nullptr);
+
+ ScanRange* scan_range;
+ while ((scan_range = state.unstarted_scan_ranges()->Dequeue()) != NULL) {
+ scan_range->Cancel(status);
+ }
WriteRange* write_range;
- while ((write_range = disk_state.unstarted_write_ranges()->Dequeue()) != nullptr) {
+ while ((write_range = state.unstarted_write_ranges()->Dequeue()) != NULL) {
write_callbacks.push_back(write_range->callback_);
}
}
- // Clear out the lists of scan ranges.
- while (ready_to_start_ranges_.Dequeue() != nullptr);
- while (cached_ranges_.Dequeue() != nullptr);
- // Ensure that the reader is scheduled on all disks (it may already be scheduled on
- // some). The disk threads will notice that the context is cancelled and do any
- // required cleanup for the disk state.
+ ScanRange* range = NULL;
+ while ((range = ready_to_start_ranges_.Dequeue()) != NULL) {
+ range->Cancel(status);
+ }
+ while ((range = blocked_ranges_.Dequeue()) != NULL) {
+ range->Cancel(status);
+ }
+ while ((range = cached_ranges_.Dequeue()) != NULL) {
+ range->Cancel(status);
+ }
+
+ // Schedule reader on all disks. The disks will notice it is cancelled and do any
+ // required cleanup
for (int i = 0; i < disk_states_.size(); ++i) {
- disk_states_[i].ScheduleContext(lock, this, i);
+ RequestContext::PerDiskState& state = disk_states_[i];
+ state.ScheduleContext(this, i);
}
}
for (const WriteRange::WriteDoneCallback& write_callback: write_callbacks) {
- write_callback(Status::CANCELLED);
+ write_callback(status_);
}
// Signal reader and unblock the GetNext/Read thread. That read will fail with
@@ -141,7 +93,7 @@ void RequestContext::Cancel() {
}
void RequestContext::CancelAndMarkInactive() {
- Cancel();
+ Cancel(Status::CANCELLED);
boost::unique_lock<boost::mutex> l(lock_);
DCHECK_NE(state_, Inactive);
@@ -150,76 +102,54 @@ void RequestContext::CancelAndMarkInactive() {
// Wait until the ranges finish up.
while (num_disks_with_ranges_ > 0) disks_complete_cond_var_.Wait(l);
- // Validate that no ranges are active.
- DCHECK_EQ(0, active_scan_ranges_.size()) << endl << DebugString();
-
- // Validate that no threads are active and the context is not queued.
- for (const PerDiskState& disk_state : disk_states_) {
- DCHECK_EQ(0, disk_state.in_flight_ranges()->size()) << endl << DebugString();
- DCHECK_EQ(0, disk_state.unstarted_scan_ranges()->size()) << endl << DebugString();
- DCHECK_EQ(0, disk_state.num_threads_in_op()) << endl << DebugString();
- DCHECK(!disk_state.is_on_queue()) << endl << DebugString();
- }
+ // Validate that no buffers were leaked from this context.
+ DCHECK_EQ(num_buffers_in_reader_.Load(), 0) << endl << DebugString();
+ DCHECK_EQ(num_used_buffers_.Load(), 0) << endl << DebugString();
DCHECK(Validate()) << endl << DebugString();
state_ = Inactive;
}
-void RequestContext::AddRangeToDisk(const unique_lock<mutex>& lock,
- RequestRange* range, ScheduleMode schedule_mode) {
- DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
- DCHECK_EQ(state_, Active) << DebugString();
- PerDiskState* disk_state = &disk_states_[range->disk_id()];
- if (disk_state->done()) {
- DCHECK_EQ(disk_state->num_remaining_ranges(), 0);
- disk_state->set_done(false);
+void RequestContext::AddRequestRange(
+ RequestRange* range, bool schedule_immediately) {
+ // DCHECK(lock_.is_locked()); // TODO: boost should have this API
+ RequestContext::PerDiskState& state = disk_states_[range->disk_id()];
+ if (state.done()) {
+ DCHECK_EQ(state.num_remaining_ranges(), 0);
+ state.set_done(false);
++num_disks_with_ranges_;
}
+
+ bool schedule_context;
if (range->request_type() == RequestType::READ) {
ScanRange* scan_range = static_cast<ScanRange*>(range);
- if (schedule_mode == ScheduleMode::IMMEDIATELY) {
- ScheduleScanRange(lock, scan_range);
- } else if (schedule_mode == ScheduleMode::UPON_GETNEXT) {
- disk_state->unstarted_scan_ranges()->Enqueue(scan_range);
+ if (schedule_immediately) {
+ ScheduleScanRange(scan_range);
+ } else {
+ state.unstarted_scan_ranges()->Enqueue(scan_range);
num_unstarted_scan_ranges_.Add(1);
- // If there's no 'next_scan_range_to_start', schedule this RequestContext so that
- // one of the 'unstarted_scan_ranges' will become the 'next_scan_range_to_start'.
- if (disk_state->next_scan_range_to_start() == nullptr) {
- disk_state->ScheduleContext(lock, this, range->disk_id());
- }
}
+ // If next_scan_range_to_start is NULL, schedule this RequestContext so that it will
+ // be set. If it's not NULL, this context will be scheduled when GetNextRange() is
+ // invoked.
+ schedule_context = state.next_scan_range_to_start() == NULL;
} else {
DCHECK(range->request_type() == RequestType::WRITE);
- DCHECK(schedule_mode == ScheduleMode::IMMEDIATELY) << static_cast<int>(schedule_mode);
+ DCHECK(!schedule_immediately);
WriteRange* write_range = static_cast<WriteRange*>(range);
- disk_state->unstarted_write_ranges()->Enqueue(write_range);
+ state.unstarted_write_ranges()->Enqueue(write_range);
- // Ensure that the context is scheduled so that the write range gets picked up.
- // ScheduleContext() has no effect if already scheduled, so this is safe to do always.
- disk_state->ScheduleContext(lock, this, range->disk_id());
+ // ScheduleContext() has no effect if the context is already scheduled,
+ // so this is safe.
+ schedule_context = true;
}
- ++disk_state->num_remaining_ranges();
-}
-
-void RequestContext::AddActiveScanRangeLocked(
- const unique_lock<mutex>& lock, ScanRange* range) {
- DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
- DCHECK(state_ == Active);
- active_scan_ranges_.insert(range);
-}
-void RequestContext::RemoveActiveScanRange(ScanRange* range) {
- unique_lock<mutex> lock(lock_);
- RemoveActiveScanRangeLocked(lock, range);
+ if (schedule_context) state.ScheduleContext(this, range->disk_id());
+ ++state.num_remaining_ranges();
}
-void RequestContext::RemoveActiveScanRangeLocked(
- const unique_lock<mutex>& lock, ScanRange* range) {
- DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
- active_scan_ranges_.erase(range);
-}
-
-RequestContext::RequestContext(DiskIoMgr* parent, int num_disks)
- : parent_(parent), disk_states_(num_disks) {}
+RequestContext::RequestContext(
+ DiskIoMgr* parent, int num_disks, MemTracker* tracker)
+ : parent_(parent), mem_tracker_(tracker), disk_states_(num_disks) {}
// Dumps out request context information. Lock should be taken by caller
string RequestContext::DebugString() const {
@@ -229,9 +159,13 @@ string RequestContext::DebugString() const {
if (state_ == RequestContext::Cancelled) ss << "Cancelled";
if (state_ == RequestContext::Active) ss << "Active";
if (state_ != RequestContext::Inactive) {
- ss << " #disk_with_ranges=" << num_disks_with_ranges_
- << " #disks=" << num_disks_with_ranges_
- << " #active scan ranges=" << active_scan_ranges_.size();
+ ss << " status_=" << (status_.ok() ? "OK" : status_.GetDetail())
+ << " #ready_buffers=" << num_ready_buffers_.Load()
+ << " #used_buffers=" << num_used_buffers_.Load()
+ << " #num_buffers_in_reader=" << num_buffers_in_reader_.Load()
+ << " #finished_scan_ranges=" << num_finished_ranges_.Load()
+ << " #disk_with_ranges=" << num_disks_with_ranges_
+ << " #disks=" << num_disks_with_ranges_;
for (int i = 0; i < disk_states_.size(); ++i) {
ss << endl << " " << i << ": "
<< "is_on_queue=" << disk_states_[i].is_on_queue()
@@ -254,6 +188,16 @@ bool RequestContext::Validate() const {
return false;
}
+ if (num_used_buffers_.Load() < 0) {
+ LOG(WARNING) << "num_used_buffers_ < 0: #used=" << num_used_buffers_.Load();
+ return false;
+ }
+
+ if (num_ready_buffers_.Load() < 0) {
+ LOG(WARNING) << "num_ready_buffers_ < 0: #used=" << num_ready_buffers_.Load();
+ return false;
+ }
+
int total_unstarted_ranges = 0;
for (int i = 0; i < disk_states_.size(); ++i) {
const PerDiskState& state = disk_states_[i];
@@ -331,8 +275,8 @@ bool RequestContext::Validate() const {
LOG(WARNING) << "Reader cancelled but has ready to start ranges.";
return false;
}
- if (!active_scan_ranges_.empty()) {
- LOG(WARNING) << "Reader cancelled but has active ranges.";
+ if (!blocked_ranges_.empty()) {
+ LOG(WARNING) << "Reader cancelled but has blocked ranges.";
return false;
}
}
@@ -340,11 +284,10 @@ bool RequestContext::Validate() const {
return true;
}
-void RequestContext::PerDiskState::ScheduleContext(const unique_lock<mutex>& context_lock,
+void RequestContext::PerDiskState::ScheduleContext(
RequestContext* context, int disk_id) {
- DCHECK(context_lock.mutex() == &context->lock_ && context_lock.owns_lock());
- if (is_on_queue_.Load() == 0 && !done_) {
- is_on_queue_.Store(1);
+ if (!is_on_queue_ && !done_) {
+ is_on_queue_ = true;
context->parent_->disk_queues_[disk_id]->EnqueueContext(context);
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/request-context.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/request-context.h b/be/src/runtime/io/request-context.h
index b028596..fd68669 100644
--- a/be/src/runtime/io/request-context.h
+++ b/be/src/runtime/io/request-context.h
@@ -23,13 +23,10 @@
namespace impala {
namespace io {
-
-// Mode argument for AddRangeToDisk().
-enum class ScheduleMode {
- IMMEDIATELY, UPON_GETNEXT, BY_CALLER
-};
/// A request context is used to group together I/O requests belonging to a client of the
-/// I/O manager for management and scheduling.
+/// I/O manager for management and scheduling. For most I/O manager clients it is an
+/// opaque pointer, but some clients may need to include this header, e.g. to make the
+/// unique_ptr<DiskIoRequestContext> destructor work correctly.
///
/// Implementation Details
/// ======================
@@ -37,109 +34,56 @@ enum class ScheduleMode {
/// maintains state across all disks as well as per disk state.
/// The unit for an IO request is a RequestRange, which may be a ScanRange or a
/// WriteRange.
-/// A scan range for the reader is on one of six states:
-/// 1) PerDiskState's 'unstarted_scan_ranges_': This range has only been queued
+/// A scan range for the reader is on one of five states:
+/// 1) PerDiskState's unstarted_ranges: This range has only been queued
/// and nothing has been read from it.
-/// 2) RequestContext's 'ready_to_start_ranges_': This range is about to be started.
-/// As soon as the reader picks it up, it will move to the 'in_flight_ranges_'
+/// 2) RequestContext's ready_to_start_ranges_: This range is about to be started.
+/// As soon as the reader picks it up, it will move to the in_flight_ranges
/// queue.
-/// 3) PerDiskState's 'in_flight_ranges_': This range is being processed and will
+/// 3) PerDiskState's in_flight_ranges: This range is being processed and will
/// be read from the next time a disk thread picks it up in GetNextRequestRange()
-/// 4) The ScanRange is blocked waiting for buffers because it does not have any unused
-/// buffers to read data into. It is unblocked when a client adds new buffers via
-/// AllocateBuffersForRange() or returns existing buffers via ReturnBuffer().
-/// ScanRanges in this state are identified by 'blocked_on_buffer_' == true.
-/// 5) ScanRange is cached and in the 'cached_ranges_' queue.
-/// 6) Inactive - either all the data for the range was returned or the range was
-/// cancelled. I.e. ScanRange::eosr_ is true or ScanRange::cancel_status_ != OK.
-///
-/// If the scan range is read and does not get blocked waiting for buffers, the
+/// 4) ScanRange's outgoing ready buffers is full. We can't read for this range
+/// anymore. We need the caller to pull a buffer off which will put this in
+/// the in_flight_ranges queue. These ranges are in the RequestContext's
+/// blocked_ranges_ queue.
+/// 5) ScanRange is cached and in the cached_ranges_ queue.
+//
+/// If the scan range is read and does not get blocked on the outgoing queue, the
/// transitions are: 1 -> 2 -> 3.
/// If the scan range does get blocked, the transitions are
/// 1 -> 2 -> 3 -> (4 -> 3)*
-///
-/// In the case of a cached scan range, the range is immediately put in 'cached_ranges_'.
+//
+/// In the case of a cached scan range, the range is immediately put in cached_ranges_.
/// When the caller asks for the next range to process, we first pull ranges from
-/// the 'cache_ranges_' queue. If the range was cached, the range is removed and
+/// the cache_ranges_ queue. If the range was cached, the range is removed and
/// done (ranges are either entirely cached or not at all). If the cached read attempt
/// fails, we put the range in state 1.
-///
-/// All scan ranges in states 1-5 are tracked in 'active_scan_ranges_' so that they can be
-/// cancelled when the RequestContext is cancelled. Scan ranges are removed from
-/// 'active_scan_ranges_' during their transition to state 6.
-///
-/// A write range for a context may be in one of two queues:
-/// 1) 'unstarted_write_ranges_': Ranges that have been queued but not processed.
-/// 2) 'in_flight_ranges_': The write range is ready to be processed by the next disk
-/// thread that picks it up in GetNextRequestRange().
-///
+//
+/// A write range for a context may be in one of two lists:
+/// 1) unstarted_write_ranges_ : Ranges that have been queued but not processed.
+/// 2) in_flight_ranges_: The write range is ready to be processed by the next disk thread
+/// that picks it up in GetNextRequestRange().
+//
/// AddWriteRange() adds WriteRanges for a disk.
/// It is the responsibility of the client to pin the data to be written via a WriteRange
/// in memory. After a WriteRange has been written, a callback is invoked to inform the
/// client that the write has completed.
-///
+//
/// An important assumption is that write does not exceed the maximum read size and that
/// the entire range is written when the write request is handled. (In other words, writes
/// are not broken up.)
-///
+//
/// When a RequestContext is processed by a disk thread in GetNextRequestRange(),
/// a write range is always removed from the list of unstarted write ranges and appended
/// to the in_flight_ranges_ queue. This is done to alternate reads and writes - a read
-/// that is scheduled (by calling GetNextUnstartedRange()) is always followed by a write
-/// (if one exists). And since at most one WriteRange can be present in in_flight_ranges_
-/// at any time (once a write range is returned from GetNetxRequestRange() it is completed
-/// and not re-enqueued), a scan range scheduled via a call to GetNextUnstartedRange() can
-/// be queued up behind at most one write range.
+/// that is scheduled (by calling GetNextRange()) is always followed by a write (if one
+/// exists). And since at most one WriteRange can be present in in_flight_ranges_ at any
+/// time (once a write range is returned from GetNetxRequestRange() it is completed an
+/// not re-enqueued), a scan range scheduled via a call to GetNextRange() can be queued up
+/// behind at most one write range.
class RequestContext {
public:
- ~RequestContext() {
- DCHECK_EQ(state_, Inactive) << "Must be unregistered. " << DebugString();
- }
-
- /// Cancel the context asynchronously. All outstanding requests are cancelled
- /// asynchronously. This does not need to be called if the context finishes normally.
- /// Calling GetNext() on any scan ranges belonging to this RequestContext will return
- /// CANCELLED (or another error, if an error was encountered for that scan range before
- /// it is cancelled).
- void Cancel();
-
- bool IsCancelled() {
- boost::unique_lock<boost::mutex> lock(lock_);
- return state_ == Cancelled;
- }
-
- int64_t bytes_read_local() const { return bytes_read_local_.Load(); }
- int64_t bytes_read_short_circuit() const { return bytes_read_short_circuit_.Load(); }
- int64_t bytes_read_dn_cache() const { return bytes_read_dn_cache_.Load(); }
- int num_remote_ranges() const { return num_remote_ranges_.Load(); }
- int64_t unexpected_remote_bytes() const { return unexpected_remote_bytes_.Load(); }
-
- int cached_file_handles_hit_count() const {
- return cached_file_handles_hit_count_.Load();
- }
-
- int cached_file_handles_miss_count() const {
- return cached_file_handles_miss_count_.Load();
- }
-
- void set_bytes_read_counter(RuntimeProfile::Counter* bytes_read_counter) {
- bytes_read_counter_ = bytes_read_counter;
- }
-
- void set_read_timer(RuntimeProfile::Counter* read_timer) { read_timer_ = read_timer; }
-
- void set_open_file_timer(RuntimeProfile::Counter* open_file_timer) {
- open_file_timer_ = open_file_timer;
- }
-
- void set_active_read_thread_counter(
- RuntimeProfile::Counter* active_read_thread_counter) {
- active_read_thread_counter_ = active_read_thread_counter;
- }
-
- void set_disks_accessed_bitmap(RuntimeProfile::Counter* disks_accessed_bitmap) {
- disks_accessed_bitmap_ = disks_accessed_bitmap;
- }
+ ~RequestContext() { DCHECK_EQ(state_, Inactive) << "Must be unregistered."; }
private:
DISALLOW_COPY_AND_ASSIGN(RequestContext);
@@ -162,19 +106,13 @@ class RequestContext {
Inactive,
};
- RequestContext(DiskIoMgr* parent, int num_disks);
-
- /// Cleans up a buffer. If the buffer was allocated with AllocateBuffersForRange(),
- /// frees the buffer. Otherwise (e.g. a client or HDFS cache buffer), just prepares the
- /// descriptor to be destroyed. After this is called, buffer->buffer() is NULL.
- /// Does not acquire 'lock_'.
- void FreeBuffer(BufferDescriptor* buffer);
+ RequestContext(DiskIoMgr* parent, int num_disks, MemTracker* tracker);
/// Decrements the number of active disks for this reader. If the disk count
/// goes to 0, the disk complete condition variable is signaled.
- /// 'lock_' must be held via 'lock'.
- void DecrementDiskRefCount(const boost::unique_lock<boost::mutex>& lock) {
- DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
+ /// Reader lock must be taken before this call.
+ void DecrementDiskRefCount() {
+ // boost doesn't let us dcheck that the reader lock is taken
DCHECK_GT(num_disks_with_ranges_, 0);
if (--num_disks_with_ranges_ == 0) {
disks_complete_cond_var_.NotifyAll();
@@ -191,48 +129,25 @@ class RequestContext {
/// Adds range to in_flight_ranges, scheduling this reader on the disk threads
/// if necessary.
- /// 'lock_' must be held via 'lock'. Only valid to call if this context is active.
- void ScheduleScanRange(const boost::unique_lock<boost::mutex>& lock, ScanRange* range) {
- DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
+ /// Reader lock must be taken before this.
+ void ScheduleScanRange(ScanRange* range) {
DCHECK_EQ(state_, Active);
- DCHECK(range != nullptr);
+ DCHECK(range != NULL);
RequestContext::PerDiskState& state = disk_states_[range->disk_id()];
state.in_flight_ranges()->Enqueue(range);
- state.ScheduleContext(lock, this, range->disk_id());
+ state.ScheduleContext(this, range->disk_id());
}
+ /// Cancels the context with status code 'status'
+ void Cancel(const Status& status);
+
/// Cancel the context if not already cancelled, wait for all scan ranges to finish
/// and mark the context as inactive, after which it cannot be used.
void CancelAndMarkInactive();
- /// Adds a request range to the appropriate disk state. 'schedule_mode' controls which
- /// queue the range is placed in. This RequestContext is scheduled on the disk state
- /// if required by 'schedule_mode'.
- ///
- /// Write ranges must always have 'schedule_mode' IMMEDIATELY and are added to the
- /// 'unstarted_write_ranges_' queue, from which they will be asynchronously moved to the
- /// 'in_flight_ranges_' queue.
- ///
- /// Scan ranges can have different 'schedule_mode' values. If IMMEDIATELY, the range is
- /// immediately added to the 'in_flight_ranges_' queue where it will be processed
- /// asynchronously by disk threads. If UPON_GETNEXT, the range is added to the
- /// 'unstarted_ranges_' queue, from which it can be returned to a client by
- /// DiskIoMgr::GetNextUnstartedRange(). If BY_CALLER, the scan range is not added to
- /// any queues. The range will be scheduled later as a separate step, e.g. when it is
- /// unblocked by adding buffers to it. Caller must hold 'lock_' via 'lock'.
- void AddRangeToDisk(const boost::unique_lock<boost::mutex>& lock, RequestRange* range,
- ScheduleMode schedule_mode);
-
- /// Adds an active range to 'active_scan_ranges_'
- void AddActiveScanRangeLocked(
- const boost::unique_lock<boost::mutex>& lock, ScanRange* range);
-
- /// Removes the range from 'active_scan_ranges_'. Called by ScanRange after eos or
- /// cancellation. If calling the Locked version, the caller must hold
- /// 'lock_'. Otherwise the function will acquire 'lock_'.
- void RemoveActiveScanRange(ScanRange* range);
- void RemoveActiveScanRangeLocked(
- const boost::unique_lock<boost::mutex>& lock, ScanRange* range);
+ /// Adds request range to disk queue for this request context. Currently,
+ /// schedule_immediately must be false is RequestRange is a write range.
+ void AddRequestRange(RequestRange* range, bool schedule_immediately);
/// Validates invariants of reader. Reader lock must be taken beforehand.
bool Validate() const;
@@ -243,6 +158,9 @@ class RequestContext {
/// Parent object
DiskIoMgr* const parent_;
+ /// Memory used for this reader. This is unowned by this object.
+ MemTracker* const mem_tracker_;
+
/// Total bytes read for this reader
RuntimeProfile::Counter* bytes_read_counter_ = nullptr;
@@ -272,6 +190,13 @@ class RequestContext {
/// Total number of bytes from remote reads that were expected to be local.
AtomicInt64 unexpected_remote_bytes_{0};
+ /// The number of buffers that have been returned to the reader (via GetNext) that the
+ /// reader has not returned. Only included for debugging and diagnostics.
+ AtomicInt32 num_buffers_in_reader_{0};
+
+ /// The number of scan ranges that have been completed for this reader.
+ AtomicInt32 num_finished_ranges_{0};
+
/// The number of scan ranges that required a remote read, updated at the end of each
/// range scan. Only used for diagnostics.
AtomicInt32 num_remote_ranges_{0};
@@ -286,6 +211,17 @@ class RequestContext {
/// Total number of file handle opens where the file handle was not in the cache
AtomicInt32 cached_file_handles_miss_count_{0};
+ /// The number of buffers that are being used for this reader. This is the sum
+ /// of all buffers in ScanRange queues and buffers currently being read into (i.e. about
+ /// to be queued). This includes both IOMgr-allocated buffers and client-provided
+ /// buffers.
+ AtomicInt32 num_used_buffers_{0};
+
+ /// The total number of ready buffers across all ranges. Ready buffers are buffers
+ /// that have been read from disk but not retrieved by the caller.
+ /// This is the sum of all queued buffers in all ranges for this reader context.
+ AtomicInt32 num_ready_buffers_{0};
+
/// All fields below are accessed by multiple threads and the lock needs to be
/// taken before accessing them. Must be acquired before ScanRange::lock_ if both
/// are held simultaneously.
@@ -294,16 +230,8 @@ class RequestContext {
/// Current state of the reader
State state_ = Active;
- /// Scan ranges that have been added to the IO mgr for this context. Ranges can only
- /// be added when 'state_' is Active. When this context is cancelled, Cancel() is
- /// called for all the active ranges. If a client attempts to add a range while
- /// 'state_' is Cancelled, the range is not added to this list and Status::CANCELLED
- /// is returned to the client. This ensures that all active ranges are cancelled as a
- /// result of RequestContext cancellation.
- /// Ranges can be cancelled or hit eos non-atomically with their removal from this set,
- /// so eos or cancelled ranges may be temporarily present here. Cancelling these ranges
- /// a second time or cancelling after eos is safe and has no effect.
- boost::unordered_set<ScanRange*> active_scan_ranges_;
+ /// Status of this reader. Set to non-ok if cancelled.
+ Status status_;
/// The number of disks with scan ranges remaining (always equal to the sum of
/// disks with ranges).
@@ -312,18 +240,21 @@ class RequestContext {
/// This is the list of ranges that are expected to be cached on the DN.
/// When the reader asks for a new range (GetNextScanRange()), we first
/// return ranges from this list.
- InternalList<ScanRange> cached_ranges_;
+ InternalQueue<ScanRange> cached_ranges_;
/// A list of ranges that should be returned in subsequent calls to
- /// GetNextUnstartedRange().
+ /// GetNextRange.
/// There is a trade-off with when to populate this list. Populating it on
- /// demand means consumers need to wait (happens in DiskIoMgr::GetNextUnstartedRange()).
+ /// demand means consumers need to wait (happens in DiskIoMgr::GetNextRange()).
/// Populating it preemptively means we make worse scheduling decisions.
/// We currently populate one range per disk.
/// TODO: think about this some more.
- InternalList<ScanRange> ready_to_start_ranges_;
+ InternalQueue<ScanRange> ready_to_start_ranges_;
ConditionVariable ready_to_start_ranges_cv_; // used with lock_
+ /// Ranges that are blocked due to back pressure on outgoing buffers.
+ InternalQueue<ScanRange> blocked_ranges_;
+
/// Condition variable for UnregisterContext() to wait for all disks to complete
ConditionVariable disks_complete_cond_var_;
@@ -342,9 +273,21 @@ class RequestContext {
next_scan_range_to_start_ = range;
}
- bool is_on_queue() const { return is_on_queue_.Load() != 0; }
+ /// We need to have a memory barrier to prevent this load from being reordered
+ /// with num_threads_in_op(), since these variables are set without the reader
+ /// lock taken
+ bool is_on_queue() const {
+ bool b = is_on_queue_;
+ __sync_synchronize();
+ return b;
+ }
- int num_threads_in_op() const { return num_threads_in_op_.Load(); }
+ int num_threads_in_op() const {
+ int v = num_threads_in_op_.Load();
+ // TODO: determine whether this barrier is necessary for any callsites.
+ AtomicUtil::MemoryBarrier();
+ return v;
+ }
const InternalQueue<ScanRange>* unstarted_scan_ranges() const {
return &unstarted_scan_ranges_;
@@ -363,41 +306,26 @@ class RequestContext {
InternalQueue<RequestRange>* in_flight_ranges() { return &in_flight_ranges_; }
/// Schedules the request context on this disk if it's not already on the queue.
- /// context->lock_ must be held by the caller via 'context_lock'.
- void ScheduleContext(const boost::unique_lock<boost::mutex>& context_lock,
- RequestContext* context, int disk_id);
-
- /// Increment the count of disk threads that have a reference to this context. These
- /// threads do not hold any locks while reading from HDFS, so we need to prevent the
- /// RequestContext from being destroyed underneath them.
- ///
- /// The caller does not need to hold 'lock_', so this can execute concurrently with
- /// itself and DecrementDiskThread().
- void IncrementDiskThreadAndDequeue() {
- /// Incrementing 'num_threads_in_op_' first so that there is no window when other
- /// threads see 'is_on_queue_ == num_threads_in_op_ == 0' and think there are no
- /// references left to this context.
+ /// Context lock must be taken before this.
+ void ScheduleContext(RequestContext* context, int disk_id);
+
+ /// Increment the ref count on reader. We need to track the number of threads per
+ /// reader per disk that are in the unlocked hdfs read code section. This is updated
+ /// by multiple threads without a lock so we need to use an atomic int.
+ void IncrementRequestThreadAndDequeue() {
num_threads_in_op_.Add(1);
- is_on_queue_.Store(0);
+ is_on_queue_ = false;
}
- /// Decrement the count of disks threads with a reference to this context. Does final
- /// cleanup if the context is cancelled and this is the last thread for the disk.
- /// context->lock_ must be held by the caller via 'context_lock'.
- void DecrementDiskThread(const boost::unique_lock<boost::mutex>& context_lock,
- RequestContext* context) {
- DCHECK(context_lock.mutex() == &context->lock_ && context_lock.owns_lock());
- num_threads_in_op_.Add(-1);
-
- if (context->state_ != Cancelled) {
- DCHECK_EQ(context->state_, Active);
- return;
- }
- // The state is cancelled, check to see if we're the last thread to touch the
- // context on this disk. We need to load 'is_on_queue_' and 'num_threads_in_op_'
- // in this order to avoid a race with IncrementDiskThreadAndDequeue().
- if (is_on_queue_.Load() == 0 && num_threads_in_op_.Load() == 0 && !done_) {
- context->DecrementDiskRefCount(context_lock);
+ void DecrementRequestThread() { num_threads_in_op_.Add(-1); }
+
+ /// Decrement request thread count and do final cleanup if this is the last
+ /// thread. RequestContext lock must be taken before this.
+ void DecrementRequestThreadAndCheckDone(RequestContext* context) {
+ num_threads_in_op_.Add(-1); // Also acts as a barrier.
+ if (!is_on_queue_ && num_threads_in_op_.Load() == 0 && !done_) {
+ // This thread is the last one for this reader on this disk, do final cleanup
+ context->DecrementDiskRefCount();
done_ = true;
}
}
@@ -410,12 +338,7 @@ class RequestContext {
bool done_ = true;
/// For each disk, keeps track if the context is on this disk's queue, indicating
- /// the disk must do some work for this context. 1 means that the context is on the
- /// disk queue, 0 means that it's not on the queue (either because it has on ranges
- /// active for the disk or because a disk thread dequeued the context and is
- /// currently processing a request).
- ///
- /// The disk needs to do work in 4 cases:
+ /// the disk must do some work for this context. The disk needs to do work in 4 cases:
/// 1) in_flight_ranges is not empty, the disk needs to read for this reader.
/// 2) next_range_to_start is NULL, the disk needs to prepare a scan range to be
/// read next.
@@ -426,15 +349,7 @@ class RequestContext {
/// useful that can be done. If there's nothing useful, the disk queue will wake up
/// and then remove the reader from the queue. Doing this causes thrashing of the
/// threads.
- ///
- /// This variable is important during context cancellation because it indicates
- /// whether a queue has a reference to the context that must be released before
- /// the context is considered unregistered. Atomically set to false after
- /// incrementing 'num_threads_in_op_' when dequeueing so that there is no window
- /// when other threads see 'is_on_queue_ == num_threads_in_op_ == 0' and think there
- /// are no references left to this context.
- /// TODO: this could be combined with 'num_threads_in_op_' to be a single refcount.
- AtomicInt32 is_on_queue_{0};
+ bool is_on_queue_ = false;
/// For each disks, the number of request ranges that have not been fully read.
/// In the non-cancellation path, this will hit 0, and done will be set to true
@@ -448,7 +363,7 @@ class RequestContext {
/// Queue of pending IO requests for this disk in the order that they will be
/// processed. A ScanRange is added to this queue when it is returned in
- /// GetNextUnstartedRange(), or when it is added with schedule_mode == IMMEDIATELY.
+ /// GetNextRange(), or when it is added with schedule_immediately = true.
/// A WriteRange is added to this queue from unstarted_write_ranges_ for each
/// invocation of GetNextRequestRange() in WorkLoop().
/// The size of this queue is always less than or equal to num_remaining_ranges.
@@ -464,11 +379,11 @@ class RequestContext {
/// range to ready_to_start_ranges_.
ScanRange* next_scan_range_to_start_ = nullptr;
- /// For each disk, the number of disk threads issuing the underlying read/write on
- /// behalf of this context. There are a few places where we release the context lock,
- /// do some work, and then grab the lock again. Because we don't hold the lock for
- /// the entire operation, we need this ref count to keep track of which thread should
- /// do final resource cleanup during cancellation.
+ /// For each disk, the number of threads issuing the underlying read/write on behalf
+ /// of this context. There are a few places where we release the context lock, do some
+ /// work, and then grab the lock again. Because we don't hold the lock for the
+ /// entire operation, we need this ref count to keep track of which thread should do
+ /// final resource cleanup during cancellation.
/// Only the thread that sees the count at 0 should do the final cleanup.
AtomicInt32 num_threads_in_op_{0};
@@ -477,8 +392,7 @@ class RequestContext {
/// unstarted_read_ranges_ and unstarted_write_ranges_ to alternate between reads
/// and writes. (Otherwise, since next_scan_range_to_start is set
/// in GetNextRequestRange() whenever it is null, repeated calls to
- /// GetNextRequestRange() and GetNextUnstartedRange() may result in only reads being
- /// processed)
+ /// GetNextRequestRange() and GetNextRange() may result in only reads being processed)
InternalQueue<WriteRange> unstarted_write_ranges_;
};
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/request-ranges.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/request-ranges.h b/be/src/runtime/io/request-ranges.h
index 0b234ac..222f847 100644
--- a/be/src/runtime/io/request-ranges.h
+++ b/be/src/runtime/io/request-ranges.h
@@ -23,15 +23,14 @@
#include <boost/thread/mutex.hpp>
-#include "common/atomic.h"
#include "common/hdfs.h"
#include "common/status.h"
-#include "runtime/bufferpool/buffer-pool.h"
#include "util/condition-variable.h"
#include "util/internal-queue.h"
-#include "util/mem-range.h"
namespace impala {
+class MemTracker;
+
namespace io {
class DiskIoMgr;
class RequestContext;
@@ -56,19 +55,24 @@ class BufferDescriptor {
/// Returns the offset within the scan range that this buffer starts at
int64_t scan_range_offset() const { return scan_range_offset_; }
+ /// Transfer ownership of buffer memory from 'mem_tracker_' to 'dst' and set
+ /// 'mem_tracker_' to 'dst'. 'mem_tracker_' and 'dst' must be non-NULL. Does not
+ /// check memory limits on 'dst': the caller should check the memory limit if a
+ /// different memory limit may apply to 'dst'. If the buffer was a client-provided
+ /// buffer, transferring is not allowed.
+ /// TODO: IMPALA-3209: revisit this as part of scanner memory usage revamp.
+ void TransferOwnership(MemTracker* dst);
+
private:
friend class DiskIoMgr;
friend class ScanRange;
friend class RequestContext;
- /// Create a buffer descriptor for a new reader, range and data buffer.
- BufferDescriptor(DiskIoMgr* io_mgr, RequestContext* reader,
- ScanRange* scan_range, uint8_t* buffer, int64_t buffer_len);
-
- /// Create a buffer descriptor allocated from the buffer pool.
+ /// Create a buffer descriptor for a new reader, range and data buffer. The buffer
+ /// memory should already be accounted against 'mem_tracker'.
BufferDescriptor(DiskIoMgr* io_mgr, RequestContext* reader,
- ScanRange* scan_range, BufferPool::ClientHandle* bp_client,
- BufferPool::BufferHandle handle);
+ ScanRange* scan_range, uint8_t* buffer, int64_t buffer_len,
+ MemTracker* mem_tracker);
/// Return true if this is a cached buffer owned by HDFS.
bool is_cached() const;
@@ -82,11 +86,14 @@ class BufferDescriptor {
/// Reader that this buffer is for.
RequestContext* const reader_;
+ /// The current tracker this buffer is associated with. After initialisation,
+ /// NULL for cached buffers and non-NULL for all other buffers.
+ MemTracker* mem_tracker_;
+
/// Scan range that this buffer is for. Non-NULL when initialised.
ScanRange* const scan_range_;
- /// Buffer for the read contents. Must be set to NULL in RequestContext::FreeBuffer()
- /// before destruction of the descriptor.
+ /// buffer with the read contents
uint8_t* buffer_;
/// length of buffer_. For buffers from cached reads, the length is 0.
@@ -98,12 +105,10 @@ class BufferDescriptor {
/// true if the current scan range is complete
bool eosr_ = false;
- int64_t scan_range_offset_ = 0;
+ /// Status of the read to this buffer. if status is not ok, 'buffer' is nullptr
+ Status status_;
- // Handle to an allocated buffer and the client used to allocate it buffer. Only used
- // for non-external buffers.
- BufferPool::ClientHandle* bp_client_ = nullptr;
- BufferPool::BufferHandle handle_;
+ int64_t scan_range_offset_ = 0;
};
/// The request type, read or write associated with a request range.
@@ -211,11 +216,11 @@ class ScanRange : public RequestRange {
/// Resets this scan range object with the scan range description. The scan range
/// is for bytes [offset, offset + len) in 'file' on 'fs' (which is nullptr for the
- /// local filesystem). The scan range must be non-empty and fall within the file bounds
- /// (len > 0 and offset >= 0 and offset + len <= file_length). 'disk_id' is the disk
- /// queue to add the range to. If 'expected_local' is true, a warning is generated if
- /// the read did not come from a local disk. 'buffer_opts' specifies buffer management
- /// options - see the DiskIoMgr class comment and the BufferOpts comments for details.
+ /// local filesystem). The scan range must fall within the file bounds (offset >= 0
+ /// and offset + len <= file_length). 'disk_id' is the disk queue to add the range
+ /// to. If 'expected_local' is true, a warning is generated if the read did not
+ /// come from a local disk. 'buffer_opts' specifies buffer management options -
+ /// see the DiskIoMgr class comment and the BufferOpts comments for details.
/// 'meta_data' is an arbitrary client-provided pointer for any auxiliary data.
void Reset(hdfsFS fs, const char* file, int64_t len, int64_t offset, int disk_id,
bool expected_local, const BufferOpts& buffer_opts, void* meta_data = nullptr);
@@ -231,17 +236,10 @@ class ScanRange : public RequestRange {
/// Only one thread can be in GetNext() at any time.
Status GetNext(std::unique_ptr<BufferDescriptor>* buffer) WARN_UNUSED_RESULT;
- /// Returns the buffer to the scan range. This must be called for every buffer
- /// returned by GetNext(). After calling this, the buffer descriptor is invalid
- /// and cannot be accessed.
- void ReturnBuffer(std::unique_ptr<BufferDescriptor> buffer);
-
- /// Cancel this scan range. This cleans up all queued buffers and wakes up any threads
- /// blocked on GetNext(). Status is a non-ok status with the reason the range was
- /// cancelled, e.g. CANCELLED if the range was cancelled because it was not needed, or
- /// another error if an error was encountered while scanning the range. Status is
- /// returned to the any callers of GetNext(). If a thread is currently blocked in
- /// GetNext(), it is woken up.
+ /// Cancel this scan range. This cleans up all queued buffers and
+ /// wakes up any threads blocked on GetNext().
+ /// Status is the reason the range was cancelled. Must not be ok().
+ /// Status is returned to the user in GetNext().
void Cancel(const Status& status);
/// return a descriptive string for debug.
@@ -257,13 +255,18 @@ class ScanRange : public RequestRange {
/// Initialize internal fields
void InitInternal(DiskIoMgr* io_mgr, RequestContext* reader);
- /// Enqueues a ready buffer with valid data for this range. This does not block.
+ /// Enqueues a buffer for this range. This does not block.
+ /// Returns true if this scan range has hit the queue capacity, false otherwise.
/// The caller passes ownership of buffer to the scan range and it is not
/// valid to access buffer after this call. The reader lock must be held by the
- /// caller. Returns false if the scan range was cancelled.
- bool EnqueueReadyBuffer(const boost::unique_lock<boost::mutex>& reader_lock,
+ /// caller.
+ bool EnqueueBuffer(const boost::unique_lock<boost::mutex>& reader_lock,
std::unique_ptr<BufferDescriptor> buffer);
+ /// Cleanup any queued buffers (i.e. due to cancellation). This cannot
+ /// be called with any locks taken.
+ void CleanupQueuedBuffers();
+
/// Validates the internal state of this range. lock_ must be taken
/// before calling this.
bool Validate();
@@ -280,10 +283,6 @@ class ScanRange : public RequestRange {
/// exclusive use by this scan range. The scan range is the exclusive owner of the
/// file handle, and the file handle is destroyed in Close().
/// All local OS files are opened using normal OS file APIs.
- ///
- /// If an error is encountered during opening, returns a status describing the error.
- /// If the scan range was cancelled, returns the reason for cancellation. Otherwise, on
- /// success, returns OK.
Status Open(bool use_file_handle_cache) WARN_UNUSED_RESULT;
/// Closes the file for this range. This function only modifies state in this range.
@@ -291,10 +290,6 @@ class ScanRange : public RequestRange {
/// Reads from this range into 'buffer', which has length 'buffer_len' bytes. Returns
/// the number of bytes read. The read position in this scan range is updated.
- ///
- /// If an error is encountered during reading, returns a status describing the error.
- /// If the scan range was cancelled, returns the reason for cancellation. Otherwise, on
- /// success, returns OK.
Status Read(uint8_t* buffer, int64_t buffer_len, int64_t* bytes_read,
bool* eosr) WARN_UNUSED_RESULT;
@@ -312,58 +307,6 @@ class ScanRange : public RequestRange {
Status ReadFromCache(const boost::unique_lock<boost::mutex>& reader_lock,
bool* read_succeeded) WARN_UNUSED_RESULT;
- /// Add buffers for the range to read data into and schedule the range if blocked.
- /// If 'returned' is true, the buffers returned from GetNext() that are being recycled
- /// via ReturnBuffer(). Otherwise the buffers are newly allocated buffers to be added.
- void AddUnusedBuffers(
- std::vector<std::unique_ptr<BufferDescriptor>>&& buffers, bool returned);
-
- /// Remove a buffer from 'unused_iomgr_buffers_' and update
- /// 'unused_iomgr_buffer_bytes_'. If 'unused_iomgr_buffers_' is empty, return NULL.
- /// 'lock_' must be held by the caller via 'scan_range_lock'.
- std::unique_ptr<BufferDescriptor> GetUnusedBuffer(
- const boost::unique_lock<boost::mutex>& scan_range_lock);
-
- /// Get the next buffer for this scan range for a disk thread to read into. Returns
- /// the new buffer if successful. If no buffers are available, marks the range
- /// as blocked and returns nullptr. Called must not hold 'lock_'.
- std::unique_ptr<BufferDescriptor> GetNextUnusedBufferForRange();
-
- /// Cleans up a buffer that was not returned to the client.
- /// Either ReturnBuffer() or CleanUpBuffer() is called for every BufferDescriptor.
- /// The caller must hold 'lock_' via 'scan_range_lock'.
- /// This function may acquire 'hdfs_lock_'
- void CleanUpBuffer(const boost::unique_lock<boost::mutex>& scan_range_lock,
- std::unique_ptr<BufferDescriptor> buffer);
-
- /// Same as CleanUpBuffer() except cleans up multiple buffers and caller must not
- /// hold 'lock_'.
- void CleanUpBuffers(std::vector<std::unique_ptr<BufferDescriptor>>&& buffers);
-
- /// Clean up all buffers in 'unused_iomgr_buffers_'. Only valid to call when the scan
- /// range is cancelled or at eos. The caller must hold 'lock_' via 'scan_range_lock'.
- void CleanUpUnusedBuffers(const boost::unique_lock<boost::mutex>& scan_range_lock);
-
- /// Same as Cancel() except reader_->lock must be held by the caller.
- void CancelFromReader(const boost::unique_lock<boost::mutex>& reader_lock,
- const Status& status);
-
- /// Same as Cancel() except doesn't remove the scan range from
- /// reader_->active_scan_ranges_. This is invoked by RequestContext::Cancel(),
- /// which removes the range itself to avoid invalidating its active_scan_ranges_
- /// iterator.
- void CancelInternal(const Status& status);
-
- /// Marks the scan range as blocked waiting for a buffer. Caller must not hold 'lock_'.
- void SetBlockedOnBuffer();
-
- /// Returns true if no more buffers will be returned to clients in the future,
- /// either because of hitting eosr or cancellation.
- bool all_buffers_returned(const boost::unique_lock<boost::mutex>& lock) const {
- DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
- return !cancel_status_.ok() || (eosr_queued_ && ready_buffers_.empty());
- }
-
/// Pointer to caller specified metadata. This is untouched by the io manager
/// and the caller can put whatever auxiliary data in here.
void* meta_data_ = nullptr;
@@ -380,9 +323,6 @@ class ScanRange : public RequestRange {
/// TODO: we can do more with this
bool expected_local_ = false;
- /// Last modified time of the file associated with the scan range. Set in Reset().
- int64_t mtime_;
-
/// Total number of bytes read remotely. This is necessary to maintain a count of
/// the number of remote scan ranges. Since IO statistics can be collected multiple
/// times for a scan range, it is necessary to keep some state about whether this
@@ -429,10 +369,6 @@ class ScanRange : public RequestRange {
struct hadoopRzBuffer* cached_buffer_ = nullptr;
};
- /// The number of buffers that have been returned to a client via GetNext() that have
- /// not yet been returned with ReturnBuffer().
- AtomicInt32 num_buffers_in_reader_{0};
-
/// Lock protecting fields below.
/// This lock should not be taken during Open()/Read()/Close().
/// If RequestContext::lock_ and this lock need to be held simultaneously,
@@ -442,40 +378,25 @@ class ScanRange : public RequestRange {
/// Number of bytes read so far for this scan range
int bytes_read_;
- /// Buffers to read into, used if the 'external_buffer_tag_' is NO_BUFFER. These are
- /// initially populated when the client calls AllocateBuffersForRange() and
- /// and are used to read scanned data into. Buffers are taken from this vector for
- /// every read and added back, if needed, when the client calls ReturnBuffer().
- std::vector<std::unique_ptr<BufferDescriptor>> unused_iomgr_buffers_;
-
- /// Total number of bytes of buffers in 'unused_iomgr_buffers_'.
- int64_t unused_iomgr_buffer_bytes_ = 0;
-
- /// Number of bytes of buffers returned from GetNextUnusedBufferForRange(). Used to
- /// infer how many bytes of buffers need to be held onto to read the rest of the scan
- /// range.
- int64_t iomgr_buffer_bytes_returned_ = 0;
+ /// Status for this range. This is non-ok if is_cancelled_ is true.
+ /// Note: an individual range can fail without the RequestContext being
+ /// cancelled. This allows us to skip individual ranges.
+ Status status_;
/// If true, the last buffer for this scan range has been queued.
- /// If this is true and 'ready_buffers_' is empty, then no more buffers will be
- /// returned to the caller by this scan range.
bool eosr_queued_ = false;
- /// If true, this scan range is not scheduled because a buffer is not available for
- /// the next I/O in the range. This can happen when the scan range is initially created
- /// or if the buffers added to the range have all been filled with data an not yet
- /// returned.
- bool blocked_on_buffer_ = false;
+ /// If true, the last buffer for this scan range has been returned.
+ bool eosr_returned_ = false;
- /// IO buffers that are queued for this scan range. When Cancel() is called
- /// this is drained by the cancelling thread. I.e. this is always empty if
- /// 'cancel_status_' is not OK.
- std::deque<std::unique_ptr<BufferDescriptor>> ready_buffers_;
+ /// If true, this scan range has been removed from the reader's in_flight_ranges
+ /// queue because the ready_buffers_ queue is full.
+ bool blocked_on_queue_ = false;
- /// Condition variable for threads in GetNext() that are waiting for the next buffer.
- /// Signalled when a buffer is enqueued in 'ready_buffers_' or the scan range is
- /// cancelled.
+ /// IO buffers that are queued for this scan range.
+ /// Condition variable for GetNext
ConditionVariable buffer_ready_cv_;
+ std::deque<std::unique_ptr<BufferDescriptor>> ready_buffers_;
/// Lock that should be taken during hdfs calls. Only one thread (the disk reading
/// thread) calls into hdfs at a time so this lock does not have performance impact.
@@ -485,16 +406,11 @@ class ScanRange : public RequestRange {
/// If this lock and lock_ need to be taken, lock_ must be taken first.
boost::mutex hdfs_lock_;
- /// If non-OK, this scan range has been cancelled. This status is the reason for
- /// cancellation - CANCELLED if cancelled without error, or another status if an
- /// error caused cancellation. Note that a range can be cancelled without cancelling
- /// the owning context. This means that ranges can be cancelled or hit errors without
- /// aborting all scan ranges.
- //
- /// Writers must hold both 'lock_' and 'hdfs_lock_'. Readers must hold either 'lock_'
- /// or 'hdfs_lock_'. This prevents the range from being cancelled while any thread
- /// is inside a critical section.
- Status cancel_status_;
+ /// If true, this scan range has been cancelled.
+ bool is_cancelled_ = false;
+
+ /// Last modified time of the file associated with the scan range
+ int64_t mtime_;
};
/// Used to specify data to be written to a file and offset.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/scan-range.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/scan-range.cc b/be/src/runtime/io/scan-range.cc
index 4f7c38b..bd89846 100644
--- a/be/src/runtime/io/scan-range.cc
+++ b/be/src/runtime/io/scan-range.cc
@@ -42,27 +42,35 @@ DEFINE_int64(adls_read_chunk_size, 128 * 1024, "The maximum read chunk size to u
// any time and only one thread will remove from the queue. This is to guarantee
// that buffers are queued and read in file order.
-bool ScanRange::EnqueueReadyBuffer(
+bool ScanRange::EnqueueBuffer(
const unique_lock<mutex>& reader_lock, unique_ptr<BufferDescriptor> buffer) {
DCHECK(reader_lock.mutex() == &reader_->lock_ && reader_lock.owns_lock());
- DCHECK(buffer->buffer_ != nullptr) << "Cannot enqueue freed buffer";
{
unique_lock<mutex> scan_range_lock(lock_);
DCHECK(Validate()) << DebugString();
+ DCHECK(!eosr_returned_);
DCHECK(!eosr_queued_);
- if (!cancel_status_.ok()) {
- // This range has been cancelled, no need to enqueue the buffer.
- CleanUpBuffer(scan_range_lock, move(buffer));
+ if (is_cancelled_) {
+ // Return the buffer, this range has been cancelled
+ if (buffer->buffer_ != nullptr) {
+ io_mgr_->num_buffers_in_readers_.Add(1);
+ reader_->num_buffers_in_reader_.Add(1);
+ }
+ reader_->num_used_buffers_.Add(-1);
+ io_mgr_->ReturnBuffer(move(buffer));
return false;
}
- // Clean up any surplus buffers. E.g. we may have allocated too many if the file was
- // shorter than expected.
- if (buffer->eosr()) CleanUpUnusedBuffers(scan_range_lock);
+ reader_->num_ready_buffers_.Add(1);
eosr_queued_ = buffer->eosr();
ready_buffers_.emplace_back(move(buffer));
+
+ DCHECK_LE(ready_buffers_.size(), DiskIoMgr::SCAN_RANGE_READY_BUFFER_LIMIT);
+ blocked_on_queue_ = ready_buffers_.size() == DiskIoMgr::SCAN_RANGE_READY_BUFFER_LIMIT;
}
+
buffer_ready_cv_.NotifyOne();
- return true;
+
+ return blocked_on_queue_;
}
Status ScanRange::GetNext(unique_ptr<BufferDescriptor>* buffer) {
@@ -70,225 +78,123 @@ Status ScanRange::GetNext(unique_ptr<BufferDescriptor>* buffer) {
bool eosr;
{
unique_lock<mutex> scan_range_lock(lock_);
+ if (eosr_returned_) return Status::OK();
DCHECK(Validate()) << DebugString();
- // No more buffers to return - return the cancel status or OK if not cancelled.
- if (all_buffers_returned(scan_range_lock)) return cancel_status_;
- while (ready_buffers_.empty() && cancel_status_.ok()) {
+ while (ready_buffers_.empty() && !is_cancelled_) {
buffer_ready_cv_.Wait(scan_range_lock);
}
- /// Propagate cancellation to the client if it happened while we were waiting.
- RETURN_IF_ERROR(cancel_status_);
+
+ if (is_cancelled_) {
+ DCHECK(!status_.ok());
+ return status_;
+ }
// Remove the first ready buffer from the queue and return it
DCHECK(!ready_buffers_.empty());
+ DCHECK_LE(ready_buffers_.size(), DiskIoMgr::SCAN_RANGE_READY_BUFFER_LIMIT);
*buffer = move(ready_buffers_.front());
ready_buffers_.pop_front();
+ eosr_returned_ = (*buffer)->eosr();
eosr = (*buffer)->eosr();
- DCHECK(!eosr || unused_iomgr_buffers_.empty()) << DebugString();
- }
-
- // Update tracking counters. The buffer has now moved from the IoMgr to the caller.
- if (eosr) reader_->RemoveActiveScanRange(this);
- num_buffers_in_reader_.Add(1);
- return Status::OK();
-}
-
-void ScanRange::ReturnBuffer(unique_ptr<BufferDescriptor> buffer_desc) {
- vector<unique_ptr<BufferDescriptor>> buffers;
- buffers.emplace_back(move(buffer_desc));
- AddUnusedBuffers(move(buffers), true);
-}
-
-void ScanRange::AddUnusedBuffers(vector<unique_ptr<BufferDescriptor>>&& buffers,
- bool returned) {
- DCHECK_GT(buffers.size(), 0);
- /// Keep track of whether the range was unblocked in this function. If so, we need
- /// to schedule it so it resumes progress.
- bool unblocked = false;
- {
- unique_lock<mutex> scan_range_lock(lock_);
- if (returned) {
- // Buffers were in reader but now aren't.
- num_buffers_in_reader_.Add(-buffers.size());
- }
-
- for (unique_ptr<BufferDescriptor>& buffer : buffers) {
- // We should not hold onto the buffers in the following cases:
- // 1. the scan range is using external buffers, e.g. cached buffers.
- // 2. the scan range is cancelled
- // 3. the scan range already hit eosr
- // 4. we already have enough buffers to read the remainder of the scan range.
- if (external_buffer_tag_ != ExternalBufferTag::NO_BUFFER
- || !cancel_status_.ok()
- || eosr_queued_
- || unused_iomgr_buffer_bytes_ >= len_ - iomgr_buffer_bytes_returned_) {
- CleanUpBuffer(scan_range_lock, move(buffer));
- } else {
- unused_iomgr_buffer_bytes_ += buffer->buffer_len();
- unused_iomgr_buffers_.emplace_back(move(buffer));
- if (blocked_on_buffer_) {
- blocked_on_buffer_ = false;
- unblocked = true;
- }
- }
- }
- }
- // Must drop the ScanRange lock before acquiring the RequestContext lock.
- if (unblocked) {
- unique_lock<mutex> reader_lock(reader_->lock_);
- // Reader may have been cancelled after we dropped 'scan_range_lock' above.
- if (reader_->state_ == RequestContext::Cancelled) {
- DCHECK(!cancel_status_.ok());
- } else {
- reader_->ScheduleScanRange(reader_lock, this);
- }
}
-}
-
-unique_ptr<BufferDescriptor> ScanRange::GetUnusedBuffer(
- const unique_lock<mutex>& scan_range_lock) {
- DCHECK(scan_range_lock.mutex() == &lock_ && scan_range_lock.owns_lock());
- if (unused_iomgr_buffers_.empty()) return nullptr;
- unique_ptr<BufferDescriptor> result = move(unused_iomgr_buffers_.back());
- unused_iomgr_buffers_.pop_back();
- unused_iomgr_buffer_bytes_ -= result->buffer_len();
- return result;
-}
-unique_ptr<BufferDescriptor> ScanRange::GetNextUnusedBufferForRange() {
- unique_lock<mutex> lock(lock_);
- unique_ptr<BufferDescriptor> buffer_desc = GetUnusedBuffer(lock);
- if (buffer_desc == nullptr) {
- blocked_on_buffer_ = true;
- } else {
- iomgr_buffer_bytes_returned_ += buffer_desc->buffer_len();
+ // Update tracking counters. The buffer has now moved from the IoMgr to the
+ // caller.
+ io_mgr_->num_buffers_in_readers_.Add(1);
+ reader_->num_buffers_in_reader_.Add(1);
+ reader_->num_ready_buffers_.Add(-1);
+ reader_->num_used_buffers_.Add(-1);
+ if (eosr) reader_->num_finished_ranges_.Add(1);
+
+ Status status = (*buffer)->status_;
+ if (!status.ok()) {
+ io_mgr_->ReturnBuffer(move(*buffer));
+ return status;
}
- return buffer_desc;
-}
-void ScanRange::SetBlockedOnBuffer() {
- unique_lock<mutex> lock(lock_);
- blocked_on_buffer_ = true;
-}
+ unique_lock<mutex> reader_lock(reader_->lock_);
-void ScanRange::CleanUpBuffer(
- const boost::unique_lock<boost::mutex>& scan_range_lock,
- unique_ptr<BufferDescriptor> buffer_desc) {
- DCHECK(scan_range_lock.mutex() == &lock_ && scan_range_lock.owns_lock());
- DCHECK(buffer_desc != nullptr);
- DCHECK_EQ(this, buffer_desc->scan_range_);
- buffer_desc->reader_->FreeBuffer(buffer_desc.get());
-
- if (all_buffers_returned(scan_range_lock) && num_buffers_in_reader_.Load() == 0) {
- // Close the scan range if there are no more buffers in the reader and no more buffers
- // will be returned to readers in future. Close() is idempotent so it is ok to call
- // multiple times during cleanup so long as the range is actually finished.
- Close();
+ DCHECK(reader_->Validate()) << endl << reader_->DebugString();
+ if (reader_->state_ == RequestContext::Cancelled) {
+ reader_->blocked_ranges_.Remove(this);
+ Cancel(reader_->status_);
+ io_mgr_->ReturnBuffer(move(*buffer));
+ return status_;
}
-}
-
-void ScanRange::CleanUpBuffers(vector<unique_ptr<BufferDescriptor>>&& buffers) {
- unique_lock<mutex> lock(lock_);
- for (unique_ptr<BufferDescriptor>& buffer : buffers) CleanUpBuffer(lock, move(buffer));
-}
-void ScanRange::CleanUpUnusedBuffers(const unique_lock<mutex>& scan_range_lock) {
- while (!unused_iomgr_buffers_.empty()) {
- CleanUpBuffer(scan_range_lock, GetUnusedBuffer(scan_range_lock));
+ {
+ // Check to see if we can re-schedule a blocked range. Note that EnqueueBuffer()
+ // may have been called after we released 'lock_' above so we need to re-check
+ // whether the queue is full.
+ unique_lock<mutex> scan_range_lock(lock_);
+ if (blocked_on_queue_
+ && ready_buffers_.size() < DiskIoMgr::SCAN_RANGE_READY_BUFFER_LIMIT
+ && !eosr_queued_) {
+ blocked_on_queue_ = false;
+ // This scan range was blocked and is no longer, add it to the reader
+ // queue again.
+ reader_->blocked_ranges_.Remove(this);
+ reader_->ScheduleScanRange(this);
+ }
}
+ return Status::OK();
}
void ScanRange::Cancel(const Status& status) {
// Cancelling a range that was never started, ignore.
if (io_mgr_ == nullptr) return;
- CancelInternal(status);
- reader_->RemoveActiveScanRange(this);
-}
-void ScanRange::CancelFromReader(const boost::unique_lock<boost::mutex>& reader_lock,
- const Status& status) {
- DCHECK(reader_lock.mutex() == &reader_->lock_ && reader_lock.owns_lock());
- CancelInternal(status);
- reader_->RemoveActiveScanRangeLocked(reader_lock, this);
-}
-
-void ScanRange::CancelInternal(const Status& status) {
- DCHECK(io_mgr_ != nullptr);
DCHECK(!status.ok());
{
- // Grab both locks to make sure that we don't change 'cancel_status_' while other
- // threads are in critical sections.
+ // Grab both locks to make sure that all working threads see is_cancelled_.
unique_lock<mutex> scan_range_lock(lock_);
- {
- unique_lock<mutex> hdfs_lock(hdfs_lock_);
- DCHECK(Validate()) << DebugString();
- // If already cancelled, preserve the original reason for cancellation. The first
- // thread to set 'cancel_status_' does the cleanup below.
- RETURN_VOID_IF_ERROR(cancel_status_);
- cancel_status_ = status;
- }
-
- /// Clean up 'ready_buffers_' while still holding 'lock_' to prevent other threads
- /// from seeing inconsistent state.
- while (!ready_buffers_.empty()) {
- CleanUpBuffer(scan_range_lock, move(ready_buffers_.front()));
- ready_buffers_.pop_front();
- }
-
- /// Clean up buffers that we don't need any more because we won't read any more data.
- CleanUpUnusedBuffers(scan_range_lock);
+ unique_lock<mutex> hdfs_lock(hdfs_lock_);
+ DCHECK(Validate()) << DebugString();
+ if (is_cancelled_) return;
+ is_cancelled_ = true;
+ status_ = status;
}
buffer_ready_cv_.NotifyAll();
+ CleanupQueuedBuffers();
// For cached buffers, we can't close the range until the cached buffer is returned.
- // Close() is called from ScanRange::CleanUpBufferLocked().
+ // Close() is called from DiskIoMgr::ReturnBuffer().
if (external_buffer_tag_ != ExternalBufferTag::CACHED_BUFFER) Close();
}
+void ScanRange::CleanupQueuedBuffers() {
+ DCHECK(is_cancelled_);
+ io_mgr_->num_buffers_in_readers_.Add(ready_buffers_.size());
+ reader_->num_buffers_in_reader_.Add(ready_buffers_.size());
+ reader_->num_used_buffers_.Add(-ready_buffers_.size());
+ reader_->num_ready_buffers_.Add(-ready_buffers_.size());
+
+ while (!ready_buffers_.empty()) {
+ io_mgr_->ReturnBuffer(move(ready_buffers_.front()));
+ ready_buffers_.pop_front();
+ }
+}
+
string ScanRange::DebugString() const {
stringstream ss;
ss << "file=" << file_ << " disk_id=" << disk_id_ << " offset=" << offset_
<< " len=" << len_ << " bytes_read=" << bytes_read_
- << " cancel_status=" << cancel_status_.GetDetail()
<< " buffer_queue=" << ready_buffers_.size()
- << " num_buffers_in_readers=" << num_buffers_in_reader_.Load()
- << " unused_iomgr_buffers=" << unused_iomgr_buffers_.size()
- << " unused_iomgr_buffer_bytes=" << unused_iomgr_buffer_bytes_
- << " blocked_on_buffer=" << blocked_on_buffer_
<< " hdfs_file=" << exclusive_hdfs_fh_;
return ss.str();
}
bool ScanRange::Validate() {
if (bytes_read_ > len_) {
- LOG(ERROR) << "Bytes read tracking is wrong. Shouldn't read past the scan range."
+ LOG(WARNING) << "Bytes read tracking is wrong. Shouldn't read past the scan range."
<< " bytes_read_=" << bytes_read_ << " len_=" << len_;
return false;
}
- if (!cancel_status_.ok() && !ready_buffers_.empty()) {
- LOG(ERROR) << "Cancelled range should not have queued buffers " << DebugString();
- return false;
- }
- int64_t unused_iomgr_buffer_bytes = 0;
- for (auto& buffer : unused_iomgr_buffers_)
- unused_iomgr_buffer_bytes += buffer->buffer_len();
- if (unused_iomgr_buffer_bytes != unused_iomgr_buffer_bytes_) {
- LOG(ERROR) << "unused_iomgr_buffer_bytes_ incorrect actual: "
- << unused_iomgr_buffer_bytes_
- << " vs. expected: " << unused_iomgr_buffer_bytes;
- return false;
- }
- bool is_finished = !cancel_status_.ok() || eosr_queued_;
- if (is_finished && !unused_iomgr_buffers_.empty()) {
- LOG(ERROR) << "Held onto too many buffers " << unused_iomgr_buffers_.size()
- << " bytes: " << unused_iomgr_buffer_bytes_
- << " cancel_status: " << cancel_status_.GetDetail()
- << " eosr_queued: " << eosr_queued_;
- return false;
- }
- if (!is_finished && blocked_on_buffer_ && !unused_iomgr_buffers_.empty()) {
- LOG(ERROR) << "Blocked despite having buffers: " << DebugString();
+ if (eosr_returned_ && !eosr_queued_) {
+ LOG(WARNING) << "Returned eosr to reader before finishing reading the scan range"
+ << " eosr_returned_=" << eosr_returned_
+ << " eosr_queued_=" << eosr_queued_;
return false;
}
return true;
@@ -297,14 +203,13 @@ bool ScanRange::Validate() {
ScanRange::ScanRange()
: RequestRange(RequestType::READ),
num_remote_bytes_(0),
- external_buffer_tag_(ExternalBufferTag::NO_BUFFER) {}
+ external_buffer_tag_(ExternalBufferTag::NO_BUFFER),
+ mtime_(-1) {}
ScanRange::~ScanRange() {
DCHECK(exclusive_hdfs_fh_ == nullptr) << "File was not closed.";
DCHECK(external_buffer_tag_ != ExternalBufferTag::CACHED_BUFFER)
<< "Cached buffer was not released.";
- DCHECK_EQ(0, ready_buffers_.size());
- DCHECK_EQ(0, num_buffers_in_reader_.Load());
}
void ScanRange::Reset(hdfsFS fs, const char* file, int64_t len, int64_t offset,
@@ -340,22 +245,24 @@ void ScanRange::Reset(hdfsFS fs, const char* file, int64_t len, int64_t offset,
void ScanRange::InitInternal(DiskIoMgr* io_mgr, RequestContext* reader) {
DCHECK(exclusive_hdfs_fh_ == nullptr);
DCHECK(local_file_ == nullptr);
+ // Reader must provide MemTracker or a buffer.
+ DCHECK(external_buffer_tag_ == ExternalBufferTag::CLIENT_BUFFER
+ || reader->mem_tracker_ != nullptr);
io_mgr_ = io_mgr;
reader_ = reader;
local_file_ = nullptr;
exclusive_hdfs_fh_ = nullptr;
bytes_read_ = 0;
- unused_iomgr_buffer_bytes_ = 0;
- iomgr_buffer_bytes_returned_ = 0;
- cancel_status_ = Status::OK();
- eosr_queued_ = false;
- blocked_on_buffer_ = false;
+ is_cancelled_ = false;
+ eosr_queued_= false;
+ eosr_returned_= false;
+ blocked_on_queue_ = false;
DCHECK(Validate()) << DebugString();
}
Status ScanRange::Open(bool use_file_handle_cache) {
unique_lock<mutex> hdfs_lock(hdfs_lock_);
- RETURN_IF_ERROR(cancel_status_);
+ if (is_cancelled_) return Status::CANCELLED;
if (fs_ != nullptr) {
if (exclusive_hdfs_fh_ != nullptr) return Status::OK();
@@ -395,7 +302,9 @@ Status ScanRange::Open(bool use_file_handle_cache) {
"for file: $1: $2", offset_, file_, GetStrErrMsg()));
}
}
- ImpaladMetrics::IO_MGR_NUM_OPEN_FILES->Increment(1L);
+ if (ImpaladMetrics::IO_MGR_NUM_OPEN_FILES != nullptr) {
+ ImpaladMetrics::IO_MGR_NUM_OPEN_FILES->Increment(1L);
+ }
return Status::OK();
}
@@ -447,7 +356,9 @@ void ScanRange::Close() {
local_file_ = nullptr;
closed_file = true;
}
- if (closed_file) ImpaladMetrics::IO_MGR_NUM_OPEN_FILES->Increment(-1L);
+ if (closed_file && ImpaladMetrics::IO_MGR_NUM_OPEN_FILES != nullptr) {
+ ImpaladMetrics::IO_MGR_NUM_OPEN_FILES->Increment(-1L);
+ }
}
int64_t ScanRange::MaxReadChunkSize() const {
@@ -475,7 +386,7 @@ int64_t ScanRange::MaxReadChunkSize() const {
Status ScanRange::Read(
uint8_t* buffer, int64_t buffer_len, int64_t* bytes_read, bool* eosr) {
unique_lock<mutex> hdfs_lock(hdfs_lock_);
- RETURN_IF_ERROR(cancel_status_);
+ if (is_cancelled_) return Status::CANCELLED;
*eosr = false;
*bytes_read = 0;
@@ -622,7 +533,7 @@ Status ScanRange::ReadFromCache(
{
unique_lock<mutex> hdfs_lock(hdfs_lock_);
- RETURN_IF_ERROR(cancel_status_);
+ if (is_cancelled_) return Status::CANCELLED;
DCHECK(exclusive_hdfs_fh_ != nullptr);
DCHECK(external_buffer_tag_ == ExternalBufferTag::NO_BUFFER);
@@ -657,16 +568,20 @@ Status ScanRange::ReadFromCache(
}
// Create a single buffer desc for the entire scan range and enqueue that.
- // The memory is owned by the HDFS java client, not the Impala backend.
+ // 'mem_tracker' is nullptr because the memory is owned by the HDFS java client,
+ // not the Impala backend.
unique_ptr<BufferDescriptor> desc = unique_ptr<BufferDescriptor>(new BufferDescriptor(
- io_mgr_, reader_, this, reinterpret_cast<uint8_t*>(buffer), 0));
+ io_mgr_, reader_, this, reinterpret_cast<uint8_t*>(buffer), 0, nullptr));
desc->len_ = bytes_read;
desc->scan_range_offset_ = 0;
desc->eosr_ = true;
bytes_read_ = bytes_read;
- EnqueueReadyBuffer(reader_lock, move(desc));
- COUNTER_ADD_IF_NOT_NULL(reader_->bytes_read_counter_, bytes_read);
+ EnqueueBuffer(reader_lock, move(desc));
+ if (reader_->bytes_read_counter_ != nullptr) {
+ COUNTER_ADD(reader_->bytes_read_counter_, bytes_read);
+ }
*read_succeeded = true;
+ reader_->num_used_buffers_.Add(1);
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/mem-tracker.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/mem-tracker.h b/be/src/runtime/mem-tracker.h
index 8d28f8f..10a3424 100644
--- a/be/src/runtime/mem-tracker.h
+++ b/be/src/runtime/mem-tracker.h
@@ -107,6 +107,7 @@ class MemTracker {
/// destruction to prevent other threads from getting a reference to the MemTracker
/// via its parent. Only used to deregister the query-level MemTracker from the
/// global hierarchy.
+ /// TODO: IMPALA-3200: this is also used by BufferedBlockMgr, which will be deleted.
void CloseAndUnregisterFromParent();
/// Include counters from a ReservationTracker in logs and other diagnostics.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/test-env.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/test-env.cc b/be/src/runtime/test-env.cc
index 0d6c6f0..770eaba 100644
--- a/be/src/runtime/test-env.cc
+++ b/be/src/runtime/test-env.cc
@@ -49,7 +49,7 @@ Status TestEnv::Init() {
exec_env_.reset(new ExecEnv);
// Populate the ExecEnv state that the backend tests need.
exec_env_->mem_tracker_.reset(new MemTracker(-1, "Process"));
- RETURN_IF_ERROR(exec_env_->disk_io_mgr()->Init());
+ RETURN_IF_ERROR(exec_env_->disk_io_mgr()->Init(exec_env_->process_mem_tracker()));
exec_env_->metrics_.reset(new MetricGroup("test-env-metrics"));
exec_env_->tmp_file_mgr_.reset(new TmpFileMgr);
if (have_tmp_file_mgr_args_) {
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/tmp-file-mgr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/tmp-file-mgr-test.cc b/be/src/runtime/tmp-file-mgr-test.cc
index 9161b63..3091c58 100644
--- a/be/src/runtime/tmp-file-mgr-test.cc
+++ b/be/src/runtime/tmp-file-mgr-test.cc
@@ -25,7 +25,6 @@
#include <gtest/gtest.h>
#include "common/init.h"
-#include "runtime/io/request-context.h"
#include "runtime/test-env.h"
#include "runtime/tmp-file-mgr-internal.h"
#include "runtime/tmp-file-mgr.h"
@@ -135,7 +134,7 @@ class TmpFileMgrTest : public ::testing::Test {
/// Helper to cancel the FileGroup RequestContext.
static void CancelIoContext(TmpFileMgr::FileGroup* group) {
- group->io_ctx_->Cancel();
+ group->io_mgr_->CancelContext(group->io_ctx_.get());
}
/// Helper to get the # of bytes allocated by the group. Validates that the sum across
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/tmp-file-mgr.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/tmp-file-mgr.cc b/be/src/runtime/tmp-file-mgr.cc
index e0c58d4..3807670 100644
--- a/be/src/runtime/tmp-file-mgr.cc
+++ b/be/src/runtime/tmp-file-mgr.cc
@@ -243,7 +243,7 @@ TmpFileMgr::FileGroup::FileGroup(TmpFileMgr* tmp_file_mgr, DiskIoMgr* io_mgr,
next_allocation_index_(0),
free_ranges_(64) {
DCHECK(tmp_file_mgr != nullptr);
- io_ctx_ = io_mgr_->RegisterContext();
+ io_ctx_ = io_mgr_->RegisterContext(nullptr);
}
TmpFileMgr::FileGroup::~FileGroup() {
@@ -396,10 +396,7 @@ Status TmpFileMgr::FileGroup::ReadAsync(WriteHandle* handle, MemRange buffer) {
BufferOpts::ReadInto(buffer.data(), buffer.len()));
read_counter_->Add(1);
bytes_read_counter_->Add(buffer.len());
- bool needs_buffers;
- RETURN_IF_ERROR(io_mgr_->StartScanRange(
- io_ctx_.get(), handle->read_range_, &needs_buffers));
- DCHECK(!needs_buffers) << "Already provided a buffer";
+ RETURN_IF_ERROR(io_mgr_->AddScanRange(io_ctx_.get(), handle->read_range_, true));
return Status::OK();
}
@@ -429,7 +426,7 @@ Status TmpFileMgr::FileGroup::WaitForAsyncRead(WriteHandle* handle, MemRange buf
}
exit:
// Always return the buffer before exiting to avoid leaking it.
- if (io_mgr_buffer != nullptr) handle->read_range_->ReturnBuffer(move(io_mgr_buffer));
+ if (io_mgr_buffer != nullptr) io_mgr_->ReturnBuffer(move(io_mgr_buffer));
handle->read_range_ = nullptr;
return status;
}
@@ -525,20 +522,11 @@ TmpFileMgr::WriteHandle::WriteHandle(
is_cancelled_(false),
write_in_flight_(false) {}
-TmpFileMgr::WriteHandle::~WriteHandle() {
- DCHECK(!write_in_flight_);
- DCHECK(read_range_ == nullptr);
-}
-
string TmpFileMgr::WriteHandle::TmpFilePath() const {
if (file_ == nullptr) return "";
return file_->path();
}
-int64_t TmpFileMgr::WriteHandle::len() const {
- return write_range_->len();
-}
-
Status TmpFileMgr::WriteHandle::Write(DiskIoMgr* io_mgr, RequestContext* io_ctx,
File* file, int64_t offset, MemRange buffer,
WriteRange::WriteDoneCallback callback) {
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/tmp-file-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/tmp-file-mgr.h b/be/src/runtime/tmp-file-mgr.h
index 55901f4..95072ae 100644
--- a/be/src/runtime/tmp-file-mgr.h
+++ b/be/src/runtime/tmp-file-mgr.h
@@ -28,6 +28,7 @@
#include "common/object-pool.h"
#include "common/status.h"
#include "gen-cpp/Types_types.h" // for TUniqueId
+#include "runtime/io/request-ranges.h"
#include "util/collection-metrics.h"
#include "util/condition-variable.h"
#include "util/mem-range.h"
@@ -36,12 +37,6 @@
#include "util/spinlock.h"
namespace impala {
-namespace io {
- class DiskIoMgr;
- class RequestContext;
- class ScanRange;
- class WriteRange;
-}
/// TmpFileMgr provides an abstraction for management of temporary (a.k.a. scratch) files
/// on the filesystem and I/O to and from them. TmpFileMgr manages multiple scratch
@@ -89,7 +84,6 @@ class TmpFileMgr {
/// Needs to be public for TmpFileMgrTest.
typedef int DeviceId;
- /// Same typedef as io::WriteRange::WriteDoneCallback.
typedef std::function<void(const Status&)> WriteDoneCallback;
/// Represents a group of temporary files - one per disk with a scratch directory. The
@@ -283,7 +277,10 @@ class TmpFileMgr {
public:
/// The write must be destroyed by passing it to FileGroup - destroying it before
/// the write completes is an error.
- ~WriteHandle();
+ ~WriteHandle() {
+ DCHECK(!write_in_flight_);
+ DCHECK(read_range_ == nullptr);
+ }
/// Cancel any in-flight read synchronously.
void CancelRead();
@@ -293,7 +290,7 @@ class TmpFileMgr {
std::string TmpFilePath() const;
/// The length of the write range in bytes.
- int64_t len() const;
+ int64_t len() const { return write_range_->len(); }
std::string DebugString();
@@ -308,7 +305,7 @@ class TmpFileMgr {
/// failure and 'is_cancelled_' is set to true on failure.
Status Write(io::DiskIoMgr* io_mgr, io::RequestContext* io_ctx, File* file,
int64_t offset, MemRange buffer,
- WriteDoneCallback callback) WARN_UNUSED_RESULT;
+ io::WriteRange::WriteDoneCallback callback) WARN_UNUSED_RESULT;
/// Retry the write after the initial write failed with an error, instead writing to
/// 'offset' of 'file'. 'write_in_flight_' must be true before calling.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/util/bit-util-test.cc
----------------------------------------------------------------------
diff --git a/be/src/util/bit-util-test.cc b/be/src/util/bit-util-test.cc
index 6f70727..5f8d443 100644
--- a/be/src/util/bit-util-test.cc
+++ b/be/src/util/bit-util-test.cc
@@ -258,17 +258,6 @@ TEST(BitUtil, Log2) {
EXPECT_EQ(BitUtil::Log2CeilingNonZero64(ULLONG_MAX), 64);
}
-TEST(BitUtil, RoundToPowerOfTwo) {
- EXPECT_EQ(16, BitUtil::RoundUpToPowerOfTwo(9));
- EXPECT_EQ(16, BitUtil::RoundUpToPowerOfTwo(15));
- EXPECT_EQ(16, BitUtil::RoundUpToPowerOfTwo(16));
- EXPECT_EQ(32, BitUtil::RoundUpToPowerOfTwo(17));
- EXPECT_EQ(8, BitUtil::RoundDownToPowerOfTwo(9));
- EXPECT_EQ(8, BitUtil::RoundDownToPowerOfTwo(15));
- EXPECT_EQ(16, BitUtil::RoundDownToPowerOfTwo(16));
- EXPECT_EQ(16, BitUtil::RoundDownToPowerOfTwo(17));
-}
-
TEST(BitUtil, RoundUpToPowerOf2) {
EXPECT_EQ(BitUtil::RoundUpToPowerOf2(7, 8), 8);
EXPECT_EQ(BitUtil::RoundUpToPowerOf2(8, 8), 8);
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/util/bit-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/bit-util.h b/be/src/util/bit-util.h
index 8a65509..5c9a29b 100644
--- a/be/src/util/bit-util.h
+++ b/be/src/util/bit-util.h
@@ -98,12 +98,6 @@ class BitUtil {
return v;
}
- /// Returns the largest power of two <= v.
- static inline int64_t RoundDownToPowerOfTwo(int64_t v) {
- int64_t v_rounded_up = RoundUpToPowerOfTwo(v);
- return v_rounded_up == v ? v : v_rounded_up / 2;
- }
-
/// Returns 'value' rounded up to the nearest multiple of 'factor' when factor is
/// a power of two
static inline int64_t RoundUpToPowerOf2(int64_t value, int64_t factor) {
@@ -111,7 +105,7 @@ class BitUtil {
return (value + (factor - 1)) & ~(factor - 1);
}
- static inline int64_t RoundDownToPowerOf2(int64_t value, int64_t factor) {
+ static inline int RoundDownToPowerOf2(int value, int factor) {
DCHECK((factor > 0) && ((factor & (factor - 1)) == 0));
return value & ~(factor - 1);
}
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/util/impalad-metrics.cc
----------------------------------------------------------------------
diff --git a/be/src/util/impalad-metrics.cc b/be/src/util/impalad-metrics.cc
index 815e4af..c2a2644 100644
--- a/be/src/util/impalad-metrics.cc
+++ b/be/src/util/impalad-metrics.cc
@@ -46,6 +46,12 @@ const char* ImpaladMetricKeys::HASH_TABLE_TOTAL_BYTES =
"impala-server.hash-table.total-bytes";
const char* ImpaladMetricKeys::IO_MGR_NUM_OPEN_FILES =
"impala-server.io-mgr.num-open-files";
+const char* ImpaladMetricKeys::IO_MGR_NUM_BUFFERS =
+ "impala-server.io-mgr.num-buffers";
+const char* ImpaladMetricKeys::IO_MGR_TOTAL_BYTES =
+ "impala-server.io-mgr.total-bytes";
+const char* ImpaladMetricKeys::IO_MGR_NUM_UNUSED_BUFFERS =
+ "impala-server.io-mgr.num-unused-buffers";
const char* ImpaladMetricKeys::IO_MGR_BYTES_READ =
"impala-server.io-mgr.bytes-read";
const char* ImpaladMetricKeys::IO_MGR_LOCAL_BYTES_READ =
@@ -205,8 +211,11 @@ void ImpaladMetrics::CreateMetrics(MetricGroup* m) {
ImpaladMetricKeys::NUM_FILES_OPEN_FOR_INSERT, 0);
// Initialize IO mgr metrics
- IO_MGR_NUM_OPEN_FILES = m->AddGauge(
- ImpaladMetricKeys::IO_MGR_NUM_OPEN_FILES, 0);
+ IO_MGR_NUM_OPEN_FILES = m->AddGauge(ImpaladMetricKeys::IO_MGR_NUM_OPEN_FILES, 0);
+ IO_MGR_NUM_BUFFERS = m->AddGauge(ImpaladMetricKeys::IO_MGR_NUM_BUFFERS, 0);
+ IO_MGR_TOTAL_BYTES = m->AddGauge(ImpaladMetricKeys::IO_MGR_TOTAL_BYTES, 0);
+ IO_MGR_NUM_UNUSED_BUFFERS = m->AddGauge(
+ ImpaladMetricKeys::IO_MGR_NUM_UNUSED_BUFFERS, 0);
IO_MGR_NUM_CACHED_FILE_HANDLES = m->AddGauge(
ImpaladMetricKeys::IO_MGR_NUM_CACHED_FILE_HANDLES, 0);
IO_MGR_NUM_FILE_HANDLES_OUTSTANDING = m->AddGauge(
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/util/impalad-metrics.h
----------------------------------------------------------------------
diff --git a/be/src/util/impalad-metrics.h b/be/src/util/impalad-metrics.h
index 7de7aa8..b49caf9 100644
--- a/be/src/util/impalad-metrics.h
+++ b/be/src/util/impalad-metrics.h
@@ -67,6 +67,15 @@ class ImpaladMetricKeys {
/// Number of files currently opened by the io mgr
static const char* IO_MGR_NUM_OPEN_FILES;
+ /// Number of IO buffers allocated by the io mgr
+ static const char* IO_MGR_NUM_BUFFERS;
+
+ /// Number of bytes used by IO buffers (used and unused).
+ static const char* IO_MGR_TOTAL_BYTES;
+
+ /// Number of IO buffers that are currently unused (and can be GC'ed)
+ static const char* IO_MGR_NUM_UNUSED_BUFFERS;
+
/// Total number of bytes read by the io mgr
static const char* IO_MGR_BYTES_READ;
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/common/thrift/PlanNodes.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/PlanNodes.thrift b/common/thrift/PlanNodes.thrift
index 1ab05a0..c5df1cd 100644
--- a/common/thrift/PlanNodes.thrift
+++ b/common/thrift/PlanNodes.thrift
@@ -239,9 +239,6 @@ struct THdfsScanNode {
// The byte offset of the slot for Parquet metadata if Parquet count star optimization
// is enabled.
10: optional i32 parquet_count_star_slot_offset
-
- // The ideal memory reservation in bytes to process an input split.
- 11: optional i64 ideal_scan_range_reservation
}
struct TDataSourceScanNode {
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java b/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
index aae3863..4f0a0e1 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
@@ -163,25 +163,6 @@ public class SlotDescriptor {
}
/**
- * Checks if this descriptor describes an array "pos" pseudo-column.
- *
- * Note: checking whether the column is null distinguishes between top-level columns
- * and nested types. This check more specifically looks just for a reference to the
- * "pos" field of an array type.
- */
- public boolean isArrayPosRef() {
- if (parent_ == null) return false;
- Type parentType = parent_.getType();
- if (parentType instanceof CollectionStructType) {
- if (((CollectionStructType)parentType).isArrayStruct() &&
- label_.equals(Path.ARRAY_POS_FIELD_NAME)) {
- return true;
- }
- }
- return false;
- }
-
- /**
* Assembles the absolute materialized path to this slot starting from the schema
* root. The materialized path points to the first non-struct schema element along the
* path starting from the parent's tuple path to this slot's path.
[9/9] impala git commit: Revert IMPALA-4835 and dependent changes
Posted by lv...@apache.org.
Revert IMPALA-4835 and dependent changes
Revert "IMPALA-6585: increase test_low_mem_limit_q21 limit"
This reverts commit 25bcb258dfd712f1514cf188206667a5e6be0e26.
Revert "IMPALA-6588: don't add empty list of ranges in text scan"
This reverts commit d57fbec6f67b83227b4c6117476da8f7d75fc4f6.
Revert "IMPALA-4835: Part 3: switch I/O buffers to buffer pool"
This reverts commit 24b4ed0b29a44090350e630d625291c01b753a36.
Revert "IMPALA-4835: Part 2: Allocate scan range buffers upfront"
This reverts commit 5699b59d0c5cbe37e888a367adb42fa12dfb0916.
Revert "IMPALA-4835: Part 1: simplify I/O mgr mem mgmt and cancellation"
This reverts commit 65680dc42107db4ff2273c635cedf83d20f0ea94.
Change-Id: Ie5ca451cd96602886b0a8ecaa846957df0269cbb
Reviewed-on: http://gerrit.cloudera.org:8080/9480
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Tested-by: Impala Public Jenkins
Reviewed-on: http://gerrit.cloudera.org:8080/9485
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/e5689fb5
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/e5689fb5
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/e5689fb5
Branch: refs/heads/2.x
Commit: e5689fb5c6a011af9e526effa9aa5399b194e977
Parents: 42c604e
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Fri Mar 2 16:09:25 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Sat Mar 3 22:07:15 2018 +0000
----------------------------------------------------------------------
be/src/exec/CMakeLists.txt | 1 -
be/src/exec/base-sequence-scanner.cc | 1 -
be/src/exec/base-sequence-scanner.h | 3 +-
be/src/exec/hdfs-lzo-text-scanner.cc | 1 -
be/src/exec/hdfs-parquet-scanner-test.cc | 96 -
be/src/exec/hdfs-parquet-scanner.cc | 234 +--
be/src/exec/hdfs-parquet-scanner.h | 44 +-
be/src/exec/hdfs-scan-node-base.cc | 95 +-
be/src/exec/hdfs-scan-node-base.h | 3 -
be/src/exec/hdfs-scan-node-mt.cc | 20 +-
be/src/exec/hdfs-scan-node.cc | 172 +-
be/src/exec/hdfs-scan-node.h | 66 +-
be/src/exec/hdfs-text-scanner.cc | 6 +-
be/src/exec/parquet-column-readers.cc | 108 +-
be/src/exec/parquet-column-readers.h | 88 +-
be/src/exec/scanner-context.cc | 42 +-
be/src/exec/scanner-context.h | 54 +-
be/src/runtime/bufferpool/buffer-pool.h | 1 -
.../bufferpool/reservation-tracker-test.cc | 8 +-
be/src/runtime/bufferpool/reservation-util.cc | 2 +-
be/src/runtime/exec-env.cc | 7 +-
be/src/runtime/io/disk-io-mgr-internal.h | 16 -
be/src/runtime/io/disk-io-mgr-stress-test.cc | 43 +-
be/src/runtime/io/disk-io-mgr-stress.cc | 89 +-
be/src/runtime/io/disk-io-mgr-stress.h | 26 +-
be/src/runtime/io/disk-io-mgr-test.cc | 849 ++++----
be/src/runtime/io/disk-io-mgr.cc | 716 +++++--
be/src/runtime/io/disk-io-mgr.h | 383 ++--
be/src/runtime/io/request-context.cc | 239 +--
be/src/runtime/io/request-context.h | 318 ++-
be/src/runtime/io/request-ranges.h | 196 +-
be/src/runtime/io/scan-range.cc | 309 ++-
be/src/runtime/mem-tracker.h | 1 +
be/src/runtime/test-env.cc | 2 +-
be/src/runtime/tmp-file-mgr-test.cc | 3 +-
be/src/runtime/tmp-file-mgr.cc | 18 +-
be/src/runtime/tmp-file-mgr.h | 17 +-
be/src/util/bit-util-test.cc | 11 -
be/src/util/bit-util.h | 8 +-
be/src/util/impalad-metrics.cc | 13 +-
be/src/util/impalad-metrics.h | 9 +
common/thrift/PlanNodes.thrift | 3 -
.../apache/impala/analysis/SlotDescriptor.java | 19 -
.../org/apache/impala/analysis/SlotRef.java | 20 +
.../org/apache/impala/planner/HdfsScanNode.java | 167 +-
.../java/org/apache/impala/util/BitUtil.java | 6 -
.../org/apache/impala/util/BitUtilTest.java | 6 -
.../queries/PlannerTest/constant-folding.test | 42 +-
.../queries/PlannerTest/disable-codegen.test | 20 +-
.../PlannerTest/fk-pk-join-detection.test | 78 +-
.../queries/PlannerTest/max-row-size.test | 80 +-
.../PlannerTest/min-max-runtime-filters.test | 6 +-
.../queries/PlannerTest/mt-dop-validation.test | 40 +-
.../queries/PlannerTest/parquet-filtering.test | 42 +-
.../queries/PlannerTest/partition-pruning.test | 4 +-
.../PlannerTest/resource-requirements.test | 1814 ++++--------------
.../PlannerTest/sort-expr-materialization.test | 32 +-
.../PlannerTest/spillable-buffer-sizing.test | 192 +-
.../queries/PlannerTest/tablesample.test | 44 +-
.../queries/PlannerTest/union.test | 8 +-
.../admission-reject-min-reservation.test | 12 +-
.../queries/QueryTest/analytic-fns.test | 5 +-
.../queries/QueryTest/codegen-mem-limit.test | 5 +-
.../QueryTest/disk-spill-encryption.test | 2 +-
.../queries/QueryTest/explain-level0.test | 4 +-
.../queries/QueryTest/explain-level1.test | 4 +-
.../queries/QueryTest/explain-level2.test | 14 +-
.../queries/QueryTest/explain-level3.test | 14 +-
.../queries/QueryTest/nested-types-tpch.test | 6 +-
.../queries/QueryTest/runtime_row_filters.test | 11 +-
.../queries/QueryTest/scanners.test | 7 -
.../functional-query/queries/QueryTest/set.test | 2 +-
.../queries/QueryTest/spilling-aggs.test | 19 +-
.../spilling-naaj-no-deny-reservation.test | 7 +-
.../queries/QueryTest/spilling-naaj.test | 8 +-
.../QueryTest/spilling-no-debug-action.test | 66 -
.../QueryTest/spilling-sorts-exhaustive.test | 10 +-
.../queries/QueryTest/spilling.test | 76 +-
.../queries/QueryTest/stats-extrapolation.test | 52 +-
.../tpch/queries/sort-reservation-usage.test | 9 +-
tests/common/test_dimensions.py | 16 +-
tests/custom_cluster/test_scratch_disk.py | 2 +-
tests/query_test/test_mem_usage_scaling.py | 11 +-
tests/query_test/test_query_mem_limit.py | 4 +-
tests/query_test/test_scanners.py | 25 +-
tests/query_test/test_scanners_fuzz.py | 9 +-
tests/query_test/test_sort.py | 16 +-
tests/query_test/test_spilling.py | 5 -
88 files changed, 2799 insertions(+), 4563 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt
index c1f91d6..aab1383 100644
--- a/be/src/exec/CMakeLists.txt
+++ b/be/src/exec/CMakeLists.txt
@@ -108,4 +108,3 @@ ADD_BE_TEST(parquet-version-test)
ADD_BE_TEST(row-batch-list-test)
ADD_BE_TEST(incr-stats-util-test)
ADD_BE_TEST(hdfs-avro-scanner-test)
-ADD_BE_TEST(hdfs-parquet-scanner-test)
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/base-sequence-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/base-sequence-scanner.cc b/be/src/exec/base-sequence-scanner.cc
index 9d95b0b..9cb6330 100644
--- a/be/src/exec/base-sequence-scanner.cc
+++ b/be/src/exec/base-sequence-scanner.cc
@@ -46,7 +46,6 @@ static const int MIN_SYNC_READ_SIZE = 64 * 1024; // bytes
Status BaseSequenceScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
const vector<HdfsFileDesc*>& files) {
- DCHECK(!files.empty());
// Issue just the header range for each file. When the header is complete,
// we'll issue the splits for that file. Splits cannot be processed until the
// header is parsed (the header object is then shared across splits for that file).
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/base-sequence-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/base-sequence-scanner.h b/be/src/exec/base-sequence-scanner.h
index 3c2326e..887ff6f 100644
--- a/be/src/exec/base-sequence-scanner.h
+++ b/be/src/exec/base-sequence-scanner.h
@@ -47,8 +47,7 @@ class ScannerContext;
/// situation, causing the block to be incorrectly skipped.
class BaseSequenceScanner : public HdfsScanner {
public:
- /// Issue the initial ranges for all sequence container files. 'files' must not be
- /// empty.
+ /// Issue the initial ranges for all sequence container files.
static Status IssueInitialRanges(HdfsScanNodeBase* scan_node,
const std::vector<HdfsFileDesc*>& files)
WARN_UNUSED_RESULT;
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-lzo-text-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-lzo-text-scanner.cc b/be/src/exec/hdfs-lzo-text-scanner.cc
index 8af89f2..88ae295 100644
--- a/be/src/exec/hdfs-lzo-text-scanner.cc
+++ b/be/src/exec/hdfs-lzo-text-scanner.cc
@@ -62,7 +62,6 @@ HdfsScanner* HdfsLzoTextScanner::GetHdfsLzoTextScanner(
Status HdfsLzoTextScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
const vector<HdfsFileDesc*>& files) {
- DCHECK(!files.empty());
if (LzoIssueInitialRanges == NULL) {
lock_guard<SpinLock> l(lzo_load_lock_);
if (library_load_status_.ok()) {
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-parquet-scanner-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-scanner-test.cc b/be/src/exec/hdfs-parquet-scanner-test.cc
deleted file mode 100644
index cbc6e76..0000000
--- a/be/src/exec/hdfs-parquet-scanner-test.cc
+++ /dev/null
@@ -1,96 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "exec/hdfs-parquet-scanner.h"
-#include "testutil/gtest-util.h"
-
-#include "common/names.h"
-
-static const int64_t MIN_BUFFER_SIZE = 64 * 1024;
-static const int64_t MAX_BUFFER_SIZE = 8 * 1024 * 1024;
-
-namespace impala {
-
-class HdfsParquetScannerTest : public testing::Test {
- protected:
- void TestDivideReservation(const vector<int64_t>& col_range_lengths,
- int64_t total_col_reservation, const vector<int64_t>& expected_reservations);
-};
-
-/// Test that DivideReservationBetweenColumns() returns 'expected_reservations' for
-/// inputs 'col_range_lengths' and 'total_col_reservation'.
-void HdfsParquetScannerTest::TestDivideReservation(const vector<int64_t>& col_range_lengths,
- int64_t total_col_reservation, const vector<int64_t>& expected_reservations) {
- vector<pair<int, int64_t>> reservations =
- HdfsParquetScanner::DivideReservationBetweenColumnsHelper(
- MIN_BUFFER_SIZE, MAX_BUFFER_SIZE, col_range_lengths, total_col_reservation);
- for (int i = 0; i < reservations.size(); ++i) {
- LOG(INFO) << i << " " << reservations[i].first << " " << reservations[i].second;
- }
- EXPECT_EQ(reservations.size(), expected_reservations.size());
- vector<bool> present(expected_reservations.size(), false);
- for (auto& reservation: reservations) {
- // Ensure that each appears exactly once.
- EXPECT_FALSE(present[reservation.first]);
- present[reservation.first] = true;
- EXPECT_EQ(expected_reservations[reservation.first], reservation.second)
- << reservation.first;
- }
-}
-
-TEST_F(HdfsParquetScannerTest, DivideReservation) {
- // Test a long scan ranges with lots of memory - should allocate 3 max-size
- // buffers per range.
- TestDivideReservation({100 * 1024 * 1024}, 50 * 1024 * 1024, {3 * MAX_BUFFER_SIZE});
- TestDivideReservation({100 * 1024 * 1024, 50 * 1024 * 1024}, 100 * 1024 * 1024,
- {3 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE});
-
- // Long scan ranges, not enough memory for 3 buffers each. Should only allocate
- // max-sized buffers, preferring the longer scan range.
- TestDivideReservation({50 * 1024 * 1024, 100 * 1024 * 1024}, 5 * MAX_BUFFER_SIZE,
- {2 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE});
- TestDivideReservation({50 * 1024 * 1024, 100 * 1024 * 1024},
- 5 * MAX_BUFFER_SIZE + MIN_BUFFER_SIZE,
- {2 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE});
- TestDivideReservation({50 * 1024 * 1024, 100 * 1024 * 1024}, 6 * MAX_BUFFER_SIZE - 1,
- {2 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE});
-
- // Test a short range with lots of memory - should round up buffer size.
- TestDivideReservation({100 * 1024}, 50 * 1024 * 1024, {128 * 1024});
-
- // Test a range << MIN_BUFFER_SIZE - should round up to buffer size.
- TestDivideReservation({13}, 50 * 1024 * 1024, {MIN_BUFFER_SIZE});
-
- // Test long ranges with limited memory.
- TestDivideReservation({100 * 1024 * 1024}, 100 * 1024, {MIN_BUFFER_SIZE});
- TestDivideReservation({100 * 1024 * 1024}, MIN_BUFFER_SIZE, {MIN_BUFFER_SIZE});
- TestDivideReservation({100 * 1024 * 1024}, 2 * MIN_BUFFER_SIZE, {2 * MIN_BUFFER_SIZE});
- TestDivideReservation({100 * 1024 * 1024}, MAX_BUFFER_SIZE - 1, {MAX_BUFFER_SIZE / 2});
- TestDivideReservation({100 * 1024 * 1024, 1024 * 1024, MIN_BUFFER_SIZE},
- 3 * MIN_BUFFER_SIZE, {MIN_BUFFER_SIZE, MIN_BUFFER_SIZE, MIN_BUFFER_SIZE});
-
- // Test a mix of scan range lengths larger than and smaller than the max I/O buffer
- // size. Long ranges get allocated most memory.
- TestDivideReservation(
- {15145047, 5019635, 5019263, 15145047, 15145047, 5019635, 5019263, 317304},
- 25165824,
- {8388608, 2097152, 524288, 8388608, 4194304, 1048576, 262144, 262144});
-}
-
-}
-
-IMPALA_TEST_MAIN();
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-parquet-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-scanner.cc b/be/src/exec/hdfs-parquet-scanner.cc
index 574ddb0..e279369 100644
--- a/be/src/exec/hdfs-parquet-scanner.cc
+++ b/be/src/exec/hdfs-parquet-scanner.cc
@@ -17,7 +17,6 @@
#include "exec/hdfs-parquet-scanner.h"
-#include <algorithm>
#include <queue>
#include <gutil/strings/substitute.h>
@@ -28,7 +27,6 @@
#include "exec/parquet-column-stats.h"
#include "exec/scanner-context.inline.h"
#include "runtime/collection-value-builder.h"
-#include "runtime/exec-env.h"
#include "runtime/io/disk-io-mgr.h"
#include "runtime/runtime-state.h"
#include "runtime/runtime-filter.inline.h"
@@ -37,7 +35,6 @@
#include "common/names.h"
using std::move;
-using std::sort;
using namespace impala;
using namespace impala::io;
@@ -50,6 +47,10 @@ constexpr int BATCHES_PER_FILTER_SELECTIVITY_CHECK = 16;
static_assert(BitUtil::IsPowerOf2(BATCHES_PER_FILTER_SELECTIVITY_CHECK),
"BATCHES_PER_FILTER_SELECTIVITY_CHECK must be a power of two");
+// Max dictionary page header size in bytes. This is an estimate and only needs to be an
+// upper bound.
+const int MAX_DICT_HEADER_SIZE = 100;
+
// Max entries in the dictionary before switching to PLAIN encoding. If a dictionary
// has fewer entries, then the entire column is dictionary encoded. This threshold
// is guaranteed to be true for Impala versions 2.9 or below.
@@ -68,7 +69,6 @@ const string PARQUET_MEM_LIMIT_EXCEEDED =
Status HdfsParquetScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
const vector<HdfsFileDesc*>& files) {
- DCHECK(!files.empty());
vector<ScanRange*> footer_ranges;
for (int i = 0; i < files.size(); ++i) {
// If the file size is less than 12 bytes, it is an invalid Parquet file.
@@ -98,7 +98,7 @@ Status HdfsParquetScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
static_cast<ScanRangeMetadata*>(split->meta_data());
// Each split is processed by first issuing a scan range for the file footer, which
// is done here, followed by scan ranges for the columns of each row group within
- // the actual split (see InitScalarColumns()). The original split is stored in the
+ // the actual split (in InitColumns()). The original split is stored in the
// metadata associated with the footer range.
ScanRange* footer_range;
if (footer_split != nullptr) {
@@ -121,13 +121,9 @@ Status HdfsParquetScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
}
}
}
- // We may not have any scan ranges if this scan node does not have the footer split for
- // any Parquet file.
- if (footer_ranges.size() > 0) {
- // The threads that process the footer will also do the scan, so we mark all the files
- // as complete here.
- RETURN_IF_ERROR(scan_node->AddDiskIoRanges(footer_ranges, files.size()));
- }
+ // The threads that process the footer will also do the scan, so we mark all the files
+ // as complete here.
+ RETURN_IF_ERROR(scan_node->AddDiskIoRanges(footer_ranges, files.size()));
return Status::OK();
}
@@ -232,14 +228,9 @@ Status HdfsParquetScanner::Open(ScannerContext* context) {
// Release I/O buffers immediately to make sure they are cleaned up
// in case we return a non-OK status anywhere below.
- int64_t stream_reservation = stream_->reservation();
- stream_ = nullptr;
context_->ReleaseCompletedResources(true);
context_->ClearStreams();
RETURN_IF_ERROR(footer_status);
- // The scanner-wide stream was used only to read the file footer. Each column has added
- // its own stream. We can use the reservation from 'stream_' for the columns now.
- total_col_reservation_ = stream_reservation;
// Parse the file schema into an internal representation for schema resolution.
schema_resolver_.reset(new ParquetSchemaResolver(*scan_node_->hdfs_table(),
@@ -256,6 +247,10 @@ Status HdfsParquetScanner::Open(ScannerContext* context) {
template_tuple_ = template_tuple_map_[scan_node_->tuple_desc()];
RETURN_IF_ERROR(InitDictFilterStructures());
+
+ // The scanner-wide stream was used only to read the file footer. Each column has added
+ // its own stream.
+ stream_ = nullptr;
return Status::OK();
}
@@ -679,13 +674,15 @@ Status HdfsParquetScanner::NextRowGroup() {
}
InitCollectionColumns();
- RETURN_IF_ERROR(InitScalarColumns());
- // Start scanning dictionary filtering column readers, so we can read the dictionary
- // pages in EvalDictionaryFilters().
- RETURN_IF_ERROR(BaseScalarColumnReader::StartScans(dict_filterable_readers_));
+ // Prepare dictionary filtering columns for first read
+ // This must be done before dictionary filtering, because this code initializes
+ // the column offsets and streams needed to read the dictionaries.
+ // TODO: Restructure the code so that the dictionary can be read without the rest
+ // of the column.
+ RETURN_IF_ERROR(InitScalarColumns(row_group_idx_, dict_filterable_readers_));
- // StartScans() may have allocated resources to scan columns. If we skip this row
+ // InitColumns() may have allocated resources to scan columns. If we skip this row
// group below, we must call ReleaseSkippedRowGroupResources() before continuing.
// If there is a dictionary-encoded column where every value is eliminated
@@ -706,10 +703,10 @@ Status HdfsParquetScanner::NextRowGroup() {
}
// At this point, the row group has passed any filtering criteria
- // Start scanning non-dictionary filtering column readers and initialize their
- // dictionaries.
- RETURN_IF_ERROR(BaseScalarColumnReader::StartScans(non_dict_filterable_readers_));
- status = BaseScalarColumnReader::InitDictionaries(non_dict_filterable_readers_);
+ // Prepare non-dictionary filtering column readers for first read and
+ // initialize their dictionaries.
+ RETURN_IF_ERROR(InitScalarColumns(row_group_idx_, non_dict_filterable_readers_));
+ status = InitDictionaries(non_dict_filterable_readers_);
if (!status.ok()) {
// Either return an error or skip this row group if it is ok to ignore errors
RETURN_IF_ERROR(state_->LogOrReturnError(status.msg()));
@@ -744,6 +741,7 @@ Status HdfsParquetScanner::NextRowGroup() {
break;
}
}
+
DCHECK(parse_status_.ok());
return Status::OK();
}
@@ -801,7 +799,6 @@ void HdfsParquetScanner::PartitionReaders(
} else {
BaseScalarColumnReader* scalar_reader =
static_cast<BaseScalarColumnReader*>(reader);
- scalar_readers_.push_back(scalar_reader);
if (can_eval_dict_filters && IsDictFilterable(scalar_reader)) {
dict_filterable_readers_.push_back(scalar_reader);
} else {
@@ -993,7 +990,7 @@ Status HdfsParquetScanner::EvalDictionaryFilters(const parquet::RowGroup& row_gr
// Any columns that were not 100% dictionary encoded need to initialize
// their dictionaries here.
- RETURN_IF_ERROR(BaseScalarColumnReader::InitDictionaries(deferred_dict_init_list));
+ RETURN_IF_ERROR(InitDictionaries(deferred_dict_init_list));
return Status::OK();
}
@@ -1442,15 +1439,12 @@ Status HdfsParquetScanner::ProcessFooter() {
BufferOpts::ReadInto(metadata_buffer.buffer(), metadata_size));
unique_ptr<BufferDescriptor> io_buffer;
- bool needs_buffers;
- RETURN_IF_ERROR(io_mgr->StartScanRange(
- scan_node_->reader_context(), metadata_range, &needs_buffers));
- DCHECK(!needs_buffers) << "Already provided a buffer";
- RETURN_IF_ERROR(metadata_range->GetNext(&io_buffer));
+ RETURN_IF_ERROR(
+ io_mgr->Read(scan_node_->reader_context(), metadata_range, &io_buffer));
DCHECK_EQ(io_buffer->buffer(), metadata_buffer.buffer());
DCHECK_EQ(io_buffer->len(), metadata_size);
DCHECK(io_buffer->eosr());
- metadata_range->ReturnBuffer(move(io_buffer));
+ io_mgr->ReturnBuffer(move(io_buffer));
}
// Deserialize file header
@@ -1650,16 +1644,23 @@ void HdfsParquetScanner::InitCollectionColumns() {
}
}
-Status HdfsParquetScanner::InitScalarColumns() {
+Status HdfsParquetScanner::InitScalarColumns(
+ int row_group_idx, const vector<BaseScalarColumnReader*>& column_readers) {
int64_t partition_id = context_->partition_descriptor()->id();
const HdfsFileDesc* file_desc = scan_node_->GetFileDesc(partition_id, filename());
DCHECK(file_desc != nullptr);
- parquet::RowGroup& row_group = file_metadata_.row_groups[row_group_idx_];
+ parquet::RowGroup& row_group = file_metadata_.row_groups[row_group_idx];
+ // All the scan ranges (one for each column).
+ vector<ScanRange*> col_ranges;
// Used to validate that the number of values in each reader in column_readers_ at the
// same SchemaElement is the same.
unordered_map<const parquet::SchemaElement*, int> num_values_map;
- for (BaseScalarColumnReader* scalar_reader : scalar_readers_) {
+ // Used to validate we issued the right number of scan ranges
+ int num_scalar_readers = 0;
+
+ for (BaseScalarColumnReader* scalar_reader: column_readers) {
+ ++num_scalar_readers;
const parquet::ColumnChunk& col_chunk = row_group.columns[scalar_reader->col_idx()];
auto num_values_it = num_values_map.find(&scalar_reader->schema_element());
int num_values = -1;
@@ -1668,115 +1669,78 @@ Status HdfsParquetScanner::InitScalarColumns() {
} else {
num_values_map[&scalar_reader->schema_element()] = col_chunk.meta_data.num_values;
}
+ int64_t col_start = col_chunk.meta_data.data_page_offset;
+
if (num_values != -1 && col_chunk.meta_data.num_values != num_values) {
// TODO: improve this error message by saying which columns are different,
// and also specify column in other error messages as appropriate
return Status(TErrorCode::PARQUET_NUM_COL_VALS_ERROR, scalar_reader->col_idx(),
col_chunk.meta_data.num_values, num_values, filename());
}
- RETURN_IF_ERROR(scalar_reader->Reset(*file_desc, col_chunk, row_group_idx_));
- }
- RETURN_IF_ERROR(
- DivideReservationBetweenColumns(scalar_readers_, total_col_reservation_));
- return Status::OK();
-}
-Status HdfsParquetScanner::DivideReservationBetweenColumns(
- const vector<BaseScalarColumnReader*>& column_readers,
- int64_t reservation_to_distribute) {
- DiskIoMgr* io_mgr = ExecEnv::GetInstance()->disk_io_mgr();
- const int64_t min_buffer_size = io_mgr->min_buffer_size();
- const int64_t max_buffer_size = io_mgr->max_buffer_size();
- // The HdfsScanNode reservation calculation in the planner ensures that we have
- // reservation for at least one buffer per column.
- if (reservation_to_distribute < min_buffer_size * column_readers.size()) {
- return Status(TErrorCode::INTERNAL_ERROR,
- Substitute("Not enough reservation in Parquet scanner for file '$0'. Need at "
- "least $1 bytes per column for $2 columns but had $3 bytes",
- filename(), min_buffer_size, column_readers.size(),
- reservation_to_distribute));
- }
+ RETURN_IF_ERROR(ParquetMetadataUtils::ValidateRowGroupColumn(file_metadata_,
+ filename(), row_group_idx, scalar_reader->col_idx(),
+ scalar_reader->schema_element(), state_));
- vector<int64_t> col_range_lengths(column_readers.size());
- for (int i = 0; i < column_readers.size(); ++i) {
- col_range_lengths[i] = column_readers[i]->scan_range()->len();
- }
- vector<pair<int, int64_t>> tmp_reservations = DivideReservationBetweenColumnsHelper(
- min_buffer_size, max_buffer_size, col_range_lengths, reservation_to_distribute);
- for (auto& tmp_reservation : tmp_reservations) {
- column_readers[tmp_reservation.first]->set_io_reservation(tmp_reservation.second);
- }
- return Status::OK();
-}
+ if (col_chunk.meta_data.__isset.dictionary_page_offset) {
+ // Already validated in ValidateColumnOffsets()
+ DCHECK_LT(col_chunk.meta_data.dictionary_page_offset, col_start);
+ col_start = col_chunk.meta_data.dictionary_page_offset;
+ }
+ int64_t col_len = col_chunk.meta_data.total_compressed_size;
+ if (col_len <= 0) {
+ return Status(Substitute("File '$0' contains invalid column chunk size: $1",
+ filename(), col_len));
+ }
+ int64_t col_end = col_start + col_len;
+
+ // Already validated in ValidateColumnOffsets()
+ DCHECK_GT(col_end, 0);
+ DCHECK_LT(col_end, file_desc->file_length);
+ if (file_version_.application == "parquet-mr" && file_version_.VersionLt(1, 2, 9)) {
+ // The Parquet MR writer had a bug in 1.2.8 and below where it didn't include the
+ // dictionary page header size in total_compressed_size and total_uncompressed_size
+ // (see IMPALA-694). We pad col_len to compensate.
+ int64_t bytes_remaining = file_desc->file_length - col_end;
+ int64_t pad = min<int64_t>(MAX_DICT_HEADER_SIZE, bytes_remaining);
+ col_len += pad;
+ }
-vector<pair<int, int64_t>> HdfsParquetScanner::DivideReservationBetweenColumnsHelper(
- int64_t min_buffer_size, int64_t max_buffer_size,
- const vector<int64_t>& col_range_lengths, int64_t reservation_to_distribute) {
- // Pair of (column index, reservation allocated).
- vector<pair<int, int64_t>> tmp_reservations;
- for (int i = 0; i < col_range_lengths.size(); ++i) tmp_reservations.emplace_back(i, 0);
-
- // Sort in descending order of length, breaking ties by index so that larger columns
- // get allocated reservation first. It is common to have dramatically different column
- // sizes in a single file because of different value sizes and compressibility. E.g.
- // consider a large STRING "comment" field versus a highly compressible
- // dictionary-encoded column with only a few distinct values. We want to give max-sized
- // buffers to large columns first to maximize the size of I/Os that we do while reading
- // this row group.
- sort(tmp_reservations.begin(), tmp_reservations.end(),
- [&col_range_lengths](const pair<int, int64_t>& left, const pair<int, int64_t>& right) {
- int64_t left_len = col_range_lengths[left.first];
- int64_t right_len = col_range_lengths[right.first];
- return left_len != right_len ? left_len > right_len : left.first < right.first;
- });
-
- // Set aside the minimum reservation per column.
- reservation_to_distribute -= min_buffer_size * col_range_lengths.size();
-
- // Allocate reservations to columns by repeatedly allocating either a max-sized buffer
- // or a large enough buffer to fit the remaining data for each column. Do this
- // round-robin up to the ideal number of I/O buffers.
- for (int i = 0; i < DiskIoMgr::IDEAL_MAX_SIZED_BUFFERS_PER_SCAN_RANGE; ++i) {
- for (auto& tmp_reservation : tmp_reservations) {
- // Add back the reservation we set aside above.
- if (i == 0) reservation_to_distribute += min_buffer_size;
-
- int64_t bytes_left_in_range =
- col_range_lengths[tmp_reservation.first] - tmp_reservation.second;
- int64_t bytes_to_add;
- if (bytes_left_in_range >= max_buffer_size) {
- if (reservation_to_distribute >= max_buffer_size) {
- bytes_to_add = max_buffer_size;
- } else if (i == 0) {
- DCHECK_EQ(0, tmp_reservation.second);
- // Ensure this range gets at least one buffer on the first iteration.
- bytes_to_add = BitUtil::RoundDownToPowerOfTwo(reservation_to_distribute);
- } else {
- DCHECK_GT(tmp_reservation.second, 0);
- // We need to read more than the max buffer size, but can't allocate a
- // max-sized buffer. Stop adding buffers to this column: we prefer to use
- // the existing max-sized buffers without small buffers mixed in so that
- // we will alway do max-sized I/Os, which make efficient use of I/O devices.
- bytes_to_add = 0;
- }
- } else if (bytes_left_in_range > 0 &&
- reservation_to_distribute >= min_buffer_size) {
- // Choose a buffer size that will fit the rest of the bytes left in the range.
- bytes_to_add = max(min_buffer_size, BitUtil::RoundUpToPowerOfTwo(bytes_left_in_range));
- // But don't add more reservation than is available.
- bytes_to_add =
- min(bytes_to_add, BitUtil::RoundDownToPowerOfTwo(reservation_to_distribute));
- } else {
- bytes_to_add = 0;
- }
- DCHECK(bytes_to_add == 0 || bytes_to_add >= min_buffer_size) << bytes_to_add;
- reservation_to_distribute -= bytes_to_add;
- tmp_reservation.second += bytes_to_add;
- DCHECK_GE(reservation_to_distribute, 0);
- DCHECK_GT(tmp_reservation.second, 0);
+ // TODO: this will need to change when we have co-located files and the columns
+ // are different files.
+ if (!col_chunk.file_path.empty() && col_chunk.file_path != filename()) {
+ return Status(Substitute("Expected parquet column file path '$0' to match "
+ "filename '$1'", col_chunk.file_path, filename()));
}
+
+ const ScanRange* split_range =
+ static_cast<ScanRangeMetadata*>(metadata_range_->meta_data())->original_split;
+
+ // Determine if the column is completely contained within a local split.
+ bool col_range_local = split_range->expected_local()
+ && col_start >= split_range->offset()
+ && col_end <= split_range->offset() + split_range->len();
+ ScanRange* col_range = scan_node_->AllocateScanRange(metadata_range_->fs(),
+ filename(), col_len, col_start, partition_id, split_range->disk_id(),
+ col_range_local,
+ BufferOpts(split_range->try_cache(), file_desc->mtime));
+ col_ranges.push_back(col_range);
+
+ // Get the stream that will be used for this column
+ ScannerContext::Stream* stream = context_->AddStream(col_range);
+ DCHECK(stream != nullptr);
+
+ RETURN_IF_ERROR(scalar_reader->Reset(&col_chunk.meta_data, stream));
}
- return tmp_reservations;
+ DCHECK_EQ(col_ranges.size(), num_scalar_readers);
+
+ // Issue all the column chunks to the io mgr and have them scheduled immediately.
+ // This means these ranges aren't returned via DiskIoMgr::GetNextRange and
+ // instead are scheduled to be read immediately.
+ RETURN_IF_ERROR(scan_node_->runtime_state()->io_mgr()->AddScanRanges(
+ scan_node_->reader_context(), col_ranges, true));
+
+ return Status::OK();
}
Status HdfsParquetScanner::InitDictionaries(
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-parquet-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-scanner.h b/be/src/exec/hdfs-parquet-scanner.h
index 92f2550..f0043b5 100644
--- a/be/src/exec/hdfs-parquet-scanner.h
+++ b/be/src/exec/hdfs-parquet-scanner.h
@@ -34,7 +34,7 @@ class CollectionValueBuilder;
struct HdfsFileDesc;
/// Internal schema representation and resolution.
-struct SchemaNode;
+class SchemaNode;
/// Class that implements Parquet definition and repetition level decoding.
class ParquetLevelDecoder;
@@ -69,8 +69,8 @@ class BoolColumnReader;
/// the split size, the mid point guarantees that we have at least 50% of the row group in
/// the current split. ProcessSplit() then computes the column ranges for these row groups
/// and submits them to the IoMgr for immediate scheduling (so they don't surface in
-/// DiskIoMgr::GetNextUnstartedRange()). Scheduling them immediately also guarantees they
-/// are all read at once.
+/// DiskIoMgr::GetNextRange()). Scheduling them immediately also guarantees they are all
+/// read at once.
///
/// Like the other scanners, each parquet scanner object is one to one with a
/// ScannerContext. Unlike the other scanners though, the context will have multiple
@@ -328,7 +328,7 @@ class HdfsParquetScanner : public HdfsScanner {
virtual ~HdfsParquetScanner() {}
/// Issue just the footer range for each file. We'll then parse the footer and pick
- /// out the columns we want. 'files' must not be empty.
+ /// out the columns we want.
static Status IssueInitialRanges(HdfsScanNodeBase* scan_node,
const std::vector<HdfsFileDesc*>& files)
WARN_UNUSED_RESULT;
@@ -361,7 +361,6 @@ class HdfsParquetScanner : public HdfsScanner {
template<typename InternalType, parquet::Type::type PARQUET_TYPE, bool MATERIALIZED>
friend class ScalarColumnReader;
friend class BoolColumnReader;
- friend class HdfsParquetScannerTest;
/// Size of the file footer. This is a guess. If this value is too little, we will
/// need to issue another read.
@@ -430,7 +429,7 @@ class HdfsParquetScanner : public HdfsScanner {
/// Number of scratch batches processed so far.
int64_t row_batches_produced_;
- /// Column reader for each top-level materialized slot in the output tuple.
+ /// Column reader for each materialized columns for this file.
std::vector<ParquetColumnReader*> column_readers_;
/// Column readers will write slot values into this scratch batch for
@@ -446,9 +445,6 @@ class HdfsParquetScanner : public HdfsScanner {
/// Scan range for the metadata.
const io::ScanRange* metadata_range_;
- /// Reservation available for scanning columns, in bytes.
- int64_t total_col_reservation_ = 0;
-
/// Pool to copy dictionary page buffer into. This pool is shared across all the
/// pages in a column chunk.
boost::scoped_ptr<MemPool> dictionary_pool_;
@@ -465,9 +461,6 @@ class HdfsParquetScanner : public HdfsScanner {
/// or nested within a collection.
std::vector<BaseScalarColumnReader*> non_dict_filterable_readers_;
- /// Flattened list of all scalar column readers in column_readers_.
- std::vector<BaseScalarColumnReader*> scalar_readers_;
-
/// Flattened collection column readers that point to readers in column_readers_.
std::vector<CollectionColumnReader*> collection_readers_;
@@ -634,24 +627,12 @@ class HdfsParquetScanner : public HdfsScanner {
WARN_UNUSED_RESULT;
/// Walks file_metadata_ and initiates reading the materialized columns. This
- /// initializes 'scalar_readers_' and divides reservation between the columns but
- /// does not start any scan ranges.
- Status InitScalarColumns() WARN_UNUSED_RESULT;
-
- /// Decides how to divide 'reservation_to_distribute' bytes of reservation between the
- /// columns. Sets the reservation on each corresponding reader in 'column_readers'.
- Status DivideReservationBetweenColumns(
- const std::vector<BaseScalarColumnReader*>& column_readers,
- int64_t reservation_to_distribute);
-
- /// Helper for DivideReservationBetweenColumns. Implements the core algorithm for
- /// dividing a reservation of 'reservation_to_distribute' bytes between columns with
- /// scan range lengths 'col_range_lengths' given a min and max buffer size. Returns
- /// a vector with an entry per column with the index into 'col_range_lengths' and the
- /// amount of reservation in bytes to give to that column.
- static std::vector<std::pair<int, int64_t>> DivideReservationBetweenColumnsHelper(
- int64_t min_buffer_size, int64_t max_buffer_size,
- const std::vector<int64_t>& col_range_lengths, int64_t reservation_to_distribute);
+ /// initializes 'column_readers' and issues the reads for the columns. 'column_readers'
+ /// includes a mix of scalar readers from multiple schema nodes (i.e., readers of
+ /// top-level scalar columns and readers of scalar columns within a collection node).
+ Status InitScalarColumns(
+ int row_group_idx, const std::vector<BaseScalarColumnReader*>& column_readers)
+ WARN_UNUSED_RESULT;
/// Initializes the column readers in collection_readers_.
void InitCollectionColumns();
@@ -687,8 +668,7 @@ class HdfsParquetScanner : public HdfsScanner {
/// Partitions the readers into scalar and collection readers. The collection readers
/// are flattened into collection_readers_. The scalar readers are partitioned into
/// dict_filterable_readers_ and non_dict_filterable_readers_ depending on whether
- /// dictionary filtering is enabled and the reader can be dictionary filtered. All
- /// scalar readers are also flattened into scalar_readers_.
+ /// dictionary filtering is enabled and the reader can be dictionary filtered.
void PartitionReaders(const vector<ParquetColumnReader*>& readers,
bool can_eval_dict_filters);
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-scan-node-base.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-base.cc b/be/src/exec/hdfs-scan-node-base.cc
index 0dfd6f0..80cb6c5 100644
--- a/be/src/exec/hdfs-scan-node-base.cc
+++ b/be/src/exec/hdfs-scan-node-base.cc
@@ -35,7 +35,6 @@
#include "exprs/scalar-expr-evaluator.h"
#include "exprs/scalar-expr.h"
#include "runtime/descriptors.h"
-#include "runtime/exec-env.h"
#include "runtime/hdfs-fs-cache.h"
#include "runtime/io/disk-io-mgr.h"
#include "runtime/io/request-context.h"
@@ -68,7 +67,6 @@ const int UNEXPECTED_REMOTE_BYTES_WARN_THRESHOLD = 64 * 1024 * 1024;
HdfsScanNodeBase::HdfsScanNodeBase(ObjectPool* pool, const TPlanNode& tnode,
const DescriptorTbl& descs)
: ScanNode(pool, tnode, descs),
- ideal_scan_range_reservation_(tnode.hdfs_scan_node.ideal_scan_range_reservation),
min_max_tuple_id_(tnode.hdfs_scan_node.__isset.min_max_tuple_id ?
tnode.hdfs_scan_node.min_max_tuple_id : -1),
skip_header_line_count_(tnode.hdfs_scan_node.__isset.skip_header_line_count ?
@@ -85,7 +83,6 @@ HdfsScanNodeBase::HdfsScanNodeBase(ObjectPool* pool, const TPlanNode& tnode,
&tnode.hdfs_scan_node.dictionary_filter_conjuncts : nullptr),
disks_accessed_bitmap_(TUnit::UNIT, 0),
active_hdfs_read_thread_counter_(TUnit::UNIT, 0) {
- DCHECK_GE(ideal_scan_range_reservation_, resource_profile_.min_reservation);
}
HdfsScanNodeBase::~HdfsScanNodeBase() {
@@ -115,6 +112,7 @@ Status HdfsScanNodeBase::Init(const TPlanNode& tnode, RuntimeState* state) {
RETURN_IF_ERROR(ScalarExpr::Create(tnode.hdfs_scan_node.min_max_conjuncts,
*min_max_row_desc, state, &min_max_conjuncts_));
}
+
return Status::OK();
}
@@ -248,16 +246,6 @@ Status HdfsScanNodeBase::Prepare(RuntimeState* state) {
ImpaladMetrics::NUM_RANGES_PROCESSED->Increment(scan_range_params_->size());
ImpaladMetrics::NUM_RANGES_MISSING_VOLUME_ID->Increment(num_ranges_missing_volume_id);
- // Check if reservation was enough to allocate at least one buffer. The
- // reservation calculation in HdfsScanNode.java should guarantee this.
- // Hitting this error indicates a misconfiguration or bug.
- int64_t min_buffer_size = ExecEnv::GetInstance()->disk_io_mgr()->min_buffer_size();
- if (scan_range_params_->size() > 0
- && resource_profile_.min_reservation < min_buffer_size) {
- return Status(TErrorCode::INTERNAL_ERROR,
- Substitute("HDFS scan min reservation $0 must be >= min buffer size $1",
- resource_profile_.min_reservation, min_buffer_size));
- }
// Add per volume stats to the runtime profile
PerVolumeStats per_volume_stats;
stringstream str;
@@ -344,18 +332,7 @@ Status HdfsScanNodeBase::Open(RuntimeState* state) {
partition_desc->partition_key_value_evals(), scan_node_pool_.get(), state);
}
- RETURN_IF_ERROR(ClaimBufferReservation(state));
- // We got the minimum reservation. Now try to get ideal reservation.
- if (resource_profile_.min_reservation != ideal_scan_range_reservation_) {
- bool increased = buffer_pool_client_.IncreaseReservation(
- ideal_scan_range_reservation_ - resource_profile_.min_reservation);
- VLOG_FILE << "Increasing reservation from minimum "
- << resource_profile_.min_reservation << "B to ideal "
- << ideal_scan_range_reservation_ << "B "
- << (increased ? "succeeded" : "failed");
- }
-
- reader_context_ = runtime_state_->io_mgr()->RegisterContext();
+ reader_context_ = runtime_state_->io_mgr()->RegisterContext(mem_tracker());
// Initialize HdfsScanNode specific counters
// TODO: Revisit counters and move the counters specific to multi-threaded scans
@@ -376,11 +353,14 @@ Status HdfsScanNodeBase::Open(RuntimeState* state) {
num_scanner_threads_started_counter_ =
ADD_COUNTER(runtime_profile(), NUM_SCANNER_THREADS_STARTED, TUnit::UNIT);
- reader_context_->set_bytes_read_counter(bytes_read_counter());
- reader_context_->set_read_timer(read_timer());
- reader_context_->set_open_file_timer(open_file_timer());
- reader_context_->set_active_read_thread_counter(&active_hdfs_read_thread_counter_);
- reader_context_->set_disks_accessed_bitmap(&disks_accessed_bitmap_);
+ runtime_state_->io_mgr()->set_bytes_read_counter(
+ reader_context_.get(), bytes_read_counter());
+ runtime_state_->io_mgr()->set_read_timer(reader_context_.get(), read_timer());
+ runtime_state_->io_mgr()->set_open_file_timer(reader_context_.get(), open_file_timer());
+ runtime_state_->io_mgr()->set_active_read_thread_counter(
+ reader_context_.get(), &active_hdfs_read_thread_counter_);
+ runtime_state_->io_mgr()->set_disks_access_bitmap(
+ reader_context_.get(), &disks_accessed_bitmap_);
average_scanner_thread_concurrency_ = runtime_profile()->AddSamplingCounter(
AVERAGE_SCANNER_THREAD_CONCURRENCY, &active_scanner_thread_counter_);
@@ -474,27 +454,18 @@ Status HdfsScanNodeBase::IssueInitialScanRanges(RuntimeState* state) {
}
}
- // Issue initial ranges for all file types. Only call functions for file types that
- // actually exist - trying to add empty lists of ranges can result in spurious
- // CANCELLED errors - see IMPALA-6564.
- for (const auto& entry : matching_per_type_files) {
- if (entry.second.empty()) continue;
- switch (entry.first) {
- case THdfsFileFormat::PARQUET:
- RETURN_IF_ERROR(HdfsParquetScanner::IssueInitialRanges(this, entry.second));
- break;
- case THdfsFileFormat::TEXT:
- RETURN_IF_ERROR(HdfsTextScanner::IssueInitialRanges(this, entry.second));
- break;
- case THdfsFileFormat::SEQUENCE_FILE:
- case THdfsFileFormat::RC_FILE:
- case THdfsFileFormat::AVRO:
- RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this, entry.second));
- break;
- default:
- DCHECK(false) << "Unexpected file type " << entry.first;
- }
- }
+ // Issue initial ranges for all file types.
+ RETURN_IF_ERROR(HdfsParquetScanner::IssueInitialRanges(this,
+ matching_per_type_files[THdfsFileFormat::PARQUET]));
+ RETURN_IF_ERROR(HdfsTextScanner::IssueInitialRanges(this,
+ matching_per_type_files[THdfsFileFormat::TEXT]));
+ RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this,
+ matching_per_type_files[THdfsFileFormat::SEQUENCE_FILE]));
+ RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this,
+ matching_per_type_files[THdfsFileFormat::RC_FILE]));
+ RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this,
+ matching_per_type_files[THdfsFileFormat::AVRO]));
+
return Status::OK();
}
@@ -553,8 +524,6 @@ ScanRange* HdfsScanNodeBase::AllocateScanRange(hdfsFS fs, const char* file,
Status HdfsScanNodeBase::AddDiskIoRanges(
const vector<ScanRange*>& ranges, int num_files_queued) {
- DCHECK(!progress_.done()) << "Don't call AddScanRanges() after all ranges finished.";
- DCHECK_GT(ranges.size(), 0);
RETURN_IF_ERROR(runtime_state_->io_mgr()->AddScanRanges(reader_context_.get(), ranges));
num_unqueued_files_.Add(-num_files_queued);
DCHECK_GE(num_unqueued_files_.Load(), 0);
@@ -857,14 +826,20 @@ void HdfsScanNodeBase::StopAndFinalizeCounters() {
Substitute("Codegen enabled: $0 out of $1", num_enabled, total));
if (reader_context_ != nullptr) {
- bytes_read_local_->Set(reader_context_->bytes_read_local());
- bytes_read_short_circuit_->Set(reader_context_->bytes_read_short_circuit());
- bytes_read_dn_cache_->Set(reader_context_->bytes_read_dn_cache());
- num_remote_ranges_->Set(reader_context_->num_remote_ranges());
- unexpected_remote_bytes_->Set(reader_context_->unexpected_remote_bytes());
- cached_file_handles_hit_count_->Set(reader_context_->cached_file_handles_hit_count());
+ bytes_read_local_->Set(
+ runtime_state_->io_mgr()->bytes_read_local(reader_context_.get()));
+ bytes_read_short_circuit_->Set(
+ runtime_state_->io_mgr()->bytes_read_short_circuit(reader_context_.get()));
+ bytes_read_dn_cache_->Set(
+ runtime_state_->io_mgr()->bytes_read_dn_cache(reader_context_.get()));
+ num_remote_ranges_->Set(static_cast<int64_t>(
+ runtime_state_->io_mgr()->num_remote_ranges(reader_context_.get())));
+ unexpected_remote_bytes_->Set(
+ runtime_state_->io_mgr()->unexpected_remote_bytes(reader_context_.get()));
+ cached_file_handles_hit_count_->Set(
+ runtime_state_->io_mgr()->cached_file_handles_hit_count(reader_context_.get()));
cached_file_handles_miss_count_->Set(
- reader_context_->cached_file_handles_miss_count());
+ runtime_state_->io_mgr()->cached_file_handles_miss_count(reader_context_.get()));
if (unexpected_remote_bytes_->value() >= UNEXPECTED_REMOTE_BYTES_WARN_THRESHOLD) {
runtime_state_->LogError(ErrorMsg(TErrorCode::GENERAL, Substitute(
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-scan-node-base.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-base.h b/be/src/exec/hdfs-scan-node-base.h
index 3a9c37f..70fbac2 100644
--- a/be/src/exec/hdfs-scan-node-base.h
+++ b/be/src/exec/hdfs-scan-node-base.h
@@ -322,9 +322,6 @@ class HdfsScanNodeBase : public ScanNode {
friend class ScannerContext;
friend class HdfsScanner;
- /// Ideal reservation to process each input split, computed by the planner.
- const int64_t ideal_scan_range_reservation_;
-
/// Tuple id of the tuple used to evaluate conjuncts on parquet::Statistics.
const int min_max_tuple_id_;
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-scan-node-mt.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-mt.cc b/be/src/exec/hdfs-scan-node-mt.cc
index f4948d9..7ea4d80 100644
--- a/be/src/exec/hdfs-scan-node-mt.cc
+++ b/be/src/exec/hdfs-scan-node-mt.cc
@@ -26,7 +26,6 @@
#include "gen-cpp/PlanNodes_types.h"
-using namespace impala::io;
using std::stringstream;
namespace impala {
@@ -77,27 +76,20 @@ Status HdfsScanNodeMt::GetNext(RuntimeState* state, RowBatch* row_batch, bool* e
scanner_->Close(row_batch);
scanner_.reset();
}
- DiskIoMgr* io_mgr = runtime_state_->io_mgr();
- bool needs_buffers;
- RETURN_IF_ERROR(io_mgr->GetNextUnstartedRange(
- reader_context_.get(), &scan_range_, &needs_buffers));
- if (scan_range_ == nullptr) {
+ RETURN_IF_ERROR(
+ runtime_state_->io_mgr()->GetNextRange(reader_context_.get(), &scan_range_));
+ if (scan_range_ == NULL) {
*eos = true;
StopAndFinalizeCounters();
return Status::OK();
}
- int64_t scanner_reservation = buffer_pool_client_.GetReservation();
- if (needs_buffers) {
- RETURN_IF_ERROR(io_mgr->AllocateBuffersForRange(reader_context_.get(),
- &buffer_pool_client_, scan_range_, scanner_reservation));
- }
ScanRangeMetadata* metadata =
static_cast<ScanRangeMetadata*>(scan_range_->meta_data());
int64_t partition_id = metadata->partition_id;
HdfsPartitionDescriptor* partition = hdfs_table_->GetPartition(partition_id);
- scanner_ctx_.reset(new ScannerContext(runtime_state_, this, &buffer_pool_client_,
- partition, filter_ctxs(), expr_results_pool()));
- scanner_ctx_->AddStream(scan_range_, scanner_reservation);
+ scanner_ctx_.reset(new ScannerContext(
+ runtime_state_, this, partition, scan_range_, filter_ctxs(),
+ expr_results_pool()));
Status status = CreateAndOpenScanner(partition, scanner_ctx_.get(), &scanner_);
if (!status.ok()) {
DCHECK(scanner_ == NULL);
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-scan-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node.cc b/be/src/exec/hdfs-scan-node.cc
index a95e47a..710a8af 100644
--- a/be/src/exec/hdfs-scan-node.cc
+++ b/be/src/exec/hdfs-scan-node.cc
@@ -26,7 +26,6 @@
#include "exec/scanner-context.h"
#include "runtime/descriptors.h"
#include "runtime/fragment-instance-state.h"
-#include "runtime/io/request-context.h"
#include "runtime/runtime-filter.inline.h"
#include "runtime/runtime-state.h"
#include "runtime/mem-tracker.h"
@@ -46,6 +45,16 @@ DECLARE_bool(skip_file_runtime_filtering);
using namespace impala;
using namespace impala::io;
+// Amount of memory that we approximate a scanner thread will use not including IoBuffers.
+// The memory used does not vary considerably between file formats (just a couple of MBs).
+// This value is conservative and taken from running against the tpch lineitem table.
+// TODO: revisit how we do this.
+const int SCANNER_THREAD_MEM_USAGE = 32 * 1024 * 1024;
+
+// Estimated upper bound on the compression ratio of compressed text files. Used to
+// estimate scanner thread memory usage.
+const int COMPRESSED_TEXT_COMPRESSION_RATIO = 11;
+
// Amount of time to block waiting for GetNext() to release scanner threads between
// checking if a scanner thread should yield itself back to the global thread pool.
const int SCANNER_THREAD_WAIT_TIME_MS = 20;
@@ -53,6 +62,11 @@ const int SCANNER_THREAD_WAIT_TIME_MS = 20;
HdfsScanNode::HdfsScanNode(ObjectPool* pool, const TPlanNode& tnode,
const DescriptorTbl& descs)
: HdfsScanNodeBase(pool, tnode, descs),
+ ranges_issued_barrier_(1),
+ scanner_thread_bytes_required_(0),
+ done_(false),
+ all_ranges_started_(false),
+ thread_avail_cb_id_(-1),
max_num_scanner_threads_(CpuInfo::num_cores()) {
}
@@ -153,6 +167,36 @@ Status HdfsScanNode::Init(const TPlanNode& tnode, RuntimeState* state) {
Status HdfsScanNode::Prepare(RuntimeState* state) {
SCOPED_TIMER(runtime_profile_->total_time_counter());
RETURN_IF_ERROR(HdfsScanNodeBase::Prepare(state));
+
+ // Compute the minimum bytes required to start a new thread. This is based on the
+ // file format.
+ // The higher the estimate, the less likely it is the query will fail but more likely
+ // the query will be throttled when it does not need to be.
+ // TODO: how many buffers should we estimate per range. The IoMgr will throttle down to
+ // one but if there are already buffers queued before memory pressure was hit, we can't
+ // reclaim that memory.
+ if (per_type_files_[THdfsFileFormat::PARQUET].size() > 0) {
+ // Parquet files require buffers per column
+ scanner_thread_bytes_required_ =
+ materialized_slots_.size() * 3 * runtime_state_->io_mgr()->max_read_buffer_size();
+ } else {
+ scanner_thread_bytes_required_ =
+ 3 * runtime_state_->io_mgr()->max_read_buffer_size();
+ }
+ // scanner_thread_bytes_required_ now contains the IoBuffer requirement.
+ // Next we add in the other memory the scanner thread will use.
+ // e.g. decompression buffers, tuple buffers, etc.
+ // For compressed text, we estimate this based on the file size (since the whole file
+ // will need to be decompressed at once). For all other formats, we use a constant.
+ // TODO: can we do something better?
+ int64_t scanner_thread_mem_usage = SCANNER_THREAD_MEM_USAGE;
+ for (HdfsFileDesc* file: per_type_files_[THdfsFileFormat::TEXT]) {
+ if (file->file_compression != THdfsCompression::NONE) {
+ int64_t bytes_required = file->file_length * COMPRESSED_TEXT_COMPRESSION_RATIO;
+ scanner_thread_mem_usage = ::max(bytes_required, scanner_thread_mem_usage);
+ }
+ }
+ scanner_thread_bytes_required_ += scanner_thread_mem_usage;
row_batches_get_timer_ = ADD_TIMER(runtime_profile(), "RowBatchQueueGetWaitTime");
row_batches_put_timer_ = ADD_TIMER(runtime_profile(), "RowBatchQueuePutWaitTime");
return Status::OK();
@@ -174,9 +218,10 @@ Status HdfsScanNode::Open(RuntimeState* state) {
max_num_scanner_threads_ = runtime_state_->query_options().num_scanner_threads;
}
DCHECK_GT(max_num_scanner_threads_, 0);
- spare_reservation_.Store(buffer_pool_client_.GetReservation());
+
thread_avail_cb_id_ = runtime_state_->resource_pool()->AddThreadAvailableCb(
bind<void>(mem_fn(&HdfsScanNode::ThreadTokenAvailableCb), this, _1));
+
return Status::OK();
}
@@ -217,28 +262,37 @@ Status HdfsScanNode::AddDiskIoRanges(const vector<ScanRange*>& ranges,
return Status::OK();
}
-bool HdfsScanNode::EnoughReservationForExtraThread(const unique_lock<mutex>& lock) {
- DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
- if (spare_reservation_.Load() >= ideal_scan_range_reservation_) return true;
- int64_t increase = ideal_scan_range_reservation_ - spare_reservation_.Load();
- if (!buffer_pool_client_.IncreaseReservation(increase)) return false;
- spare_reservation_.Add(increase);
- return true;
-}
-
-int64_t HdfsScanNode::DeductReservationForScannerThread(const unique_lock<mutex>& lock,
- bool first_thread) {
- DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
- int64_t amount;
- if (first_thread) {
- amount = spare_reservation_.Load() >= ideal_scan_range_reservation_ ?
- ideal_scan_range_reservation_ : resource_profile_.min_reservation;
- } else {
- amount = ideal_scan_range_reservation_;
+// For controlling the amount of memory used for scanners, we approximate the
+// scanner mem usage based on scanner_thread_bytes_required_, rather than the
+// consumption in the scan node's mem tracker. The problem with the scan node
+// trackers is that it does not account for the memory the scanner will use.
+// For example, if there is 110 MB of memory left (based on the mem tracker)
+// and we estimate that a scanner will use 100MB, we want to make sure to only
+// start up one additional thread. However, after starting the first thread, the
+// mem tracker value will not change immediately (it takes some time before the
+// scanner is running and starts using memory). Therefore we just use the estimate
+// based on the number of running scanner threads.
+bool HdfsScanNode::EnoughMemoryForScannerThread(bool new_thread) {
+ int64_t committed_scanner_mem =
+ active_scanner_thread_counter_.value() * scanner_thread_bytes_required_;
+ int64_t tracker_consumption = mem_tracker()->consumption();
+ int64_t est_additional_scanner_mem = committed_scanner_mem - tracker_consumption;
+ if (est_additional_scanner_mem < 0) {
+ // This is the case where our estimate was too low. Expand the estimate based
+ // on the usage.
+ int64_t avg_consumption =
+ tracker_consumption / active_scanner_thread_counter_.value();
+ // Take the average and expand it by 50%. Some scanners will not have hit their
+ // steady state mem usage yet.
+ // TODO: how can we scale down if we've overestimated.
+ // TODO: better heuristic?
+ scanner_thread_bytes_required_ = static_cast<int64_t>(avg_consumption * 1.5);
+ est_additional_scanner_mem = 0;
}
- int64_t remainder = spare_reservation_.Add(-amount);
- DCHECK_GE(remainder, 0);
- return amount;
+
+ // If we are starting a new thread, take that into account now.
+ if (new_thread) est_additional_scanner_mem += scanner_thread_bytes_required_;
+ return est_additional_scanner_mem < mem_tracker()->SpareCapacity();
}
void HdfsScanNode::ThreadTokenAvailableCb(ThreadResourceMgr::ResourcePool* pool) {
@@ -268,45 +322,36 @@ void HdfsScanNode::ThreadTokenAvailableCb(ThreadResourceMgr::ResourcePool* pool)
// TODO: This still leans heavily on starvation-free locks, come up with a more
// correct way to communicate between this method and ScannerThreadHelper
unique_lock<mutex> lock(lock_);
-
- const int64_t num_active_scanner_threads = active_scanner_thread_counter_.value();
- const bool first_thread = num_active_scanner_threads == 0;
// Cases 1, 2, 3.
if (done_ || all_ranges_started_ ||
- num_active_scanner_threads >= progress_.remaining()) {
+ active_scanner_thread_counter_.value() >= progress_.remaining()) {
break;
}
// Cases 5 and 6.
- if (!first_thread &&
+ if (active_scanner_thread_counter_.value() > 0 &&
(materialized_row_batches_->Size() >= max_materialized_row_batches_ ||
- !EnoughReservationForExtraThread(lock))) {
+ !EnoughMemoryForScannerThread(true))) {
break;
}
// Case 7 and 8.
- if (num_active_scanner_threads >= max_num_scanner_threads_ ||
- !pool->TryAcquireThreadToken()) {
+ bool is_reserved = false;
+ if (active_scanner_thread_counter_.value() >= max_num_scanner_threads_ ||
+ !pool->TryAcquireThreadToken(&is_reserved)) {
break;
}
- // Deduct the reservation. We haven't dropped the lock since the
- // first_thread/EnoughReservationForExtraThread() checks so spare reservation
- // must be available.
- int64_t scanner_thread_reservation =
- DeductReservationForScannerThread(lock, first_thread);
COUNTER_ADD(&active_scanner_thread_counter_, 1);
string name = Substitute("scanner-thread (finst:$0, plan-node-id:$1, thread-idx:$2)",
PrintId(runtime_state_->fragment_instance_id()), id(),
num_scanner_threads_started_counter_->value());
- auto fn = [this, scanner_thread_reservation]() {
- this->ScannerThread(scanner_thread_reservation);
- };
+
+ auto fn = [this]() { this->ScannerThread(); };
std::unique_ptr<Thread> t;
status =
Thread::Create(FragmentInstanceState::FINST_THREAD_GROUP_NAME, name, fn, &t, true);
if (!status.ok()) {
- ReturnReservationFromScannerThread(scanner_thread_reservation);
COUNTER_ADD(&active_scanner_thread_counter_, -1);
// Release the token and skip running callbacks to find a replacement. Skipping
// serves two purposes. First, it prevents a mutual recursion between this function
@@ -327,10 +372,9 @@ void HdfsScanNode::ThreadTokenAvailableCb(ThreadResourceMgr::ResourcePool* pool)
}
}
-void HdfsScanNode::ScannerThread(int64_t scanner_thread_reservation) {
+void HdfsScanNode::ScannerThread() {
SCOPED_THREAD_COUNTER_MEASUREMENT(scanner_thread_counters());
SCOPED_THREAD_COUNTER_MEASUREMENT(runtime_state_->total_thread_statistics());
- DiskIoMgr* io_mgr = runtime_state_->io_mgr();
// Make thread-local copy of filter contexts to prune scan ranges, and to pass to the
// scanner for finer-grained filtering. Use a thread-local MemPool for the filter
@@ -356,7 +400,8 @@ void HdfsScanNode::ScannerThread(int64_t scanner_thread_reservation) {
// this thread.
unique_lock<mutex> l(lock_);
if (active_scanner_thread_counter_.value() > 1) {
- if (runtime_state_->resource_pool()->optional_exceeded()) {
+ if (runtime_state_->resource_pool()->optional_exceeded() ||
+ !EnoughMemoryForScannerThread(false)) {
// We can't break here. We need to update the counter with the lock held or else
// all threads might see active_scanner_thread_counter_.value > 1
COUNTER_ADD(&active_scanner_thread_counter_, -1);
@@ -376,29 +421,21 @@ void HdfsScanNode::ScannerThread(int64_t scanner_thread_reservation) {
// to return if there's an error.
ranges_issued_barrier_.Wait(SCANNER_THREAD_WAIT_TIME_MS, &unused);
- // Take a snapshot of num_unqueued_files_ before calling GetNextUnstartedRange().
+ ScanRange* scan_range;
+ // Take a snapshot of num_unqueued_files_ before calling GetNextRange().
// We don't want num_unqueued_files_ to go to zero between the return from
- // GetNextUnstartedRange() and the check for when all ranges are complete.
+ // GetNextRange() and the check for when all ranges are complete.
int num_unqueued_files = num_unqueued_files_.Load();
// TODO: the Load() acts as an acquire barrier. Is this needed? (i.e. any earlier
// stores that need to complete?)
AtomicUtil::MemoryBarrier();
- ScanRange* scan_range;
- bool needs_buffers;
Status status =
- io_mgr->GetNextUnstartedRange(reader_context_.get(), &scan_range, &needs_buffers);
+ runtime_state_->io_mgr()->GetNextRange(reader_context_.get(), &scan_range);
- if (status.ok() && scan_range != nullptr) {
- if (needs_buffers) {
- status = io_mgr->AllocateBuffersForRange(
- reader_context_.get(), &buffer_pool_client_, scan_range,
- scanner_thread_reservation);
- }
- if (status.ok()) {
- // Got a scan range. Process the range end to end (in this thread).
- status = ProcessSplit(filter_status.ok() ? filter_ctxs : vector<FilterContext>(),
- &expr_results_pool, scan_range, scanner_thread_reservation);
- }
+ if (status.ok() && scan_range != NULL) {
+ // Got a scan range. Process the range end to end (in this thread).
+ status = ProcessSplit(filter_status.ok() ? filter_ctxs : vector<FilterContext>(),
+ &expr_results_pool, scan_range);
}
if (!status.ok()) {
@@ -428,8 +465,8 @@ void HdfsScanNode::ScannerThread(int64_t scanner_thread_reservation) {
// TODO: Based on the usage pattern of all_ranges_started_, it looks like it is not
// needed to acquire the lock in x86.
unique_lock<mutex> l(lock_);
- // All ranges have been queued and GetNextUnstartedRange() returned NULL. This means
- // that every range is either done or being processed by another thread.
+ // All ranges have been queued and GetNextRange() returned NULL. This means that
+ // every range is either done or being processed by another thread.
all_ranges_started_ = true;
break;
}
@@ -437,7 +474,6 @@ void HdfsScanNode::ScannerThread(int64_t scanner_thread_reservation) {
COUNTER_ADD(&active_scanner_thread_counter_, -1);
exit:
- ReturnReservationFromScannerThread(scanner_thread_reservation);
runtime_state_->resource_pool()->ReleaseThreadToken(false);
for (auto& ctx: filter_ctxs) ctx.expr_eval->Close(runtime_state_);
filter_mem_pool.FreeAll();
@@ -445,8 +481,7 @@ exit:
}
Status HdfsScanNode::ProcessSplit(const vector<FilterContext>& filter_ctxs,
- MemPool* expr_results_pool, ScanRange* scan_range,
- int64_t scanner_thread_reservation) {
+ MemPool* expr_results_pool, ScanRange* scan_range) {
DCHECK(scan_range != NULL);
ScanRangeMetadata* metadata = static_cast<ScanRangeMetadata*>(scan_range->meta_data());
@@ -471,9 +506,8 @@ Status HdfsScanNode::ProcessSplit(const vector<FilterContext>& filter_ctxs,
return Status::OK();
}
- ScannerContext context(runtime_state_, this, &buffer_pool_client_, partition,
- filter_ctxs, expr_results_pool);
- context.AddStream(scan_range, scanner_thread_reservation);
+ ScannerContext context(
+ runtime_state_, this, partition, scan_range, filter_ctxs, expr_results_pool);
scoped_ptr<HdfsScanner> scanner;
Status status = CreateAndOpenScanner(partition, &context, &scanner);
if (!status.ok()) {
@@ -515,7 +549,9 @@ Status HdfsScanNode::ProcessSplit(const vector<FilterContext>& filter_ctxs,
void HdfsScanNode::SetDoneInternal() {
if (done_) return;
done_ = true;
- if (reader_context_ != nullptr) reader_context_->Cancel();
+ if (reader_context_ != nullptr) {
+ runtime_state_->io_mgr()->CancelContext(reader_context_.get());
+ }
materialized_row_batches_->Shutdown();
}
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-scan-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node.h b/be/src/exec/hdfs-scan-node.h
index 81e826e..a1c97cf 100644
--- a/be/src/exec/hdfs-scan-node.h
+++ b/be/src/exec/hdfs-scan-node.h
@@ -27,7 +27,6 @@
#include <boost/scoped_ptr.hpp>
#include <boost/thread/mutex.hpp>
-#include "common/atomic.h"
#include "exec/filter-context.h"
#include "exec/hdfs-scan-node-base.h"
#include "runtime/io/disk-io-mgr.h"
@@ -59,14 +58,8 @@ class TPlanNode;
/// 5. The scanner finishes the scan range and informs the scan node so it can track
/// end of stream.
///
-/// Buffer management:
-/// ------------------
-/// The different scanner threads all allocate I/O buffers from the node's Buffer Pool
-/// client. The scan node ensures that enough reservation is available to start a
-/// scanner thread before launching each one with (see
-/// EnoughReservationForExtraThread()), after which the scanner thread is responsible
-/// for staying within the reservation handed off to it.
-///
+/// TODO: This class allocates a bunch of small utility objects that should be
+/// recycled.
/// TODO: Remove this class once the fragment-based multi-threaded execution is
/// fully functional.
class HdfsScanNode : public HdfsScanNodeBase {
@@ -107,7 +100,12 @@ class HdfsScanNode : public HdfsScanNodeBase {
private:
/// Released when initial ranges are issued in the first call to GetNext().
- CountingBarrier ranges_issued_barrier_{1};
+ CountingBarrier ranges_issued_barrier_;
+
+ /// The estimated memory required to start up a new scanner thread. If the memory
+ /// left (due to limits) is less than this value, we won't start up optional
+ /// scanner threads.
+ int64_t scanner_thread_bytes_required_;
/// Thread group for all scanner worker threads
ThreadGroup scanner_threads_;
@@ -132,16 +130,16 @@ class HdfsScanNode : public HdfsScanNodeBase {
/// are finished, an error/cancellation occurred, or the limit was reached.
/// Setting this to true triggers the scanner threads to clean up.
/// This should not be explicitly set. Instead, call SetDone().
- bool done_ = false;
+ bool done_;
/// Set to true if all ranges have started. Some of the ranges may still be in flight
/// being processed by scanner threads, but no new ScannerThreads should be started.
- bool all_ranges_started_ = false;
+ bool all_ranges_started_;
/// The id of the callback added to the thread resource manager when thread token
/// is available. Used to remove the callback before this scan node is destroyed.
/// -1 if no callback is registered.
- int thread_avail_cb_id_ = -1;
+ int thread_avail_cb_id_;
/// Maximum number of scanner threads. Set to 'NUM_SCANNER_THREADS' if that query
/// option is set. Otherwise, it's set to the number of cpu cores. Scanner threads
@@ -149,14 +147,6 @@ class HdfsScanNode : public HdfsScanNodeBase {
/// the number of cores.
int max_num_scanner_threads_;
- /// Amount of the 'buffer_pool_client_' reservation that is not allocated to scanner
- /// threads. Doled out to scanner threads when they are started and returned when
- /// those threads no longer need it. Can be atomically incremented without holding
- /// 'lock_' but 'lock_' is held when decrementing to ensure that the check for
- /// reservation and the actual deduction are atomic with respect to other threads
- /// trying to claim reservation.
- AtomicInt64 spare_reservation_{0};
-
/// The wait time for fetching a row batch from the row batch queue.
RuntimeProfile::Counter* row_batches_get_timer_;
@@ -170,35 +160,21 @@ class HdfsScanNode : public HdfsScanNodeBase {
/// Main function for scanner thread. This thread pulls the next range to be
/// processed from the IoMgr and then processes the entire range end to end.
/// This thread terminates when all scan ranges are complete or an error occurred.
- /// The caller must have reserved 'scanner_thread_reservation' bytes of memory for
- /// this thread with DeductReservationForScannerThread().
- void ScannerThread(int64_t scanner_thread_reservation);
+ void ScannerThread();
/// Process the entire scan range with a new scanner object. Executed in scanner
/// thread. 'filter_ctxs' is a clone of the class-wide filter_ctxs_, used to filter rows
/// in this split.
Status ProcessSplit(const std::vector<FilterContext>& filter_ctxs,
- MemPool* expr_results_pool, io::ScanRange* scan_range,
- int64_t scanner_thread_reservation) WARN_UNUSED_RESULT;
-
- /// Return true if there is enough reservation to start an extra scanner thread.
- /// Tries to increase reservation if enough is not already available in
- /// 'spare_reservation_'. 'lock_' must be held via 'lock'
- bool EnoughReservationForExtraThread(const boost::unique_lock<boost::mutex>& lock);
-
- /// Deduct reservation to start a new scanner thread from 'spare_reservation_'. If
- /// 'first_thread' is true, this is the first thread to be started and only the
- /// minimum reservation is required to be available. Otherwise
- /// EnoughReservationForExtra() thread must have returned true in the current
- /// critical section so that 'ideal_scan_range_bytes_' is available for the extra
- /// thread. Returns the amount deducted. 'lock_' must be held via 'lock'.
- int64_t DeductReservationForScannerThread(const boost::unique_lock<boost::mutex>& lock,
- bool first_thread);
-
- /// Called by scanner thread to return or all of its reservation that is not needed.
- void ReturnReservationFromScannerThread(int64_t bytes) {
- spare_reservation_.Add(bytes);
- }
+ MemPool* expr_results_pool, io::ScanRange* scan_range) WARN_UNUSED_RESULT;
+
+ /// Returns true if there is enough memory (against the mem tracker limits) to
+ /// have a scanner thread.
+ /// If new_thread is true, the calculation is for starting a new scanner thread.
+ /// If false, it determines whether there's adequate memory for the existing
+ /// set of scanner threads.
+ /// lock_ must be taken before calling this.
+ bool EnoughMemoryForScannerThread(bool new_thread);
/// Checks for eos conditions and returns batches from materialized_row_batches_.
Status GetNextInternal(RuntimeState* state, RowBatch* row_batch, bool* eos)
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-text-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-text-scanner.cc b/be/src/exec/hdfs-text-scanner.cc
index 2a6a912..253bcc8 100644
--- a/be/src/exec/hdfs-text-scanner.cc
+++ b/be/src/exec/hdfs-text-scanner.cc
@@ -144,10 +144,8 @@ Status HdfsTextScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
DCHECK(false);
}
}
- if (compressed_text_scan_ranges.size() > 0) {
- RETURN_IF_ERROR(scan_node->AddDiskIoRanges(compressed_text_scan_ranges,
- compressed_text_files));
- }
+ RETURN_IF_ERROR(scan_node->AddDiskIoRanges(compressed_text_scan_ranges,
+ compressed_text_files));
if (lzo_text_files.size() > 0) {
// This will dlopen the lzo binary and can fail if the lzo binary is not present.
RETURN_IF_ERROR(HdfsLzoTextScanner::IssueInitialRanges(scan_node, lzo_text_files));
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/parquet-column-readers.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/parquet-column-readers.cc b/be/src/exec/parquet-column-readers.cc
index 153b24a..2939691 100644
--- a/be/src/exec/parquet-column-readers.cc
+++ b/be/src/exec/parquet-column-readers.cc
@@ -30,7 +30,6 @@
#include "exec/scanner-context.inline.h"
#include "rpc/thrift-util.h"
#include "runtime/collection-value-builder.h"
-#include "runtime/exec-env.h"
#include "runtime/tuple-row.h"
#include "runtime/tuple.h"
#include "runtime/runtime-state.h"
@@ -50,10 +49,6 @@ DEFINE_bool(convert_legacy_hive_parquet_utc_timestamps, false,
static const int BITPACKED_DEPRECATION_WARNING_FREQUENCY = 100;
-// Max dictionary page header size in bytes. This is an estimate and only needs to be an
-// upper bound.
-static const int MAX_DICT_HEADER_SIZE = 100;
-
// Max data page header size in bytes. This is an estimate and only needs to be an upper
// bound. It is theoretically possible to have a page header of any size due to string
// value statistics, but in practice we'll have trouble reading string values this large.
@@ -73,8 +68,6 @@ static int debug_count = 0;
#define SHOULD_TRIGGER_DEBUG_ACTION(x) (false)
#endif
-using namespace impala::io;
-
namespace impala {
const string PARQUET_COL_MEM_LIMIT_EXCEEDED =
@@ -854,99 +847,8 @@ static bool RequiresSkippedDictionaryHeaderCheck(
return v.VersionEq(1,1,0) || (v.VersionEq(1,2,0) && v.is_impala_internal);
}
-Status BaseScalarColumnReader::Reset(const HdfsFileDesc& file_desc,
- const parquet::ColumnChunk& col_chunk, int row_group_idx) {
- num_buffered_values_ = 0;
- data_ = nullptr;
- data_end_ = nullptr;
- stream_ = nullptr;
- io_reservation_ = 0;
- metadata_ = &col_chunk.meta_data;
- num_values_read_ = 0;
- def_level_ = -1;
- // See ColumnReader constructor.
- rep_level_ = max_rep_level() == 0 ? 0 : -1;
- pos_current_value_ = -1;
-
- if (metadata_->codec != parquet::CompressionCodec::UNCOMPRESSED) {
- RETURN_IF_ERROR(Codec::CreateDecompressor(
- nullptr, false, PARQUET_TO_IMPALA_CODEC[metadata_->codec], &decompressor_));
- }
- int64_t col_start = col_chunk.meta_data.data_page_offset;
-
- RETURN_IF_ERROR(ParquetMetadataUtils::ValidateRowGroupColumn(parent_->file_metadata_,
- parent_->filename(), row_group_idx, col_idx(), schema_element(),
- parent_->state_));
-
- if (col_chunk.meta_data.__isset.dictionary_page_offset) {
- // Already validated in ValidateColumnOffsets()
- DCHECK_LT(col_chunk.meta_data.dictionary_page_offset, col_start);
- col_start = col_chunk.meta_data.dictionary_page_offset;
- }
- int64_t col_len = col_chunk.meta_data.total_compressed_size;
- if (col_len <= 0) {
- return Status(Substitute("File '$0' contains invalid column chunk size: $1",
- filename(), col_len));
- }
- int64_t col_end = col_start + col_len;
-
- // Already validated in ValidateColumnOffsets()
- DCHECK_GT(col_end, 0);
- DCHECK_LT(col_end, file_desc.file_length);
- const ParquetFileVersion& file_version = parent_->file_version_;
- if (file_version.application == "parquet-mr" && file_version.VersionLt(1, 2, 9)) {
- // The Parquet MR writer had a bug in 1.2.8 and below where it didn't include the
- // dictionary page header size in total_compressed_size and total_uncompressed_size
- // (see IMPALA-694). We pad col_len to compensate.
- int64_t bytes_remaining = file_desc.file_length - col_end;
- int64_t pad = min<int64_t>(MAX_DICT_HEADER_SIZE, bytes_remaining);
- col_len += pad;
- }
-
- // TODO: this will need to change when we have co-located files and the columns
- // are different files.
- if (!col_chunk.file_path.empty() && col_chunk.file_path != filename()) {
- return Status(Substitute("Expected parquet column file path '$0' to match "
- "filename '$1'", col_chunk.file_path, filename()));
- }
-
- const ScanRange* metadata_range = parent_->metadata_range_;
- int64_t partition_id = parent_->context_->partition_descriptor()->id();
- const ScanRange* split_range =
- static_cast<ScanRangeMetadata*>(metadata_range->meta_data())->original_split;
- // Determine if the column is completely contained within a local split.
- bool col_range_local = split_range->expected_local()
- && col_start >= split_range->offset()
- && col_end <= split_range->offset() + split_range->len();
- scan_range_ = parent_->scan_node_->AllocateScanRange(metadata_range->fs(),
- filename(), col_len, col_start, partition_id, split_range->disk_id(),
- col_range_local,
- BufferOpts(split_range->try_cache(), file_desc.mtime));
- ClearDictionaryDecoder();
- return Status::OK();
-}
-
-Status BaseScalarColumnReader::StartScan() {
- DCHECK(scan_range_ != nullptr) << "Must Reset() before starting scan.";
- DiskIoMgr* io_mgr = ExecEnv::GetInstance()->disk_io_mgr();
- ScannerContext* context = parent_->context_;
- DCHECK_GT(io_reservation_, 0);
- bool needs_buffers;
- RETURN_IF_ERROR(io_mgr->StartScanRange(
- parent_->scan_node_->reader_context(), scan_range_, &needs_buffers));
- if (needs_buffers) {
- RETURN_IF_ERROR(io_mgr->AllocateBuffersForRange(
- parent_->scan_node_->reader_context(), context->bp_client(),
- scan_range_, io_reservation_));
- }
- stream_ = parent_->context_->AddStream(scan_range_, io_reservation_);
- DCHECK(stream_ != nullptr);
- return Status::OK();
-}
-
Status BaseScalarColumnReader::ReadPageHeader(bool peek,
parquet::PageHeader* next_page_header, uint32_t* next_header_size, bool* eos) {
- DCHECK(stream_ != nullptr);
*eos = false;
uint8_t* buffer;
@@ -1030,7 +932,7 @@ Status BaseScalarColumnReader::InitDictionary() {
bool eos;
parquet::PageHeader next_page_header;
uint32_t next_header_size;
- DCHECK(stream_ != nullptr);
+
DCHECK(!HasDictionaryDecoder());
RETURN_IF_ERROR(ReadPageHeader(true /* peek */, &next_page_header,
@@ -1142,14 +1044,6 @@ Status BaseScalarColumnReader::InitDictionary() {
return Status::OK();
}
-Status BaseScalarColumnReader::InitDictionaries(
- const vector<BaseScalarColumnReader*> readers) {
- for (BaseScalarColumnReader* reader : readers) {
- RETURN_IF_ERROR(reader->InitDictionary());
- }
- return Status::OK();
-}
-
Status BaseScalarColumnReader::ReadDataPage() {
// We're about to move to the next data page. The previous data page is
// now complete, free up any memory allocated for it. If the data page contained
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/parquet-column-readers.h
----------------------------------------------------------------------
diff --git a/be/src/exec/parquet-column-readers.h b/be/src/exec/parquet-column-readers.h
index bfb20ec..3a8ad70 100644
--- a/be/src/exec/parquet-column-readers.h
+++ b/be/src/exec/parquet-column-readers.h
@@ -326,29 +326,42 @@ class BaseScalarColumnReader : public ParquetColumnReader {
BaseScalarColumnReader(HdfsParquetScanner* parent, const SchemaNode& node,
const SlotDescriptor* slot_desc)
: ParquetColumnReader(parent, node, slot_desc),
+ data_(NULL),
+ data_end_(NULL),
+ def_levels_(true),
+ rep_levels_(false),
+ page_encoding_(parquet::Encoding::PLAIN_DICTIONARY),
+ num_buffered_values_(0),
+ num_values_read_(0),
+ metadata_(NULL),
+ stream_(NULL),
data_page_pool_(new MemPool(parent->scan_node_->mem_tracker())) {
DCHECK_GE(node_.col_idx, 0) << node_.DebugString();
}
virtual ~BaseScalarColumnReader() { }
- /// Resets the reader for each row group in the file and creates the scan
- /// range for the column, but does not start it. To start scanning,
- /// set_io_reservation() must be called to assign reservation to this
- /// column, followed by StartScan().
- Status Reset(const HdfsFileDesc& file_desc, const parquet::ColumnChunk& col_chunk,
- int row_group_idx);
-
- /// Starts the column scan range. The reader must be Reset() and have a
- /// reservation assigned via set_io_reservation(). This must be called
- /// before any of the column data can be read (including dictionary and
- /// data pages). Returns an error status if there was an error starting the
- /// scan or allocating buffers for it.
- Status StartScan();
-
- /// Helper to start scans for multiple columns at once.
- static Status StartScans(const std::vector<BaseScalarColumnReader*> readers) {
- for (BaseScalarColumnReader* reader : readers) RETURN_IF_ERROR(reader->StartScan());
+ /// This is called once for each row group in the file.
+ Status Reset(const parquet::ColumnMetaData* metadata, ScannerContext::Stream* stream) {
+ DCHECK(stream != NULL);
+ DCHECK(metadata != NULL);
+
+ num_buffered_values_ = 0;
+ data_ = NULL;
+ data_end_ = NULL;
+ stream_ = stream;
+ metadata_ = metadata;
+ num_values_read_ = 0;
+ def_level_ = -1;
+ // See ColumnReader constructor.
+ rep_level_ = max_rep_level() == 0 ? 0 : -1;
+ pos_current_value_ = -1;
+
+ if (metadata_->codec != parquet::CompressionCodec::UNCOMPRESSED) {
+ RETURN_IF_ERROR(Codec::CreateDecompressor(
+ NULL, false, PARQUET_TO_IMPALA_CODEC[metadata_->codec], &decompressor_));
+ }
+ ClearDictionaryDecoder();
return Status::OK();
}
@@ -363,27 +376,22 @@ class BaseScalarColumnReader : public ParquetColumnReader {
if (dict_decoder != nullptr) dict_decoder->Close();
}
- io::ScanRange* scan_range() const { return scan_range_; }
int64_t total_len() const { return metadata_->total_compressed_size; }
int col_idx() const { return node_.col_idx; }
THdfsCompression::type codec() const {
if (metadata_ == NULL) return THdfsCompression::NONE;
return PARQUET_TO_IMPALA_CODEC[metadata_->codec];
}
- void set_io_reservation(int bytes) { io_reservation_ = bytes; }
/// Reads the next definition and repetition levels for this column. Initializes the
/// next data page if necessary.
virtual bool NextLevels() { return NextLevels<true>(); }
- /// Check the data stream to see if there is a dictionary page. If there is,
- /// use that page to initialize dict_decoder_ and advance the data stream
- /// past the dictionary page.
+ // Check the data stream to see if there is a dictionary page. If there is,
+ // use that page to initialize dict_decoder_ and advance the data stream
+ // past the dictionary page.
Status InitDictionary();
- /// Convenience function to initialize multiple dictionaries.
- static Status InitDictionaries(const std::vector<BaseScalarColumnReader*> readers);
-
// Returns the dictionary or NULL if the dictionary doesn't exist
virtual DictDecoderBase* GetDictionaryDecoder() { return nullptr; }
@@ -409,45 +417,33 @@ class BaseScalarColumnReader : public ParquetColumnReader {
// fit in as few cache lines as possible.
/// Pointer to start of next value in data page
- uint8_t* data_ = nullptr;
+ uint8_t* data_;
/// End of the data page.
- const uint8_t* data_end_ = nullptr;
+ const uint8_t* data_end_;
/// Decoder for definition levels.
- ParquetLevelDecoder def_levels_{true};
+ ParquetLevelDecoder def_levels_;
/// Decoder for repetition levels.
- ParquetLevelDecoder rep_levels_{false};
+ ParquetLevelDecoder rep_levels_;
/// Page encoding for values of the current data page. Cached here for perf. Set in
/// InitDataPage().
- parquet::Encoding::type page_encoding_ = parquet::Encoding::PLAIN_DICTIONARY;
+ parquet::Encoding::type page_encoding_;
/// Num values remaining in the current data page
- int num_buffered_values_ = 0;
+ int num_buffered_values_;
// Less frequently used members that are not accessed in inner loop should go below
// here so they do not occupy precious cache line space.
/// The number of values seen so far. Updated per data page.
- int64_t num_values_read_ = 0;
-
- /// Metadata for the column for the current row group.
- const parquet::ColumnMetaData* metadata_ = nullptr;
+ int64_t num_values_read_;
+ const parquet::ColumnMetaData* metadata_;
boost::scoped_ptr<Codec> decompressor_;
-
- /// The scan range for the column's data. Initialized for each row group by Reset().
- io::ScanRange* scan_range_ = nullptr;
-
- // Stream used to read data from 'scan_range_'. Initialized by StartScan().
- ScannerContext::Stream* stream_ = nullptr;
-
- /// Reservation in bytes to use for I/O buffers in 'scan_range_'/'stream_'. Must be set
- /// with set_io_reservation() before 'stream_' is initialized. Reset for each row group
- /// by Reset().
- int64_t io_reservation_ = 0;
+ ScannerContext::Stream* stream_;
/// Pool to allocate storage for data pages from - either decompression buffers for
/// compressed data pages or copies of the data page with var-len data to attach to
[7/9] impala git commit: Revert IMPALA-4835 and dependent changes
Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr-test.cc b/be/src/runtime/io/disk-io-mgr-test.cc
index 3a0f727..b03ec31 100644
--- a/be/src/runtime/io/disk-io-mgr-test.cc
+++ b/be/src/runtime/io/disk-io-mgr-test.cc
@@ -22,16 +22,12 @@
#include "codegen/llvm-codegen.h"
#include "common/init.h"
-#include "runtime/bufferpool/buffer-pool.h"
-#include "runtime/bufferpool/reservation-tracker.h"
+#include "runtime/io/request-context.h"
#include "runtime/io/disk-io-mgr-stress.h"
#include "runtime/io/disk-io-mgr.h"
-#include "runtime/io/request-context.h"
-#include "runtime/test-env.h"
+#include "runtime/mem-tracker.h"
#include "runtime/thread-resource-mgr.h"
-#include "service/fe-support.h"
#include "testutil/gtest-util.h"
-#include "testutil/rand-util.h"
#include "util/condition-variable.h"
#include "util/cpu-info.h"
#include "util/disk-info.h"
@@ -39,20 +35,13 @@
#include "common/names.h"
-using std::mt19937;
-using std::uniform_int_distribution;
-using std::uniform_real_distribution;
-
-DECLARE_int64(min_buffer_size);
DECLARE_int32(num_remote_hdfs_io_threads);
DECLARE_int32(num_s3_io_threads);
DECLARE_int32(num_adls_io_threads);
-const int MIN_BUFFER_SIZE = 128;
+const int MIN_BUFFER_SIZE = 512;
const int MAX_BUFFER_SIZE = 1024;
-const int64_t LARGE_RESERVATION_LIMIT = 4L * 1024L * 1024L * 1024L;
-const int64_t LARGE_INITIAL_RESERVATION = 128L * 1024L * 1024L;
-const int64_t BUFFER_POOL_CAPACITY = LARGE_RESERVATION_LIMIT;
+const int LARGE_MEM_LIMIT = 1024 * 1024 * 1024;
namespace impala {
namespace io {
@@ -60,41 +49,14 @@ namespace io {
class DiskIoMgrTest : public testing::Test {
public:
- virtual void SetUp() {
- test_env_.reset(new TestEnv);
- // Tests try to allocate arbitrarily small buffers. Ensure Buffer Pool allows it.
- test_env_->SetBufferPoolArgs(1, BUFFER_POOL_CAPACITY);
- ASSERT_OK(test_env_->Init());
- RandTestUtil::SeedRng("DISK_IO_MGR_TEST_SEED", &rng_);
- }
+ virtual void SetUp() {}
virtual void TearDown() {
- root_reservation_.Close();
pool_.Clear();
- test_env_.reset();
- }
-
- /// Initialises 'root_reservation_'. The reservation is automatically closed in
- /// TearDown().
- void InitRootReservation(int64_t reservation_limit) {
- root_reservation_.InitRootTracker(
- RuntimeProfile::Create(&pool_, "root"), reservation_limit);
}
-
- /// Initialise 'client' with the given reservation limit. The client reservation is a
- /// child of 'root_reservation_'.
- void RegisterBufferPoolClient(int64_t reservation_limit, int64_t initial_reservation,
- BufferPool::ClientHandle* client) {
- ASSERT_OK(buffer_pool()->RegisterClient("", nullptr, &root_reservation_, nullptr,
- reservation_limit, RuntimeProfile::Create(&pool_, ""), client));
- if (initial_reservation > 0) {
- ASSERT_TRUE(client->IncreaseReservation(initial_reservation));
- }
- }
-
void WriteValidateCallback(int num_writes, WriteRange** written_range,
- DiskIoMgr* io_mgr, RequestContext* reader, BufferPool::ClientHandle* client,
- int32_t* data, Status expected_status, const Status& status) {
+ DiskIoMgr* io_mgr, RequestContext* reader, int32_t* data,
+ Status expected_status, const Status& status) {
if (expected_status.code() == TErrorCode::CANCELLED) {
EXPECT_TRUE(status.ok() || status.IsCancelled()) << "Error: " << status.GetDetail();
} else {
@@ -104,8 +66,8 @@ class DiskIoMgrTest : public testing::Test {
ScanRange* scan_range = pool_.Add(new ScanRange());
scan_range->Reset(nullptr, (*written_range)->file(), (*written_range)->len(),
(*written_range)->offset(), 0, false, BufferOpts::Uncached());
- ValidateSyncRead(io_mgr, reader, client, scan_range,
- reinterpret_cast<const char*>(data), sizeof(int32_t));
+ ValidateSyncRead(io_mgr, reader, scan_range, reinterpret_cast<const char*>(data),
+ sizeof(int32_t));
}
{
@@ -126,13 +88,9 @@ class DiskIoMgrTest : public testing::Test {
protected:
void CreateTempFile(const char* filename, const char* data) {
- CreateTempFile(filename, data, strlen(data));
- }
-
- void CreateTempFile(const char* filename, const char* data, int64_t data_bytes) {
FILE* file = fopen(filename, "w");
EXPECT_TRUE(file != nullptr);
- fwrite(data, 1, data_bytes, file);
+ fwrite(data, 1, strlen(data), file);
fclose(file);
}
@@ -157,22 +115,15 @@ class DiskIoMgrTest : public testing::Test {
}
static void ValidateSyncRead(DiskIoMgr* io_mgr, RequestContext* reader,
- BufferPool::ClientHandle* client, ScanRange* range, const char* expected,
- int expected_len = -1) {
+ ScanRange* range, const char* expected, int expected_len = -1) {
unique_ptr<BufferDescriptor> buffer;
- bool needs_buffers;
- ASSERT_OK(io_mgr->StartScanRange(reader, range, &needs_buffers));
- if (needs_buffers) {
- ASSERT_OK(io_mgr->AllocateBuffersForRange(
- reader, client, range, io_mgr->max_buffer_size()));
- }
- ASSERT_OK(range->GetNext(&buffer));
+ ASSERT_OK(io_mgr->Read(reader, range, &buffer));
ASSERT_TRUE(buffer != nullptr);
EXPECT_EQ(buffer->len(), range->len());
if (expected_len < 0) expected_len = strlen(expected);
int cmp = memcmp(buffer->buffer(), expected, expected_len);
EXPECT_TRUE(cmp == 0);
- range->ReturnBuffer(move(buffer));
+ io_mgr->ReturnBuffer(move(buffer));
}
static void ValidateScanRange(DiskIoMgr* io_mgr, ScanRange* range,
@@ -185,13 +136,13 @@ class DiskIoMgrTest : public testing::Test {
Status status = range->GetNext(&buffer);
ASSERT_TRUE(status.ok() || status.code() == expected_status.code());
if (buffer == nullptr || !status.ok()) {
- if (buffer != nullptr) range->ReturnBuffer(move(buffer));
+ if (buffer != nullptr) io_mgr->ReturnBuffer(move(buffer));
break;
}
ASSERT_LE(buffer->len(), expected_len);
memcpy(result + range->offset() + buffer->scan_range_offset(),
buffer->buffer(), buffer->len());
- range->ReturnBuffer(move(buffer));
+ io_mgr->ReturnBuffer(move(buffer));
}
ValidateEmptyOrCorrect(expected, result, expected_len);
}
@@ -199,46 +150,35 @@ class DiskIoMgrTest : public testing::Test {
// Continues pulling scan ranges from the io mgr until they are all done.
// Updates num_ranges_processed with the number of ranges seen by this thread.
static void ScanRangeThread(DiskIoMgr* io_mgr, RequestContext* reader,
- BufferPool::ClientHandle* client, const char* expected_result, int expected_len,
- const Status& expected_status, int max_ranges, AtomicInt32* num_ranges_processed) {
+ const char* expected_result, int expected_len, const Status& expected_status,
+ int max_ranges, AtomicInt32* num_ranges_processed) {
int num_ranges = 0;
while (max_ranges == 0 || num_ranges < max_ranges) {
ScanRange* range;
- bool needs_buffers;
- Status status = io_mgr->GetNextUnstartedRange(reader, &range, &needs_buffers);
+ Status status = io_mgr->GetNextRange(reader, &range);
ASSERT_TRUE(status.ok() || status.code() == expected_status.code());
if (range == nullptr) break;
- if (needs_buffers) {
- ASSERT_OK(io_mgr->AllocateBuffersForRange(
- reader, client, range, io_mgr->max_buffer_size() * 3));
- }
ValidateScanRange(io_mgr, range, expected_result, expected_len, expected_status);
num_ranges_processed->Add(1);
++num_ranges;
}
}
- ScanRange* InitRange(ObjectPool* pool, const char* file_path, int offset, int len,
+ ScanRange* AllocateRange() {
+ return pool_.Add(new ScanRange);
+ }
+
+ ScanRange* InitRange(const char* file_path, int offset, int len,
int disk_id, int64_t mtime, void* meta_data = nullptr, bool is_cached = false) {
- ScanRange* range = pool->Add(new ScanRange);
+ ScanRange* range = AllocateRange();
range->Reset(nullptr, file_path, len, offset, disk_id, true,
BufferOpts(is_cached, mtime), meta_data);
EXPECT_EQ(mtime, range->mtime());
return range;
}
- /// Convenience function to get a reference to the buffer pool.
- BufferPool* buffer_pool() const { return ExecEnv::GetInstance()->buffer_pool(); }
-
- boost::scoped_ptr<TestEnv> test_env_;
-
- /// Per-test random number generator. Seeded before every test.
- mt19937 rng_;
-
ObjectPool pool_;
- ReservationTracker root_reservation_;
-
mutex written_mutex_;
ConditionVariable writes_done_;
int num_ranges_written_;
@@ -249,7 +189,7 @@ class DiskIoMgrTest : public testing::Test {
// by reading the data back via a separate IoMgr instance. All writes are expected to
// complete successfully.
TEST_F(DiskIoMgrTest, SingleWriter) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
num_ranges_written_ = 0;
string tmp_file = "/tmp/disk_io_mgr_test.txt";
int num_ranges = 100;
@@ -263,27 +203,25 @@ TEST_F(DiskIoMgrTest, SingleWriter) {
}
scoped_ptr<DiskIoMgr> read_io_mgr(new DiskIoMgr(1, 1, 1, 1, 10));
- BufferPool::ClientHandle read_client;
- RegisterBufferPoolClient(
- LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
- ASSERT_OK(read_io_mgr->Init());
- unique_ptr<RequestContext> reader = read_io_mgr->RegisterContext();
+ MemTracker reader_mem_tracker(LARGE_MEM_LIMIT);
+ ASSERT_OK(read_io_mgr->Init(&reader_mem_tracker));
+ unique_ptr<RequestContext> reader =
+ read_io_mgr->RegisterContext(&reader_mem_tracker);
for (int num_threads_per_disk = 1; num_threads_per_disk <= 5; ++num_threads_per_disk) {
for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
- // Pool for temporary objects from this iteration only.
- ObjectPool tmp_pool;
+ pool_.Clear(); // Destroy scan ranges from previous iterations.
DiskIoMgr io_mgr(num_disks, num_threads_per_disk, num_threads_per_disk, 1, 10);
- ASSERT_OK(io_mgr.Init());
- unique_ptr<RequestContext> writer = io_mgr.RegisterContext();
+ ASSERT_OK(io_mgr.Init(&mem_tracker));
+ unique_ptr<RequestContext> writer = io_mgr.RegisterContext(&mem_tracker);
for (int i = 0; i < num_ranges; ++i) {
- int32_t* data = tmp_pool.Add(new int32_t);
+ int32_t* data = pool_.Add(new int32_t);
*data = rand();
- WriteRange** new_range = tmp_pool.Add(new WriteRange*);
- WriteRange::WriteDoneCallback callback = bind(
- mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, num_ranges, new_range,
- read_io_mgr.get(), reader.get(), &read_client, data, Status::OK(), _1);
- *new_range = tmp_pool.Add(
- new WriteRange(tmp_file, cur_offset, num_ranges % num_disks, callback));
+ WriteRange** new_range = pool_.Add(new WriteRange*);
+ WriteRange::WriteDoneCallback callback =
+ bind(mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, num_ranges,
+ new_range, read_io_mgr.get(), reader.get(), data, Status::OK(), _1);
+ *new_range = pool_.Add(new WriteRange(
+ tmp_file, cur_offset, num_ranges % num_disks, callback));
(*new_range)->SetData(reinterpret_cast<uint8_t*>(data), sizeof(int32_t));
EXPECT_OK(io_mgr.AddWriteRange(writer.get(), *new_range));
cur_offset += sizeof(int32_t);
@@ -299,26 +237,27 @@ TEST_F(DiskIoMgrTest, SingleWriter) {
}
read_io_mgr->UnregisterContext(reader.get());
- buffer_pool()->DeregisterClient(&read_client);
+ read_io_mgr.reset();
}
// Perform invalid writes (e.g. file in non-existent directory, negative offset) and
// validate that an error status is returned via the write callback.
TEST_F(DiskIoMgrTest, InvalidWrite) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
num_ranges_written_ = 0;
string tmp_file = "/non-existent/file.txt";
DiskIoMgr io_mgr(1, 1, 1, 1, 10);
- ASSERT_OK(io_mgr.Init());
- unique_ptr<RequestContext> writer = io_mgr.RegisterContext();
+ ASSERT_OK(io_mgr.Init(&mem_tracker));
+ unique_ptr<RequestContext> writer = io_mgr.RegisterContext(nullptr);
int32_t* data = pool_.Add(new int32_t);
*data = rand();
// Write to file in non-existent directory.
WriteRange** new_range = pool_.Add(new WriteRange*);
WriteRange::WriteDoneCallback callback =
- bind(mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, 2, new_range, nullptr,
- nullptr, nullptr, data, Status(TErrorCode::DISK_IO_ERROR, "Test Failure"), _1);
+ bind(mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, 2, new_range,
+ (DiskIoMgr*)nullptr, (RequestContext*)nullptr, data,
+ Status(TErrorCode::DISK_IO_ERROR, "Test Failure"), _1);
*new_range = pool_.Add(new WriteRange(tmp_file, rand(), 0, callback));
(*new_range)->SetData(reinterpret_cast<uint8_t*>(data), sizeof(int32_t));
@@ -333,9 +272,9 @@ TEST_F(DiskIoMgrTest, InvalidWrite) {
}
new_range = pool_.Add(new WriteRange*);
- callback = bind(mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, 2, new_range,
- nullptr, nullptr, nullptr, data,
- Status(TErrorCode::DISK_IO_ERROR, "Test Failure"), _1);
+ callback = bind(mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, 2,
+ new_range, (DiskIoMgr*)nullptr, (RequestContext*)nullptr,
+ data, Status(TErrorCode::DISK_IO_ERROR, "Test Failure"), _1);
*new_range = pool_.Add(new WriteRange(tmp_file, -1, 0, callback));
(*new_range)->SetData(reinterpret_cast<uint8_t*>(data), sizeof(int32_t));
@@ -353,7 +292,7 @@ TEST_F(DiskIoMgrTest, InvalidWrite) {
// AddWriteRange() is expected to succeed before the cancel and fail after it.
// The writes themselves may finish with status cancelled or ok.
TEST_F(DiskIoMgrTest, SingleWriterCancel) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
num_ranges_written_ = 0;
string tmp_file = "/tmp/disk_io_mgr_test.txt";
int num_ranges = 100;
@@ -368,33 +307,30 @@ TEST_F(DiskIoMgrTest, SingleWriterCancel) {
}
scoped_ptr<DiskIoMgr> read_io_mgr(new DiskIoMgr(1, 1, 1, 1, 10));
- BufferPool::ClientHandle read_client;
- RegisterBufferPoolClient(
- LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
- ASSERT_OK(read_io_mgr->Init());
- unique_ptr<RequestContext> reader = read_io_mgr->RegisterContext();
+ MemTracker reader_mem_tracker(LARGE_MEM_LIMIT);
+ ASSERT_OK(read_io_mgr->Init(&reader_mem_tracker));
+ unique_ptr<RequestContext> reader =
+ read_io_mgr->RegisterContext(&reader_mem_tracker);
for (int num_threads_per_disk = 1; num_threads_per_disk <= 5; ++num_threads_per_disk) {
for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
- // Pool for temporary objects from this iteration only.
- ObjectPool tmp_pool;
+ pool_.Clear(); // Destroy scan ranges from previous iterations.
DiskIoMgr io_mgr(num_disks, num_threads_per_disk, num_threads_per_disk, 1, 10);
- ASSERT_OK(io_mgr.Init());
- unique_ptr<RequestContext> writer = io_mgr.RegisterContext();
+ ASSERT_OK(io_mgr.Init(&mem_tracker));
+ unique_ptr<RequestContext> writer = io_mgr.RegisterContext(&mem_tracker);
Status validate_status = Status::OK();
for (int i = 0; i < num_ranges; ++i) {
if (i == num_ranges_before_cancel) {
- writer->Cancel();
+ io_mgr.CancelContext(writer.get());
validate_status = Status::CANCELLED;
}
- int32_t* data = tmp_pool.Add(new int32_t);
+ int32_t* data = pool_.Add(new int32_t);
*data = rand();
- WriteRange** new_range = tmp_pool.Add(new WriteRange*);
- WriteRange::WriteDoneCallback callback =
- bind(mem_fn(&DiskIoMgrTest::WriteValidateCallback), this,
- num_ranges_before_cancel, new_range, read_io_mgr.get(), reader.get(),
- &read_client, data, Status::CANCELLED, _1);
- *new_range = tmp_pool.Add(
- new WriteRange(tmp_file, cur_offset, num_ranges % num_disks, callback));
+ WriteRange** new_range = pool_.Add(new WriteRange*);
+ WriteRange::WriteDoneCallback callback = bind(
+ mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, num_ranges_before_cancel,
+ new_range, read_io_mgr.get(), reader.get(), data, Status::CANCELLED, _1);
+ *new_range = pool_.Add(new WriteRange(
+ tmp_file, cur_offset, num_ranges % num_disks, callback));
(*new_range)->SetData(reinterpret_cast<uint8_t*>(data), sizeof(int32_t));
cur_offset += sizeof(int32_t);
Status add_status = io_mgr.AddWriteRange(writer.get(), *new_range);
@@ -411,13 +347,13 @@ TEST_F(DiskIoMgrTest, SingleWriterCancel) {
}
read_io_mgr->UnregisterContext(reader.get());
- buffer_pool()->DeregisterClient(&read_client);
+ read_io_mgr.reset();
}
// Basic test with a single reader, testing multiple threads, disks and a different
// number of buffers.
TEST_F(DiskIoMgrTest, SingleReader) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
const char* data = "abcdefghijklm";
int len = strlen(data);
@@ -431,7 +367,7 @@ TEST_F(DiskIoMgrTest, SingleReader) {
for (int num_threads_per_disk = 1; num_threads_per_disk <= 5; ++num_threads_per_disk) {
for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
for (int num_read_threads = 1; num_read_threads <= 5; ++num_read_threads) {
- ObjectPool tmp_pool;
+ ObjectPool pool;
LOG(INFO) << "Starting test with num_threads_per_disk=" << num_threads_per_disk
<< " num_disk=" << num_disks
<< " num_read_threads=" << num_read_threads;
@@ -439,40 +375,38 @@ TEST_F(DiskIoMgrTest, SingleReader) {
if (++iters % 5000 == 0) LOG(ERROR) << "Starting iteration " << iters;
DiskIoMgr io_mgr(num_disks, num_threads_per_disk, num_threads_per_disk, 1, 1);
- ASSERT_OK(io_mgr.Init());
- BufferPool::ClientHandle read_client;
- RegisterBufferPoolClient(
- LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
- unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+ ASSERT_OK(io_mgr.Init(&mem_tracker));
+ MemTracker reader_mem_tracker;
+ unique_ptr<RequestContext> reader =
+ io_mgr.RegisterContext(&reader_mem_tracker);
vector<ScanRange*> ranges;
for (int i = 0; i < len; ++i) {
int disk_id = i % num_disks;
- ranges.push_back(InitRange(&tmp_pool, tmp_file, 0, len, disk_id, stat_val.st_mtime));
+ ranges.push_back(InitRange(tmp_file, 0, len, disk_id, stat_val.st_mtime));
}
ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges));
AtomicInt32 num_ranges_processed;
thread_group threads;
for (int i = 0; i < num_read_threads; ++i) {
- threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(),
- &read_client, data, len, Status::OK(), 0, &num_ranges_processed));
+ threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(), data, len,
+ Status::OK(), 0, &num_ranges_processed));
}
threads.join_all();
EXPECT_EQ(num_ranges_processed.Load(), ranges.size());
io_mgr.UnregisterContext(reader.get());
- EXPECT_EQ(read_client.GetUsedReservation(), 0);
- buffer_pool()->DeregisterClient(&read_client);
+ EXPECT_EQ(reader_mem_tracker.consumption(), 0);
}
}
}
- EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+ EXPECT_EQ(mem_tracker.consumption(), 0);
}
// This test issues adding additional scan ranges while there are some still in flight.
TEST_F(DiskIoMgrTest, AddScanRangeTest) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
const char* data = "abcdefghijklm";
int len = strlen(data);
@@ -485,19 +419,17 @@ TEST_F(DiskIoMgrTest, AddScanRangeTest) {
int64_t iters = 0;
for (int num_threads_per_disk = 1; num_threads_per_disk <= 5; ++num_threads_per_disk) {
for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
- // Pool for temporary objects from this iteration only.
- ObjectPool tmp_pool;
+ pool_.Clear(); // Destroy scan ranges from previous iterations.
LOG(INFO) << "Starting test with num_threads_per_disk=" << num_threads_per_disk
<< " num_disk=" << num_disks;
if (++iters % 5000 == 0) LOG(ERROR) << "Starting iteration " << iters;
DiskIoMgr io_mgr(num_disks, num_threads_per_disk, num_threads_per_disk, 1, 1);
- ASSERT_OK(io_mgr.Init());
- BufferPool::ClientHandle read_client;
- RegisterBufferPoolClient(
- LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
- unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+ ASSERT_OK(io_mgr.Init(&mem_tracker));
+ MemTracker reader_mem_tracker;
+ unique_ptr<RequestContext> reader =
+ io_mgr.RegisterContext(&reader_mem_tracker);
vector<ScanRange*> ranges_first_half;
vector<ScanRange*> ranges_second_half;
@@ -505,10 +437,10 @@ TEST_F(DiskIoMgrTest, AddScanRangeTest) {
int disk_id = i % num_disks;
if (i > len / 2) {
ranges_second_half.push_back(
- InitRange(&tmp_pool, tmp_file, i, 1, disk_id, stat_val.st_mtime));
+ InitRange(tmp_file, i, 1, disk_id, stat_val.st_mtime));
} else {
ranges_first_half.push_back(
- InitRange(&tmp_pool, tmp_file, i, 1, disk_id, stat_val.st_mtime));
+ InitRange(tmp_file, i, 1, disk_id, stat_val.st_mtime));
}
}
AtomicInt32 num_ranges_processed;
@@ -517,8 +449,8 @@ TEST_F(DiskIoMgrTest, AddScanRangeTest) {
ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges_first_half));
// Read a couple of them
- ScanRangeThread(&io_mgr, reader.get(), &read_client, data, strlen(data),
- Status::OK(), 2, &num_ranges_processed);
+ ScanRangeThread(&io_mgr, reader.get(), data, strlen(data), Status::OK(), 2,
+ &num_ranges_processed);
// Issue second half
ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges_second_half));
@@ -526,26 +458,24 @@ TEST_F(DiskIoMgrTest, AddScanRangeTest) {
// Start up some threads and then cancel
thread_group threads;
for (int i = 0; i < 3; ++i) {
- threads.add_thread(
- new thread(ScanRangeThread, &io_mgr, reader.get(), &read_client, data,
- strlen(data), Status::CANCELLED, 0, &num_ranges_processed));
+ threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(), data,
+ strlen(data), Status::CANCELLED, 0, &num_ranges_processed));
}
threads.join_all();
EXPECT_EQ(num_ranges_processed.Load(), len);
io_mgr.UnregisterContext(reader.get());
- EXPECT_EQ(read_client.GetUsedReservation(), 0);
- buffer_pool()->DeregisterClient(&read_client);
+ EXPECT_EQ(reader_mem_tracker.consumption(), 0);
}
}
- EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+ EXPECT_EQ(mem_tracker.consumption(), 0);
}
// Test to make sure that sync reads and async reads work together
// Note: this test is constructed so the number of buffers is greater than the
// number of scan ranges.
TEST_F(DiskIoMgrTest, SyncReadTest) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
const char* data = "abcdefghijklm";
int len = strlen(data);
@@ -558,8 +488,7 @@ TEST_F(DiskIoMgrTest, SyncReadTest) {
int64_t iters = 0;
for (int num_threads_per_disk = 1; num_threads_per_disk <= 5; ++num_threads_per_disk) {
for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
- // Pool for temporary objects from this iteration only.
- ObjectPool tmp_pool;
+ pool_.Clear(); // Destroy scan ranges from previous iterations.
LOG(INFO) << "Starting test with num_threads_per_disk=" << num_threads_per_disk
<< " num_disk=" << num_disks;
@@ -567,56 +496,54 @@ TEST_F(DiskIoMgrTest, SyncReadTest) {
DiskIoMgr io_mgr(num_disks, num_threads_per_disk, num_threads_per_disk,
MIN_BUFFER_SIZE, MAX_BUFFER_SIZE);
- ASSERT_OK(io_mgr.Init());
- BufferPool::ClientHandle read_client;
- RegisterBufferPoolClient(
- LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
- unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+ ASSERT_OK(io_mgr.Init(&mem_tracker));
+ MemTracker reader_mem_tracker;
+ unique_ptr<RequestContext> reader =
+ io_mgr.RegisterContext(&reader_mem_tracker);
ScanRange* complete_range =
- InitRange(&tmp_pool, tmp_file, 0, strlen(data), 0, stat_val.st_mtime);
+ InitRange(tmp_file, 0, strlen(data), 0, stat_val.st_mtime);
// Issue some reads before the async ones are issued
- ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
- ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
+ ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
+ ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
+
vector<ScanRange*> ranges;
for (int i = 0; i < len; ++i) {
int disk_id = i % num_disks;
- ranges.push_back(
- InitRange(&tmp_pool, tmp_file, 0, len, disk_id, stat_val.st_mtime));
+ ranges.push_back(InitRange(tmp_file, 0, len, disk_id, stat_val.st_mtime));
}
ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges));
AtomicInt32 num_ranges_processed;
thread_group threads;
for (int i = 0; i < 5; ++i) {
- threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(),
- &read_client, data, strlen(data), Status::OK(), 0, &num_ranges_processed));
+ threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(), data,
+ strlen(data), Status::OK(), 0, &num_ranges_processed));
}
// Issue some more sync ranges
for (int i = 0; i < 5; ++i) {
sched_yield();
- ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
+ ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
}
threads.join_all();
- ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
- ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
+ ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
+ ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
EXPECT_EQ(num_ranges_processed.Load(), ranges.size());
io_mgr.UnregisterContext(reader.get());
- EXPECT_EQ(read_client.GetUsedReservation(), 0);
- buffer_pool()->DeregisterClient(&read_client);
+ EXPECT_EQ(reader_mem_tracker.consumption(), 0);
}
}
- EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+ EXPECT_EQ(mem_tracker.consumption(), 0);
}
// Tests a single reader cancelling half way through scan ranges.
TEST_F(DiskIoMgrTest, SingleReaderCancel) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
const char* data = "abcdefghijklm";
int len = strlen(data);
@@ -629,25 +556,22 @@ TEST_F(DiskIoMgrTest, SingleReaderCancel) {
int64_t iters = 0;
for (int num_threads_per_disk = 1; num_threads_per_disk <= 5; ++num_threads_per_disk) {
for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
- // Pool for temporary objects from this iteration only.
- ObjectPool tmp_pool;
+ pool_.Clear(); // Destroy scan ranges from previous iterations.
LOG(INFO) << "Starting test with num_threads_per_disk=" << num_threads_per_disk
<< " num_disk=" << num_disks;
if (++iters % 5000 == 0) LOG(ERROR) << "Starting iteration " << iters;
DiskIoMgr io_mgr(num_disks, num_threads_per_disk, num_threads_per_disk, 1, 1);
- ASSERT_OK(io_mgr.Init());
- BufferPool::ClientHandle read_client;
- RegisterBufferPoolClient(
- LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
- unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+ ASSERT_OK(io_mgr.Init(&mem_tracker));
+ MemTracker reader_mem_tracker;
+ unique_ptr<RequestContext> reader =
+ io_mgr.RegisterContext(&reader_mem_tracker);
vector<ScanRange*> ranges;
for (int i = 0; i < len; ++i) {
int disk_id = i % num_disks;
- ranges.push_back(
- InitRange(&tmp_pool, tmp_file, 0, len, disk_id, stat_val.st_mtime));
+ ranges.push_back(InitRange(tmp_file, 0, len, disk_id, stat_val.st_mtime));
}
ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges));
@@ -655,112 +579,94 @@ TEST_F(DiskIoMgrTest, SingleReaderCancel) {
int num_succesful_ranges = ranges.size() / 2;
// Read half the ranges
for (int i = 0; i < num_succesful_ranges; ++i) {
- ScanRangeThread(&io_mgr, reader.get(), &read_client, data, strlen(data),
- Status::OK(), 1, &num_ranges_processed);
+ ScanRangeThread(&io_mgr, reader.get(), data, strlen(data), Status::OK(), 1,
+ &num_ranges_processed);
}
EXPECT_EQ(num_ranges_processed.Load(), num_succesful_ranges);
// Start up some threads and then cancel
thread_group threads;
for (int i = 0; i < 3; ++i) {
- threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(),
- &read_client, data, strlen(data), Status::CANCELLED, 0,
- &num_ranges_processed));
+ threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(), data,
+ strlen(data), Status::CANCELLED, 0, &num_ranges_processed));
}
- reader->Cancel();
+ io_mgr.CancelContext(reader.get());
sched_yield();
threads.join_all();
- EXPECT_TRUE(reader->IsCancelled());
+ EXPECT_TRUE(io_mgr.context_status(reader.get()).IsCancelled());
io_mgr.UnregisterContext(reader.get());
- EXPECT_EQ(read_client.GetUsedReservation(), 0);
- buffer_pool()->DeregisterClient(&read_client);
+ EXPECT_EQ(reader_mem_tracker.consumption(), 0);
}
}
- EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+ EXPECT_EQ(mem_tracker.consumption(), 0);
}
-// Test readers running with different amounts of memory and getting blocked on scan
-// ranges that have run out of buffers.
-TEST_F(DiskIoMgrTest, MemScarcity) {
+// Test when the reader goes over the mem limit
+TEST_F(DiskIoMgrTest, MemLimits) {
const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
- // File is 2.5 max buffers so that we can scan file without returning buffers
- // when we get the max reservation below.
- const int64_t DATA_BYTES = MAX_BUFFER_SIZE * 5 / 2;
- char data[DATA_BYTES];
- for (int i = 0; i < DATA_BYTES; ++i) {
- data[i] = uniform_int_distribution<uint8_t>(0, 255)(rng_);
- }
- CreateTempFile(tmp_file, data, DATA_BYTES);
+ const char* data = "abcdefghijklm";
+ int len = strlen(data);
+ CreateTempFile(tmp_file, data);
// Get mtime for file
struct stat stat_val;
stat(tmp_file, &stat_val);
- const int RESERVATION_LIMIT_NUM_BUFFERS = 20;
- const int64_t RESERVATION_LIMIT = RESERVATION_LIMIT_NUM_BUFFERS * MAX_BUFFER_SIZE;
- InitRootReservation(RESERVATION_LIMIT);
-
- thread_group threads;
- // Allocate enough ranges so that the total buffers exceeds the limit.
+ const int mem_limit_num_buffers = 2;
+ // Allocate enough ranges so that the total buffers exceeds the mem limit.
const int num_ranges = 25;
{
+ MemTracker root_mem_tracker(mem_limit_num_buffers * MAX_BUFFER_SIZE);
DiskIoMgr io_mgr(1, 1, 1, MIN_BUFFER_SIZE, MAX_BUFFER_SIZE);
- ASSERT_OK(io_mgr.Init());
- BufferPool::ClientHandle read_client;
- RegisterBufferPoolClient(RESERVATION_LIMIT, RESERVATION_LIMIT, &read_client);
- unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+ ASSERT_OK(io_mgr.Init(&root_mem_tracker));
+ MemTracker reader_mem_tracker(-1, "Reader", &root_mem_tracker);
+ unique_ptr<RequestContext> reader = io_mgr.RegisterContext(&reader_mem_tracker);
vector<ScanRange*> ranges;
for (int i = 0; i < num_ranges; ++i) {
- ranges.push_back(InitRange(&pool_, tmp_file, 0, DATA_BYTES, 0, stat_val.st_mtime));
+ ranges.push_back(InitRange(tmp_file, 0, len, 0, stat_val.st_mtime));
}
ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges));
- // Keep starting new ranges without returning buffers until we run out of
- // reservation.
- while (read_client.GetUnusedReservation() >= MIN_BUFFER_SIZE) {
+
+ // Don't return buffers to force memory pressure
+ vector<unique_ptr<BufferDescriptor>> buffers;
+
+ AtomicInt32 num_ranges_processed;
+ ScanRangeThread(&io_mgr, reader.get(), data, strlen(data), Status::MemLimitExceeded(),
+ 1, &num_ranges_processed);
+
+ char result[strlen(data) + 1];
+ // Keep reading new ranges without returning buffers. This forces us
+ // to go over the limit eventually.
+ while (true) {
+ memset(result, 0, strlen(data) + 1);
ScanRange* range = nullptr;
- bool needs_buffers;
- ASSERT_OK(io_mgr.GetNextUnstartedRange(reader.get(), &range, &needs_buffers));
+ Status status = io_mgr.GetNextRange(reader.get(), &range);
+ ASSERT_TRUE(status.ok() || status.IsMemLimitExceeded());
if (range == nullptr) break;
- ASSERT_TRUE(needs_buffers);
- // Pick a random amount of memory to reserve.
- int64_t max_bytes_to_alloc = uniform_int_distribution<int64_t>(MIN_BUFFER_SIZE,
- min<int64_t>(read_client.GetUnusedReservation(), MAX_BUFFER_SIZE * 3))(rng_);
- ASSERT_OK(io_mgr.AllocateBuffersForRange(
- reader.get(), &read_client, range, max_bytes_to_alloc));
- // Start a thread fetching from the range. The thread will either finish the
- // range or be cancelled.
- threads.add_thread(new thread([&data, DATA_BYTES, range] {
- // Don't return buffers to force memory pressure.
- vector<unique_ptr<BufferDescriptor>> buffers;
- int64_t data_offset = 0;
- Status status;
- while (true) {
- unique_ptr<BufferDescriptor> buffer;
- status = range->GetNext(&buffer);
- ASSERT_TRUE(status.ok() || status.IsCancelled()) << status.GetDetail();
- if (status.IsCancelled() || buffer == nullptr) break;
- EXPECT_EQ(0, memcmp(data + data_offset, buffer->buffer(), buffer->len()));
- data_offset += buffer->len();
- buffers.emplace_back(move(buffer));
- }
- if (status.ok()) ASSERT_EQ(DATA_BYTES, data_offset);
- for (auto& buffer : buffers) range->ReturnBuffer(move(buffer));
- }));
- // Let the thread start running before starting the next.
- SleepForMs(10);
+
+ while (true) {
+ unique_ptr<BufferDescriptor> buffer;
+ Status status = range->GetNext(&buffer);
+ ASSERT_TRUE(status.ok() || status.IsMemLimitExceeded());
+ if (buffer == nullptr) break;
+ memcpy(result + range->offset() + buffer->scan_range_offset(),
+ buffer->buffer(), buffer->len());
+ buffers.push_back(move(buffer));
+ }
+ ValidateEmptyOrCorrect(data, result, strlen(data));
}
- // Let the threads run for a bit then cancel everything.
- SleepForMs(500);
- reader->Cancel();
- // Wait until the threads have returned their buffers before unregistering.
- threads.join_all();
+
+ for (int i = 0; i < buffers.size(); ++i) {
+ io_mgr.ReturnBuffer(move(buffers[i]));
+ }
+
+ EXPECT_TRUE(io_mgr.context_status(reader.get()).IsMemLimitExceeded());
io_mgr.UnregisterContext(reader.get());
- EXPECT_EQ(read_client.GetUsedReservation(), 0);
- buffer_pool()->DeregisterClient(&read_client);
+ EXPECT_EQ(reader_mem_tracker.consumption(), 0);
}
}
@@ -769,7 +675,7 @@ TEST_F(DiskIoMgrTest, MemScarcity) {
// only tests the fallback mechanism.
// TODO: we can fake the cached read path without HDFS
TEST_F(DiskIoMgrTest, CachedReads) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
const char* data = "abcdefghijklm";
int len = strlen(data);
@@ -783,55 +689,52 @@ TEST_F(DiskIoMgrTest, CachedReads) {
{
DiskIoMgr io_mgr(num_disks, 1, 1, MIN_BUFFER_SIZE, MAX_BUFFER_SIZE);
- ASSERT_OK(io_mgr.Init());
- BufferPool::ClientHandle read_client;
- RegisterBufferPoolClient(
- LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
- unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+ ASSERT_OK(io_mgr.Init(&mem_tracker));
+ MemTracker reader_mem_tracker;
+ unique_ptr<RequestContext> reader = io_mgr.RegisterContext(&reader_mem_tracker);
ScanRange* complete_range =
- InitRange(&pool_, tmp_file, 0, strlen(data), 0, stat_val.st_mtime, nullptr, true);
+ InitRange(tmp_file, 0, strlen(data), 0, stat_val.st_mtime, nullptr, true);
// Issue some reads before the async ones are issued
- ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
- ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
+ ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
+ ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
vector<ScanRange*> ranges;
for (int i = 0; i < len; ++i) {
int disk_id = i % num_disks;
ranges.push_back(
- InitRange(&pool_, tmp_file, 0, len, disk_id, stat_val.st_mtime, nullptr, true));
+ InitRange(tmp_file, 0, len, disk_id, stat_val.st_mtime, nullptr, true));
}
ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges));
AtomicInt32 num_ranges_processed;
thread_group threads;
for (int i = 0; i < 5; ++i) {
- threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(), &read_client,
- data, strlen(data), Status::OK(), 0, &num_ranges_processed));
+ threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(), data,
+ strlen(data), Status::OK(), 0, &num_ranges_processed));
}
// Issue some more sync ranges
for (int i = 0; i < 5; ++i) {
sched_yield();
- ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
+ ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
}
threads.join_all();
- ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
- ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
+ ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
+ ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
EXPECT_EQ(num_ranges_processed.Load(), ranges.size());
io_mgr.UnregisterContext(reader.get());
- EXPECT_EQ(read_client.GetUsedReservation(), 0);
- buffer_pool()->DeregisterClient(&read_client);
+ EXPECT_EQ(reader_mem_tracker.consumption(), 0);
}
- EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+ EXPECT_EQ(mem_tracker.consumption(), 0);
}
TEST_F(DiskIoMgrTest, MultipleReaderWriter) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
const int ITERATIONS = 1;
const char* data = "abcdefghijklmnopqrstuvwxyz";
const int num_contexts = 5;
@@ -853,22 +756,17 @@ TEST_F(DiskIoMgrTest, MultipleReaderWriter) {
int64_t iters = 0;
vector<unique_ptr<RequestContext>> contexts(num_contexts);
- unique_ptr<BufferPool::ClientHandle[]> clients(
- new BufferPool::ClientHandle[num_contexts]);
Status status;
for (int iteration = 0; iteration < ITERATIONS; ++iteration) {
for (int threads_per_disk = 1; threads_per_disk <= 5; ++threads_per_disk) {
for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
- // Pool for temporary objects from this iteration only.
- ObjectPool tmp_pool;
DiskIoMgr io_mgr(num_disks, threads_per_disk, threads_per_disk, MIN_BUFFER_SIZE,
MAX_BUFFER_SIZE);
- ASSERT_OK(io_mgr.Init());
+ ASSERT_OK(io_mgr.Init(&mem_tracker));
for (int file_index = 0; file_index < num_contexts; ++file_index) {
- RegisterBufferPoolClient(
- LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &clients[file_index]);
- contexts[file_index] = io_mgr.RegisterContext();
+ contexts[file_index] = io_mgr.RegisterContext(&mem_tracker);
}
+ pool_.Clear();
int read_offset = 0;
int write_offset = 0;
while (read_offset < file_size) {
@@ -880,11 +778,11 @@ TEST_F(DiskIoMgrTest, MultipleReaderWriter) {
int num_scan_ranges = min<int>(num_reads_queued, write_offset - read_offset);
for (int i = 0; i < num_scan_ranges; ++i) {
ranges.push_back(InitRange(
- &tmp_pool, file_name.c_str(), read_offset, 1, i % num_disks, stat_val.st_mtime));
- threads.add_thread(new thread(ScanRangeThread, &io_mgr,
- contexts[context_index].get(), &clients[context_index],
- reinterpret_cast<const char*>(data + (read_offset % strlen(data))),
- 1, Status::OK(), num_scan_ranges, &num_ranges_processed));
+ file_name.c_str(), read_offset, 1, i % num_disks, stat_val.st_mtime));
+ threads.add_thread(
+ new thread(ScanRangeThread, &io_mgr, contexts[context_index].get(),
+ reinterpret_cast<const char*>(data + (read_offset % strlen(data))),
+ 1, Status::OK(), num_scan_ranges, &num_ranges_processed));
++read_offset;
}
@@ -894,7 +792,7 @@ TEST_F(DiskIoMgrTest, MultipleReaderWriter) {
WriteRange::WriteDoneCallback callback =
bind(mem_fn(&DiskIoMgrTest::WriteCompleteCallback),
this, num_write_ranges, _1);
- WriteRange* new_range = tmp_pool.Add(new WriteRange(
+ WriteRange* new_range = pool_.Add(new WriteRange(
file_name, write_offset, i % num_disks, callback));
new_range->SetData(
reinterpret_cast<const uint8_t*>(data + (write_offset % strlen(data))),
@@ -911,9 +809,9 @@ TEST_F(DiskIoMgrTest, MultipleReaderWriter) {
threads.join_all();
} // for (int context_index
} // while (read_offset < file_size)
+
for (int file_index = 0; file_index < num_contexts; ++file_index) {
io_mgr.UnregisterContext(contexts[file_index].get());
- buffer_pool()->DeregisterClient(&clients[file_index]);
}
} // for (int num_disks
} // for (int threads_per_disk
@@ -922,19 +820,23 @@ TEST_F(DiskIoMgrTest, MultipleReaderWriter) {
// This test will test multiple concurrent reads each reading a different file.
TEST_F(DiskIoMgrTest, MultipleReader) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
const int NUM_READERS = 5;
const int DATA_LEN = 50;
const int ITERATIONS = 25;
const int NUM_THREADS_PER_READER = 3;
- vector<string> file_names(NUM_READERS);
- vector<int64_t> mtimes(NUM_READERS);
- vector<string> data(NUM_READERS);
- unique_ptr<BufferPool::ClientHandle[]> clients(
- new BufferPool::ClientHandle[NUM_READERS]);
- vector<unique_ptr<RequestContext>> readers(NUM_READERS);
- vector<char*> results(NUM_READERS);
+ vector<string> file_names;
+ vector<int64_t> mtimes;
+ vector<string> data;
+ vector<unique_ptr<RequestContext>> readers;
+ vector<char*> results;
+
+ file_names.resize(NUM_READERS);
+ readers.resize(NUM_READERS);
+ mtimes.resize(NUM_READERS);
+ data.resize(NUM_READERS);
+ results.resize(NUM_READERS);
// Initialize data for each reader. The data will be
// 'abcd...' for reader one, 'bcde...' for reader two (wrapping around at 'z')
@@ -965,25 +867,22 @@ TEST_F(DiskIoMgrTest, MultipleReader) {
for (int iteration = 0; iteration < ITERATIONS; ++iteration) {
for (int threads_per_disk = 1; threads_per_disk <= 5; ++threads_per_disk) {
for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
- // Pool for temporary objects from this iteration only.
- ObjectPool tmp_pool;
+ pool_.Clear(); // Destroy scan ranges from previous iterations.
LOG(INFO) << "Starting test with num_threads_per_disk=" << threads_per_disk
<< " num_disk=" << num_disks;
if (++iters % 2500 == 0) LOG(ERROR) << "Starting iteration " << iters;
DiskIoMgr io_mgr(num_disks, threads_per_disk, threads_per_disk, MIN_BUFFER_SIZE,
MAX_BUFFER_SIZE);
- ASSERT_OK(io_mgr.Init());
+ EXPECT_OK(io_mgr.Init(&mem_tracker));
for (int i = 0; i < NUM_READERS; ++i) {
- RegisterBufferPoolClient(
- LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &clients[i]);
- readers[i] = io_mgr.RegisterContext();
+ readers[i] = io_mgr.RegisterContext(&mem_tracker);
vector<ScanRange*> ranges;
for (int j = 0; j < DATA_LEN; ++j) {
int disk_id = j % num_disks;
- ranges.push_back(InitRange(&tmp_pool, file_names[i].c_str(), j, 1, disk_id, mtimes[i]));
+ ranges.push_back(InitRange(file_names[i].c_str(), j, 1, disk_id, mtimes[i]));
}
ASSERT_OK(io_mgr.AddScanRanges(readers[i].get(), ranges));
}
@@ -993,20 +892,18 @@ TEST_F(DiskIoMgrTest, MultipleReader) {
for (int i = 0; i < NUM_READERS; ++i) {
for (int j = 0; j < NUM_THREADS_PER_READER; ++j) {
threads.add_thread(new thread(ScanRangeThread, &io_mgr, readers[i].get(),
- &clients[i], data[i].c_str(), data[i].size(), Status::OK(), 0,
- &num_ranges_processed));
+ data[i].c_str(), data[i].size(), Status::OK(), 0, &num_ranges_processed));
}
}
threads.join_all();
EXPECT_EQ(num_ranges_processed.Load(), DATA_LEN * NUM_READERS);
for (int i = 0; i < NUM_READERS; ++i) {
io_mgr.UnregisterContext(readers[i].get());
- buffer_pool()->DeregisterClient(&clients[i]);
}
}
}
}
- EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+ EXPECT_EQ(mem_tracker.consumption(), 0);
}
// Stress test for multiple clients with cancellation
@@ -1018,148 +915,115 @@ TEST_F(DiskIoMgrTest, StressTest) {
test.Run(2); // In seconds
}
-// IMPALA-2366: handle partial read where range goes past end of file.
-TEST_F(DiskIoMgrTest, PartialRead) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
- const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
- const char* data = "the quick brown fox jumped over the lazy dog";
- int len = strlen(data);
- int read_len = len + 1000; // Read past end of file.
- // Test with various buffer sizes to exercise different code paths, e.g.
- // * the truncated data ends exactly on a buffer boundary
- // * the data is split between many buffers
- // * the data fits in one buffer
- const int64_t MIN_BUFFER_SIZE = 2;
- vector<int64_t> max_buffer_sizes{4, 16, 32, 128, 1024, 4096};
- CreateTempFile(tmp_file, data);
-
- // Get mtime for file
- struct stat stat_val;
- stat(tmp_file, &stat_val);
-
- for (int64_t max_buffer_size : max_buffer_sizes) {
- DiskIoMgr io_mgr(1, 1, 1, MIN_BUFFER_SIZE, max_buffer_size);
- ASSERT_OK(io_mgr.Init());
- unique_ptr<RequestContext> reader;
- reader = io_mgr.RegisterContext();
-
- BufferPool::ClientHandle read_client;
- RegisterBufferPoolClient(
- LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
-
- // We should not read past the end of file.
- ScanRange* range = InitRange(&pool_, tmp_file, 0, read_len, 0, stat_val.st_mtime);
- unique_ptr<BufferDescriptor> buffer;
- bool needs_buffers;
- ASSERT_OK(io_mgr.StartScanRange(reader.get(), range, &needs_buffers));
- if (needs_buffers) {
- ASSERT_OK(io_mgr.AllocateBuffersForRange(
- reader.get(), &read_client, range, 3 * max_buffer_size));
- }
-
- int64_t bytes_read = 0;
- bool eosr = false;
- do {
- ASSERT_OK(range->GetNext(&buffer));
- ASSERT_GE(buffer->buffer_len(), MIN_BUFFER_SIZE);
- ASSERT_LE(buffer->buffer_len(), max_buffer_size);
- ASSERT_LE(buffer->len(), len - bytes_read);
- ASSERT_TRUE(memcmp(buffer->buffer(), data + bytes_read, buffer->len()) == 0);
- bytes_read += buffer->len();
- eosr = buffer->eosr();
- // Should see eosr if we've read past the end of the file. If the data is an exact
- // multiple of the max buffer size then we may read to the end of the file without
- // noticing that it is eosr. Eosr will be returned on the next read in that case.
- ASSERT_TRUE(bytes_read < len || buffer->eosr()
- || (buffer->len() == max_buffer_size && len % max_buffer_size == 0))
- << "max_buffer_size " << max_buffer_size << " bytes_read " << bytes_read
- << "len " << len << " buffer->len() " << buffer->len()
- << " buffer->buffer_len() " << buffer->buffer_len();
- ASSERT_TRUE(buffer->len() > 0 || buffer->eosr());
- range->ReturnBuffer(move(buffer));
- } while (!eosr);
-
- io_mgr.UnregisterContext(reader.get());
- EXPECT_EQ(read_client.GetUsedReservation(), 0);
- buffer_pool()->DeregisterClient(&read_client);
- }
-}
-
-// Test zero-length scan range.
-TEST_F(DiskIoMgrTest, ZeroLengthScanRange) {
- const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
- const char* data = "the quick brown fox jumped over the lazy dog";
- const int64_t MIN_BUFFER_SIZE = 2;
- const int64_t MAX_BUFFER_SIZE = 1024;
- CreateTempFile(tmp_file, data);
-
- // Get mtime for file
- struct stat stat_val;
- stat(tmp_file, &stat_val);
-
- DiskIoMgr io_mgr(1, 1, 1, MIN_BUFFER_SIZE, MAX_BUFFER_SIZE);
-
- ASSERT_OK(io_mgr.Init());
- unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
-
- ScanRange* range = InitRange(&pool_, tmp_file, 0, 0, 0, stat_val.st_mtime);
- bool needs_buffers;
- Status status = io_mgr.StartScanRange(reader.get(), range, &needs_buffers);
- ASSERT_EQ(TErrorCode::DISK_IO_ERROR, status.code());
-
- status = io_mgr.AddScanRanges(reader.get(), vector<ScanRange*>({range}));
- ASSERT_EQ(TErrorCode::DISK_IO_ERROR, status.code());
-
+TEST_F(DiskIoMgrTest, Buffers) {
+ // Test default min/max buffer size
+ int min_buffer_size = 1024;
+ int max_buffer_size = 8 * 1024 * 1024; // 8 MB
+ MemTracker root_mem_tracker(max_buffer_size * 2);
+
+ DiskIoMgr io_mgr(1, 1, 1, min_buffer_size, max_buffer_size);
+ ASSERT_OK(io_mgr.Init(&root_mem_tracker));
+ ASSERT_EQ(root_mem_tracker.consumption(), 0);
+
+ MemTracker reader_mem_tracker(-1, "Reader", &root_mem_tracker);
+ unique_ptr<RequestContext> reader;
+ reader = io_mgr.RegisterContext(&reader_mem_tracker);
+
+ ScanRange* dummy_range = InitRange("dummy", 0, 0, 0, 0);
+
+ // buffer length should be rounded up to min buffer size
+ int64_t buffer_len = 1;
+ unique_ptr<BufferDescriptor> buffer_desc;
+ buffer_desc = io_mgr.GetFreeBuffer(reader.get(), dummy_range, buffer_len);
+ EXPECT_TRUE(buffer_desc->buffer() != nullptr);
+ EXPECT_EQ(min_buffer_size, buffer_desc->buffer_len());
+ EXPECT_EQ(1, io_mgr.num_allocated_buffers_.Load());
+ io_mgr.FreeBufferMemory(buffer_desc.get());
+ io_mgr.ReturnBuffer(move(buffer_desc));
+ EXPECT_EQ(min_buffer_size, root_mem_tracker.consumption());
+
+ // reuse buffer
+ buffer_len = min_buffer_size;
+ buffer_desc = io_mgr.GetFreeBuffer(reader.get(), dummy_range, buffer_len);
+ EXPECT_TRUE(buffer_desc->buffer() != nullptr);
+ EXPECT_EQ(min_buffer_size, buffer_desc->buffer_len());
+ EXPECT_EQ(1, io_mgr.num_allocated_buffers_.Load());
+ io_mgr.FreeBufferMemory(buffer_desc.get());
+ io_mgr.ReturnBuffer(move(buffer_desc));
+ EXPECT_EQ(min_buffer_size, root_mem_tracker.consumption());
+
+ // bump up to next buffer size
+ buffer_len = min_buffer_size + 1;
+ buffer_desc = io_mgr.GetFreeBuffer(reader.get(), dummy_range, buffer_len);
+ EXPECT_TRUE(buffer_desc->buffer() != nullptr);
+ EXPECT_EQ(min_buffer_size * 2, buffer_desc->buffer_len());
+ EXPECT_EQ(2, io_mgr.num_allocated_buffers_.Load());
+ EXPECT_EQ(min_buffer_size * 3, root_mem_tracker.consumption());
+
+ // gc unused buffer
+ io_mgr.GcIoBuffers();
+ EXPECT_EQ(1, io_mgr.num_allocated_buffers_.Load());
+ EXPECT_EQ(min_buffer_size * 2, root_mem_tracker.consumption());
+
+ io_mgr.FreeBufferMemory(buffer_desc.get());
+ io_mgr.ReturnBuffer(move(buffer_desc));
+
+ // max buffer size
+ buffer_len = max_buffer_size;
+ buffer_desc = io_mgr.GetFreeBuffer(reader.get(), dummy_range, buffer_len);
+ EXPECT_TRUE(buffer_desc->buffer() != nullptr);
+ EXPECT_EQ(max_buffer_size, buffer_desc->buffer_len());
+ EXPECT_EQ(2, io_mgr.num_allocated_buffers_.Load());
+ io_mgr.FreeBufferMemory(buffer_desc.get());
+ io_mgr.ReturnBuffer(move(buffer_desc));
+ EXPECT_EQ(min_buffer_size * 2 + max_buffer_size, root_mem_tracker.consumption());
+
+ // gc buffers
+ io_mgr.GcIoBuffers();
+ EXPECT_EQ(io_mgr.num_allocated_buffers_.Load(), 0);
+ EXPECT_EQ(root_mem_tracker.consumption(), 0);
io_mgr.UnregisterContext(reader.get());
}
-// Test what happens if don't call AllocateBuffersForRange() after trying to start a
-// range.
-TEST_F(DiskIoMgrTest, SkipAllocateBuffers) {
+// IMPALA-2366: handle partial read where range goes past end of file.
+TEST_F(DiskIoMgrTest, PartialRead) {
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
const char* data = "the quick brown fox jumped over the lazy dog";
int len = strlen(data);
- const int64_t MIN_BUFFER_SIZE = 2;
- const int64_t MAX_BUFFER_SIZE = 1024;
+ int read_len = len + 1000; // Read past end of file.
CreateTempFile(tmp_file, data);
// Get mtime for file
struct stat stat_val;
stat(tmp_file, &stat_val);
- DiskIoMgr io_mgr(1, 1, 1, MIN_BUFFER_SIZE, MAX_BUFFER_SIZE);
+ scoped_ptr<DiskIoMgr> io_mgr(new DiskIoMgr(1, 1, 1, read_len, read_len));
- ASSERT_OK(io_mgr.Init());
+ ASSERT_OK(io_mgr->Init(&mem_tracker));
MemTracker reader_mem_tracker;
- unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+ unique_ptr<RequestContext> reader;
+ reader = io_mgr->RegisterContext(&reader_mem_tracker);
// We should not read past the end of file.
- vector<ScanRange*> ranges;
- for (int i = 0; i < 4; ++i) {
- ranges.push_back(InitRange(&pool_, tmp_file, 0, len, 0, stat_val.st_mtime));
- }
- bool needs_buffers;
- // Test StartScanRange().
- ASSERT_OK(io_mgr.StartScanRange(reader.get(), ranges[0], &needs_buffers));
- EXPECT_TRUE(needs_buffers);
- ASSERT_OK(io_mgr.StartScanRange(reader.get(), ranges[1], &needs_buffers));
- EXPECT_TRUE(needs_buffers);
-
- // Test AddScanRanges()/GetNextUnstartedRange().
- ASSERT_OK(
- io_mgr.AddScanRanges(reader.get(), vector<ScanRange*>({ranges[2], ranges[3]})));
-
- // Cancel two directly, cancel the other two indirectly via the context.
- ranges[0]->Cancel(Status::CANCELLED);
- ranges[2]->Cancel(Status::CANCELLED);
- reader->Cancel();
+ ScanRange* range = InitRange(tmp_file, 0, read_len, 0, stat_val.st_mtime);
+ unique_ptr<BufferDescriptor> buffer;
+ ASSERT_OK(io_mgr->Read(reader.get(), range, &buffer));
+ ASSERT_TRUE(buffer->eosr());
+ ASSERT_EQ(len, buffer->len());
+ ASSERT_TRUE(memcmp(buffer->buffer(), data, len) == 0);
+ io_mgr->ReturnBuffer(move(buffer));
- io_mgr.UnregisterContext(reader.get());
+ io_mgr->UnregisterContext(reader.get());
+ pool_.Clear();
+ io_mgr.reset();
+ EXPECT_EQ(reader_mem_tracker.consumption(), 0);
+ EXPECT_EQ(mem_tracker.consumption(), 0);
}
// Test reading into a client-allocated buffer.
TEST_F(DiskIoMgrTest, ReadIntoClientBuffer) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
const char* data = "the quick brown fox jumped over the lazy dog";
int len = strlen(data);
@@ -1168,19 +1032,19 @@ TEST_F(DiskIoMgrTest, ReadIntoClientBuffer) {
scoped_ptr<DiskIoMgr> io_mgr(new DiskIoMgr(1, 1, 1, read_len, read_len));
- ASSERT_OK(io_mgr->Init());
- // Reader doesn't need to provide client if it's providing buffers.
- unique_ptr<RequestContext> reader = io_mgr->RegisterContext();
+ ASSERT_OK(io_mgr->Init(&mem_tracker));
+ // Reader doesn't need to provide mem tracker if it's providing buffers.
+ MemTracker* reader_mem_tracker = nullptr;
+ unique_ptr<RequestContext> reader;
+ reader = io_mgr->RegisterContext(reader_mem_tracker);
for (int buffer_len : vector<int>({len - 1, len, len + 1})) {
vector<uint8_t> client_buffer(buffer_len);
int scan_len = min(len, buffer_len);
- ScanRange* range = pool_.Add(new ScanRange);
+ ScanRange* range = AllocateRange();
range->Reset(nullptr, tmp_file, scan_len, 0, 0, true,
BufferOpts::ReadInto(client_buffer.data(), buffer_len));
- bool needs_buffers;
- ASSERT_OK(io_mgr->StartScanRange(reader.get(), range, &needs_buffers));
- ASSERT_FALSE(needs_buffers);
+ ASSERT_OK(io_mgr->AddScanRange(reader.get(), range, true));
unique_ptr<BufferDescriptor> io_buffer;
ASSERT_OK(range->GetNext(&io_buffer));
@@ -1190,133 +1054,76 @@ TEST_F(DiskIoMgrTest, ReadIntoClientBuffer) {
ASSERT_EQ(memcmp(io_buffer->buffer(), data, scan_len), 0);
// DiskIoMgr should not have allocated memory.
- EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
- range->ReturnBuffer(move(io_buffer));
+ EXPECT_EQ(mem_tracker.consumption(), 0);
+ io_mgr->ReturnBuffer(move(io_buffer));
}
io_mgr->UnregisterContext(reader.get());
- EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+ pool_.Clear();
+ io_mgr.reset();
+ EXPECT_EQ(mem_tracker.consumption(), 0);
}
// Test reading into a client-allocated buffer where the read fails.
TEST_F(DiskIoMgrTest, ReadIntoClientBufferError) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
const char* tmp_file = "/file/that/does/not/exist";
const int SCAN_LEN = 128;
scoped_ptr<DiskIoMgr> io_mgr(new DiskIoMgr(1, 1, 1, SCAN_LEN, SCAN_LEN));
- ASSERT_OK(io_mgr->Init());
+ ASSERT_OK(io_mgr->Init(&mem_tracker));
+ // Reader doesn't need to provide mem tracker if it's providing buffers.
+ MemTracker* reader_mem_tracker = nullptr;
+ unique_ptr<RequestContext> reader;
vector<uint8_t> client_buffer(SCAN_LEN);
for (int i = 0; i < 1000; ++i) {
- // Reader doesn't need to provide mem tracker if it's providing buffers.
- BufferPool::ClientHandle read_client;
- RegisterBufferPoolClient(
- LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
- unique_ptr<RequestContext> reader = io_mgr->RegisterContext();
- ScanRange* range = pool_.Add(new ScanRange);
+ reader = io_mgr->RegisterContext(reader_mem_tracker);
+ ScanRange* range = AllocateRange();
range->Reset(nullptr, tmp_file, SCAN_LEN, 0, 0, true,
BufferOpts::ReadInto(client_buffer.data(), SCAN_LEN));
- bool needs_buffers;
- ASSERT_OK(io_mgr->StartScanRange(reader.get(), range, &needs_buffers));
- ASSERT_FALSE(needs_buffers);
+ ASSERT_OK(io_mgr->AddScanRange(reader.get(), range, true));
/// Also test the cancellation path. Run multiple iterations since it is racy whether
/// the read fails before the cancellation.
- if (i >= 1) reader->Cancel();
+ if (i >= 1) io_mgr->CancelContext(reader.get());
unique_ptr<BufferDescriptor> io_buffer;
ASSERT_FALSE(range->GetNext(&io_buffer).ok());
// DiskIoMgr should not have allocated memory.
- EXPECT_EQ(read_client.GetUsedReservation(), 0);
+ EXPECT_EQ(mem_tracker.consumption(), 0);
io_mgr->UnregisterContext(reader.get());
- EXPECT_EQ(read_client.GetUsedReservation(), 0);
- buffer_pool()->DeregisterClient(&read_client);
}
- EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+ pool_.Clear();
+ io_mgr.reset();
+ EXPECT_EQ(mem_tracker.consumption(), 0);
}
// Test to verify configuration parameters for number of I/O threads per disk.
TEST_F(DiskIoMgrTest, VerifyNumThreadsParameter) {
- InitRootReservation(LARGE_RESERVATION_LIMIT);
const int num_io_threads_for_remote_disks = FLAGS_num_remote_hdfs_io_threads
+ FLAGS_num_s3_io_threads + FLAGS_num_adls_io_threads;
// Verify num_io_threads_per_rotational_disk and num_io_threads_per_solid_state_disk.
// Since we do not have control over which disk is used, we check for either type
// (rotational/solid state)
+ MemTracker mem_tracker(LARGE_MEM_LIMIT);
const int num_io_threads_per_rotational_or_ssd = 2;
DiskIoMgr io_mgr(1, num_io_threads_per_rotational_or_ssd,
num_io_threads_per_rotational_or_ssd, 1, 10);
- ASSERT_OK(io_mgr.Init());
+ ASSERT_OK(io_mgr.Init(&mem_tracker));
const int num_io_threads = io_mgr.disk_thread_group_.Size();
ASSERT_TRUE(num_io_threads ==
num_io_threads_per_rotational_or_ssd + num_io_threads_for_remote_disks);
}
-
-// Test to verify that the correct buffer sizes are chosen given different
-// of scan range lengths and max_bytes values.
-TEST_F(DiskIoMgrTest, BufferSizeSelection) {
- DiskIoMgr io_mgr(1, 1, 1, MIN_BUFFER_SIZE, MAX_BUFFER_SIZE);
- ASSERT_OK(io_mgr.Init());
-
- // Scan range doesn't fit in max_bytes - allocate as many max-sized buffers as possible.
- EXPECT_EQ(vector<int64_t>(3, MAX_BUFFER_SIZE),
- io_mgr.ChooseBufferSizes(10 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE));
- EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE}),
- io_mgr.ChooseBufferSizes(10 * MAX_BUFFER_SIZE, MAX_BUFFER_SIZE));
- EXPECT_EQ(vector<int64_t>(4, MAX_BUFFER_SIZE),
- io_mgr.ChooseBufferSizes(10 * MAX_BUFFER_SIZE, 4 * MAX_BUFFER_SIZE));
-
- // Scan range fits in max_bytes - allocate as many max-sized buffers as possible, then
- // a smaller buffer to fit the remainder.
- EXPECT_EQ(vector<int64_t>(2, MAX_BUFFER_SIZE),
- io_mgr.ChooseBufferSizes(2 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE));
- EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE, MAX_BUFFER_SIZE, MIN_BUFFER_SIZE}),
- io_mgr.ChooseBufferSizes(2 * MAX_BUFFER_SIZE + 1, 3 * MAX_BUFFER_SIZE));
- EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE, MAX_BUFFER_SIZE, 2 * MIN_BUFFER_SIZE}),
- io_mgr.ChooseBufferSizes(
- 2 * MAX_BUFFER_SIZE + MIN_BUFFER_SIZE + 1, 3 * MAX_BUFFER_SIZE));
- EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE, MAX_BUFFER_SIZE, 2 * MIN_BUFFER_SIZE}),
- io_mgr.ChooseBufferSizes(
- 2 * MAX_BUFFER_SIZE + 2 * MIN_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE));
-
- // Scan range is smaller than max buffer size - allocate a single buffer that fits
- // the range.
- EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE}),
- io_mgr.ChooseBufferSizes(MAX_BUFFER_SIZE - 1, 3 * MAX_BUFFER_SIZE));
- EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE / 2}),
- io_mgr.ChooseBufferSizes(MAX_BUFFER_SIZE - 1, MAX_BUFFER_SIZE / 2));
- EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE / 2}),
- io_mgr.ChooseBufferSizes(MAX_BUFFER_SIZE / 2 - 1, 3 * MAX_BUFFER_SIZE));
- EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE / 2}),
- io_mgr.ChooseBufferSizes(MAX_BUFFER_SIZE / 2- 1, MAX_BUFFER_SIZE / 2));
- EXPECT_EQ(vector<int64_t>({MIN_BUFFER_SIZE}),
- io_mgr.ChooseBufferSizes(MIN_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE));
- EXPECT_EQ(vector<int64_t>({MIN_BUFFER_SIZE}),
- io_mgr.ChooseBufferSizes(MIN_BUFFER_SIZE, MIN_BUFFER_SIZE));
-
- // Scan range is smaller than max buffer size and max bytes is smaller still -
- // should allocate a single smaller buffer.
- EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE / 4}),
- io_mgr.ChooseBufferSizes(MAX_BUFFER_SIZE / 2, MAX_BUFFER_SIZE / 2 - 1));
-
- // Non power-of-two size > max buffer size.
- EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE, MIN_BUFFER_SIZE}),
- io_mgr.ChooseBufferSizes(MAX_BUFFER_SIZE + 7, 3 * MAX_BUFFER_SIZE));
- // Non power-of-two size < min buffer size.
- EXPECT_EQ(vector<int64_t>({MIN_BUFFER_SIZE}),
- io_mgr.ChooseBufferSizes(MIN_BUFFER_SIZE - 7, 3 * MAX_BUFFER_SIZE));
-}
}
}
int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
impala::InitCommonRuntime(argc, argv, true, impala::TestInfo::BE_TEST);
- impala::InitFeSupport();
return RUN_ALL_TESTS();
}
[8/9] impala git commit: Revert IMPALA-4835 and dependent changes
Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/scanner-context.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/scanner-context.cc b/be/src/exec/scanner-context.cc
index c669e65..abdde07 100644
--- a/be/src/exec/scanner-context.cc
+++ b/be/src/exec/scanner-context.cc
@@ -41,15 +41,14 @@ static const int64_t INIT_READ_PAST_SIZE_BYTES = 64 * 1024;
const int64_t ScannerContext::Stream::OUTPUT_BUFFER_BYTES_LEFT_INIT;
ScannerContext::ScannerContext(RuntimeState* state, HdfsScanNodeBase* scan_node,
- BufferPool::ClientHandle* bp_client, HdfsPartitionDescriptor* partition_desc,
- const vector<FilterContext>& filter_ctxs,
- MemPool* expr_results_pool)
+ HdfsPartitionDescriptor* partition_desc, ScanRange* scan_range,
+ const vector<FilterContext>& filter_ctxs, MemPool* expr_results_pool)
: state_(state),
scan_node_(scan_node),
- bp_client_(bp_client),
partition_desc_(partition_desc),
filter_ctxs_(filter_ctxs),
expr_results_pool_(expr_results_pool) {
+ AddStream(scan_range);
}
ScannerContext::~ScannerContext() {
@@ -67,20 +66,19 @@ void ScannerContext::ClearStreams() {
}
ScannerContext::Stream::Stream(ScannerContext* parent, ScanRange* scan_range,
- int64_t reservation, const HdfsFileDesc* file_desc)
+ const HdfsFileDesc* file_desc)
: parent_(parent),
scan_range_(scan_range),
file_desc_(file_desc),
- reservation_(reservation),
file_len_(file_desc->file_length),
next_read_past_size_bytes_(INIT_READ_PAST_SIZE_BYTES),
boundary_pool_(new MemPool(parent->scan_node_->mem_tracker())),
boundary_buffer_(new StringBuffer(boundary_pool_.get())) {
}
-ScannerContext::Stream* ScannerContext::AddStream(ScanRange* range, int64_t reservation) {
- streams_.emplace_back(new Stream(this, range, reservation,
- scan_node_->GetFileDesc(partition_desc_->id(), range->file())));
+ScannerContext::Stream* ScannerContext::AddStream(ScanRange* range) {
+ streams_.emplace_back(new Stream(
+ this, range, scan_node_->GetFileDesc(partition_desc_->id(), range->file())));
return streams_.back().get();
}
@@ -103,7 +101,6 @@ void ScannerContext::Stream::ReleaseCompletedResources(bool done) {
Status ScannerContext::Stream::GetNextBuffer(int64_t read_past_size) {
DCHECK_EQ(0, io_buffer_bytes_left_);
- DiskIoMgr* io_mgr = ExecEnv::GetInstance()->disk_io_mgr();
if (UNLIKELY(parent_->cancelled())) return Status::CANCELLED;
if (io_buffer_ != nullptr) ReturnIoBuffer();
@@ -124,7 +121,7 @@ Status ScannerContext::Stream::GetNextBuffer(int64_t read_past_size) {
SCOPED_TIMER(parent_->state_->total_storage_wait_timer());
int64_t read_past_buffer_size = 0;
- int64_t max_buffer_size = io_mgr->max_buffer_size();
+ int64_t max_buffer_size = parent_->state_->io_mgr()->max_read_buffer_size();
if (!read_past_size_cb_.empty()) read_past_buffer_size = read_past_size_cb_(offset);
if (read_past_buffer_size <= 0) {
// Either no callback was set or the callback did not return an estimate. Use
@@ -136,7 +133,6 @@ Status ScannerContext::Stream::GetNextBuffer(int64_t read_past_size) {
read_past_buffer_size = ::max(read_past_buffer_size, read_past_size);
read_past_buffer_size = ::min(read_past_buffer_size, file_bytes_remaining);
read_past_buffer_size = ::min(read_past_buffer_size, max_buffer_size);
- read_past_buffer_size = ::min(read_past_buffer_size, reservation_);
// We're reading past the scan range. Be careful not to read past the end of file.
DCHECK_GE(read_past_buffer_size, 0);
if (read_past_buffer_size == 0) {
@@ -147,23 +143,8 @@ Status ScannerContext::Stream::GetNextBuffer(int64_t read_past_size) {
ScanRange* range = parent_->scan_node_->AllocateScanRange(
scan_range_->fs(), filename(), read_past_buffer_size, offset, partition_id,
scan_range_->disk_id(), false, BufferOpts::Uncached());
- bool needs_buffers;
- RETURN_IF_ERROR(io_mgr->StartScanRange(
- parent_->scan_node_->reader_context(), range, &needs_buffers));
- if (needs_buffers) {
- // Allocate fresh buffers. The buffers for 'scan_range_' should be released now
- // since we hit EOS.
- if (reservation_ < io_mgr->min_buffer_size()) {
- return Status(Substitute("Could not read past end of scan range in file '$0'. "
- "Reservation provided $1 was < the minimum I/O buffer size",
- reservation_, io_mgr->min_buffer_size()));
- }
- RETURN_IF_ERROR(io_mgr->AllocateBuffersForRange(
- parent_->scan_node_->reader_context(), parent_->bp_client_, range,
- reservation_));
- }
- RETURN_IF_ERROR(range->GetNext(&io_buffer_));
- DCHECK(io_buffer_->eosr());
+ RETURN_IF_ERROR(parent_->state_->io_mgr()->Read(
+ parent_->scan_node_->reader_context(), range, &io_buffer_));
}
DCHECK(io_buffer_ != nullptr);
@@ -343,8 +324,7 @@ Status ScannerContext::Stream::CopyIoToBoundary(int64_t num_bytes) {
void ScannerContext::Stream::ReturnIoBuffer() {
DCHECK(io_buffer_ != nullptr);
- ScanRange* range = io_buffer_->scan_range();
- range->ReturnBuffer(move(io_buffer_));
+ ExecEnv::GetInstance()->disk_io_mgr()->ReturnBuffer(move(io_buffer_));
io_buffer_pos_ = nullptr;
io_buffer_bytes_left_ = 0;
}
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/scanner-context.h
----------------------------------------------------------------------
diff --git a/be/src/exec/scanner-context.h b/be/src/exec/scanner-context.h
index 6292486..a131d3f 100644
--- a/be/src/exec/scanner-context.h
+++ b/be/src/exec/scanner-context.h
@@ -27,7 +27,6 @@
#include "common/compiler-util.h"
#include "common/status.h"
#include "exec/filter-context.h"
-#include "runtime/bufferpool/buffer-pool.h"
#include "runtime/io/request-ranges.h"
namespace impala {
@@ -85,12 +84,10 @@ class TupleRow;
class ScannerContext {
public:
/// Create a scanner context with the parent scan_node (where materialized row batches
- /// get pushed to) and the scan range to process. Buffers are allocated using
- /// 'bp_client'.
- ScannerContext(RuntimeState* state, HdfsScanNodeBase* scan_node,
- BufferPool::ClientHandle* bp_client,
- HdfsPartitionDescriptor* partition_desc,
- const std::vector<FilterContext>& filter_ctxs,
+ /// get pushed to) and the scan range to process.
+ /// This context starts with 1 stream.
+ ScannerContext(RuntimeState*, HdfsScanNodeBase*, HdfsPartitionDescriptor*,
+ io::ScanRange* scan_range, const std::vector<FilterContext>& filter_ctxs,
MemPool* expr_results_pool);
/// Destructor verifies that all stream objects have been released.
~ScannerContext();
@@ -153,7 +150,6 @@ class ScannerContext {
const char* filename() { return scan_range_->file(); }
const io::ScanRange* scan_range() { return scan_range_; }
const HdfsFileDesc* file_desc() { return file_desc_; }
- int64_t reservation() const { return reservation_; }
/// Returns the buffer's current offset in the file.
int64_t file_offset() const { return scan_range_->offset() + total_bytes_returned_; }
@@ -215,15 +211,9 @@ class ScannerContext {
private:
friend class ScannerContext;
- ScannerContext* const parent_;
- io::ScanRange* const scan_range_;
- const HdfsFileDesc* const file_desc_;
-
- /// Reservation given to this stream for allocating I/O buffers. The reservation is
- /// shared with 'scan_range_', so the context must be careful not to use this until
- /// all of 'scan_ranges_'s buffers have been freed. Must be >= the minimum IoMgr
- /// buffer size to allow reading past the end of 'scan_range_'.
- const int64_t reservation_;
+ ScannerContext* parent_;
+ io::ScanRange* scan_range_;
+ const HdfsFileDesc* file_desc_;
/// Total number of bytes returned from GetBytes()
int64_t total_bytes_returned_ = 0;
@@ -282,8 +272,7 @@ class ScannerContext {
/// output_buffer_bytes_left_ will be set to something else.
static const int64_t OUTPUT_BUFFER_BYTES_LEFT_INIT = 0;
- /// Private constructor. See AddStream() for public API.
- Stream(ScannerContext* parent, io::ScanRange* scan_range, int64_t reservation,
+ Stream(ScannerContext* parent, io::ScanRange* scan_range,
const HdfsFileDesc* file_desc);
/// GetBytes helper to handle the slow path.
@@ -366,37 +355,24 @@ class ScannerContext {
/// size to 0.
void ClearStreams();
- /// Add a stream to this ScannerContext for 'range'. 'range' must already have any
- /// buffers that it needs allocated. 'reservation' is the amount of reservation that
- /// is given to this stream for allocating I/O buffers. The reservation is shared with
- /// 'range', so the context must be careful not to use this until all of 'range's
- /// buffers have been freed. Must be >= the minimum IoMgr buffer size o allow reading
- /// past the end of 'range'.
- ///
- /// Returns the added stream. The returned stream is owned by this context.
- Stream* AddStream(io::ScanRange* range, int64_t reservation);
+ /// Add a stream to this ScannerContext for 'range'. The stream is owned by this
+ /// context.
+ Stream* AddStream(io::ScanRange* range);
/// Returns true if RuntimeState::is_cancelled() is true, or if scan node is not
/// multi-threaded and is done (finished, cancelled or reached it's limit).
/// In all other cases returns false.
bool cancelled() const;
- BufferPool::ClientHandle* bp_client() const { return bp_client_; }
- HdfsPartitionDescriptor* partition_descriptor() const { return partition_desc_; }
+ HdfsPartitionDescriptor* partition_descriptor() { return partition_desc_; }
const std::vector<FilterContext>& filter_ctxs() const { return filter_ctxs_; }
MemPool* expr_results_pool() const { return expr_results_pool_; }
private:
friend class Stream;
- RuntimeState* const state_;
- HdfsScanNodeBase* const scan_node_;
-
- /// Buffer pool client used to allocate I/O buffers. This is accessed by multiple
- /// threads in the multi-threaded scan node, so those threads must take care to only
- /// call thread-safe BufferPool methods with this client.
- BufferPool::ClientHandle* const bp_client_;
-
- HdfsPartitionDescriptor* const partition_desc_;
+ RuntimeState* state_;
+ HdfsScanNodeBase* scan_node_;
+ HdfsPartitionDescriptor* partition_desc_;
/// Vector of streams. Non-columnar formats will always have one stream per context.
std::vector<std::unique_ptr<Stream>> streams_;
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/bufferpool/buffer-pool.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/bufferpool/buffer-pool.h b/be/src/runtime/bufferpool/buffer-pool.h
index d14da63..285aacb 100644
--- a/be/src/runtime/bufferpool/buffer-pool.h
+++ b/be/src/runtime/bufferpool/buffer-pool.h
@@ -37,7 +37,6 @@
namespace impala {
-class MemTracker;
class ReservationTracker;
class RuntimeProfile;
class SystemAllocator;
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/bufferpool/reservation-tracker-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/bufferpool/reservation-tracker-test.cc b/be/src/runtime/bufferpool/reservation-tracker-test.cc
index e441402..c46c5ea 100644
--- a/be/src/runtime/bufferpool/reservation-tracker-test.cc
+++ b/be/src/runtime/bufferpool/reservation-tracker-test.cc
@@ -522,15 +522,15 @@ TEST_F(ReservationTrackerTest, TransferReservation) {
TEST_F(ReservationTrackerTest, ReservationUtil) {
const int64_t MEG = 1024 * 1024;
const int64_t GIG = 1024 * 1024 * 1024;
- EXPECT_EQ(32 * MEG, ReservationUtil::RESERVATION_MEM_MIN_REMAINING);
+ EXPECT_EQ(75 * MEG, ReservationUtil::RESERVATION_MEM_MIN_REMAINING);
EXPECT_EQ(0, ReservationUtil::GetReservationLimitFromMemLimit(0));
EXPECT_EQ(0, ReservationUtil::GetReservationLimitFromMemLimit(-1));
- EXPECT_EQ(0, ReservationUtil::GetReservationLimitFromMemLimit(32 * MEG));
+ EXPECT_EQ(0, ReservationUtil::GetReservationLimitFromMemLimit(75 * MEG));
EXPECT_EQ(8 * GIG, ReservationUtil::GetReservationLimitFromMemLimit(10 * GIG));
- EXPECT_EQ(32 * MEG, ReservationUtil::GetMinMemLimitFromReservation(0));
- EXPECT_EQ(32 * MEG, ReservationUtil::GetMinMemLimitFromReservation(-1));
+ EXPECT_EQ(75 * MEG, ReservationUtil::GetMinMemLimitFromReservation(0));
+ EXPECT_EQ(75 * MEG, ReservationUtil::GetMinMemLimitFromReservation(-1));
EXPECT_EQ(500 * MEG, ReservationUtil::GetMinMemLimitFromReservation(400 * MEG));
EXPECT_EQ(5 * GIG, ReservationUtil::GetMinMemLimitFromReservation(4 * GIG));
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/bufferpool/reservation-util.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/bufferpool/reservation-util.cc b/be/src/runtime/bufferpool/reservation-util.cc
index a27ab9d..85718ab 100644
--- a/be/src/runtime/bufferpool/reservation-util.cc
+++ b/be/src/runtime/bufferpool/reservation-util.cc
@@ -24,7 +24,7 @@ namespace impala {
// Most operators that accumulate memory use reservations, so the majority of memory
// should be allocated to buffer reservations, as a heuristic.
const double ReservationUtil::RESERVATION_MEM_FRACTION = 0.8;
-const int64_t ReservationUtil::RESERVATION_MEM_MIN_REMAINING = 32 * 1024 * 1024;
+const int64_t ReservationUtil::RESERVATION_MEM_MIN_REMAINING = 75 * 1024 * 1024;
int64_t ReservationUtil::GetReservationLimitFromMemLimit(int64_t mem_limit) {
int64_t max_reservation = std::min<int64_t>(
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/exec-env.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/exec-env.cc b/be/src/runtime/exec-env.cc
index 2f19786..51eece5 100644
--- a/be/src/runtime/exec-env.cc
+++ b/be/src/runtime/exec-env.cc
@@ -36,9 +36,9 @@
#include "runtime/client-cache.h"
#include "runtime/coordinator.h"
#include "runtime/data-stream-mgr.h"
+#include "runtime/io/disk-io-mgr.h"
#include "runtime/hbase-table-factory.h"
#include "runtime/hdfs-fs-cache.h"
-#include "runtime/io/disk-io-mgr.h"
#include "runtime/krpc-data-stream-mgr.h"
#include "runtime/lib-cache.h"
#include "runtime/mem-tracker.h"
@@ -367,7 +367,10 @@ Status ExecEnv::Init() {
LOG(INFO) << "Buffer pool limit: "
<< PrettyPrinter::Print(buffer_pool_limit, TUnit::BYTES);
- RETURN_IF_ERROR(disk_io_mgr_->Init());
+ RETURN_IF_ERROR(disk_io_mgr_->Init(mem_tracker_.get()));
+
+ mem_tracker_->AddGcFunction(
+ [this](int64_t bytes_to_free) { disk_io_mgr_->GcIoBuffers(bytes_to_free); });
// Start services in order to ensure that dependencies between them are met
if (enable_webserver_) {
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr-internal.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr-internal.h b/be/src/runtime/io/disk-io-mgr-internal.h
index 2d32487..3fc3895 100644
--- a/be/src/runtime/io/disk-io-mgr-internal.h
+++ b/be/src/runtime/io/disk-io-mgr-internal.h
@@ -35,25 +35,9 @@
#include "util/filesystem-util.h"
#include "util/hdfs-util.h"
#include "util/impalad-metrics.h"
-#include "util/runtime-profile-counters.h"
/// This file contains internal structures shared between submodules of the IoMgr. Users
/// of the IoMgr do not need to include this file.
-
-// Macros to work around counters sometimes not being provided.
-// TODO: fix things so that counters are always non-NULL.
-#define COUNTER_ADD_IF_NOT_NULL(c, v) \
- do { \
- ::impala::RuntimeProfile::Counter* __ctr__ = (c); \
- if (__ctr__ != nullptr) __ctr__->Add(v); \
- } while (false);
-
-#define COUNTER_BITOR_IF_NOT_NULL(c, v) \
- do { \
- ::impala::RuntimeProfile::Counter* __ctr__ = (c); \
- if (__ctr__ != nullptr) __ctr__->BitOr(v); \
- } while (false);
-
namespace impala {
namespace io {
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr-stress-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr-stress-test.cc b/be/src/runtime/io/disk-io-mgr-stress-test.cc
index 2ec1d09..45b36ed 100644
--- a/be/src/runtime/io/disk-io-mgr-stress-test.cc
+++ b/be/src/runtime/io/disk-io-mgr-stress-test.cc
@@ -15,13 +15,8 @@
// specific language governing permissions and limitations
// under the License.
-#include <gflags/gflags.h>
-
-#include "common/init.h"
#include "runtime/io/disk-io-mgr-stress.h"
-#include "common/init.h"
-#include "runtime/test-env.h"
-#include "service/fe-support.h"
+#include "util/cpu-info.h"
#include "util/string-parser.h"
#include "common/names.h"
@@ -33,32 +28,34 @@ using namespace impala::io;
// can be passed to control how long to run this test (0 for forever).
// TODO: make these configurable once we decide how to run BE tests with args
-constexpr int DEFAULT_DURATION_SEC = 1;
+const int DEFAULT_DURATION_SEC = 1;
const int NUM_DISKS = 5;
const int NUM_THREADS_PER_DISK = 5;
const int NUM_CLIENTS = 10;
const bool TEST_CANCELLATION = true;
-const int64_t BUFFER_POOL_CAPACITY = 1024L * 1024L * 1024L * 4L;
-
-DEFINE_int64(duration_sec, DEFAULT_DURATION_SEC,
- "Disk I/O Manager stress test duration in seconds. 0 means run indefinitely.");
int main(int argc, char** argv) {
- impala::InitCommonRuntime(argc, argv, true, impala::TestInfo::BE_TEST);
- impala::InitFeSupport();
-
- if (FLAGS_duration_sec != 0) {
- printf("Running stress test for %ld seconds.\n", FLAGS_duration_sec);
+ google::InitGoogleLogging(argv[0]);
+ CpuInfo::Init();
+ OsInfo::Init();
+ impala::InitThreading();
+ int duration_sec = DEFAULT_DURATION_SEC;
+
+ if (argc == 2) {
+ StringParser::ParseResult status;
+ duration_sec = StringParser::StringToInt<int>(argv[1], strlen(argv[1]), &status);
+ if (status != StringParser::PARSE_SUCCESS) {
+ printf("Invalid arg: %s\n", argv[1]);
+ return 1;
+ }
+ }
+ if (duration_sec != 0) {
+ printf("Running stress test for %d seconds.\n", duration_sec);
} else {
printf("Running stress test indefinitely.\n");
}
-
- TestEnv test_env;
- // Tests try to allocate arbitrarily small buffers. Ensure Buffer Pool allows it.
- test_env.SetBufferPoolArgs(DiskIoMgrStress::MIN_READ_BUFFER_SIZE, BUFFER_POOL_CAPACITY);
- Status status = test_env.Init();
- CHECK(status.ok()) << status.GetDetail();
DiskIoMgrStress test(NUM_DISKS, NUM_THREADS_PER_DISK, NUM_CLIENTS, TEST_CANCELLATION);
- test.Run(FLAGS_duration_sec);
+ test.Run(duration_sec);
+
return 0;
}
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr-stress.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr-stress.cc b/be/src/runtime/io/disk-io-mgr-stress.cc
index 3fd33de..8815357 100644
--- a/be/src/runtime/io/disk-io-mgr-stress.cc
+++ b/be/src/runtime/io/disk-io-mgr-stress.cc
@@ -19,8 +19,6 @@
#include "runtime/io/disk-io-mgr-stress.h"
-#include "runtime/bufferpool/reservation-tracker.h"
-#include "runtime/exec-env.h"
#include "runtime/io/request-context.h"
#include "util/time.h"
@@ -29,20 +27,18 @@
using namespace impala;
using namespace impala::io;
-constexpr float DiskIoMgrStress::ABORT_CHANCE;
-const int DiskIoMgrStress::MIN_READ_LEN;
-const int DiskIoMgrStress::MAX_READ_LEN;
+static const float ABORT_CHANCE = .10f;
+static const int MIN_READ_LEN = 1;
+static const int MAX_READ_LEN = 20;
-const int DiskIoMgrStress::MIN_FILE_LEN;
-const int DiskIoMgrStress::MAX_FILE_LEN;
+static const int MIN_FILE_LEN = 10;
+static const int MAX_FILE_LEN = 1024;
// Make sure this is between MIN/MAX FILE_LEN to test more cases
-const int DiskIoMgrStress::MIN_READ_BUFFER_SIZE;
-const int DiskIoMgrStress::MAX_READ_BUFFER_SIZE;
+static const int MIN_READ_BUFFER_SIZE = 64;
+static const int MAX_READ_BUFFER_SIZE = 128;
-const int DiskIoMgrStress::MAX_BUFFER_BYTES_PER_SCAN_RANGE;
-
-const int DiskIoMgrStress::CANCEL_READER_PERIOD_MS;
+static const int CANCEL_READER_PERIOD_MS = 20; // in ms
static void CreateTempFile(const char* filename, const char* data) {
FILE* file = fopen(filename, "w");
@@ -51,7 +47,7 @@ static void CreateTempFile(const char* filename, const char* data) {
fclose(file);
}
-string DiskIoMgrStress::GenerateRandomData() {
+string GenerateRandomData() {
int rand_len = rand() % (MAX_FILE_LEN - MIN_FILE_LEN) + MIN_FILE_LEN;
stringstream ss;
for (int i = 0; i < rand_len; ++i) {
@@ -63,8 +59,6 @@ string DiskIoMgrStress::GenerateRandomData() {
struct DiskIoMgrStress::Client {
boost::mutex lock;
- /// Pool for objects that is cleared when the client is (re-)initialized in NewClient().
- ObjectPool obj_pool;
unique_ptr<RequestContext> reader;
int file_idx;
vector<ScanRange*> scan_ranges;
@@ -83,7 +77,7 @@ DiskIoMgrStress::DiskIoMgrStress(int num_disks, int num_threads_per_disk,
io_mgr_.reset(new DiskIoMgr(num_disks, num_threads_per_disk, num_threads_per_disk,
MIN_READ_BUFFER_SIZE, MAX_READ_BUFFER_SIZE));
- Status status = io_mgr_->Init();
+ Status status = io_mgr_->Init(&mem_tracker_);
CHECK(status.ok());
// Initialize some data files. It doesn't really matter how many there are.
@@ -98,7 +92,6 @@ DiskIoMgrStress::DiskIoMgrStress(int num_disks, int num_threads_per_disk,
clients_ = new Client[num_clients_];
client_mem_trackers_.resize(num_clients_);
- buffer_pool_clients_.reset(new BufferPool::ClientHandle[num_clients_]);
for (int i = 0; i < num_clients_; ++i) {
NewClient(i);
}
@@ -117,16 +110,9 @@ void DiskIoMgrStress::ClientThread(int client_id) {
while (!eos) {
ScanRange* range;
- bool needs_buffers;
- Status status =
- io_mgr_->GetNextUnstartedRange(client->reader.get(), &range, &needs_buffers);
+ Status status = io_mgr_->GetNextRange(client->reader.get(), &range);
CHECK(status.ok() || status.IsCancelled());
if (range == NULL) break;
- if (needs_buffers) {
- status = io_mgr_->AllocateBuffersForRange(client->reader.get(),
- &buffer_pool_clients_[client_id], range, MAX_BUFFER_BYTES_PER_SCAN_RANGE);
- CHECK(status.ok()) << status.GetDetail();
- }
while (true) {
unique_ptr<BufferDescriptor> buffer;
@@ -151,7 +137,7 @@ void DiskIoMgrStress::ClientThread(int client_id) {
// Copy the bytes from this read into the result buffer.
memcpy(read_buffer + file_offset, buffer->buffer(), buffer->len());
- range->ReturnBuffer(move(buffer));
+ io_mgr_->ReturnBuffer(move(buffer));
bytes_read += len;
CHECK_GE(bytes_read, 0);
@@ -173,7 +159,6 @@ void DiskIoMgrStress::ClientThread(int client_id) {
// Unregister the old client and get a new one
unique_lock<mutex> lock(client->lock);
io_mgr_->UnregisterContext(client->reader.get());
- client->reader.reset();
NewClient(client_id);
}
@@ -185,9 +170,11 @@ void DiskIoMgrStress::ClientThread(int client_id) {
// Cancel a random reader
void DiskIoMgrStress::CancelRandomReader() {
if (!includes_cancellation_) return;
- Client* rand_client = &clients_[rand() % num_clients_];
- unique_lock<mutex> lock(rand_client->lock);
- rand_client->reader->Cancel();
+
+ int rand_client = rand() % num_clients_;
+
+ unique_lock<mutex> lock(clients_[rand_client].lock);
+ io_mgr_->CancelContext(clients_[rand_client].reader.get());
}
void DiskIoMgrStress::Run(int sec) {
@@ -212,18 +199,10 @@ void DiskIoMgrStress::Run(int sec) {
for (int i = 0; i < num_clients_; ++i) {
unique_lock<mutex> lock(clients_[i].lock);
- if (clients_[i].reader != NULL) clients_[i].reader->Cancel();
+ if (clients_[i].reader != NULL) io_mgr_->CancelContext(clients_[i].reader.get());
}
- readers_.join_all();
- for (int i = 0; i < num_clients_; ++i) {
- if (clients_[i].reader != nullptr) {
- io_mgr_->UnregisterContext(clients_[i].reader.get());
- }
- ExecEnv::GetInstance()->buffer_pool()->DeregisterClient(&buffer_pool_clients_[i]);
- client_mem_trackers_[i]->Close();
- }
- mem_tracker_.Close();
+ readers_.join_all();
}
// Initialize a client to read one of the files at random. The scan ranges are
@@ -244,41 +223,25 @@ void DiskIoMgrStress::NewClient(int i) {
}
}
- // Clean up leftover state from the previous client (if any).
+ for (int i = 0; i < client.scan_ranges.size(); ++i) {
+ delete client.scan_ranges[i];
+ }
client.scan_ranges.clear();
- ExecEnv* exec_env = ExecEnv::GetInstance();
- exec_env->buffer_pool()->DeregisterClient(&buffer_pool_clients_[i]);
- if (client_mem_trackers_[i] != nullptr) client_mem_trackers_[i]->Close();
- client.obj_pool.Clear();
int assigned_len = 0;
while (assigned_len < file_len) {
int range_len = rand() % (MAX_READ_LEN - MIN_READ_LEN) + MIN_READ_LEN;
range_len = min(range_len, file_len - assigned_len);
- ScanRange* range = client.obj_pool.Add(new ScanRange);
+ ScanRange* range = new ScanRange();
range->Reset(NULL, files_[client.file_idx].filename.c_str(), range_len, assigned_len,
0, false, BufferOpts::Uncached());
client.scan_ranges.push_back(range);
assigned_len += range_len;
}
- string client_name = Substitute("Client $0", i);
- client_mem_trackers_[i].reset(new MemTracker(-1, client_name, &mem_tracker_));
- Status status = exec_env->buffer_pool()->RegisterClient(client_name, nullptr,
- exec_env->buffer_reservation(), client_mem_trackers_[i].get(),
- numeric_limits<int64_t>::max(), RuntimeProfile::Create(&client.obj_pool, client_name),
- &buffer_pool_clients_[i]);
- CHECK(status.ok());
- // Reserve enough memory for 3 buffers per range, which should be enough to guarantee
- // progress.
- CHECK(buffer_pool_clients_[i].IncreaseReservationToFit(
- MAX_BUFFER_BYTES_PER_SCAN_RANGE * client.scan_ranges.size()))
- << buffer_pool_clients_[i].DebugString() << "\n"
- << exec_env->buffer_pool()->DebugString() << "\n"
- << exec_env->buffer_reservation()->DebugString();
-
- client.reader = io_mgr_->RegisterContext();
- status = io_mgr_->AddScanRanges(client.reader.get(), client.scan_ranges);
+ client_mem_trackers_[i].reset(new MemTracker(-1, "", &mem_tracker_));
+ client.reader = io_mgr_->RegisterContext(client_mem_trackers_[i].get());
+ Status status = io_mgr_->AddScanRanges(client.reader.get(), client.scan_ranges);
CHECK(status.ok());
}
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr-stress.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr-stress.h b/be/src/runtime/io/disk-io-mgr-stress.h
index 574b58c..b872694 100644
--- a/be/src/runtime/io/disk-io-mgr-stress.h
+++ b/be/src/runtime/io/disk-io-mgr-stress.h
@@ -22,11 +22,8 @@
#include <memory>
#include <vector>
#include <boost/scoped_ptr.hpp>
-#include <boost/thread/condition_variable.hpp>
#include <boost/thread/thread.hpp>
-#include "common/object-pool.h"
-#include "runtime/bufferpool/buffer-pool.h"
#include "runtime/io/disk-io-mgr.h"
#include "runtime/mem-tracker.h"
#include "runtime/thread-resource-mgr.h"
@@ -46,29 +43,15 @@ class DiskIoMgrStress {
/// Run the test for 'sec'. If 0, run forever
void Run(int sec);
- static constexpr float ABORT_CHANCE = .10f;
- static const int MIN_READ_LEN = 1;
- static const int MAX_READ_LEN = 20;
-
- static const int MIN_FILE_LEN = 10;
- static const int MAX_FILE_LEN = 1024;
-
- // Make sure this is between MIN/MAX FILE_LEN to test more cases
- static const int MIN_READ_BUFFER_SIZE = 64;
- static const int MAX_READ_BUFFER_SIZE = 128;
-
- // Maximum bytes to allocate per scan range.
- static const int MAX_BUFFER_BYTES_PER_SCAN_RANGE = MAX_READ_BUFFER_SIZE * 3;
-
- static const int CANCEL_READER_PERIOD_MS = 20;
private:
struct Client;
struct File {
std::string filename;
- std::string data; // the data in the file, used to validate
+ std::string data; // the data in the file, used to validate
};
+
/// Files used for testing. These are created at startup and recycled
/// during the test
std::vector<File> files_;
@@ -89,9 +72,6 @@ class DiskIoMgrStress {
/// Client MemTrackers, one per client.
std::vector<std::unique_ptr<MemTracker>> client_mem_trackers_;
- /// Buffer pool clients, one per client.
- std::unique_ptr<BufferPool::ClientHandle[]> buffer_pool_clients_;
-
/// If true, tests cancelling readers
bool includes_cancellation_;
@@ -108,8 +88,6 @@ class DiskIoMgrStress {
/// Possibly cancels a random reader.
void CancelRandomReader();
-
- static std::string GenerateRandomData();
};
}
}
[3/9] impala git commit: Revert IMPALA-4835 and dependent changes
Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
index 003e68f..261e8fe 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
@@ -1,795 +1,24 @@
# Parquet scan
select * from tpch_parquet.lineitem
---- PLAN
-Max Per-Host Resource Reservation: Memory=72.00MB
-Per-Host Resource Estimates: Memory=80.00MB
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=80.00MB mem-reservation=72.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_parquet.lineitem]
- partitions=1/1 files=3 size=193.71MB
- stored statistics:
- table: rows=6001215 size=193.71MB
- columns: all
- extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
- tuple-ids=0 row-size=263B cardinality=6001215
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=72.00MB
-Per-Host Resource Estimates: Memory=80.00MB
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=263B cardinality=6001215
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=80.00MB mem-reservation=72.00MB
-00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
- stored statistics:
- table: rows=6001215 size=193.71MB
- columns: all
- extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
- tuple-ids=0 row-size=263B cardinality=6001215
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=144.00MB
-Per-Host Resource Estimates: Memory=160.00MB
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=263B cardinality=6001215
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=160.00MB mem-reservation=144.00MB
-00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
- stored statistics:
- table: rows=6001215 size=193.71MB
- columns: all
- extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
- tuple-ids=0 row-size=263B cardinality=6001215
-====
-# Single column parquet scan - memory reservation is reduced compared to multi-column
-# scan.
-select l_comment from tpch_parquet.lineitem
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=80.00MB
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_parquet.lineitem]
- partitions=1/1 files=3 size=193.71MB
- stored statistics:
- table: rows=6001215 size=193.71MB
- columns: all
- extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=42B cardinality=6001215
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=80.00MB
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=42B cardinality=6001215
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB
-00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
- stored statistics:
- table: rows=6001215 size=193.71MB
- columns: all
- extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=42B cardinality=6001215
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=160.00MB
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=42B cardinality=6001215
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=160.00MB mem-reservation=16.00MB
-00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
- stored statistics:
- table: rows=6001215 size=193.71MB
- columns: all
- extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=42B cardinality=6001215
-====
-# Small parquet files - memory reservation is reduced because of small file size.
-select string_col from functional_parquet.alltypes;
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=16.00MB mem-reservation=8.00KB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [functional_parquet.alltypes]
- partitions=24/24 files=24 size=174.62KB
- stored statistics:
- table: rows=unavailable size=unavailable
- partitions: 0/24 rows=unavailable
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=8.00KB
- tuple-ids=0 row-size=16B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=16B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=8.00KB
-00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
- partitions=24/24 files=24 size=174.62KB
- stored statistics:
- table: rows=unavailable size=unavailable
- partitions: 0/24 rows=unavailable
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=8.00KB
- tuple-ids=0 row-size=16B cardinality=unavailable
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00KB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=16B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=16.00KB
-00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
- partitions=24/24 files=24 size=174.62KB
- stored statistics:
- table: rows=unavailable size=unavailable
- partitions: 0/24 rows=unavailable
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=8.00KB
- tuple-ids=0 row-size=16B cardinality=unavailable
-====
-# Multi-column parquet scan with small files - memory reservation is reduced because of
-# small file size but a minimum amount is reserved per column.
-select int_col, float_col, string_col from functional_parquet.alltypes;
----- PLAN
-Max Per-Host Resource Reservation: Memory=24.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=16.00MB mem-reservation=24.00KB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [functional_parquet.alltypes]
- partitions=24/24 files=24 size=174.62KB
- stored statistics:
- table: rows=unavailable size=unavailable
- partitions: 0/24 rows=unavailable
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=24.00KB
- tuple-ids=0 row-size=24B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=24.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=24B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=24.00KB
-00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
- partitions=24/24 files=24 size=174.62KB
- stored statistics:
- table: rows=unavailable size=unavailable
- partitions: 0/24 rows=unavailable
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=24.00KB
- tuple-ids=0 row-size=24B cardinality=unavailable
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=48.00KB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=24B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=48.00KB
-00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
- partitions=24/24 files=24 size=174.62KB
- stored statistics:
- table: rows=unavailable size=unavailable
- partitions: 0/24 rows=unavailable
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=24.00KB
- tuple-ids=0 row-size=24B cardinality=unavailable
-====
-# Parquet scan with no materialized columns. Need reservation to scan levels to determine
-# row count.
-select 'foo' from functional_parquet.alltypes
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00KB
-Per-Host Resource Estimates: Memory=10.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=1.00MB mem-reservation=8.00KB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [functional_parquet.alltypes]
- partitions=24/24 files=24 size=174.62KB
- stored statistics:
- table: rows=unavailable size=unavailable
- partitions: 0/24 rows=unavailable
- columns: all
- extrapolated-rows=disabled
- mem-estimate=1.00MB mem-reservation=8.00KB
- tuple-ids=0 row-size=0B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00KB
-Per-Host Resource Estimates: Memory=10.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=0B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=1.00MB mem-reservation=8.00KB
-00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
- partitions=24/24 files=24 size=174.62KB
- stored statistics:
- table: rows=unavailable size=unavailable
- partitions: 0/24 rows=unavailable
- columns: all
- extrapolated-rows=disabled
- mem-estimate=1.00MB mem-reservation=8.00KB
- tuple-ids=0 row-size=0B cardinality=unavailable
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00KB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=0B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=16.00KB
-00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
- partitions=24/24 files=24 size=174.62KB
- stored statistics:
- table: rows=unavailable size=unavailable
- partitions: 0/24 rows=unavailable
- columns: all
- extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=8.00KB
- tuple-ids=0 row-size=0B cardinality=unavailable
-====
-# Parquet nested types, unnested in scan - should reserve memory for each column.
-select o_orderkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk
-from tpch_nested_parquet.customer.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=48.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=48.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_nested_parquet.customer.c_orders]
- partitions=1/1 files=4 size=292.36MB
- stored statistics:
- table: rows=150000 size=292.36MB
- columns: all
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=48.00MB
- tuple-ids=0 row-size=80B cardinality=1500000
-====
-# Parquet nested types, unnested in scan - don't reserve extra memory for "pos" virtual
-# column that piggy-backs on another column.
-select o_orderkey, pos
-from tpch_nested_parquet.customer.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_nested_parquet.customer.c_orders]
- partitions=1/1 files=4 size=292.36MB
- stored statistics:
- table: rows=150000 size=292.36MB
- columns: all
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=16B cardinality=1500000
-====
-# Parquet nested types, unnested in scan - reserve memory for "pos" virtual column if it
-# is the only column materialized.
-select pos
-from tpch_nested_parquet.customer.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_nested_parquet.customer.c_orders]
- partitions=1/1 files=4 size=292.36MB
- stored statistics:
- table: rows=150000 size=292.36MB
- columns: all
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=8B cardinality=1500000
-====
-# Parquet nested types, left nested in scan - should reserve memory for each scalar
-# column in the nested collection plus the top-level column.
-select c_custkey, o_orderkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority,
- o_clerk
-from tpch_nested_parquet.customer c, c.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=56.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_nested_parquet.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=56.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:SUBPLAN
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=1,0 row-size=104B cardinality=1500000
-|
-|--04:NESTED LOOP JOIN [CROSS JOIN]
-| | mem-estimate=24B mem-reservation=0B
-| | tuple-ids=1,0 row-size=104B cardinality=10
-| |
-| |--02:SINGULAR ROW SRC
-| | parent-subplan=01
-| | mem-estimate=0B mem-reservation=0B
-| | tuple-ids=0 row-size=24B cardinality=1
-| |
-| 03:UNNEST [c.c_orders]
-| parent-subplan=01
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=1 row-size=0B cardinality=10
-|
-00:SCAN HDFS [tpch_nested_parquet.customer c]
- partitions=1/1 files=4 size=292.36MB
- predicates: !empty(c.c_orders)
- stored statistics:
- table: rows=150000 size=292.36MB
- columns missing stats: c_orders
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=56.00MB
- tuple-ids=0 row-size=24B cardinality=150000
-====
-# Parquet nested types, left nested in scan - should reserve memory for each scalar
-# column (excluding pos) in the nested collection plus the top-level column.
-select c_custkey, o_orderkey, pos
-from tpch_nested_parquet.customer c, c.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_nested_parquet.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=16.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:SUBPLAN
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=1,0 row-size=40B cardinality=1500000
-|
-|--04:NESTED LOOP JOIN [CROSS JOIN]
-| | mem-estimate=24B mem-reservation=0B
-| | tuple-ids=1,0 row-size=40B cardinality=10
-| |
-| |--02:SINGULAR ROW SRC
-| | parent-subplan=01
-| | mem-estimate=0B mem-reservation=0B
-| | tuple-ids=0 row-size=24B cardinality=1
-| |
-| 03:UNNEST [c.c_orders]
-| parent-subplan=01
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=1 row-size=0B cardinality=10
-|
-00:SCAN HDFS [tpch_nested_parquet.customer c]
- partitions=1/1 files=4 size=292.36MB
- predicates: !empty(c.c_orders)
- stored statistics:
- table: rows=150000 size=292.36MB
- columns missing stats: c_orders
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=16.00MB
- tuple-ids=0 row-size=24B cardinality=150000
-====
-# Parquet nested types, left nested in scan - should reserve memory for virtual
-# pos column in the nested collection because nothing else is materialized from
-# the collection, plus the top-level column.
-select c_custkey, pos
-from tpch_nested_parquet.customer c, c.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_nested_parquet.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=16.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:SUBPLAN
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=1,0 row-size=32B cardinality=1500000
-|
-|--04:NESTED LOOP JOIN [CROSS JOIN]
-| | mem-estimate=24B mem-reservation=0B
-| | tuple-ids=1,0 row-size=32B cardinality=10
-| |
-| |--02:SINGULAR ROW SRC
-| | parent-subplan=01
-| | mem-estimate=0B mem-reservation=0B
-| | tuple-ids=0 row-size=24B cardinality=1
-| |
-| 03:UNNEST [c.c_orders]
-| parent-subplan=01
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=1 row-size=0B cardinality=10
-|
-00:SCAN HDFS [tpch_nested_parquet.customer c]
- partitions=1/1 files=4 size=292.36MB
- predicates: !empty(c.c_orders)
- stored statistics:
- table: rows=150000 size=292.36MB
- columns missing stats: c_orders
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=16.00MB
- tuple-ids=0 row-size=24B cardinality=150000
-====
-# Parquet nested types, left nested in scan - should reserve memory for nested column
-# plus top-level column even though nested fields are not materialized because the
-# column levels still need to be scanned to determine the # of elements in the collection.
-select c_custkey
-from tpch_nested_parquet.customer c, c.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_nested_parquet.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=16.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:SUBPLAN
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=1,0 row-size=24B cardinality=1500000
-|
-|--04:NESTED LOOP JOIN [CROSS JOIN]
-| | mem-estimate=24B mem-reservation=0B
-| | tuple-ids=1,0 row-size=24B cardinality=10
-| |
-| |--02:SINGULAR ROW SRC
-| | parent-subplan=01
-| | mem-estimate=0B mem-reservation=0B
-| | tuple-ids=0 row-size=24B cardinality=1
-| |
-| 03:UNNEST [c.c_orders]
-| parent-subplan=01
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=1 row-size=0B cardinality=10
-|
-00:SCAN HDFS [tpch_nested_parquet.customer c]
- partitions=1/1 files=4 size=292.36MB
- predicates: !empty(c.c_orders)
- stored statistics:
- table: rows=150000 size=292.36MB
- columns missing stats: c_orders
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=16.00MB
- tuple-ids=0 row-size=24B cardinality=150000
-====
-# Parquet nested types, left nested in scan - should reserve memory for nested column
-# only when no top-level column is materialized.
-select o_orderkey
-from tpch_nested_parquet.customer c, c.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_nested_parquet.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:SUBPLAN
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=1,0 row-size=24B cardinality=1500000
-|
-|--04:NESTED LOOP JOIN [CROSS JOIN]
-| | mem-estimate=16B mem-reservation=0B
-| | tuple-ids=1,0 row-size=24B cardinality=10
-| |
-| |--02:SINGULAR ROW SRC
-| | parent-subplan=01
-| | mem-estimate=0B mem-reservation=0B
-| | tuple-ids=0 row-size=16B cardinality=1
-| |
-| 03:UNNEST [c.c_orders]
-| parent-subplan=01
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=1 row-size=0B cardinality=10
-|
-00:SCAN HDFS [tpch_nested_parquet.customer c]
- partitions=1/1 files=4 size=292.36MB
- predicates: !empty(c.c_orders)
- stored statistics:
- table: rows=150000 size=292.36MB
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=16B cardinality=150000
-====
-# Parquet nested types with two levels of nesting materialized in scan. Should
-# reserve memory for columns at all three levels
-select c_custkey, o_orderkey, l_comment
-from tpch_nested_parquet.customer c, c.c_orders o, o.o_lineitems
----- PLAN
-Max Per-Host Resource Reservation: Memory=24.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_nested_parquet.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=24.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:SUBPLAN
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=2,1,0 row-size=64B cardinality=15000000
-|
-|--08:NESTED LOOP JOIN [CROSS JOIN]
-| | mem-estimate=24B mem-reservation=0B
-| | tuple-ids=2,1,0 row-size=64B cardinality=100
-| |
-| |--02:SINGULAR ROW SRC
-| | parent-subplan=01
-| | mem-estimate=0B mem-reservation=0B
-| | tuple-ids=0 row-size=24B cardinality=1
-| |
-| 04:SUBPLAN
-| | mem-estimate=0B mem-reservation=0B
-| | tuple-ids=2,1 row-size=40B cardinality=100
-| |
-| |--07:NESTED LOOP JOIN [CROSS JOIN]
-| | | mem-estimate=24B mem-reservation=0B
-| | | tuple-ids=2,1 row-size=40B cardinality=10
-| | |
-| | |--05:SINGULAR ROW SRC
-| | | parent-subplan=04
-| | | mem-estimate=0B mem-reservation=0B
-| | | tuple-ids=1 row-size=24B cardinality=1
-| | |
-| | 06:UNNEST [o.o_lineitems]
-| | parent-subplan=04
-| | mem-estimate=0B mem-reservation=0B
-| | tuple-ids=2 row-size=0B cardinality=10
-| |
-| 03:UNNEST [c.c_orders o]
-| parent-subplan=01
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=1 row-size=0B cardinality=10
-|
-00:SCAN HDFS [tpch_nested_parquet.customer c]
- partitions=1/1 files=4 size=292.36MB
- predicates: !empty(c.c_orders)
- predicates on o: !empty(o.o_lineitems)
- stored statistics:
- table: rows=150000 size=292.36MB
- columns missing stats: c_orders
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=24.00MB
- tuple-ids=0 row-size=24B cardinality=150000
-====
-# Text scan
-select * from tpch.lineitem
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch.lineitem]
- partitions=1/1 files=1 size=718.94MB
- stored statistics:
- table: rows=6001215 size=718.94MB
- columns: all
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=263B cardinality=6001215
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=263B cardinality=6001215
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-00:SCAN HDFS [tpch.lineitem, RANDOM]
- partitions=1/1 files=1 size=718.94MB
- stored statistics:
- table: rows=6001215 size=718.94MB
- columns: all
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=263B cardinality=6001215
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=176.00MB
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=263B cardinality=6001215
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
-00:SCAN HDFS [tpch.lineitem, RANDOM]
- partitions=1/1 files=1 size=718.94MB
- stored statistics:
- table: rows=6001215 size=718.94MB
- columns: all
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=263B cardinality=6001215
-====
-# Single column text scan - memory reservation is same as multi-column scan.
-select l_comment from tpch_parquet.lineitem
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=80.00MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB
+| Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
00:SCAN HDFS [tpch_parquet.lineitem]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=42B cardinality=6001215
+ mem-estimate=80.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=80.00MB
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -799,20 +28,20 @@ PLAN-ROOT SINK
|
01:EXCHANGE [UNPARTITIONED]
| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=42B cardinality=6001215
+| tuple-ids=0 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB
+Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=42B cardinality=6001215
+ mem-estimate=80.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=160.00MB
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -822,44 +51,41 @@ PLAN-ROOT SINK
|
01:EXCHANGE [UNPARTITIONED]
| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=42B cardinality=6001215
+| tuple-ids=0 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=160.00MB mem-reservation=16.00MB
+Per-Host Resources: mem-estimate=160.00MB mem-reservation=0B
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=42B cardinality=6001215
+ mem-estimate=80.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
====
-# Text scan on small files - memory reservation is reduced.
-select * from functional.alltypes
+# Text scan
+select * from tpch.lineitem;
---- PLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-Codegen disabled by planner
+Max Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=88.00MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
+| Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
-00:SCAN HDFS [functional.alltypes]
- partitions=24/24 files=24 size=478.45KB
+00:SCAN HDFS [tpch.lineitem]
+ partitions=1/1 files=1 size=718.94MB
stored statistics:
- table: rows=7300 size=478.45KB
- partitions: 24/24 rows=7300
+ table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=32.00KB
- tuple-ids=0 row-size=97B cardinality=7300
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-Codegen disabled by planner
+Max Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=88.00MB
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -868,314 +94,21 @@ PLAN-ROOT SINK
|
01:EXCHANGE [UNPARTITIONED]
| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=97B cardinality=7300
+| tuple-ids=0 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
-00:SCAN HDFS [functional.alltypes, RANDOM]
- partitions=24/24 files=24 size=478.45KB
- stored statistics:
- table: rows=7300 size=478.45KB
- partitions: 24/24 rows=7300
- columns: all
- extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=32.00KB
- tuple-ids=0 row-size=97B cardinality=7300
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=64.00KB
-Per-Host Resource Estimates: Memory=32.00MB
-Codegen disabled by planner
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=97B cardinality=7300
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=64.00KB
-00:SCAN HDFS [functional.alltypes, RANDOM]
- partitions=24/24 files=24 size=478.45KB
+Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
+00:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
stored statistics:
- table: rows=7300 size=478.45KB
- partitions: 24/24 rows=7300
+ table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=32.00KB
- tuple-ids=0 row-size=97B cardinality=7300
-====
-# Avro scan.
-select * from tpch_avro.orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_avro.orders
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_avro.orders]
- partitions=1/1 files=2 size=156.92MB
- stored statistics:
- table: rows=unavailable size=156.92MB
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=108B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_avro.orders
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=108B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-00:SCAN HDFS [tpch_avro.orders, RANDOM]
- partitions=1/1 files=2 size=156.92MB
- stored statistics:
- table: rows=unavailable size=156.92MB
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=108B cardinality=unavailable
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=176.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_avro.orders
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=108B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
-00:SCAN HDFS [tpch_avro.orders, RANDOM]
- partitions=1/1 files=2 size=156.92MB
- stored statistics:
- table: rows=unavailable size=156.92MB
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=108B cardinality=unavailable
-====
-# RC scan.
-select * from tpch_rc.customer
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_rc.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_rc.customer]
- partitions=1/1 files=1 size=22.48MB
- stored statistics:
- table: rows=unavailable size=22.48MB
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=98B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_rc.customer
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=98B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=8.00MB
-00:SCAN HDFS [tpch_rc.customer, RANDOM]
- partitions=1/1 files=1 size=22.48MB
- stored statistics:
- table: rows=unavailable size=22.48MB
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=98B cardinality=unavailable
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=64.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_rc.customer
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=98B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-Per-Host Resources: mem-estimate=64.00MB mem-reservation=16.00MB
-00:SCAN HDFS [tpch_rc.customer, RANDOM]
- partitions=1/1 files=1 size=22.48MB
- stored statistics:
- table: rows=unavailable size=22.48MB
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=98B cardinality=unavailable
-====
-# Seq scan.
-select * from tpcds_seq_snap.web_returns
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpcds_seq_snap.web_returns
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=16.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpcds_seq_snap.web_returns]
- partitions=1/1 files=1 size=6.55MB
- stored statistics:
- table: rows=unavailable size=6.55MB
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=104B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpcds_seq_snap.web_returns
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=104B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=8.00MB
-00:SCAN HDFS [tpcds_seq_snap.web_returns, RANDOM]
- partitions=1/1 files=1 size=6.55MB
- stored statistics:
- table: rows=unavailable size=6.55MB
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=104B cardinality=unavailable
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpcds_seq_snap.web_returns
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=104B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=16.00MB
-00:SCAN HDFS [tpcds_seq_snap.web_returns, RANDOM]
- partitions=1/1 files=1 size=6.55MB
- stored statistics:
- table: rows=unavailable size=6.55MB
- columns: unavailable
- extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=8.00MB
- tuple-ids=0 row-size=104B cardinality=unavailable
-====
-# Mixed table format scan
-select * from functional.alltypesmixedformat
----- PLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional.alltypesmixedformat
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [functional.alltypesmixedformat]
- partitions=3/3 files=3 size=58.36KB
- stored statistics:
- table: rows=unavailable size=unavailable
- partitions: 0/3 rows=unavailable
- columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col
- extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=32.00KB
- tuple-ids=0 row-size=88B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional.alltypesmixedformat
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-| mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=88B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
-00:SCAN HDFS [functional.alltypesmixedformat, RANDOM]
- partitions=3/3 files=3 size=58.36KB
- stored statistics:
- table: rows=unavailable size=unavailable
- partitions: 0/3 rows=unavailable
- columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col
- extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=32.00KB
- tuple-ids=0 row-size=88B cardinality=unavailable
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=64.00KB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional.alltypesmixedformat
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -1184,19 +117,18 @@ PLAN-ROOT SINK
|
01:EXCHANGE [UNPARTITIONED]
| mem-estimate=0B mem-reservation=0B
-| tuple-ids=0 row-size=88B cardinality=unavailable
+| tuple-ids=0 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=64.00KB
-00:SCAN HDFS [functional.alltypesmixedformat, RANDOM]
- partitions=3/3 files=3 size=58.36KB
+Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
+00:SCAN HDFS [tpch.lineitem, RANDOM]
+ partitions=1/1 files=1 size=718.94MB
stored statistics:
- table: rows=unavailable size=unavailable
- partitions: 0/3 rows=unavailable
- columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col
+ table: rows=6001215 size=718.94MB
+ columns: all
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=32.00KB
- tuple-ids=0 row-size=88B cardinality=unavailable
+ mem-estimate=88.00MB mem-reservation=0B
+ tuple-ids=0 row-size=263B cardinality=6001215
====
# HBase scan
select * from functional_hbase.alltypes
@@ -1326,11 +258,11 @@ select * from tpch.lineitem
union all
select * from tpch.lineitem
---- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=88.00MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
+| Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -1345,7 +277,7 @@ PLAN-ROOT SINK
| table: rows=6001215 size=718.94MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=88.00MB mem-reservation=8.00MB
+| mem-estimate=88.00MB mem-reservation=0B
| tuple-ids=1 row-size=263B cardinality=6001215
|
01:SCAN HDFS [tpch.lineitem]
@@ -1354,10 +286,10 @@ PLAN-ROOT SINK
table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=88.00MB
F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1370,7 +302,7 @@ PLAN-ROOT SINK
| tuple-ids=2 row-size=263B cardinality=12002430
|
F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
+Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
00:UNION
| pass-through-operands: all
| mem-estimate=0B mem-reservation=0B
@@ -1382,7 +314,7 @@ Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
| table: rows=6001215 size=718.94MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=88.00MB mem-reservation=8.00MB
+| mem-estimate=88.00MB mem-reservation=0B
| tuple-ids=1 row-size=263B cardinality=6001215
|
01:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -1391,10 +323,10 @@ Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=176.00MB
F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1407,7 +339,7 @@ PLAN-ROOT SINK
| tuple-ids=2 row-size=263B cardinality=12002430
|
F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
+Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
00:UNION
| pass-through-operands: all
| mem-estimate=0B mem-reservation=0B
@@ -1419,7 +351,7 @@ Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
| table: rows=6001215 size=718.94MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=88.00MB mem-reservation=8.00MB
+| mem-estimate=88.00MB mem-reservation=0B
| tuple-ids=1 row-size=263B cardinality=6001215
|
01:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -1428,7 +360,7 @@ Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
====
# Grouping aggregation
@@ -1436,11 +368,11 @@ select l_orderkey, count(*)
from tpch_parquet.lineitem
group by l_orderkey
---- PLAN
-Max Per-Host Resource Reservation: Memory=42.00MB
+Max Per-Host Resource Reservation: Memory=34.00MB
Per-Host Resource Estimates: Memory=114.00MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=114.00MB mem-reservation=42.00MB
+| Per-Host Resources: mem-estimate=114.00MB mem-reservation=34.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -1451,15 +383,15 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=16B cardinality=1563438
|
00:SCAN HDFS [tpch_parquet.lineitem]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=50.50MB
+Max Per-Host Resource Reservation: Memory=42.50MB
Per-Host Resource Estimates: Memory=124.00MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1484,7 +416,7 @@ Per-Host Resources: mem-estimate=10.00MB mem-reservation=8.50MB
| tuple-ids=1 row-size=16B cardinality=1563438
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=114.00MB mem-reservation=42.00MB
+Per-Host Resources: mem-estimate=114.00MB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: l_orderkey
@@ -1492,15 +424,15 @@ Per-Host Resources: mem-estimate=114.00MB mem-reservation=42.00MB
| tuple-ids=1 row-size=16B cardinality=1563438
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=93.50MB
+Max Per-Host Resource Reservation: Memory=77.50MB
Per-Host Resource Estimates: Memory=248.00MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1525,7 +457,7 @@ Per-Host Resources: mem-estimate=20.00MB mem-reservation=9.50MB
| tuple-ids=1 row-size=16B cardinality=1563438
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=228.00MB mem-reservation=84.00MB
+Per-Host Resources: mem-estimate=228.00MB mem-reservation=68.00MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: l_orderkey
@@ -1533,22 +465,22 @@ Per-Host Resources: mem-estimate=228.00MB mem-reservation=84.00MB
| tuple-ids=1 row-size=16B cardinality=1563438
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
====
# Non-grouping aggregation with zero-slot parquet scan
select count(*) from tpch_parquet.lineitem
---- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=18.00MB
+Max Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=11.00MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=18.00MB mem-reservation=8.00MB
+| Per-Host Resources: mem-estimate=11.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -1558,16 +490,16 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=8B cardinality=1
|
00:SCAN HDFS [tpch_parquet.lineitem]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=8.00MB mem-reservation=8.00MB
+ mem-estimate=1.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=28.00MB
+Max Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=21.00MB
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=10.00MB mem-reservation=0B
@@ -1584,22 +516,22 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=8B cardinality=1
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=18.00MB mem-reservation=8.00MB
+Per-Host Resources: mem-estimate=11.00MB mem-reservation=0B
01:AGGREGATE
| output: sum_init_zero(tpch_parquet.lineitem.parquet-stats: num_rows)
| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
| tuple-ids=1 row-size=8B cardinality=1
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=8.00MB mem-reservation=8.00MB
+ mem-estimate=1.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=190.00MB
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1617,19 +549,19 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=8B cardinality=1
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=180.00MB mem-reservation=16.00MB
+Per-Host Resources: mem-estimate=180.00MB mem-reservation=0B
01:AGGREGATE
| output: sum_init_zero(tpch_parquet.lineitem.parquet-stats: num_rows)
| mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
| tuple-ids=1 row-size=8B cardinality=1
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
====
# Sort
@@ -1637,11 +569,11 @@ select *
from tpch_parquet.lineitem
order by l_comment
---- PLAN
-Max Per-Host Resource Reservation: Memory=84.00MB
+Max Per-Host Resource Reservation: Memory=12.00MB
Per-Host Resource Estimates: Memory=120.00MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=120.00MB mem-reservation=84.00MB
+| Per-Host Resources: mem-estimate=120.00MB mem-reservation=12.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -1651,15 +583,15 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=84.00MB
+Max Per-Host Resource Reservation: Memory=12.00MB
Per-Host Resource Estimates: Memory=120.00MB
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1673,22 +605,22 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=120.00MB mem-reservation=84.00MB
+Per-Host Resources: mem-estimate=120.00MB mem-reservation=12.00MB
01:SORT
| order by: l_comment ASC
| mem-estimate=40.00MB mem-reservation=12.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=168.00MB
+Max Per-Host Resource Reservation: Memory=24.00MB
Per-Host Resource Estimates: Memory=240.00MB
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1702,19 +634,19 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=240.00MB mem-reservation=168.00MB
+Per-Host Resources: mem-estimate=240.00MB mem-reservation=24.00MB
01:SORT
| order by: l_comment ASC
| mem-estimate=40.00MB mem-reservation=12.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
====
# TOP-N
@@ -1723,11 +655,11 @@ from tpch_parquet.lineitem
order by l_comment
limit 100
---- PLAN
-Max Per-Host Resource Reservation: Memory=72.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=80.03MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=80.03MB mem-reservation=72.00MB
+| Per-Host Resources: mem-estimate=80.03MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -1737,15 +669,15 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=263B cardinality=100
|
00:SCAN HDFS [tpch_parquet.lineitem]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=72.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=80.03MB
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1760,22 +692,22 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=263B cardinality=100
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=80.03MB mem-reservation=72.00MB
+Per-Host Resources: mem-estimate=80.03MB mem-reservation=0B
01:TOP-N [LIMIT=100]
| order by: l_comment ASC
| mem-estimate=25.66KB mem-reservation=0B
| tuple-ids=1 row-size=263B cardinality=100
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=144.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=160.05MB
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1790,30 +722,30 @@ PLAN-ROOT SINK
| tuple-ids=1 row-size=263B cardinality=100
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=160.05MB mem-reservation=144.00MB
+Per-Host Resources: mem-estimate=160.05MB mem-reservation=0B
01:TOP-N [LIMIT=100]
| order by: l_comment ASC
| mem-estimate=25.66KB mem-reservation=0B
| tuple-ids=1 row-size=263B cardinality=100
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
====
# Broadcast Hash Join
select *
from tpch.lineitem inner join tpch.orders on l_orderkey = o_orderkey
---- PLAN
-Max Per-Host Resource Reservation: Memory=51.00MB
+Max Per-Host Resource Reservation: Memory=35.00MB
Per-Host Resource Estimates: Memory=477.41MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=477.41MB mem-reservation=51.00MB runtime-filters-memory=1.00MB
+| Per-Host Resources: mem-estimate=477.41MB mem-reservation=35.00MB runtime-filters-memory=1.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -1830,7 +762,7 @@ PLAN-ROOT SINK
| table: rows=1500000 size=162.56MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=88.00MB mem-reservation=8.00MB
+| mem-estimate=88.00MB mem-reservation=0B
| tuple-ids=1 row-size=191B cardinality=1500000
|
00:SCAN HDFS [tpch.lineitem]
@@ -1840,10 +772,10 @@ PLAN-ROOT SINK
table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=51.00MB
+Max Per-Host Resource Reservation: Memory=35.00MB
Per-Host Resource Estimates: Memory=477.41MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1856,7 +788,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=454B cardinality=5757710
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=389.41MB mem-reservation=43.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=389.41MB mem-reservation=35.00MB runtime-filters-memory=1.00MB
02:HASH JOIN [INNER JOIN, BROADCAST]
| hash predicates: l_orderkey = o_orderkey
| fk/pk conjuncts: l_orderkey = o_orderkey
@@ -1869,14 +801,14 @@ Per-Host Resources: mem-estimate=389.41MB mem-reservation=43.00MB runtime-filter
| | tuple-ids=1 row-size=191B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
+| Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch.orders, RANDOM]
| partitions=1/1 files=1 size=162.56MB
| stored statistics:
| table: rows=1500000 size=162.56MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=88.00MB mem-reservation=8.00MB
+| mem-estimate=88.00MB mem-reservation=0B
| tuple-ids=1 row-size=191B cardinality=1500000
|
00:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -1886,10 +818,10 @@ Per-Host Resources: mem-estimate=389.41MB mem-reservation=43.00MB runtime-filter
table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=102.00MB
+Max Per-Host Resource Reservation: Memory=70.00MB
Per-Host Resource Estimates: Memory=954.83MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1902,7 +834,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=454B cardinality=5757710
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=778.83MB mem-reservation=86.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=778.83MB mem-reservation=70.00MB runtime-filters-memory=1.00MB
02:HASH JOIN [INNER JOIN, BROADCAST]
| hash-table-id=00
| hash predicates: l_orderkey = o_orderkey
@@ -1923,14 +855,14 @@ Per-Host Resources: mem-estimate=778.83MB mem-reservation=86.00MB runtime-filter
| | tuple-ids=1 row-size=191B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-| Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
+| Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch.orders, RANDOM]
| partitions=1/1 files=1 size=162.56MB
| stored statistics:
| table: rows=1500000 size=162.56MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=88.00MB mem-reservation=8.00MB
+| mem-estimate=88.00MB mem-reservation=0B
| tuple-ids=1 row-size=191B cardinality=1500000
|
00:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -1940,18 +872,18 @@ Per-Host Resources: mem-estimate=778.83MB mem-reservation=86.00MB runtime-filter
table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
====
# Shuffle Hash Join
select *
from tpch.lineitem inner join /* +shuffle */ tpch.orders on l_orderkey = o_orderkey
---- PLAN
-Max Per-Host Resource Reservation: Memory=51.00MB
+Max Per-Host Resource Reservation: Memory=35.00MB
Per-Host Resource Estimates: Memory=477.41MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=477.41MB mem-reservation=51.00MB runtime-filters-memory=1.00MB
+| Per-Host Resources: mem-estimate=477.41MB mem-reservation=35.00MB runtime-filters-memory=1.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -1968,7 +900,7 @@ PLAN-ROOT SINK
| table: rows=1500000 size=162.56MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=88.00MB mem-reservation=8.00MB
+| mem-estimate=88.00MB mem-reservation=0B
| tuple-ids=1 row-size=191B cardinality=1500000
|
00:SCAN HDFS [tpch.lineitem]
@@ -1978,10 +910,10 @@ PLAN-ROOT SINK
table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=52.00MB
+Max Per-Host Resource Reservation: Memory=36.00MB
Per-Host Resource Estimates: Memory=278.14MB
F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2007,14 +939,14 @@ Per-Host Resources: mem-estimate=101.14MB mem-reservation=35.00MB runtime-filter
| | tuple-ids=1 row-size=191B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
+| Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch.orders, RANDOM]
| partitions=1/1 files=1 size=162.56MB
| stored statistics:
| table: rows=1500000 size=162.56MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=88.00MB mem-reservation=8.00MB
+| mem-estimate=88.00MB mem-reservation=0B
| tuple-ids=1 row-size=191B cardinality=1500000
|
03:EXCHANGE [HASH(l_orderkey)]
@@ -2022,7 +954,7 @@ Per-Host Resources: mem-estimate=101.14MB mem-reservation=35.00MB runtime-filter
| tuple-ids=0 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=89.00MB mem-reservation=9.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=89.00MB mem-reservation=1.00MB runtime-filters-memory=1.00MB
00:SCAN HDFS [tpch.lineitem, RANDOM]
partitions=1/1 files=1 size=718.94MB
runtime filters: RF000[bloom] -> l_orderkey
@@ -2030,10 +962,10 @@ Per-Host Resources: mem-estimate=89.00MB mem-reservation=9.00MB runtime-filters-
table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=104.00MB
+Max Per-Host Resource Reservation: Memory=72.00MB
Per-Host Resource Estimates: Memory=456.14MB
F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2067,14 +999,14 @@ Per-Host Resources: mem-estimate=102.14MB mem-reservation=70.00MB runtime-filter
| | tuple-ids=1 row-size=191B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-| Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
+| Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch.orders, RANDOM]
| partitions=1/1 files=1 size=162.56MB
| stored statistics:
| table: rows=1500000 size=162.56MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=88.00MB mem-reservation=8.00MB
+| mem-estimate=88.00MB mem-reservation=0B
| tuple-ids=1 row-size=191B cardinality=1500000
|
03:EXCHANGE [HASH(l_orderkey)]
@@ -2082,7 +1014,7 @@ Per-Host Resources: mem-estimate=102.14MB mem-reservation=70.00MB runtime-filter
| tuple-ids=0 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=178.00MB mem-reservation=18.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=178.00MB mem-reservation=2.00MB runtime-filters-memory=1.00MB
00:SCAN HDFS [tpch.lineitem, RANDOM]
partitions=1/1 files=1 size=718.94MB
runtime filters: RF000[bloom] -> l_orderkey
@@ -2090,18 +1022,18 @@ Per-Host Resources: mem-estimate=178.00MB mem-reservation=18.00MB runtime-filter
table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
====
# Nested loop join
select *
from tpch.lineitem, tpch.orders
---- PLAN
-Max Per-Host Resource Reservation: Memory=16.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=449.10MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=449.10MB mem-reservation=16.00MB
+| Per-Host Resources: mem-estimate=449.10MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -2115,7 +1047,7 @@ PLAN-ROOT SINK
| table: rows=1500000 size=162.56MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=88.00MB mem-reservation=8.00MB
+| mem-estimate=88.00MB mem-reservation=0B
| tuple-ids=1 row-size=191B cardinality=1500000
|
00:SCAN HDFS [tpch.lineitem]
@@ -2124,10 +1056,10 @@ PLAN-ROOT SINK
table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=16.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=449.10MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2140,7 +1072,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=454B cardinality=9001822500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=361.10MB mem-reservation=8.00MB
+Per-Host Resources: mem-estimate=361.10MB mem-reservation=0B
02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
| mem-estimate=273.10MB mem-reservation=0B
| tuple-ids=0,1 row-size=454B cardinality=9001822500000
@@ -2150,14 +1082,14 @@ Per-Host Resources: mem-estimate=361.10MB mem-reservation=8.00MB
| | tuple-ids=1 row-size=191B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-| Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
+| Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch.orders, RANDOM]
| partitions=1/1 files=1 size=162.56MB
| stored statistics:
| table: rows=1500000 size=162.56MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=88.00MB mem-reservation=8.00MB
+| mem-estimate=88.00MB mem-reservation=0B
| tuple-ids=1 row-size=191B cardinality=1500000
|
00:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -2166,10 +1098,10 @@ Per-Host Resources: mem-estimate=361.10MB mem-reservation=8.00MB
table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=32.00MB
+Max Per-Host Resource Reservation: Memory=0B
Per-Host Resource Estimates: Memory=898.21MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2182,7 +1114,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=454B cardinality=9001822500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=722.21MB mem-reservation=16.00MB
+Per-Host Resources: mem-estimate=722.21MB mem-reservation=0B
02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
| join table id: 00
| mem-estimate=273.10MB mem-reservation=0B
@@ -2199,14 +1131,14 @@ Per-Host Resources: mem-estimate=722.21MB mem-reservation=16.00MB
| | tuple-ids=1 row-size=191B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-| Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
+| Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch.orders, RANDOM]
| partitions=1/1 files=1 size=162.56MB
| stored statistics:
| table: rows=1500000 size=162.56MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=88.00MB mem-reservation=8.00MB
+| mem-estimate=88.00MB mem-reservation=0B
| tuple-ids=1 row-size=191B cardinality=1500000
|
00:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -2215,7 +1147,7 @@ Per-Host Resources: mem-estimate=722.21MB mem-reservation=16.00MB
table: rows=6001215 size=718.94MB
columns: all
extrapolated-rows=disabled
- mem-estimate=88.00MB mem-reservation=8.00MB
+ mem-estimate=88.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
====
# Empty set node
@@ -2264,12 +1196,12 @@ PLAN-ROOT SINK
select max(tinyint_col) over(partition by int_col)
from functional.alltypes
---- PLAN
-Max Per-Host Resource Reservation: Memory=10.03MB
+Max Per-Host Resource Reservation: Memory=10.00MB
Per-Host Resource Estimates: Memory=26.00MB
Codegen disabled by planner
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=26.00MB mem-reservation=10.03MB
+| Per-Host Resources: mem-estimate=26.00MB mem-reservation=10.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -2291,10 +1223,10 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=32.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=5B cardinality=7300
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=10.03MB
+Max Per-Host Resource Reservation: Memory=10.00MB
Per-Host Resource Estimates: Memory=26.00MB
Codegen disabled by planner
@@ -2325,7 +1257,7 @@ Per-Host Resources: mem-estimate=10.00MB mem-reservation=10.00MB
| tuple-ids=0 row-size=5B cardinality=7300
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
+Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
00:SCAN HDFS [functional.alltypes, RANDOM]
partitions=24/24 files=24 size=478.45KB
stored statistics:
@@ -2333,10 +1265,10 @@ Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=32.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=5B cardinality=7300
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=20.06MB
+Max Per-Host Resource Reservation: Memory=20.00MB
Per-Host Resource Estimates: Memory=52.00MB
Codegen disabled by planner
@@ -2367,7 +1299,7 @@ Per-Host Resources: mem-estimate=20.00MB mem-reservation=20.00MB
| tuple-ids=0 row-size=5B cardinality=7300
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=64.00KB
+Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
00:SCAN HDFS [functional.alltypes, RANDOM]
partitions=24/24 files=24 size=478.45KB
stored statistics:
@@ -2375,7 +1307,7 @@ Per-Host Resources: mem-estimate=32.00MB mem-reservation=64.00KB
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=32.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=5B cardinality=7300
====
# Pipeline of blocking operators from analytic fns. Blocking operators break
@@ -2385,11 +1317,11 @@ select *, row_number() over (order by o_totalprice) rnum_price,
row_number() over (order by o_orderpriority) rnum_priority
from tpch_parquet.orders
---- PLAN
-Max Per-Host Resource Reservation: Memory=72.00MB
-Per-Host Resource Estimates: Memory=78.00MB
+Max Per-Host Resource Reservation: Memory=36.00MB
+Per-Host Resource Estimates: Memory=70.00MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=78.00MB mem-reservation=72.00MB
+| Per-Host Resources: mem-estimate=70.00MB mem-reservation=36.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -2435,11 +1367,11 @@ PLAN-ROOT SINK
table: rows=1500000 size=54.07MB
columns: all
extrapolated-rows=disabled
- mem-estimate=48.00MB mem-reservation=48.00MB
+ mem-estimate=40.00MB mem-reservation=0B
tuple-ids=0 row-size=191B cardinality=1500000
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=92.00MB
-Per-Host Resource Estimates: Memory=110.00MB
+Max Per-Host Resource Reservation: Memory=44.00MB
+Per-Host Resource Estimates: Memory=102.00MB
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=44.00MB mem-reservation=32.00MB
@@ -2483,7 +1415,7 @@ PLAN-ROOT SINK
| tuple-ids=6 row-size=191B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-Per-Host Resources: mem-estimate=66.00MB mem-reservation=60.00MB
+Per-Host Resources: mem-estimate=58.00MB mem-reservation=12.00MB
01:SORT
| order by: o_totalprice ASC
| mem-estimate=18.00MB mem-reservation=12.00MB spill-buffer=2.00MB
@@ -2495,11 +1427,11 @@ Per-Host Resources: mem-estimate=66.00MB mem-reservation=60.00MB
table: rows=1500000 size=54.07MB
columns: all
extrapolated-rows=disabled
- mem-estimate=48.00MB mem-reservation=48.00MB
+ mem-estimate=40.00MB mem-reservation=0B
tuple-ids=0 row-size=191B cardinality=1500000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=152.00MB
-Per-Host Resource Estimates: Memory=176.00MB
+Max Per-Host Resource Reservation: Memory=56.00MB
+Per-Host Resource Estimates: Memory=160.00MB
F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=44.00MB mem-reservation=32.00MB
@@ -2543,7 +1475,7 @@ PLAN-ROOT SINK
| tuple-ids=6 row-size=191B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-Per-Host Resources: mem-estimate=132.00MB mem-reservation=120.00MB
+Per-Host Resources: mem-estimate=116.00MB mem-reservation=24.00MB
01:SORT
| order by: o_totalprice ASC
| mem-estimate=18.00MB mem-reservation=12.00MB spill-buffer=2.00MB
@@ -2555,7 +1487,7 @@ Per-Host Resources: mem-estimate=132.00MB mem-reservation=120.00MB
table: rows=1500000 size=54.07MB
columns: all
extrapolated-rows=disabled
- mem-estimate=48.00MB mem-reservation=48.00MB
+ mem-estimate=40.00MB mem-reservation=0B
tuple-ids=0 row-size=191B cardinality=1500000
====
# Union with non-trivial branches: each branch executes sequentially within fragment.
@@ -2571,11 +1503,11 @@ select l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment
from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
where l_shipmode = 'F'
---- PLAN
-Max Per-Host Resource Reservation: Memory=102.00MB
+Max Per-Host Resource Reservation: Memory=54.00MB
Per-Host Resource Estimates: Memory=142.58MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=142.58MB mem-reservation=102.00MB runtime-filters-memory=3.00MB
+| Per-Host Resources: mem-estimate=142.58MB mem-reservation=54.00MB runtime-filters-memory=3.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -2597,20 +1529,20 @@ PLAN-ROOT SINK
| | table: rows=1500000 size=54.07MB
| | columns: all
| | extrapolated-rows=disabled
-| | mem-estimate=40.00MB mem-reservation=8.00MB
+| | mem-estimate=40.00MB mem-reservation=0B
| | tuple-ids=6 row-size=8B cardinality=1500000
| |
| 08:SCAN HDFS [tpch_parquet.lineitem]
-| partitions=1/1 files=3 size=193.71MB
+| partitions=1/1 files=3 size=193.73MB
| predicates: l_shipmode = 'F'
| runtime filters: RF004[bloom] -> l_orderkey
| stored statistics:
-| table: rows=6001215 size=193.71MB
+| table: rows=6001215 size=193.73MB
| columns: all
| extrapolated-rows=disabled
| parquet statistics predicates: l_shipmode = 'F'
| parquet dictionary predicates: l_shipmode = 'F'
-| mem-estimate=80.00MB mem-reservation=48.00MB
+| mem-estimate=80.00MB mem-reservation=0B
| tuple-ids=5 row-size=91B cardinality=857316
|
|--07:HASH JOIN [INNER JOIN]
@@ -2629,17 +1561,17 @@ PLAN-ROOT SINK
| | extrapolated-rows=disabled
| | parquet statistics predicates: o_orderpriority = '2-HIGH'
| | parquet dictionary predicates: o_orderpriority = '2-HIGH'
-| | mem-estimate=40.00MB mem-reservation=16.00MB
+| | mem-estimate=40.00MB mem-reservation=0B
| | tuple-ids=4 row-size=32B cardinality=300000
| |
| 05:SCAN HDFS [tpch_parquet.lineitem]
-| partitions=1/1 files=3 size=193.71MB
+| partitions=1/1 files=3 size=193.73MB
| runtime filters: RF002[bloom] -> l_orderkey
| stored statistics:
-| table: rows=6001215 size=193.71MB
+| table: rows=6001215 size=193.73MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=80.00MB mem-reservation=40.00MB
+| mem-estimate=80.00MB mem-reservation=0B
| tuple-ids=3 row-size=70B cardinality=6001215
|
04:AGGREGATE [FINALIZE]
@@ -2660,23 +1592,23 @@ PLAN-ROOT SINK
| table: rows=1500000 size=54.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=40.00MB mem-reservation=8.00MB
+| mem-estimate=40.00MB mem-reservation=0B
| tuple-ids=1 row-size=8B cardinality=1500000
|
01:SCAN HDFS [tpch_parquet.lineitem]
- partitions=1/1 files=3 size=193.71MB
+ partitions=1/1 files=3 size=193.73MB
predicates: l_tax > 10
runtime filters: RF000[bloom] -> l_orderkey
stored statistics:
- table: rows=6001215 size=193.71MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
parquet statistics predicates: l_tax > 10
parquet dictionary predicates: l_tax > 10
- mem-estimate=80.00MB mem-reservation=48.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=78B cardinality=600122
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=187.75MB
+Max Per-Host Resource Reservation: Memory=76.75MB
Per-Host Resource Estimates: Memory=348.33MB
F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2689,7 +1621,7 @@ PLAN-ROOT SINK
| tuple-ids=7 row-size=70B cardinality=2549844
|
F08:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=99.00MB mem-reservation=67.00MB runtime-filters-memory=2.00MB
+Per-Host Resources: mem-estimate=99.00MB mem-reservation=36.00MB runtime-filters-memory=2.00MB
00:UNION
| pass-through-operands: 14
| mem-estimate=0B mem-reservation=0B
@@ -2707,27 +1639,27 @@ Per-Host Resources: mem-estimate=99.00MB mem-reservation=67.00MB runtime-filters
| | | tuple-ids=6 row-size=8B cardinality=1500000
| | |
| | F07:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-| | Per-Host Resources: mem-estimate=40.00MB mem-reservation=8.00MB
+| | Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
| | 09:SCAN HDFS [tpch_parquet.orders, RANDOM]
| | partitions=1/1 files=2 size=54.07MB
| | stored statistics:
| | table: rows=1500000 size=54.07MB
| | columns: all
| | extrapolated-rows=disabled
-| | mem-estimate=40.00MB mem-reservation=8.00MB
+| | mem-estimate=40.00MB mem-reservation=0B
| | tuple-ids=6 row-size=8B cardinality=1500000
| |
| 08:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-| partitions=1/1 files=3 size=193.71MB
+| partitions=1/1 files=3 size=193.73MB
| predicates: l_shipmode = 'F'
| runtime filters: RF004[bloom] -> l_orderkey
| stored statistics:
-| table: rows=6001215 size=193.71MB
+| table: rows=6001215 size=193.73MB
| columns: all
| extrapolated-rows=disabled
| parquet statistics predicates: l_shipmode = 'F'
| parquet dictionary predicates: l_shipmode = 'F'
-| mem-estimate=80.00MB mem-reservation=48.00MB
+| mem-estimate=80.00MB mem-reservation=0B
| tuple-ids=5 row-size=91B cardinality=857316
|
|--07:HASH JOIN [INNER JOIN, BROADCAST]
@@ -2742,7 +1674,7 @@ Per-Host Resources: mem-estimate=99.00MB mem-reservation=67.00MB runtime-filters
| | | tuple-ids=4 row-size=32B cardinality=300000
| | |
| | F05:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-| | Per-Host Resources: mem-estimate=40.00MB mem-reservation=16.00MB
+| | Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
| | 06:SCAN HDFS [tpch_parquet.orders, RANDOM]
| | partitions=1/1 files=2 size=54.07MB
| | predicates: o_orderpriority = '2-HIGH'
@@ -2752,17 +1684,17 @@ Per-Host Resources: mem-estimate=99.00MB mem-reservation=67.00MB runtime-filters
| | extrapolated-rows=disabled
| | parquet statistics predicates: o_
<TRUNCATED>
[2/9] impala git commit: Revert IMPALA-4835 and dependent changes
Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test
index b10c5cd..146524a 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test
@@ -2,7 +2,7 @@
select * from functional.alltypes order by random()
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
+| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -19,14 +19,14 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=97B cardinality=7300
====
# sort on a deterministic expr that exceeds the cost threshold
select * from functional.alltypes order by abs(id) + abs(id)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
+| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -43,14 +43,14 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=97B cardinality=7300
====
# sort on a deterministic expr that doesn't exceed the cost threshold
select * from functional.alltypes order by tinyint_col + 1
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
+| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -66,7 +66,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=97B cardinality=7300
====
# sort on multiple exprs, subset is materialized
@@ -74,7 +74,7 @@ select * from functional.alltypes
order by dayofweek(timestamp_col), true, id + 1, string_col = date_string_col, id = tinyint_col
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
+| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -91,7 +91,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=97B cardinality=7300
====
# expensive analytic order by expr gets materialized
@@ -99,7 +99,7 @@ select last_value(id) over (order by to_date(timestamp_col), bool_col is null)
from functional.alltypes
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.03MB
+| Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -123,7 +123,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=21B cardinality=7300
====
# expensive order by expr in top-n gets materialized
@@ -131,7 +131,7 @@ select id from functional.alltypes order by string_col like 'a.*b', id * bigint_
regexp_replace(string_col, 'a.*b', 'c') limit 10
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
+| Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -148,14 +148,14 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=29B cardinality=7300
====
# sort on udf, gets materialized
select * from functional.alltypes order by TestFn(double_col)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
+| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -172,14 +172,14 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=97B cardinality=7300
====
# sort expr contains SlotRefs that don't need to be materialized separately
select concat(date_string_col, string_col) c from functional.alltypes order by c
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
+| Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -196,6 +196,6 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=41B cardinality=7300
====
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
index f57e578..03273bc 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
@@ -3,7 +3,7 @@ select straight_join *
from tpch_parquet.customer
inner join tpch_parquet.nation on c_nationkey = n_nationkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=18.97MB
+Max Per-Host Resource Reservation: Memory=2.94MB
Per-Host Resource Estimates: Memory=42.94MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -16,7 +16,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=355B cardinality=150000
|
F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-Per-Host Resources: mem-estimate=26.94MB mem-reservation=18.94MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=26.94MB mem-reservation=2.94MB runtime-filters-memory=1.00MB
02:HASH JOIN [INNER JOIN, BROADCAST]
| hash predicates: c_nationkey = n_nationkey
| fk/pk conjuncts: c_nationkey = n_nationkey
@@ -29,14 +29,14 @@ Per-Host Resources: mem-estimate=26.94MB mem-reservation=18.94MB runtime-filters
| | tuple-ids=1 row-size=117B cardinality=25
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
+| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.nation, RANDOM]
| partitions=1/1 files=1 size=2.74KB
| stored statistics:
| table: rows=25 size=2.74KB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=16.00MB mem-reservation=32.00KB
+| mem-estimate=16.00MB mem-reservation=0B
| tuple-ids=1 row-size=117B cardinality=25
|
00:SCAN HDFS [tpch_parquet.customer, RANDOM]
@@ -46,10 +46,10 @@ Per-Host Resources: mem-estimate=26.94MB mem-reservation=18.94MB runtime-filters
table: rows=150000 size=12.31MB
columns: all
extrapolated-rows=disabled
- mem-estimate=24.00MB mem-reservation=16.00MB
+ mem-estimate=24.00MB mem-reservation=0B
tuple-ids=0 row-size=238B cardinality=150000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=37.94MB
+Max Per-Host Resource Reservation: Memory=5.88MB
Per-Host Resource Estimates: Memory=85.88MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -62,7 +62,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=355B cardinality=150000
|
F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-Per-Host Resources: mem-estimate=53.88MB mem-reservation=37.88MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=53.88MB mem-reservation=5.88MB runtime-filters-memory=1.00MB
02:HASH JOIN [INNER JOIN, BROADCAST]
| hash-table-id=00
| hash predicates: c_nationkey = n_nationkey
@@ -83,14 +83,14 @@ Per-Host Resources: mem-estimate=53.88MB mem-reservation=37.88MB runtime-filters
| | tuple-ids=1 row-size=117B cardinality=25
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=64.00KB
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.nation, RANDOM]
| partitions=1/1 files=1 size=2.74KB
| stored statistics:
| table: rows=25 size=2.74KB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=16.00MB mem-reservation=32.00KB
+| mem-estimate=16.00MB mem-reservation=0B
| tuple-ids=1 row-size=117B cardinality=25
|
00:SCAN HDFS [tpch_parquet.customer, RANDOM]
@@ -100,7 +100,7 @@ Per-Host Resources: mem-estimate=53.88MB mem-reservation=37.88MB runtime-filters
table: rows=150000 size=12.31MB
columns: all
extrapolated-rows=disabled
- mem-estimate=24.00MB mem-reservation=16.00MB
+ mem-estimate=24.00MB mem-reservation=0B
tuple-ids=0 row-size=238B cardinality=150000
====
# Join with large build side - should use default-sized buffers.
@@ -108,8 +108,8 @@ select straight_join *
from tpch_parquet.lineitem
left join tpch_parquet.orders on l_orderkey = o_orderkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=154.00MB
-Per-Host Resource Estimates: Memory=428.41MB
+Max Per-Host Resource Reservation: Memory=34.00MB
+Per-Host Resource Estimates: Memory=420.41MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -121,7 +121,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1N row-size=454B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=380.41MB mem-reservation=106.00MB
+Per-Host Resources: mem-estimate=380.41MB mem-reservation=34.00MB
02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
| hash predicates: l_orderkey = o_orderkey
| fk/pk conjuncts: l_orderkey = o_orderkey
@@ -133,27 +133,27 @@ Per-Host Resources: mem-estimate=380.41MB mem-reservation=106.00MB
| | tuple-ids=1 row-size=191B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-| Per-Host Resources: mem-estimate=48.00MB mem-reservation=48.00MB
+| Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.orders, RANDOM]
| partitions=1/1 files=2 size=54.07MB
| stored statistics:
| table: rows=1500000 size=54.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=48.00MB mem-reservation=48.00MB
+| mem-estimate=40.00MB mem-reservation=0B
| tuple-ids=1 row-size=191B cardinality=1500000
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.72MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.72MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=308.00MB
-Per-Host Resource Estimates: Memory=856.83MB
+Max Per-Host Resource Reservation: Memory=68.00MB
+Per-Host Resource Estimates: Memory=840.83MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -165,7 +165,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1N row-size=454B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=760.83MB mem-reservation=212.00MB
+Per-Host Resources: mem-estimate=760.83MB mem-reservation=68.00MB
02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
| hash-table-id=00
| hash predicates: l_orderkey = o_orderkey
@@ -185,23 +185,23 @@ Per-Host Resources: mem-estimate=760.83MB mem-reservation=212.00MB
| | tuple-ids=1 row-size=191B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-| Per-Host Resources: mem-estimate=96.00MB mem-reservation=96.00MB
+| Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.orders, RANDOM]
| partitions=1/1 files=2 size=54.07MB
| stored statistics:
| table: rows=1500000 size=54.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=48.00MB mem-reservation=48.00MB
+| mem-estimate=40.00MB mem-reservation=0B
| tuple-ids=1 row-size=191B cardinality=1500000
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.72MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.72MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
====
# Shuffle join with mid-sized input.
@@ -209,8 +209,8 @@ select straight_join *
from tpch_parquet.orders
join /*+shuffle*/ tpch_parquet.customer on o_custkey = c_custkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=100.00MB
-Per-Host Resource Estimates: Memory=108.00MB
+Max Per-Host Resource Reservation: Memory=36.00MB
+Per-Host Resource Estimates: Memory=100.00MB
F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -235,14 +235,14 @@ Per-Host Resources: mem-estimate=35.00MB mem-reservation=35.00MB runtime-filters
| | tuple-ids=1 row-size=238B cardinality=150000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=24.00MB mem-reservation=16.00MB
+| Per-Host Resources: mem-estimate=24.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.customer, RANDOM]
| partitions=1/1 files=1 size=12.31MB
| stored statistics:
| table: rows=150000 size=12.31MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=24.00MB mem-reservation=16.00MB
+| mem-estimate=24.00MB mem-reservation=0B
| tuple-ids=1 row-size=238B cardinality=150000
|
03:EXCHANGE [HASH(o_custkey)]
@@ -250,7 +250,7 @@ Per-Host Resources: mem-estimate=35.00MB mem-reservation=35.00MB runtime-filters
| tuple-ids=0 row-size=191B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-Per-Host Resources: mem-estimate=49.00MB mem-reservation=49.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=41.00MB mem-reservation=1.00MB runtime-filters-memory=1.00MB
00:SCAN HDFS [tpch_parquet.orders, RANDOM]
partitions=1/1 files=2 size=54.07MB
runtime filters: RF000[bloom] -> o_custkey
@@ -258,11 +258,11 @@ Per-Host Resources: mem-estimate=49.00MB mem-reservation=49.00MB runtime-filters
table: rows=1500000 size=54.07MB
columns: all
extrapolated-rows=disabled
- mem-estimate=48.00MB mem-reservation=48.00MB
+ mem-estimate=40.00MB mem-reservation=0B
tuple-ids=0 row-size=191B cardinality=1500000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=166.00MB
-Per-Host Resource Estimates: Memory=182.00MB
+Max Per-Host Resource Reservation: Memory=38.00MB
+Per-Host Resource Estimates: Memory=166.00MB
F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -295,14 +295,14 @@ Per-Host Resources: mem-estimate=36.00MB mem-reservation=36.00MB runtime-filters
| | tuple-ids=1 row-size=238B cardinality=150000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-| Per-Host Resources: mem-estimate=48.00MB mem-reservation=32.00MB
+| Per-Host Resources: mem-estimate=48.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.customer, RANDOM]
| partitions=1/1 files=1 size=12.31MB
| stored statistics:
| table: rows=150000 size=12.31MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=24.00MB mem-reservation=16.00MB
+| mem-estimate=24.00MB mem-reservation=0B
| tuple-ids=1 row-size=238B cardinality=150000
|
03:EXCHANGE [HASH(o_custkey)]
@@ -310,7 +310,7 @@ Per-Host Resources: mem-estimate=36.00MB mem-reservation=36.00MB runtime-filters
| tuple-ids=0 row-size=191B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-Per-Host Resources: mem-estimate=98.00MB mem-reservation=98.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=82.00MB mem-reservation=2.00MB runtime-filters-memory=1.00MB
00:SCAN HDFS [tpch_parquet.orders, RANDOM]
partitions=1/1 files=2 size=54.07MB
runtime filters: RF000[bloom] -> o_custkey
@@ -318,7 +318,7 @@ Per-Host Resources: mem-estimate=98.00MB mem-reservation=98.00MB runtime-filters
table: rows=1500000 size=54.07MB
columns: all
extrapolated-rows=disabled
- mem-estimate=48.00MB mem-reservation=48.00MB
+ mem-estimate=40.00MB mem-reservation=0B
tuple-ids=0 row-size=191B cardinality=1500000
====
# Broadcast join with mid-sized input - should use larger buffers than shuffle join.
@@ -326,8 +326,8 @@ select straight_join *
from tpch_parquet.orders
join /*+broadcast*/ tpch_parquet.customer on o_custkey = c_custkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=99.00MB
-Per-Host Resource Estimates: Memory=110.38MB
+Max Per-Host Resource Reservation: Memory=35.00MB
+Per-Host Resource Estimates: Memory=102.38MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -339,7 +339,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=428B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-Per-Host Resources: mem-estimate=86.38MB mem-reservation=83.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=78.38MB mem-reservation=35.00MB runtime-filters-memory=1.00MB
02:HASH JOIN [INNER JOIN, BROADCAST]
| hash predicates: o_custkey = c_custkey
| fk/pk conjuncts: o_custkey = c_custkey
@@ -352,14 +352,14 @@ Per-Host Resources: mem-estimate=86.38MB mem-reservation=83.00MB runtime-filters
| | tuple-ids=1 row-size=238B cardinality=150000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=24.00MB mem-reservation=16.00MB
+| Per-Host Resources: mem-estimate=24.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.customer, RANDOM]
| partitions=1/1 files=1 size=12.31MB
| stored statistics:
| table: rows=150000 size=12.31MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=24.00MB mem-reservation=16.00MB
+| mem-estimate=24.00MB mem-reservation=0B
| tuple-ids=1 row-size=238B cardinality=150000
|
00:SCAN HDFS [tpch_parquet.orders, RANDOM]
@@ -369,11 +369,11 @@ Per-Host Resources: mem-estimate=86.38MB mem-reservation=83.00MB runtime-filters
table: rows=1500000 size=54.07MB
columns: all
extrapolated-rows=disabled
- mem-estimate=48.00MB mem-reservation=48.00MB
+ mem-estimate=40.00MB mem-reservation=0B
tuple-ids=0 row-size=191B cardinality=1500000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=198.00MB
-Per-Host Resource Estimates: Memory=220.76MB
+Max Per-Host Resource Reservation: Memory=70.00MB
+Per-Host Resource Estimates: Memory=204.76MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -385,7 +385,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1 row-size=428B cardinality=1500000
|
F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-Per-Host Resources: mem-estimate=172.76MB mem-reservation=166.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=156.76MB mem-reservation=70.00MB runtime-filters-memory=1.00MB
02:HASH JOIN [INNER JOIN, BROADCAST]
| hash-table-id=00
| hash predicates: o_custkey = c_custkey
@@ -406,14 +406,14 @@ Per-Host Resources: mem-estimate=172.76MB mem-reservation=166.00MB runtime-filte
| | tuple-ids=1 row-size=238B cardinality=150000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-| Per-Host Resources: mem-estimate=48.00MB mem-reservation=32.00MB
+| Per-Host Resources: mem-estimate=48.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.customer, RANDOM]
| partitions=1/1 files=1 size=12.31MB
| stored statistics:
| table: rows=150000 size=12.31MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=24.00MB mem-reservation=16.00MB
+| mem-estimate=24.00MB mem-reservation=0B
| tuple-ids=1 row-size=238B cardinality=150000
|
00:SCAN HDFS [tpch_parquet.orders, RANDOM]
@@ -423,7 +423,7 @@ Per-Host Resources: mem-estimate=172.76MB mem-reservation=166.00MB runtime-filte
table: rows=1500000 size=54.07MB
columns: all
extrapolated-rows=disabled
- mem-estimate=48.00MB mem-reservation=48.00MB
+ mem-estimate=40.00MB mem-reservation=0B
tuple-ids=0 row-size=191B cardinality=1500000
====
# Join with no stats for right input - should use default buffers.
@@ -431,7 +431,7 @@ select straight_join *
from functional_parquet.alltypes
left join functional_parquet.alltypestiny on alltypes.id = alltypestiny.id
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=34.17MB
+Max Per-Host Resource Reservation: Memory=34.00MB
Per-Host Resource Estimates: Memory=2.03GB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypes, functional_parquet.alltypestiny
@@ -446,7 +446,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1N row-size=176B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.09MB
+Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.00MB
02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
| hash predicates: alltypes.id = alltypestiny.id
| fk/pk conjuncts: assumed fk/pk
@@ -458,7 +458,7 @@ Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.09MB
| | tuple-ids=1 row-size=88B cardinality=unavailable
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-| Per-Host Resources: mem-estimate=16.00MB mem-reservation=88.00KB
+| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
| 01:SCAN HDFS [functional_parquet.alltypestiny, RANDOM]
| partitions=4/4 files=4 size=9.75KB
| stored statistics:
@@ -466,7 +466,7 @@ Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.09MB
| partitions: 0/4 rows=unavailable
| columns: unavailable
| extrapolated-rows=disabled
-| mem-estimate=16.00MB mem-reservation=88.00KB
+| mem-estimate=16.00MB mem-reservation=0B
| tuple-ids=1 row-size=88B cardinality=unavailable
|
00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
@@ -476,10 +476,10 @@ Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.09MB
partitions: 0/24 rows=unavailable
columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=88.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=88B cardinality=unavailable
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=68.34MB
+Max Per-Host Resource Reservation: Memory=68.00MB
Per-Host Resource Estimates: Memory=4.06GB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypestiny
@@ -494,7 +494,7 @@ PLAN-ROOT SINK
| tuple-ids=0,1N row-size=176B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.17MB
+Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.00MB
02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
| hash-table-id=00
| hash predicates: alltypes.id = alltypestiny.id
@@ -514,7 +514,7 @@ Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.17MB
| | tuple-ids=1 row-size=88B cardinality=unavailable
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=176.00KB
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
| 01:SCAN HDFS [functional_parquet.alltypestiny, RANDOM]
| partitions=4/4 files=4 size=9.75KB
| stored statistics:
@@ -522,7 +522,7 @@ Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.17MB
| partitions: 0/4 rows=unavailable
| columns: unavailable
| extrapolated-rows=disabled
-| mem-estimate=16.00MB mem-reservation=88.00KB
+| mem-estimate=16.00MB mem-reservation=0B
| tuple-ids=1 row-size=88B cardinality=unavailable
|
00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
@@ -532,7 +532,7 @@ Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.17MB
partitions: 0/24 rows=unavailable
columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=88.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=88B cardinality=unavailable
====
# Low NDV aggregation - should scale down buffers to minimum.
@@ -540,7 +540,7 @@ select c_nationkey, avg(c_acctbal)
from tpch_parquet.customer
group by c_nationkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=19.94MB
+Max Per-Host Resource Reservation: Memory=3.94MB
Per-Host Resource Estimates: Memory=44.00MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -565,7 +565,7 @@ Per-Host Resources: mem-estimate=10.00MB mem-reservation=1.94MB
| tuple-ids=1 row-size=10B cardinality=25
|
F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-Per-Host Resources: mem-estimate=34.00MB mem-reservation=18.00MB
+Per-Host Resources: mem-estimate=34.00MB mem-reservation=2.00MB
01:AGGREGATE [STREAMING]
| output: avg(c_acctbal)
| group by: c_nationkey
@@ -578,10 +578,10 @@ Per-Host Resources: mem-estimate=34.00MB mem-reservation=18.00MB
table: rows=150000 size=12.31MB
columns: all
extrapolated-rows=disabled
- mem-estimate=24.00MB mem-reservation=16.00MB
+ mem-estimate=24.00MB mem-reservation=0B
tuple-ids=0 row-size=10B cardinality=150000
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=39.88MB
+Max Per-Host Resource Reservation: Memory=7.88MB
Per-Host Resource Estimates: Memory=88.00MB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -606,7 +606,7 @@ Per-Host Resources: mem-estimate=20.00MB mem-reservation=3.88MB
| tuple-ids=1 row-size=10B cardinality=25
|
F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-Per-Host Resources: mem-estimate=68.00MB mem-reservation=36.00MB
+Per-Host Resources: mem-estimate=68.00MB mem-reservation=4.00MB
01:AGGREGATE [STREAMING]
| output: avg(c_acctbal)
| group by: c_nationkey
@@ -619,7 +619,7 @@ Per-Host Resources: mem-estimate=68.00MB mem-reservation=36.00MB
table: rows=150000 size=12.31MB
columns: all
extrapolated-rows=disabled
- mem-estimate=24.00MB mem-reservation=16.00MB
+ mem-estimate=24.00MB mem-reservation=0B
tuple-ids=0 row-size=10B cardinality=150000
====
# Mid NDV aggregation - should scale down buffers to intermediate size.
@@ -629,7 +629,7 @@ from tpch_parquet.lineitem
group by 1, 2
having count(*) = 1
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=111.00MB
+Max Per-Host Resource Reservation: Memory=87.00MB
Per-Host Resource Estimates: Memory=227.12MB
F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -674,14 +674,14 @@ Per-Host Resources: mem-estimate=72.12MB mem-reservation=52.00MB runtime-filters
| | tuple-ids=1 row-size=25B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-| Per-Host Resources: mem-estimate=40.00MB mem-reservation=16.00MB
+| Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.orders, RANDOM]
| partitions=1/1 files=2 size=54.07MB
| stored statistics:
| table: rows=1500000 size=54.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=40.00MB mem-reservation=16.00MB
+| mem-estimate=40.00MB mem-reservation=0B
| tuple-ids=1 row-size=25B cardinality=1500000
|
04:EXCHANGE [HASH(l_orderkey)]
@@ -689,18 +689,18 @@ Per-Host Resources: mem-estimate=72.12MB mem-reservation=52.00MB runtime-filters
| tuple-ids=0 row-size=8B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=81.00MB mem-reservation=9.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=81.00MB mem-reservation=1.00MB runtime-filters-memory=1.00MB
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.72MB
+ partitions=1/1 files=3 size=193.73MB
runtime filters: RF000[bloom] -> l_orderkey
stored statistics:
- table: rows=6001215 size=193.72MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=171.00MB
+Max Per-Host Resource Reservation: Memory=123.00MB
Per-Host Resource Estimates: Memory=363.00MB
F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -753,14 +753,14 @@ Per-Host Resources: mem-estimate=87.00MB mem-reservation=87.00MB runtime-filters
| | tuple-ids=1 row-size=25B cardinality=1500000
| |
| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-| Per-Host Resources: mem-estimate=80.00MB mem-reservation=32.00MB
+| Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
| 01:SCAN HDFS [tpch_parquet.orders, RANDOM]
| partitions=1/1 files=2 size=54.07MB
| stored statistics:
| table: rows=1500000 size=54.07MB
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=40.00MB mem-reservation=16.00MB
+| mem-estimate=40.00MB mem-reservation=0B
| tuple-ids=1 row-size=25B cardinality=1500000
|
04:EXCHANGE [HASH(l_orderkey)]
@@ -768,22 +768,22 @@ Per-Host Resources: mem-estimate=87.00MB mem-reservation=87.00MB runtime-filters
| tuple-ids=0 row-size=8B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=162.00MB mem-reservation=18.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=162.00MB mem-reservation=2.00MB runtime-filters-memory=1.00MB
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.72MB
+ partitions=1/1 files=3 size=193.73MB
runtime filters: RF000[bloom] -> l_orderkey
stored statistics:
- table: rows=6001215 size=193.72MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=8.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=8B cardinality=6001215
====
# High NDV aggregation - should use default buffer size.
select distinct *
from tpch_parquet.lineitem
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=140.00MB
+Max Per-Host Resource Reservation: Memory=68.00MB
Per-Host Resource Estimates: Memory=3.31GB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -807,22 +807,22 @@ Per-Host Resources: mem-estimate=1.62GB mem-reservation=34.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=1.69GB mem-reservation=106.00MB
+Per-Host Resources: mem-estimate=1.69GB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.72MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.72MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=280.00MB
+Max Per-Host Resource Reservation: Memory=136.00MB
Per-Host Resource Estimates: Memory=6.62GB
F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -846,19 +846,19 @@ Per-Host Resources: mem-estimate=3.23GB mem-reservation=68.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=3.39GB mem-reservation=212.00MB
+Per-Host Resources: mem-estimate=3.39GB mem-reservation=68.00MB
01:AGGREGATE [STREAMING]
| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
| tuple-ids=1 row-size=263B cardinality=6001215
|
00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
- partitions=1/1 files=3 size=193.72MB
+ partitions=1/1 files=3 size=193.73MB
stored statistics:
- table: rows=6001215 size=193.72MB
+ table: rows=6001215 size=193.73MB
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=72.00MB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=263B cardinality=6001215
====
# Aggregation with unknown input - should use default buffer size.
@@ -866,7 +866,7 @@ select string_col, count(*)
from functional_parquet.alltypestiny
group by string_col
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=68.01MB
+Max Per-Host Resource Reservation: Memory=68.00MB
Per-Host Resource Estimates: Memory=272.00MB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypestiny
@@ -893,7 +893,7 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=34.00MB
| tuple-ids=1 row-size=24B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.01MB
+Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.00MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: string_col
@@ -907,10 +907,10 @@ Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.01MB
partitions: 0/4 rows=unavailable
columns: unavailable
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=8.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=16B cardinality=unavailable
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=136.02MB
+Max Per-Host Resource Reservation: Memory=136.00MB
Per-Host Resource Estimates: Memory=544.00MB
WARNING: The following tables are missing relevant table and/or column statistics.
functional_parquet.alltypestiny
@@ -937,7 +937,7 @@ Per-Host Resources: mem-estimate=256.00MB mem-reservation=68.00MB
| tuple-ids=1 row-size=24B cardinality=unavailable
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=288.00MB mem-reservation=68.02MB
+Per-Host Resources: mem-estimate=288.00MB mem-reservation=68.00MB
01:AGGREGATE [STREAMING]
| output: count(*)
| group by: string_col
@@ -951,6 +951,6 @@ Per-Host Resources: mem-estimate=288.00MB mem-reservation=68.02MB
partitions: 0/4 rows=unavailable
columns: unavailable
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=8.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=16B cardinality=unavailable
====
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
index 1844f11..48163ec 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
@@ -2,7 +2,7 @@
select * from functional.alltypes tablesample system(10) repeatable(1234)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=32.00KB
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -13,14 +13,14 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=32.00KB
+ mem-estimate=32.00MB mem-reservation=0B
tuple-ids=0 row-size=97B cardinality=730
====
# Sample 50%
select * from functional.alltypes tablesample system(50) repeatable(1234)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=80.00MB mem-reservation=32.00KB
+| Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -31,7 +31,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=80.00MB mem-reservation=32.00KB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=97B cardinality=3650
====
# Sampling and scan predicates. Scan predicates are evaluated after sampling and
@@ -40,7 +40,7 @@ select * from functional.alltypes tablesample system(50) repeatable(1234)
where id < 10
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=80.00MB mem-reservation=32.00KB
+| Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -53,7 +53,7 @@ PLAN-ROOT SINK
columns: all
extrapolated-rows=disabled
parquet dictionary predicates: id < 10
- mem-estimate=80.00MB mem-reservation=32.00KB
+ mem-estimate=80.00MB mem-reservation=0B
tuple-ids=0 row-size=97B cardinality=365
====
# Partition pruning + sampling. Partition pruning happens before sampling.
@@ -61,7 +61,7 @@ select * from functional.alltypes tablesample system(50) repeatable(1234)
where year = 2009
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=48.00MB mem-reservation=32.00KB
+| Per-Host Resources: mem-estimate=48.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -72,7 +72,7 @@ PLAN-ROOT SINK
partitions: 12/12 rows=3650
columns: all
extrapolated-rows=disabled
- mem-estimate=48.00MB mem-reservation=32.00KB
+ mem-estimate=48.00MB mem-reservation=0B
tuple-ids=0 row-size=97B cardinality=1825
====
# Edge case: sample 0%, no files should be selected
@@ -97,7 +97,7 @@ PLAN-ROOT SINK
select * from functional.alltypes tablesample system(1) repeatable(1234)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=32.00KB
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -108,7 +108,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=32.00KB
+ mem-estimate=32.00MB mem-reservation=0B
tuple-ids=0 row-size=97B cardinality=73
====
# Edge case: sample 1% and prune partitions, at least one file should be selected
@@ -116,7 +116,7 @@ select * from functional.alltypes tablesample system(1) repeatable(1234)
where year = 2010
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=32.00KB
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -127,14 +127,14 @@ PLAN-ROOT SINK
partitions: 12/12 rows=3650
columns: all
extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=32.00KB
+ mem-estimate=32.00MB mem-reservation=0B
tuple-ids=0 row-size=97B cardinality=37
====
# Edge case: sample 100%, all files should be selected
select * from functional.alltypes tablesample system (100)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
+| Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -145,25 +145,25 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=97B cardinality=7300
====
# Table that has no stats.
select id from functional_parquet.alltypes tablesample system(10) repeatable(1234)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=16.00MB mem-reservation=8.00KB
+| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
00:SCAN HDFS [functional_parquet.alltypes]
- partitions=3/24 files=3 size=22.10KB
+ partitions=3/24 files=3 size=22.07KB
stored statistics:
table: rows=unavailable size=unavailable
partitions: 0/24 rows=unavailable
columns: unavailable
extrapolated-rows=disabled
- mem-estimate=16.00MB mem-reservation=8.00KB
+ mem-estimate=16.00MB mem-reservation=0B
tuple-ids=0 row-size=4B cardinality=unavailable
====
# Sampling in a subquery.
@@ -172,7 +172,7 @@ select id from functional.alltypes t1 where exists (
where t1.id = t2.id)
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=162.94MB mem-reservation=2.98MB runtime-filters-memory=1.00MB
+| Per-Host Resources: mem-estimate=162.94MB mem-reservation=2.94MB runtime-filters-memory=1.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -189,7 +189,7 @@ PLAN-ROOT SINK
| partitions: 4/4 rows=100
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=32.00MB mem-reservation=8.00KB
+| mem-estimate=32.00MB mem-reservation=0B
| tuple-ids=1 row-size=4B cardinality=10
|
00:SCAN HDFS [functional.alltypes t1]
@@ -200,7 +200,7 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=4B cardinality=7300
====
# Sampling in WITH-clause view.
@@ -208,7 +208,7 @@ with t as (select * from functional.alltypes tablesample system(10) repeatable(1
select id from t
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=32.00MB mem-reservation=32.00KB
+| Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B
|
@@ -219,6 +219,6 @@ PLAN-ROOT SINK
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=32.00MB mem-reservation=32.00KB
+ mem-estimate=32.00MB mem-reservation=0B
tuple-ids=0 row-size=4B cardinality=730
====
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/union.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/union.test b/testdata/workloads/functional-planner/queries/PlannerTest/union.test
index 09ac11b..a0d0c26 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/union.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/union.test
@@ -3233,7 +3233,7 @@ PLAN-ROOT SINK
| tuple-ids=2 row-size=4B cardinality=7300
|
F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
+Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
00:UNION
| pass-through-operands: all
| mem-estimate=0B mem-reservation=0B
@@ -3246,7 +3246,7 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
| partitions: 24/24 rows=7300
| columns: all
| extrapolated-rows=disabled
-| mem-estimate=128.00MB mem-reservation=32.00KB
+| mem-estimate=128.00MB mem-reservation=0B
| tuple-ids=1 row-size=4B cardinality=7300
|
01:SCAN HDFS [functional.emptytable, RANDOM]
@@ -3277,7 +3277,7 @@ PLAN-ROOT SINK
| tuple-ids=2 row-size=4B cardinality=7300
|
F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
+Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
00:UNION
| pass-through-operands: all
| mem-estimate=0B mem-reservation=0B
@@ -3300,6 +3300,6 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
partitions: 24/24 rows=7300
columns: all
extrapolated-rows=disabled
- mem-estimate=128.00MB mem-reservation=32.00KB
+ mem-estimate=128.00MB mem-reservation=0B
tuple-ids=0 row-size=4B cardinality=7300
====
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
index de12db1..119c2fd 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
@@ -5,7 +5,7 @@ select distinct * from functional_parquet.alltypesagg
---- CATCH
minimum memory reservation is greater than memory available to the
query for buffer reservations. Memory reservation needed given the
- current plan: 68.09 MB. Set mem_limit to at least 100.09 MB.
+ current plan: 68.00 MB. Set mem_limit to at least 143.00 MB.
====
---- QUERY
set mem_limit=150mb;
@@ -13,14 +13,14 @@ select distinct * from functional_parquet.alltypesagg
---- CATCH
minimum memory reservation needed is greater than pool max mem resources.
Pool max mem resources: 10.00 MB.
- Cluster-wide memory reservation needed: 204.26 MB
+ Cluster-wide memory reservation needed: 204.00 MB
====
---- QUERY
set buffer_pool_limit=10mb;
select distinct * from functional_parquet.alltypesagg
---- CATCH
minimum memory reservation is greater than memory available to the query
- for buffer reservations. Increase the buffer_pool_limit to 68.09 MB.
+ for buffer reservations. Increase the buffer_pool_limit to 68.00 MB.
====
---- QUERY
set mem_limit=1024;
@@ -29,14 +29,14 @@ from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
---- CATCH
minimum memory reservation is greater than memory available to the
query for buffer reservations. Memory reservation needed given the
- current plan: 22.75 MB. Set mem_limit to at least 54.75 MB.
+ current plan: 6.75 MB. Set mem_limit to at least 81.75 MB.
====
---- QUERY
-set mem_limit=50mb;
+set mem_limit=80mb;
select count(*)
from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
---- CATCH
minimum memory reservation is greater than memory available to the
query for buffer reservations. Memory reservation needed given the
- current plan: 34.00 MB. Set mem_limit to at least 66.00 MB.
+ current plan: 18.00 MB. Set mem_limit to at least 93.00 MB.
====
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test b/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
index d725391..f8a024a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
@@ -1584,10 +1584,9 @@ int, bigint, bigint, double
====
---- QUERY
# Check that the a large analytic query can succeed with the minimum buffers (3 buffers
-# for sort, 2 buffer for analytic, plus 32KB extra for the I/O buffers required to scan
-# the < 16KB functional_parquet.alltypes files).
+# for sort, 2 buffer for analytic).
SET default_spillable_buffer_size=8m;
-SET buffer_pool_limit=47m;
+SET buffer_pool_limit=40m;
SELECT lag(-180, 13) over (ORDER BY t1.int_col ASC, t2.int_col ASC) AS int_col
FROM functional_parquet.alltypes t1 CROSS JOIN functional_parquet.alltypes t2 LIMIT 10;
---- TYPES
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test b/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test
index f7013ae..dea6066 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test
@@ -1,8 +1,9 @@
=====
---- QUERY
-set exec_single_node_rows_threshold=0;
set mem_limit=100k;
-with t as (values(1), (2), (3), (4)) select min(t1.`1` + t2.`1`) from t t1 join t t2;
+select *
+from alltypes
+where substr(string_col, 1) = "";
---- CATCH
Codegen failed to reserve
=====
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test b/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test
index 48649e5..2c4fede 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test
@@ -1,6 +1,6 @@
====
---- QUERY
-set buffer_pool_limit=90m;
+set buffer_pool_limit=10m;
set default_spillable_buffer_size=64k;
select count(*)
from (select distinct o_orderdate, o_custkey, o_comment
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
index 53afc32..92abd5f 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
@@ -5,8 +5,8 @@ explain
select *
from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.*
-row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
+'Max Per-Host Resource Reservation: Memory=35.00MB'
+'Per-Host Resource Estimates: Memory=477.41MB'
''
'PLAN-ROOT SINK'
'04:EXCHANGE [UNPARTITIONED]'
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
index 6d58d29..e65745a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
@@ -5,8 +5,8 @@ explain
select *
from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.*
-row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
+'Max Per-Host Resource Reservation: Memory=35.00MB'
+'Per-Host Resource Estimates: Memory=477.41MB'
''
'PLAN-ROOT SINK'
'|'
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
index 17fdd10..b1c58f4 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
@@ -5,8 +5,8 @@ explain
select *
from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.*
-row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
+'Max Per-Host Resource Reservation: Memory=35.00MB'
+'Per-Host Resource Estimates: Memory=477.41MB'
''
'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
'| Per-Host Resources: mem-estimate=0B mem-reservation=0B'
@@ -18,12 +18,12 @@ row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
'| tuple-ids=0,1 row-size=454B cardinality=5757710'
'|'
'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3'
-row_regex:.*Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+'Per-Host Resources: mem-estimate=389.41MB mem-reservation=35.00MB runtime-filters-memory=1.00MB'
'02:HASH JOIN [INNER JOIN, BROADCAST]'
'| hash predicates: l_orderkey = o_orderkey'
'| fk/pk conjuncts: l_orderkey = o_orderkey'
'| runtime filters: RF000[bloom] <- o_orderkey'
-row_regex:.*| mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB spill-buffer=[0-9.]*MB.*
+'| mem-estimate=300.41MB mem-reservation=34.00MB spill-buffer=2.00MB'
'| tuple-ids=0,1 row-size=454B cardinality=5757710'
'|'
'|--03:EXCHANGE [BROADCAST]'
@@ -31,14 +31,14 @@ row_regex:.*| mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB spill-buffer=[0-
'| | tuple-ids=1 row-size=191B cardinality=1500000'
'| |'
'| F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2'
-row_regex:.*| Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+'| Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B'
'| 01:SCAN HDFS [tpch.orders, RANDOM]'
row_regex:.*partitions=1/1 files=1 size=.*
'| stored statistics:'
row_regex:.*table: rows=1500000 size=.*
'| columns: all'
'| extrapolated-rows=disabled'
-row_regex:.*| mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+'| mem-estimate=88.00MB mem-reservation=0B'
'| tuple-ids=1 row-size=191B cardinality=1500000'
'|'
'00:SCAN HDFS [tpch.lineitem, RANDOM]'
@@ -48,7 +48,7 @@ row_regex:.*partitions=1/1 files=1 size=.*
row_regex:.*table: rows=6001215 size=.*
' columns: all'
' extrapolated-rows=disabled'
-row_regex:.* mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+' mem-estimate=88.00MB mem-reservation=0B'
' tuple-ids=0 row-size=263B cardinality=6001215'
====
---- QUERY
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
index 4ef3524..625e2b0 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
@@ -5,8 +5,8 @@ explain
select *
from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.*
-row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
+'Max Per-Host Resource Reservation: Memory=35.00MB'
+'Per-Host Resource Estimates: Memory=477.41MB'
''
'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
'Per-Host Resources: mem-estimate=0B mem-reservation=0B'
@@ -18,14 +18,14 @@ row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
' tuple-ids=0,1 row-size=454B cardinality=5757710'
''
'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3'
-row_regex:.*Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+'Per-Host Resources: mem-estimate=389.41MB mem-reservation=35.00MB runtime-filters-memory=1.00MB'
' DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=04, UNPARTITIONED]'
' | mem-estimate=0B mem-reservation=0B'
' 02:HASH JOIN [INNER JOIN, BROADCAST]'
' | hash predicates: l_orderkey = o_orderkey'
' | fk/pk conjuncts: l_orderkey = o_orderkey'
' | runtime filters: RF000[bloom] <- o_orderkey'
-row_regex:.* | mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB spill-buffer=[0-9.]*MB.*
+' | mem-estimate=300.41MB mem-reservation=34.00MB spill-buffer=2.00MB'
' | tuple-ids=0,1 row-size=454B cardinality=5757710'
' |'
' |--03:EXCHANGE [BROADCAST]'
@@ -39,11 +39,11 @@ row_regex:.*partitions=1/1 files=1 size=.*
row_regex:.*table: rows=6001215 size=.*
' columns: all'
' extrapolated-rows=disabled'
-row_regex:.* mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+' mem-estimate=88.00MB mem-reservation=0B'
' tuple-ids=0 row-size=263B cardinality=6001215'
''
'F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2'
-row_regex:.*Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+'Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B'
' DATASTREAM SINK [FRAGMENT=F00, EXCHANGE=03, BROADCAST]'
' | mem-estimate=0B mem-reservation=0B'
' 01:SCAN HDFS [tpch.orders, RANDOM]'
@@ -52,7 +52,7 @@ row_regex:.*partitions=1/1 files=1 size=.*
row_regex:.*table: rows=1500000 size=.*
' columns: all'
' extrapolated-rows=disabled'
-row_regex:.* mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+' mem-estimate=88.00MB mem-reservation=0B'
' tuple-ids=1 row-size=191B cardinality=1500000'
====
---- QUERY
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
index 6b213eb..c8a80b2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
@@ -162,7 +162,7 @@ BIGINT,BIGINT
---- QUERY
# IMPALA-2376
# Set memory limit low enough to get the below query to consistently fail.
-# This was originally a regression test that hit an error like:
+# TODO: change query and/or table so we can consistently check for the following error:
# Failed to allocate buffer for collection '...'.
set mem_limit=4m;
select max(cnt) from customer c,
@@ -170,7 +170,7 @@ select max(cnt) from customer c,
---- TYPES
BIGINT
---- CATCH
-Rejected query from pool default-pool: minimum memory reservation is greater than memory available to the query for buffer reservations.
+Memory limit exceeded
====
---- QUERY
# IMPALA-2473: Scan query with large row size leading to oversized batches.
@@ -237,7 +237,7 @@ bigint, bigint
# This was originally a regression test for IMPALA-5446: dropped status from
# Sorter::Reset() when sort cannot get reserved buffer. However with the
# IMPALA-3200 changes it now succeeds.
-set buffer_pool_limit=180m;
+set buffer_pool_limit=100m;
select c_custkey, c_name, c_address, c_phone, c_acctbal, c_mktsegment, c_comment,
o_orderdate, sum(o_totalprice), min(rnum)
from customer c,
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
index fdca977..3d92acb 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
@@ -345,7 +345,7 @@ where l_orderkey = 965 and l_extendedprice * l_tax = p_retailprice;
# consumption / spilling behaviour.
####################################################
-SET BUFFER_POOL_LIMIT=80m;
+SET BUFFER_POOL_LIMIT=40m;
SET RUNTIME_FILTER_MODE=GLOBAL;
SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS;
SET RUNTIME_BLOOM_FILTER_SIZE=16M;
@@ -396,14 +396,15 @@ SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS;
SET RUNTIME_FILTER_MIN_SIZE=128MB;
SET RUNTIME_FILTER_MAX_SIZE=500MB;
# Query would have been admitted if memory for runtime filters was not accounted for.
-SET BUFFER_POOL_LIMIT=290MB;
+SET BUFFER_POOL_LIMIT=140MB;
select STRAIGHT_JOIN * from alltypes a join [SHUFFLE] alltypes b
on a.month = b.id and b.int_col = -3
---- RESULTS
---- CATCH
-Rejected query from pool default-pool: minimum memory reservation is
+ImpalaBeeswaxException: INNER EXCEPTION: <class 'beeswaxd.ttypes.BeeswaxException'>
+ MESSAGE: Rejected query from pool default-pool: minimum memory reservation is
greater than memory available to the query for buffer reservations. Increase
- the buffer_pool_limit to 290.17 MB. See the query profile for more information
+ the buffer_pool_limit to 290.00 MB. See the query profile for more information
about the per-node memory requirements.
====
---- QUERY
@@ -413,7 +414,7 @@ SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS;
SET RUNTIME_FILTER_MIN_SIZE=128MB;
SET RUNTIME_FILTER_MAX_SIZE=500MB;
# This would run perfectly with just enough memory provided by the buffer pool.
-SET BUFFER_POOL_LIMIT=295MB;
+SET BUFFER_POOL_LIMIT=290MB;
select STRAIGHT_JOIN * from alltypes a join [SHUFFLE] alltypes b
on a.month = b.id and b.int_col = -3
---- RESULTS
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/scanners.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/scanners.test b/testdata/workloads/functional-query/queries/QueryTest/scanners.test
index a000a15..bd5f496 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/scanners.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/scanners.test
@@ -110,10 +110,3 @@ on (v.x = t3.bool_col) where t3.bool_col = true
---- TYPES
BIGINT
====
----- QUERY
-# Scan an empty unpartitioned table.
-select * from emptytable;
----- RESULTS
----- TYPES
-STRING,INT
-====
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/set.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/set.test b/testdata/workloads/functional-query/queries/QueryTest/set.test
index 7983224..32ad938 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/set.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/set.test
@@ -129,7 +129,7 @@ select 1
set mem_limit=1;
select count(string_col) from functional.alltypestiny
---- CATCH
-Rejected query from pool default-pool: minimum memory reservation
+Memory limit exceeded
====
---- QUERY
# Set mem_limit back to unlimited and query should succeed again.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
index 20813c7..05552d2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
@@ -1,6 +1,6 @@
====
---- QUERY
-set buffer_pool_limit=34m;
+set buffer_pool_limit=10m;
select l_orderkey, count(*)
from lineitem
group by 1
@@ -25,7 +25,7 @@ row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
====
---- QUERY
# Test query with string grouping column and string agg columns
-set buffer_pool_limit=82m;
+set buffer_pool_limit=10m;
set num_nodes=1;
select l_returnflag, l_orderkey, avg(l_tax), min(l_shipmode)
from lineitem
@@ -39,9 +39,10 @@ order by 1,2 limit 3
STRING, BIGINT, DECIMAL, STRING
---- RUNTIME_PROFILE
row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
+row_regex: .*NumRepartitions: .* \([1-9][0-9]*\)
====
---- QUERY
-set buffer_pool_limit=34m;
+set buffer_pool_limit=10m;
select l_orderkey, count(*)
from lineitem
group by 1
@@ -65,7 +66,7 @@ row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
====
---- QUERY
# Test query with string grouping column
-set buffer_pool_limit=34m;
+set buffer_pool_limit=10m;
set num_nodes=1;
select l_comment, count(*)
from lineitem
@@ -81,10 +82,11 @@ order by count(*) desc limit 5
STRING, BIGINT
---- RUNTIME_PROFILE
row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
+row_regex: .*NumRepartitions: .* \([1-9][0-9]*\)
====
---- QUERY
# Test query with string grouping column and string agg columns
-set buffer_pool_limit=82m;
+set buffer_pool_limit=10m;
set num_nodes=1;
select l_returnflag, l_orderkey, round(avg(l_tax),2), min(l_shipmode)
from lineitem
@@ -99,10 +101,11 @@ STRING, BIGINT, DECIMAL, STRING
---- RUNTIME_PROFILE
# Verify that spilling happened in the agg.
row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
+row_regex: .*NumRepartitions: .* \([1-9][0-9]*\)
====
---- QUERY
# Test with non-scalar intermediate state (avg() uses fixed intermediate value).
-set buffer_pool_limit=34m;
+set buffer_pool_limit=10m;
select l_orderkey, avg(l_orderkey)
from lineitem
group by 1
@@ -122,7 +125,7 @@ row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
====
---- QUERY
# Test aggregation spill with group_concat distinct
-set buffer_pool_limit=78m;
+set buffer_pool_limit=30m;
select l_orderkey, count(*), group_concat(distinct l_linestatus, '|')
from lineitem
group by 1
@@ -168,7 +171,7 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
# Merge aggregation requires 17 buffers and preaggregation requires 16 buffers
# plus 1mb of hash tables. The buffer size is 256k for this test.
set max_row_size=256k;
-set buffer_pool_limit="57.25M";
+set buffer_pool_limit=9472k;
select count(*)
from (select distinct * from orders) t
---- TYPES
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test
index 7678675..420e721 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test
@@ -13,7 +13,7 @@
# except:
# * even-numbered l_orderkey values, because there is a NULL on the build side.
# * l_orderkey values > 5500000, because there is a NULL on the probe side
-set buffer_pool_limit=110m;
+set buffer_pool_limit=12m;
select l_orderkey, l_partkey, l_suppkey, l_linenumber
from lineitem
where l_suppkey = 4162 and l_shipmode = 'AIR' and l_returnflag = 'A' and
@@ -54,9 +54,8 @@ order by 1,2,3,4
BIGINT,BIGINT,BIGINT,INT
====
---- QUERY
-set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
-# Execute NAAJ query 1 with a debug action so that it can not get enough memory to fit
-# the null build rows in memory.
+set buffer_pool_limit=7m;
+# Execute NAAJ query 1 without enough memory to fit the null build rows in memory.
select l_orderkey, l_partkey, l_suppkey, l_linenumber
from lineitem
where l_suppkey = 4162 and l_shipmode = 'AIR' and l_returnflag = 'A' and
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test
index 39ae2bb..4c0d790 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test
@@ -21,7 +21,7 @@
# where l_orderkey % 2 = 1 and p_retailprice = l_extendedprice * l_tax
# order by 1,2,3,4 limit 5
#
-set buffer_pool_limit=90m;
+set buffer_pool_limit=10m;
select l_orderkey, l_partkey, l_suppkey, l_linenumber
from lineitem
where if(l_orderkey % 2 = 0, NULL, l_partkey) not in (
@@ -41,7 +41,7 @@ BIGINT,BIGINT,BIGINT,INT
---- QUERY
# Execute NAAJ query 2 in-memory only without enough memory to complete.
set scratch_limit=0;
-set buffer_pool_limit=90m;
+set buffer_pool_limit=10m;
select l_orderkey, l_partkey, l_suppkey, l_linenumber
from lineitem
where if(l_orderkey % 2 = 0, NULL, l_partkey) not in (
@@ -65,7 +65,7 @@ Could not free memory by spilling to disk: scratch_limit is 0
# where p_partkey = 2 and l_partkey % 2 = 1
# order by 1,2,3,4
#
-set buffer_pool_limit=90m;
+set buffer_pool_limit=10m;
select l_orderkey, l_partkey, l_suppkey, l_linenumber
from lineitem
where l_partkey not in (
@@ -86,7 +86,7 @@ BIGINT,BIGINT,BIGINT,INT
# =======================================================================================
# This returns one less row than the previous query because l_partkey=116009 is
# replaced with a NULL.
-set buffer_pool_limit=90m;
+set buffer_pool_limit=10m;
select l_orderkey, l_partkey, l_suppkey, l_linenumber
from lineitem
where if(l_partkey % 5 != 1, NULL, l_partkey) not in (
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test
deleted file mode 100644
index f89e537..0000000
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test
+++ /dev/null
@@ -1,66 +0,0 @@
-# This file contains tests where we don't want the python test framework to supply the
-# debug_action value because the test won't succeed with all possible debug_action values.
-====
----- QUERY
-# Tests for the case where a spilled partition has 0 probe rows and so we don't build the
-# hash table in a partitioned hash join. Always runs with the minimum reservation to force
-# spilling.
-# INNER JOIN
-set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
-select straight_join count(*)
-from
-lineitem a, lineitem b
-where
-a.l_partkey = 1 and
-a.l_orderkey = b.l_orderkey;
----- TYPES
-BIGINT
----- RESULTS
-173
----- RUNTIME_PROFILE
-row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
-====
----- QUERY
-# spilled partition with 0 probe rows, NULL AWARE LEFT ANTI JOIN
-set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
-select straight_join count(*)
-from
-lineitem a
-where
-a.l_partkey not in (select l_partkey from lineitem where l_partkey > 10)
-and a.l_partkey < 1000;
----- TYPES
-BIGINT
----- RESULTS
-287
----- RUNTIME_PROFILE
-row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
-====
----- QUERY
-# spilled partition with 0 probe rows, RIGHT OUTER JOIN
-set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
-select straight_join count(*)
-from
-supplier right outer join lineitem on s_suppkey = l_suppkey
-where s_acctbal > 0 and s_acctbal < 10;
----- TYPES
-BIGINT
----- RESULTS
-12138
----- RUNTIME_PROFILE
-row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
-====
----- QUERY
-# spilled partition with 0 probe rows, RIGHT ANTI JOIN
-set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
-with x as (select * from supplier limit 10)
-select straight_join count(*)
-from
-x right anti join lineitem on s_suppkey + 100 = l_suppkey;
----- TYPES
-BIGINT
----- RESULTS
-5995258
----- RUNTIME_PROFILE
-row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
-====
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
index 7d65523..415fe5d 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
@@ -3,7 +3,7 @@
# Test sort with small char column materialized by exprs.
# Set low memory limit to force spilling.
# IMPALA-3332: comparator makes local allocations that cause runaway memory consumption.
-set buffer_pool_limit=28m;
+set buffer_pool_limit=4m;
set mem_limit=200m;
set disable_outermost_topn=1;
select cast(l_comment as char(50))
@@ -41,7 +41,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
---- QUERY
# Test sort with small input char column materialized before sort.
set mem_limit=200m;
-set buffer_pool_limit=28m;
+set buffer_pool_limit=4m;
set disable_outermost_topn=1;
select char_col
from (select cast(l_comment as char(50)) char_col
@@ -80,7 +80,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
# Test sort with large input char column materialized before sort.
# Set low memory limit to force spilling.
set mem_limit=200m;
-set buffer_pool_limit=28m;
+set buffer_pool_limit=4m;
set disable_outermost_topn=1;
select char_col
from (select cast(l_comment as char(200)) char_col
@@ -118,7 +118,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
---- QUERY
# Test sort with varchar column materialized by exprs.
# Set low memory limit to force spilling.
-set buffer_pool_limit=28m;
+set buffer_pool_limit=4m;
# IMPALA-3332: comparator makes local allocations that cause runaway memory consumption.
set mem_limit=200m;
set disable_outermost_topn=1;
@@ -158,7 +158,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
# Test sort with input varchar column materialized before sort.
# Set low memory limit to force spilling.
set mem_limit=200m;
-set buffer_pool_limit=28m;
+set buffer_pool_limit=4m;
set disable_outermost_topn=1;
select char_col
from (select cast(l_comment as varchar(50)) char_col
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/spilling.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling.test b/testdata/workloads/functional-query/queries/QueryTest/spilling.test
index 23d9271..6639d82 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling.test
@@ -1,6 +1,6 @@
====
---- QUERY
-set buffer_pool_limit=215m;
+set buffer_pool_limit=15m;
select count(l1.l_tax)
from
lineitem l1,
@@ -22,7 +22,7 @@ BIGINT
row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
====
---- QUERY
-set buffer_pool_limit=50m;
+set buffer_pool_limit=2m;
select max(t1.total_count), max(t1.l_shipinstruct), max(t1.l_comment) from
(select l_shipinstruct, l_comment, count(*) over () total_count from lineitem) t1
---- RESULTS
@@ -60,7 +60,7 @@ row_regex: .*SpilledPartitions: 0 .*
# Adding TPCH-Q21 in the spilling test to check for IMPALA-1471 (spilling left anti
# and left outer joins were returning wrong results).
# Q21 - Suppliers Who Kept Orders Waiting Query
-set buffer_pool_limit=200m;
+set buffer_pool_limit=26m;
select
s_name,
count(*) as numwait
@@ -211,7 +211,7 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
---- QUERY
# IMPALA-1346/IMPALA-1546: fix sorter memory management so that it can complete
# successfully when in same pipeline as a spilling join.
-set buffer_pool_limit=170m;
+set buffer_pool_limit=50m;
set disable_outermost_topn=1;
select * from lineitem
inner join orders on l_orderkey = o_orderkey
@@ -246,6 +246,68 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
====
---- QUERY
+# Tests for the case where a spilled partition has 0 probe rows and so we don't build the
+# hash table in a partitioned hash join.
+# INNER JOIN
+set buffer_pool_limit=10m;
+select straight_join count(*)
+from
+lineitem a, lineitem b
+where
+a.l_partkey = 1 and
+a.l_orderkey = b.l_orderkey;
+---- TYPES
+BIGINT
+---- RESULTS
+173
+---- RUNTIME_PROFILE
+row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# spilled partition with 0 probe rows, NULL AWARE LEFT ANTI JOIN
+set buffer_pool_limit=10m;
+select straight_join count(*)
+from
+lineitem a
+where
+a.l_partkey not in (select l_partkey from lineitem where l_partkey > 10)
+and a.l_partkey < 1000;
+---- TYPES
+BIGINT
+---- RESULTS
+287
+---- RUNTIME_PROFILE
+row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# spilled partition with 0 probe rows, RIGHT OUTER JOIN
+set buffer_pool_limit=10m;
+select straight_join count(*)
+from
+supplier right outer join lineitem on s_suppkey = l_suppkey
+where s_acctbal > 0 and s_acctbal < 10;
+---- TYPES
+BIGINT
+---- RESULTS
+12138
+---- RUNTIME_PROFILE
+row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# spilled partition with 0 probe rows, RIGHT ANTI JOIN
+set buffer_pool_limit=30m;
+with x as (select * from supplier limit 10)
+select straight_join count(*)
+from
+x right anti join lineitem on s_suppkey + 100 = l_suppkey;
+---- TYPES
+BIGINT
+---- RESULTS
+5995258
+---- RUNTIME_PROFILE
+row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
+====
+---- QUERY
# IMPALA-5173: spilling hash join feeding into right side of nested loop join.
# Equivalent to:
# select *
@@ -253,7 +315,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
# where l1.l_quantity = 31.0 and l1.l_tax = 0.03 and l1.l_orderkey <= 100000
# order by l_orderkey, l_partkey, l_suppkey, l_linenumber
# limit 5
-set buffer_pool_limit=177m;
+set buffer_pool_limit=9m;
set num_nodes=1;
select straight_join l.*
from
@@ -283,7 +345,7 @@ bigint,bigint,bigint,int,decimal,decimal,decimal,decimal,string,string,string,st
====
---- QUERY
# Test spilling aggregation when grouping by nondeterministic expression
-set buffer_pool_limit=79m;
+set buffer_pool_limit=5m;
set num_nodes=1;
select l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment
from tpch_parquet.lineitem
@@ -297,7 +359,7 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
---- QUERY
# Test spilling join with many duplicates in join key. We don't expect this to succeed
# with a memory constraint: see IMPALA-4857.
-set buffer_pool_limit=167m;
+set buffer_pool_limit=20m;
select *
from lineitem l1 join lineitem l2 on l1.l_linenumber = l2.l_linenumber
---- CATCH
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test b/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
index 836ed0a..7da7baf 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
@@ -15,6 +15,7 @@ explain select id from alltypes;
' partitions: 0/12 rows=unavailable'
' columns: unavailable'
' extrapolated-rows=unavailable'
+' mem-estimate=16.00MB mem-reservation=0B'
' tuple-ids=0 row-size=4B cardinality=unavailable'
---- TYPES
STRING
@@ -52,14 +53,14 @@ STRING,STRING,BIGINT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING
# Stats are available now.
explain select id from alltypes;
---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=.*
-row_regex:.*Per-Host Resource Estimates: Memory=.*
+'Max Per-Host Resource Reservation: Memory=0B'
+'Per-Host Resource Estimates: Memory=16.00MB'
'Codegen disabled by planner'
''
'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
+'| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
'PLAN-ROOT SINK'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'| mem-estimate=0B mem-reservation=0B'
'|'
'00:SCAN HDFS [$DATABASE.alltypes]'
row_regex:.*partitions=12/12 files=12 size=.*
@@ -68,7 +69,7 @@ row_regex:.*table: rows=3650 size=.*
' partitions: 0/12 rows=unavailable'
' columns: all'
' extrapolated-rows=3650'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+' mem-estimate=16.00MB mem-reservation=0B'
' tuple-ids=0 row-size=4B cardinality=3650'
---- TYPES
STRING
@@ -77,14 +78,14 @@ STRING
# Select a subset of partitions.
explain select id from alltypes where month in (1, 2, 3);
---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=.*
-row_regex:.*Per-Host Resource Estimates: Memory=.*
+'Max Per-Host Resource Reservation: Memory=0B'
+'Per-Host Resource Estimates: Memory=16.00MB'
'Codegen disabled by planner'
''
'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
'PLAN-ROOT SINK'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'| mem-estimate=0B mem-reservation=0B'
'|'
'00:SCAN HDFS [$DATABASE.alltypes]'
row_regex:.*partitions=3/12 files=3 size=.*
@@ -93,7 +94,7 @@ row_regex:.*table: rows=3650 size=.*
' partitions: 0/3 rows=unavailable'
' columns: all'
' extrapolated-rows=904'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+' mem-estimate=16.00MB mem-reservation=0B'
' tuple-ids=0 row-size=4B cardinality=904'
---- TYPES
STRING
@@ -104,13 +105,13 @@ insert into alltypes partition(year, month)
select * from functional_parquet.alltypes where year = 2009;
explain select id from alltypes;
---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=.*
-row_regex:.*Per-Host Resource Estimates: Memory=.*
+'Max Per-Host Resource Reservation: Memory=0B'
+'Per-Host Resource Estimates: Memory=16.00MB'
''
'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
+'| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
'PLAN-ROOT SINK'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'| mem-estimate=0B mem-reservation=0B'
'|'
'00:SCAN HDFS [$DATABASE.alltypes]'
row_regex:.*partitions=12/12 files=24 size=.*
@@ -119,7 +120,7 @@ row_regex:.*table: rows=3650 size=.*
' partitions: 0/12 rows=unavailable'
' columns: all'
' extrapolated-rows=7300'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+' mem-estimate=16.00MB mem-reservation=0B'
' tuple-ids=0 row-size=4B cardinality=7300'
---- TYPES
STRING
@@ -130,14 +131,14 @@ insert into alltypes partition(year, month)
select * from functional_parquet.alltypes where year = 2010;
explain select id from alltypes where year = 2010;
---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=.*
-row_regex:.*Per-Host Resource Estimates: Memory=16.00MB'
+'Max Per-Host Resource Reservation: Memory=0B'
+'Per-Host Resource Estimates: Memory=16.00MB'
'Codegen disabled by planner'
''
'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
+'| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
'PLAN-ROOT SINK'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'| mem-estimate=0B mem-reservation=0B'
'|'
'00:SCAN HDFS [$DATABASE.alltypes]'
row_regex:.*partitions=12/24 files=12 size=.*
@@ -146,7 +147,7 @@ row_regex:.*table: rows=3650 size=.*
' partitions: 0/12 rows=unavailable'
' columns: all'
' extrapolated-rows=3651'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+' mem-estimate=16.00MB mem-reservation=0B'
' tuple-ids=0 row-size=4B cardinality=3651'
---- TYPES
STRING
@@ -156,14 +157,14 @@ STRING
compute stats alltypes;
explain select id from alltypes where year = 2010;
---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=.*
-row_regex:.*Per-Host Resource Estimates: Memory=16.00MB'
+'Max Per-Host Resource Reservation: Memory=0B'
+'Per-Host Resource Estimates: Memory=16.00MB'
'Codegen disabled by planner'
''
'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
+'| Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
'PLAN-ROOT SINK'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'| mem-estimate=0B mem-reservation=0B'
'|'
'00:SCAN HDFS [$DATABASE.alltypes]'
row_regex:.*partitions=12/24 files=12 size=.*
@@ -172,7 +173,7 @@ row_regex:.*table: rows=10950 size=.*
' partitions: 0/12 rows=unavailable'
' columns: all'
' extrapolated-rows=3651'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+' mem-estimate=16.00MB mem-reservation=0B'
' tuple-ids=0 row-size=4B cardinality=3651'
---- TYPES
STRING
@@ -187,6 +188,7 @@ explain select id from alltypes;
' partitions: 0/24 rows=unavailable'
' columns: unavailable'
' extrapolated-rows=unavailable'
+' mem-estimate=16.00MB mem-reservation=0B'
' tuple-ids=0 row-size=4B cardinality=unavailable'
---- TYPES
STRING