You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by lv...@apache.org on 2018/03/03 22:33:29 UTC

[1/9] impala git commit: Revert IMPALA-4835 and dependent changes

Repository: impala
Updated Branches:
  refs/heads/2.x 42c604e9f -> e5689fb5c


http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/tpch/queries/sort-reservation-usage.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/tpch/queries/sort-reservation-usage.test b/testdata/workloads/tpch/queries/sort-reservation-usage.test
index af31ccb..92f180d 100644
--- a/testdata/workloads/tpch/queries/sort-reservation-usage.test
+++ b/testdata/workloads/tpch/queries/sort-reservation-usage.test
@@ -1,18 +1,17 @@
 ====
 ---- QUERY
 # Test that in-mem sorts incrementally give up memory when emitting output.
-# This query and scratch limit are chosen so that the query fails if the first sort
-# does not give up memory to the second sort. Scans the text formatted file so that
-# the scan uses less reservation.
+# This query and the limit is calibrated to fail if the first sort does not
+# give up memory to the second sort.
 set num_nodes=1;
 set scratch_limit=0;
-set buffer_pool_limit=35m;
+set buffer_pool_limit=15m;
 set default_spillable_buffer_size=64kb;
 SELECT *
 FROM   (SELECT
         Rank() OVER(ORDER BY  l_orderkey) AS rank,
         Rank() OVER(ORDER BY  l_partkey) AS rank2
-        FROM tpch.lineitem
+        FROM lineitem
         WHERE l_shipdate < '1992-05-09') a
 WHERE rank < 10
 ORDER BY rank;

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/common/test_dimensions.py
----------------------------------------------------------------------
diff --git a/tests/common/test_dimensions.py b/tests/common/test_dimensions.py
index 3c1b5e7..4171e1f 100644
--- a/tests/common/test_dimensions.py
+++ b/tests/common/test_dimensions.py
@@ -131,13 +131,13 @@ SINGLE_NODE_ONLY = [1]
 ALL_NODES_ONLY = [0]
 ALL_DISABLE_CODEGEN_OPTIONS = [True, False]
 
-def create_single_exec_option_dimension(num_nodes=0, disable_codegen_rows_threshold=5000):
+def create_single_exec_option_dimension():
   """Creates an exec_option dimension that will produce a single test vector"""
-  return create_exec_option_dimension(cluster_sizes=[num_nodes],
-      disable_codegen_options=[False],
-      # Make sure codegen kicks in for functional.alltypes.
-      disable_codegen_rows_threshold_options=[disable_codegen_rows_threshold],
-      batch_sizes=[0])
+  return create_exec_option_dimension(cluster_sizes=ALL_NODES_ONLY,
+                                      disable_codegen_options=[False],
+                                      # Make sure codegen kicks in for functional.alltypes.
+                                      disable_codegen_rows_threshold_options=[5000],
+                                      batch_sizes=[0])
 
 def create_exec_option_dimension(cluster_sizes=ALL_CLUSTER_SIZES,
                                  disable_codegen_options=ALL_DISABLE_CODEGEN_OPTIONS,
@@ -145,15 +145,13 @@ def create_exec_option_dimension(cluster_sizes=ALL_CLUSTER_SIZES,
                                  sync_ddl=None, exec_single_node_option=[0],
                                  # We already run with codegen on and off explicitly -
                                  # don't need automatic toggling.
-                                 disable_codegen_rows_threshold_options=[0],
-                                 debug_action_options=[None]):
+                                 disable_codegen_rows_threshold_options=[0]):
   exec_option_dimensions = {
       'abort_on_error': [1],
       'exec_single_node_rows_threshold': exec_single_node_option,
       'batch_size': batch_sizes,
       'disable_codegen': disable_codegen_options,
       'disable_codegen_rows_threshold': disable_codegen_rows_threshold_options,
-      'debug_action': debug_action_options,
       'num_nodes': cluster_sizes}
 
   if sync_ddl is not None:

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/custom_cluster/test_scratch_disk.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_scratch_disk.py b/tests/custom_cluster/test_scratch_disk.py
index 65bde66..bd3c7e4 100644
--- a/tests/custom_cluster/test_scratch_disk.py
+++ b/tests/custom_cluster/test_scratch_disk.py
@@ -39,7 +39,7 @@ class TestScratchDir(CustomClusterTestSuite):
       """
   # Buffer pool limit that is low enough to force Impala to spill to disk when executing
   # spill_query.
-  buffer_pool_limit = "45m"
+  buffer_pool_limit = "32m"
 
   def count_nonempty_dirs(self, dirs):
     count = 0

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/query_test/test_mem_usage_scaling.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_mem_usage_scaling.py b/tests/query_test/test_mem_usage_scaling.py
index 2d6c17b..25af44d 100644
--- a/tests/query_test/test_mem_usage_scaling.py
+++ b/tests/query_test/test_mem_usage_scaling.py
@@ -96,7 +96,7 @@ class TestExprMemUsage(ImpalaTestSuite):
 class TestLowMemoryLimits(ImpalaTestSuite):
   '''Super class for the memory limit tests with the TPC-H and TPC-DS queries'''
 
-  def low_memory_limit_test(self, vector, tpch_query, limit):
+  def low_memory_limit_test(self, vector, tpch_query, limit, xfail_mem_limit=None):
     mem = vector.get_value('mem_limit')
     # Mem consumption can be +-30MBs, depending on how many scanner threads are
     # running. Adding this extra mem in order to reduce false negatives in the tests.
@@ -113,11 +113,13 @@ class TestLowMemoryLimits(ImpalaTestSuite):
     try:
       self.run_test_case(tpch_query, new_vector)
     except ImpalaBeeswaxException as e:
-      if not expects_error: raise
+      if not expects_error and not xfail_mem_limit: raise
       found_expected_error = False
       for error_msg in MEM_LIMIT_ERROR_MSGS:
         if error_msg in str(e): found_expected_error = True
       assert found_expected_error, str(e)
+      if not expects_error and xfail_mem_limit:
+        pytest.xfail(xfail_mem_limit)
 
 
 @SkipIfLocal.mem_usage_different
@@ -132,7 +134,7 @@ class TestTpchMemLimitError(TestLowMemoryLimits):
                        'Q6' : 25, 'Q7' : 200, 'Q8' : 125, 'Q9' : 200, 'Q10' : 162,\
                        'Q11' : 112, 'Q12' : 150, 'Q13' : 125, 'Q14' : 125, 'Q15' : 125,\
                        'Q16' : 137, 'Q17' : 137, 'Q18' : 196, 'Q19' : 112, 'Q20' : 162,\
-                       'Q21' : 300, 'Q22' : 125}
+                       'Q21' : 187, 'Q22' : 125}
 
   @classmethod
   def get_workload(self):
@@ -175,7 +177,8 @@ class TestTpchMemLimitError(TestLowMemoryLimits):
     self.low_memory_limit_test(vector, 'tpch-q8', self.MIN_MEM_FOR_TPCH['Q8'])
 
   def test_low_mem_limit_q9(self, vector):
-    self.low_memory_limit_test(vector, 'tpch-q9', self.MIN_MEM_FOR_TPCH['Q9'])
+    self.low_memory_limit_test(vector, 'tpch-q9', self.MIN_MEM_FOR_TPCH['Q9'],
+            xfail_mem_limit="IMPALA-3328: TPC-H Q9 memory limit test is flaky")
 
   def test_low_mem_limit_q10(self, vector):
     self.low_memory_limit_test(vector, 'tpch-q10', self.MIN_MEM_FOR_TPCH['Q10'])

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/query_test/test_query_mem_limit.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_query_mem_limit.py b/tests/query_test/test_query_mem_limit.py
index 17ea9f5..2fdd6eb 100644
--- a/tests/query_test/test_query_mem_limit.py
+++ b/tests/query_test/test_query_mem_limit.py
@@ -118,9 +118,7 @@ class TestCodegenMemLimit(ImpalaTestSuite):
   @classmethod
   def add_test_dimensions(cls):
     super(TestCodegenMemLimit, cls).add_test_dimensions()
-    # Run with num_nodes=1 to avoid races between fragments allocating memory.
-    cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension(
-        num_nodes=1, disable_codegen_rows_threshold=0))
+    cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
     # Only run the query for parquet
     cls.ImpalaTestMatrix.add_constraint(
       lambda v: v.get_value('table_format').file_format == 'parquet')

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/query_test/test_scanners.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 2d0fd0c..9b252da 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -37,9 +37,7 @@ from tests.common.skip import (
     SkipIfADLS,
     SkipIfIsilon,
     SkipIfLocal)
-from tests.common.test_dimensions import (
-    create_single_exec_option_dimension,
-    create_exec_option_dimension)
+from tests.common.test_dimensions import create_single_exec_option_dimension
 from tests.common.test_result_verifier import (
     parse_column_types,
     parse_column_labels,
@@ -51,11 +49,6 @@ from tests.util.hdfs_util import NAMENODE
 from tests.util.get_parquet_metadata import get_parquet_metadata
 from tests.util.test_file_parser import QueryTestSectionReader
 
-# Test scanners with denial of reservations at varying frequency. This will affect the
-# number of scanner threads that can be spun up.
-DEBUG_ACTION_DIMS = [None,
-  '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@0.5',
-  '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0']
 
 class TestScannersAllTableFormats(ImpalaTestSuite):
   BATCH_SIZES = [0, 1, 16]
@@ -73,20 +66,18 @@ class TestScannersAllTableFormats(ImpalaTestSuite):
       cls.ImpalaTestMatrix.add_dimension(cls.create_table_info_dimension('pairwise'))
     cls.ImpalaTestMatrix.add_dimension(
         ImpalaTestDimension('batch_size', *TestScannersAllTableFormats.BATCH_SIZES))
-    cls.ImpalaTestMatrix.add_dimension(
-        ImpalaTestDimension('debug_action', *DEBUG_ACTION_DIMS))
 
   def test_scanners(self, vector):
     new_vector = deepcopy(vector)
     new_vector.get_value('exec_option')['batch_size'] = vector.get_value('batch_size')
-    new_vector.get_value('exec_option')['debug_action'] = vector.get_value('debug_action')
     self.run_test_case('QueryTest/scanners', new_vector)
 
   def test_hdfs_scanner_profile(self, vector):
-    if vector.get_value('table_format').file_format in ('kudu', 'hbase') or \
-       vector.get_value('exec_option')['num_nodes'] != 0:
+    if vector.get_value('table_format').file_format in ('kudu', 'hbase'):
       pytest.skip()
-    self.run_test_case('QueryTest/hdfs_scanner_profile', vector)
+    new_vector = deepcopy(vector)
+    new_vector.get_value('exec_option')['num_nodes'] = 0
+    self.run_test_case('QueryTest/hdfs_scanner_profile', new_vector)
 
 # Test all the scanners with a simple limit clause. The limit clause triggers
 # cancellation in the scanner code paths.
@@ -180,8 +171,6 @@ class TestWideRow(ImpalaTestSuite):
   @classmethod
   def add_test_dimensions(cls):
     super(TestWideRow, cls).add_test_dimensions()
-    cls.ImpalaTestMatrix.add_dimension(
-        create_exec_option_dimension(debug_action_options=DEBUG_ACTION_DIMS))
     # I can't figure out how to load a huge row into hbase
     cls.ImpalaTestMatrix.add_constraint(
       lambda v: v.get_value('table_format').file_format != 'hbase')
@@ -213,8 +202,6 @@ class TestWideTable(ImpalaTestSuite):
   @classmethod
   def add_test_dimensions(cls):
     super(TestWideTable, cls).add_test_dimensions()
-    cls.ImpalaTestMatrix.add_dimension(
-        create_exec_option_dimension(debug_action_options=DEBUG_ACTION_DIMS))
     cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension("num_cols", *cls.NUM_COLS))
     # To cut down on test execution time, only run in exhaustive.
     if cls.exploration_strategy() != 'exhaustive':
@@ -257,8 +244,6 @@ class TestParquet(ImpalaTestSuite):
   @classmethod
   def add_test_dimensions(cls):
     super(TestParquet, cls).add_test_dimensions()
-    cls.ImpalaTestMatrix.add_dimension(
-        create_exec_option_dimension(debug_action_options=DEBUG_ACTION_DIMS))
     cls.ImpalaTestMatrix.add_constraint(
       lambda v: v.get_value('table_format').file_format == 'parquet')
 

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/query_test/test_scanners_fuzz.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_scanners_fuzz.py b/tests/query_test/test_scanners_fuzz.py
index 0b42e5d..c336a17 100644
--- a/tests/query_test/test_scanners_fuzz.py
+++ b/tests/query_test/test_scanners_fuzz.py
@@ -48,12 +48,6 @@ class TestScannersFuzzing(ImpalaTestSuite):
   # Test a range of batch sizes to exercise different corner cases.
   BATCH_SIZES = [0, 1, 16, 10000]
 
-  # Test with denial of reservations at varying frequency. This will affect the number
-  # of scanner threads that can be spun up.
-  DEBUG_ACTION_VALUES = [None,
-    '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@0.5',
-    '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0']
-
   @classmethod
   def get_workload(cls):
     return 'functional-query'
@@ -65,8 +59,7 @@ class TestScannersFuzzing(ImpalaTestSuite):
         create_exec_option_dimension_from_dict({
           'abort_on_error' : cls.ABORT_ON_ERROR_VALUES,
           'num_nodes' : cls.NUM_NODES_VALUES,
-          'mem_limit' : cls.MEM_LIMITS,
-          'debug_action' : cls.DEBUG_ACTION_VALUES}))
+          'mem_limit' : cls.MEM_LIMITS}))
     # TODO: enable for more table formats once they consistently pass the fuzz test.
     cls.ImpalaTestMatrix.add_constraint(lambda v:
         v.get_value('table_format').file_format in ('avro', 'parquet') or

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/query_test/test_sort.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_sort.py b/tests/query_test/test_sort.py
index ea0760f..23de9f2 100644
--- a/tests/query_test/test_sort.py
+++ b/tests/query_test/test_sort.py
@@ -68,23 +68,14 @@ class TestQueryFullSort(ImpalaTestSuite):
       order by o_orderdate"""
     exec_option = copy(vector.get_value('exec_option'))
     table_format = vector.get_value('table_format')
-    exec_option['default_spillable_buffer_size'] = '8M'
-
-    # Minimum memory for different parts of the plan.
-    sort_reservation_mb = 48
-    if table_format.file_format == 'parquet':
-      scan_reservation_mb = 24
-    else:
-      scan_reservation_mb = 8
-    total_reservation_mb = sort_reservation_mb + scan_reservation_mb
 
     # The below memory value assume 8M pages.
-    # Test with unlimited and minimum memory for all file formats.
-    buffer_pool_limit_values = ['-1', '{0}M'.format(total_reservation_mb)]
+    exec_option['default_spillable_buffer_size'] = '8M'
+    buffer_pool_limit_values = ['-1', '48M'] # Unlimited and minimum memory.
     if self.exploration_strategy() == 'exhaustive' and \
         table_format.file_format == 'parquet':
       # Test some intermediate values for parquet on exhaustive.
-      buffer_pool_limit_values += ['128M', '256M']
+      buffer_pool_limit_values += ['64M', '128M', '256M']
     for buffer_pool_limit in buffer_pool_limit_values:
       exec_option['buffer_pool_limit'] = buffer_pool_limit
       result = transpose_results(self.execute_query(
@@ -93,6 +84,7 @@ class TestQueryFullSort(ImpalaTestSuite):
 
   def test_sort_join(self, vector):
     """With 200m memory limit this should be a 2-phase sort"""
+
     query = """select o1.o_orderdate, o2.o_custkey, o1.o_comment from orders o1 join
     orders o2 on (o1.o_orderkey = o2.o_orderkey) order by o1.o_orderdate limit 100000"""
 

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/tests/query_test/test_spilling.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_spilling.py b/tests/query_test/test_spilling.py
index 05ada9f..0b36429 100644
--- a/tests/query_test/test_spilling.py
+++ b/tests/query_test/test_spilling.py
@@ -95,8 +95,3 @@ class TestSpillingNoDebugActionDimensions(ImpalaTestSuite):
       setting debug_action to alternative values via query options."""
     self.run_test_case('QueryTest/spilling-query-options', vector)
 
-  def test_spilling_no_debug_action(self, vector):
-    """Spilling tests that will not succeed if run with an arbitrary debug action.
-       These tests either run with no debug action set or set their own debug action."""
-    self.run_test_case('QueryTest/spilling-no-debug-action', vector)
-


[6/9] impala git commit: Revert IMPALA-4835 and dependent changes

Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr.cc b/be/src/runtime/io/disk-io-mgr.cc
index 0d2afe2..8ff6609 100644
--- a/be/src/runtime/io/disk-io-mgr.cc
+++ b/be/src/runtime/io/disk-io-mgr.cc
@@ -15,10 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "runtime/io/disk-io-mgr.h"
-
 #include "common/global-flags.h"
-#include "runtime/exec-env.h"
+#include "runtime/io/disk-io-mgr.h"
 #include "runtime/io/disk-io-mgr-internal.h"
 #include "runtime/io/handle-cache.inline.h"
 
@@ -54,8 +52,6 @@ DEFINE_int32(num_threads_per_disk, 0, "Number of I/O threads per disk");
 static const int THREADS_PER_ROTATIONAL_DISK = 1;
 static const int THREADS_PER_SOLID_STATE_DISK = 8;
 
-const int64_t DiskIoMgr::IDEAL_MAX_SIZED_BUFFERS_PER_SCAN_RANGE;
-
 // The maximum number of the threads per rotational disk is also the max queue depth per
 // rotational disk.
 static const string num_io_threads_per_rotational_disk_help_msg = Substitute("Number of "
@@ -126,6 +122,13 @@ DEFINE_uint64(unused_file_handle_timeout_sec, 21600, "Maximum time, in seconds,
 DEFINE_uint64(num_file_handle_cache_partitions, 16, "Number of partitions used by the "
     "file handle cache.");
 
+// The IoMgr is able to run with a wide range of memory usage. If a query has memory
+// remaining less than this value, the IoMgr will stop all buffering regardless of the
+// current queue size.
+static const int LOW_MEMORY = 64 * 1024 * 1024;
+
+const int DiskIoMgr::SCAN_RANGE_READY_BUFFER_LIMIT;
+
 AtomicInt32 DiskIoMgr::next_disk_id_;
 
 namespace detail {
@@ -152,6 +155,34 @@ string DiskIoMgr::DebugString() {
   return ss.str();
 }
 
+BufferDescriptor::BufferDescriptor(DiskIoMgr* io_mgr,
+    RequestContext* reader, ScanRange* scan_range, uint8_t* buffer,
+    int64_t buffer_len, MemTracker* mem_tracker)
+  : io_mgr_(io_mgr),
+    reader_(reader),
+    mem_tracker_(mem_tracker),
+    scan_range_(scan_range),
+    buffer_(buffer),
+    buffer_len_(buffer_len) {
+  DCHECK(io_mgr != nullptr);
+  DCHECK(scan_range != nullptr);
+  DCHECK(buffer != nullptr);
+  DCHECK_GE(buffer_len, 0);
+  DCHECK_NE(scan_range->external_buffer_tag_ == ScanRange::ExternalBufferTag::NO_BUFFER,
+      mem_tracker == nullptr);
+}
+
+void BufferDescriptor::TransferOwnership(MemTracker* dst) {
+  DCHECK(dst != nullptr);
+  DCHECK(!is_client_buffer());
+  // Memory of cached buffers is not tracked against a tracker.
+  if (is_cached()) return;
+  DCHECK(mem_tracker_ != nullptr);
+  dst->Consume(buffer_len_);
+  mem_tracker_->Release(buffer_len_);
+  mem_tracker_ = dst;
+}
+
 WriteRange::WriteRange(
     const string& file, int64_t file_offset, int disk_id, WriteDoneCallback callback)
   : RequestRange(RequestType::WRITE), callback_(callback) {
@@ -192,8 +223,8 @@ DiskIoMgr::DiskIoMgr() :
     num_io_threads_per_solid_state_disk_(GetFirstPositiveVal(
         FLAGS_num_io_threads_per_solid_state_disk, FLAGS_num_threads_per_disk,
         THREADS_PER_SOLID_STATE_DISK)),
-    max_buffer_size_(BitUtil::RoundUpToPowerOfTwo(FLAGS_read_size)),
-    min_buffer_size_(BitUtil::RoundDownToPowerOfTwo(FLAGS_min_buffer_size)),
+    max_buffer_size_(FLAGS_read_size),
+    min_buffer_size_(FLAGS_min_buffer_size),
     shut_down_(false),
     total_bytes_read_counter_(TUnit::BYTES),
     read_timer_(TUnit::TIME_NS),
@@ -202,6 +233,8 @@ DiskIoMgr::DiskIoMgr() :
         FLAGS_num_file_handle_cache_partitions,
         FLAGS_unused_file_handle_timeout_sec) {
   DCHECK_LE(READ_SIZE_MIN_VALUE, FLAGS_read_size);
+  int64_t max_buffer_size_scaled = BitUtil::Ceil(max_buffer_size_, min_buffer_size_);
+  free_buffers_.resize(BitUtil::Log2Ceiling64(max_buffer_size_scaled) + 1);
   int num_local_disks = DiskInfo::num_disks();
   if (FLAGS_num_disks < 0 || FLAGS_num_disks > DiskInfo::num_disks()) {
     LOG(WARNING) << "Number of disks specified should be between 0 and the number of "
@@ -215,11 +248,11 @@ DiskIoMgr::DiskIoMgr() :
 }
 
 DiskIoMgr::DiskIoMgr(int num_local_disks, int threads_per_rotational_disk,
-    int threads_per_solid_state_disk, int64_t min_buffer_size, int64_t max_buffer_size) :
+    int threads_per_solid_state_disk, int min_buffer_size, int max_buffer_size) :
     num_io_threads_per_rotational_disk_(threads_per_rotational_disk),
     num_io_threads_per_solid_state_disk_(threads_per_solid_state_disk),
-    max_buffer_size_(BitUtil::RoundUpToPowerOfTwo(max_buffer_size)),
-    min_buffer_size_(BitUtil::RoundDownToPowerOfTwo(min_buffer_size)),
+    max_buffer_size_(max_buffer_size),
+    min_buffer_size_(min_buffer_size),
     shut_down_(false),
     total_bytes_read_counter_(TUnit::BYTES),
     read_timer_(TUnit::TIME_NS),
@@ -227,6 +260,8 @@ DiskIoMgr::DiskIoMgr(int num_local_disks, int threads_per_rotational_disk,
         FileSystemUtil::MaxNumFileHandles()),
         FLAGS_num_file_handle_cache_partitions,
         FLAGS_unused_file_handle_timeout_sec) {
+  int64_t max_buffer_size_scaled = BitUtil::Ceil(max_buffer_size_, min_buffer_size_);
+  free_buffers_.resize(BitUtil::Log2Ceiling64(max_buffer_size_scaled) + 1);
   if (num_local_disks == 0) num_local_disks = DiskInfo::num_disks();
   disk_queues_.resize(num_local_disks + REMOTE_NUM_DISKS);
   CheckSseSupport();
@@ -250,22 +285,37 @@ DiskIoMgr::~DiskIoMgr() {
   for (int i = 0; i < disk_queues_.size(); ++i) {
     if (disk_queues_[i] == nullptr) continue;
     int disk_id = disk_queues_[i]->disk_id;
-    for (RequestContext* context : disk_queues_[i]->request_contexts) {
-      unique_lock<mutex> context_lock(context->lock_);
-      DCHECK_EQ(context->disk_states_[disk_id].num_threads_in_op(), 0);
-      DCHECK(context->disk_states_[disk_id].done());
-      context->DecrementDiskRefCount(context_lock);
+    for (list<RequestContext*>::iterator it = disk_queues_[i]->request_contexts.begin();
+        it != disk_queues_[i]->request_contexts.end(); ++it) {
+      DCHECK_EQ((*it)->disk_states_[disk_id].num_threads_in_op(), 0);
+      DCHECK((*it)->disk_states_[disk_id].done());
+      (*it)->DecrementDiskRefCount();
     }
   }
 
+  DCHECK_EQ(num_buffers_in_readers_.Load(), 0);
+
+  // Delete all allocated buffers
+  int num_free_buffers = 0;
+  for (int idx = 0; idx < free_buffers_.size(); ++idx) {
+    num_free_buffers += free_buffers_[idx].size();
+  }
+  DCHECK_EQ(num_allocated_buffers_.Load(), num_free_buffers);
+  GcIoBuffers();
+
   for (int i = 0; i < disk_queues_.size(); ++i) {
     delete disk_queues_[i];
   }
 
+  if (free_buffer_mem_tracker_ != nullptr) free_buffer_mem_tracker_->Close();
   if (cached_read_options_ != nullptr) hadoopRzOptionsFree(cached_read_options_);
 }
 
-Status DiskIoMgr::Init() {
+Status DiskIoMgr::Init(MemTracker* process_mem_tracker) {
+  DCHECK(process_mem_tracker != nullptr);
+  free_buffer_mem_tracker_.reset(
+      new MemTracker(-1, "Free Disk IO Buffers", process_mem_tracker, false));
+
   for (int i = 0; i < disk_queues_.size(); ++i) {
     disk_queues_[i] = new DiskQueue(i);
     int num_threads_per_disk;
@@ -311,14 +361,101 @@ Status DiskIoMgr::Init() {
   return Status::OK();
 }
 
-unique_ptr<RequestContext> DiskIoMgr::RegisterContext() {
-  return unique_ptr<RequestContext>(new RequestContext(this, num_total_disks()));
+unique_ptr<RequestContext> DiskIoMgr::RegisterContext(MemTracker* mem_tracker) {
+  return unique_ptr<RequestContext>(
+      new RequestContext(this, num_total_disks(), mem_tracker));
 }
 
 void DiskIoMgr::UnregisterContext(RequestContext* reader) {
   reader->CancelAndMarkInactive();
 }
 
+// Cancellation requires coordination from multiple threads.  Each thread that currently
+// has a reference to the request context must notice the cancel and remove it from its
+// tracking structures.  The last thread to touch the context should deallocate (aka
+// recycle) the request context object.  Potential threads are:
+//  1. Disk threads that are currently reading for this reader.
+//  2. Caller threads that are waiting in GetNext.
+//
+// The steps are:
+// 1. Cancel will immediately set the context in the Cancelled state.  This prevents any
+// other thread from adding more ready buffers to the context (they all take a lock and
+// check the state before doing so), or any write ranges to the context.
+// 2. Cancel will call cancel on each ScanRange that is not yet complete, unblocking
+// any threads in GetNext(). The reader will see the cancelled Status returned. Cancel
+// also invokes the callback for the WriteRanges with the cancelled state.
+// 3. Disk threads notice the context is cancelled either when picking the next context
+// to process or when they try to enqueue a ready buffer.  Upon noticing the cancelled
+// state, removes the context from the disk queue.  The last thread per disk with an
+// outstanding reference to the context decrements the number of disk queues the context
+// is on.
+void DiskIoMgr::CancelContext(RequestContext* context) {
+  context->Cancel(Status::CANCELLED);
+}
+
+void DiskIoMgr::set_read_timer(RequestContext* r, RuntimeProfile::Counter* c) {
+  r->read_timer_ = c;
+}
+
+void DiskIoMgr::set_open_file_timer(RequestContext* r, RuntimeProfile::Counter* c) {
+  r->open_file_timer_ = c;
+}
+
+void DiskIoMgr::set_bytes_read_counter(RequestContext* r, RuntimeProfile::Counter* c) {
+  r->bytes_read_counter_ = c;
+}
+
+void DiskIoMgr::set_active_read_thread_counter(RequestContext* r,
+    RuntimeProfile::Counter* c) {
+  r->active_read_thread_counter_ = c;
+}
+
+void DiskIoMgr::set_disks_access_bitmap(RequestContext* r,
+    RuntimeProfile::Counter* c) {
+  r->disks_accessed_bitmap_ = c;
+}
+
+int64_t DiskIoMgr::queue_size(RequestContext* reader) const {
+  return reader->num_ready_buffers_.Load();
+}
+
+Status DiskIoMgr::context_status(RequestContext* context) const {
+  unique_lock<mutex> lock(context->lock_);
+  return context->status_;
+}
+
+int64_t DiskIoMgr::bytes_read_local(RequestContext* reader) const {
+  return reader->bytes_read_local_.Load();
+}
+
+int64_t DiskIoMgr::bytes_read_short_circuit(RequestContext* reader) const {
+  return reader->bytes_read_short_circuit_.Load();
+}
+
+int64_t DiskIoMgr::bytes_read_dn_cache(RequestContext* reader) const {
+  return reader->bytes_read_dn_cache_.Load();
+}
+
+int DiskIoMgr::num_remote_ranges(RequestContext* reader) const {
+  return reader->num_remote_ranges_.Load();
+}
+
+int64_t DiskIoMgr::unexpected_remote_bytes(RequestContext* reader) const {
+  return reader->unexpected_remote_bytes_.Load();
+}
+
+int DiskIoMgr::cached_file_handles_hit_count(RequestContext* reader) const {
+  return reader->cached_file_handles_hit_count_.Load();
+}
+
+int DiskIoMgr::cached_file_handles_miss_count(RequestContext* reader) const {
+  return reader->cached_file_handles_miss_count_.Load();
+}
+
+int64_t DiskIoMgr::GetReadThroughput() {
+  return RuntimeProfile::UnitsPerSecond(&total_bytes_read_counter_, &read_timer_);
+}
+
 Status DiskIoMgr::ValidateScanRange(ScanRange* range) {
   int disk_id = range->disk_id_;
   if (disk_id < 0 || disk_id >= disk_queues_.size()) {
@@ -329,91 +466,84 @@ Status DiskIoMgr::ValidateScanRange(ScanRange* range) {
     return Status(TErrorCode::DISK_IO_ERROR,
         Substitute("Invalid scan range. Negative offset $0", range->offset_));
   }
-  if (range->len_ <= 0) {
+  if (range->len_ < 0) {
     return Status(TErrorCode::DISK_IO_ERROR,
-        Substitute("Invalid scan range. Non-positive length $0", range->len_));
+        Substitute("Invalid scan range. Negative length $0", range->len_));
   }
   return Status::OK();
 }
 
-Status DiskIoMgr::AddScanRanges(
-    RequestContext* reader, const vector<ScanRange*>& ranges) {
-  DCHECK_GT(ranges.size(), 0);
+Status DiskIoMgr::AddScanRanges(RequestContext* reader,
+    const vector<ScanRange*>& ranges, bool schedule_immediately) {
+  if (ranges.empty()) return Status::OK();
+
   // Validate and initialize all ranges
   for (int i = 0; i < ranges.size(); ++i) {
     RETURN_IF_ERROR(ValidateScanRange(ranges[i]));
     ranges[i]->InitInternal(this, reader);
   }
 
+  // disks that this reader needs to be scheduled on.
   unique_lock<mutex> reader_lock(reader->lock_);
   DCHECK(reader->Validate()) << endl << reader->DebugString();
 
-  if (reader->state_ == RequestContext::Cancelled) return Status::CANCELLED;
+  if (reader->state_ == RequestContext::Cancelled) {
+    DCHECK(!reader->status_.ok());
+    return reader->status_;
+  }
 
   // Add each range to the queue of the disk the range is on
-  for (ScanRange* range : ranges) {
+  for (int i = 0; i < ranges.size(); ++i) {
     // Don't add empty ranges.
-    DCHECK_NE(range->len(), 0);
-    reader->AddActiveScanRangeLocked(reader_lock, range);
+    DCHECK_NE(ranges[i]->len(), 0);
+    ScanRange* range = ranges[i];
+
     if (range->try_cache_) {
-      reader->cached_ranges_.Enqueue(range);
-    } else {
-      reader->AddRangeToDisk(reader_lock, range, ScheduleMode::UPON_GETNEXT);
+      if (schedule_immediately) {
+        bool cached_read_succeeded;
+        RETURN_IF_ERROR(range->ReadFromCache(reader_lock, &cached_read_succeeded));
+        if (cached_read_succeeded) continue;
+        // Cached read failed, fall back to AddRequestRange() below.
+      } else {
+        reader->cached_ranges_.Enqueue(range);
+        continue;
+      }
     }
+    reader->AddRequestRange(range, schedule_immediately);
   }
   DCHECK(reader->Validate()) << endl << reader->DebugString();
+
   return Status::OK();
 }
 
-Status DiskIoMgr::StartScanRange(RequestContext* reader, ScanRange* range,
-    bool* needs_buffers) {
-  RETURN_IF_ERROR(ValidateScanRange(range));
-  range->InitInternal(this, reader);
-
-  unique_lock<mutex> reader_lock(reader->lock_);
-  DCHECK(reader->Validate()) << endl << reader->DebugString();
-  if (reader->state_ == RequestContext::Cancelled) return Status::CANCELLED;
-
-  DCHECK_NE(range->len(), 0);
-  if (range->try_cache_) {
-    bool cached_read_succeeded;
-    RETURN_IF_ERROR(range->ReadFromCache(reader_lock, &cached_read_succeeded));
-    if (cached_read_succeeded) {
-      DCHECK(reader->Validate()) << endl << reader->DebugString();
-      *needs_buffers = false;
-      return Status::OK();
-    }
-    // Cached read failed, fall back to normal read path.
-  }
-  // If we don't have a buffer yet, the caller must allocate buffers for the range.
-  *needs_buffers = range->external_buffer_tag_ == ScanRange::ExternalBufferTag::NO_BUFFER;
-  if (*needs_buffers) range->SetBlockedOnBuffer();
-  reader->AddActiveScanRangeLocked(reader_lock, range);
-  reader->AddRangeToDisk(reader_lock, range,
-      *needs_buffers ? ScheduleMode::BY_CALLER : ScheduleMode::IMMEDIATELY);
-  DCHECK(reader->Validate()) << endl << reader->DebugString();
-  return Status::OK();
+Status DiskIoMgr::AddScanRange(
+    RequestContext* reader, ScanRange* range, bool schedule_immediately) {
+  return AddScanRanges(reader, vector<ScanRange*>({range}), schedule_immediately);
 }
 
 // This function returns the next scan range the reader should work on, checking
 // for eos and error cases. If there isn't already a cached scan range or a scan
 // range prepared by the disk threads, the caller waits on the disk threads.
-Status DiskIoMgr::GetNextUnstartedRange(RequestContext* reader, ScanRange** range,
-    bool* needs_buffers) {
+Status DiskIoMgr::GetNextRange(RequestContext* reader, ScanRange** range) {
   DCHECK(reader != nullptr);
   DCHECK(range != nullptr);
   *range = nullptr;
-  *needs_buffers = false;
+  Status status = Status::OK();
 
   unique_lock<mutex> reader_lock(reader->lock_);
   DCHECK(reader->Validate()) << endl << reader->DebugString();
+
   while (true) {
-    if (reader->state_ == RequestContext::Cancelled) return Status::CANCELLED;
+    if (reader->state_ == RequestContext::Cancelled) {
+      DCHECK(!reader->status_.ok());
+      status = reader->status_;
+      break;
+    }
 
     if (reader->num_unstarted_scan_ranges_.Load() == 0 &&
         reader->ready_to_start_ranges_.empty() && reader->cached_ranges_.empty()) {
       // All ranges are done, just return.
-      return Status::OK();
+      break;
     }
 
     if (!reader->cached_ranges_.empty()) {
@@ -425,7 +555,7 @@ Status DiskIoMgr::GetNextUnstartedRange(RequestContext* reader, ScanRange** rang
       if (cached_read_succeeded) return Status::OK();
 
       // This range ended up not being cached. Loop again and pick up a new range.
-      reader->AddRangeToDisk(reader_lock, *range, ScheduleMode::UPON_GETNEXT);
+      reader->AddRequestRange(*range, false);
       DCHECK(reader->Validate()) << endl << reader->DebugString();
       *range = nullptr;
       continue;
@@ -441,72 +571,183 @@ Status DiskIoMgr::GetNextUnstartedRange(RequestContext* reader, ScanRange** rang
       // Set this to nullptr, the next time this disk runs for this reader, it will
       // get another range ready.
       reader->disk_states_[disk_id].set_next_scan_range_to_start(nullptr);
-      ScanRange::ExternalBufferTag buffer_tag = (*range)->external_buffer_tag_;
-      if (buffer_tag == ScanRange::ExternalBufferTag::NO_BUFFER) {
-        // We can't schedule this range until the client gives us buffers. The context
-        // must be rescheduled regardless to ensure that 'next_scan_range_to_start' is
-        // refilled.
-        reader->disk_states_[disk_id].ScheduleContext(reader_lock, reader, disk_id);
-        (*range)->SetBlockedOnBuffer();
-        *needs_buffers = true;
-      } else {
-        reader->ScheduleScanRange(reader_lock, *range);
-      }
-      return Status::OK();
+      reader->ScheduleScanRange(*range);
+      break;
     }
   }
+  return status;
 }
 
-Status DiskIoMgr::AllocateBuffersForRange(RequestContext* reader,
-    BufferPool::ClientHandle* bp_client, ScanRange* range, int64_t max_bytes) {
-  DCHECK_GE(max_bytes, min_buffer_size_);
-  DCHECK(range->external_buffer_tag_ == ScanRange::ExternalBufferTag::NO_BUFFER)
-     << static_cast<int>(range->external_buffer_tag_) << " invalid to allocate buffers "
-     << "when already reading into an external buffer";
-  BufferPool* bp = ExecEnv::GetInstance()->buffer_pool();
-  Status status;
-  vector<unique_ptr<BufferDescriptor>> buffers;
-  for (int64_t buffer_size : ChooseBufferSizes(range->len(), max_bytes)) {
-    BufferPool::BufferHandle handle;
-    status = bp->AllocateBuffer(bp_client, buffer_size, &handle);
-    if (!status.ok()) goto error;
-    buffers.emplace_back(new BufferDescriptor(
-        this, reader, range, bp_client, move(handle)));
+Status DiskIoMgr::Read(RequestContext* reader,
+    ScanRange* range, std::unique_ptr<BufferDescriptor>* buffer) {
+  DCHECK(range != nullptr);
+  DCHECK(buffer != nullptr);
+  *buffer = nullptr;
+
+  if (range->len() > max_buffer_size_
+      && range->external_buffer_tag_ != ScanRange::ExternalBufferTag::CLIENT_BUFFER) {
+    return Status(TErrorCode::DISK_IO_ERROR, Substitute("Internal error: cannot "
+        "perform sync read of '$0' bytes that is larger than the max read buffer size "
+        "'$1'.", range->len(), max_buffer_size_));
   }
-  range->AddUnusedBuffers(move(buffers), false);
+
+  vector<ScanRange*> ranges;
+  ranges.push_back(range);
+  RETURN_IF_ERROR(AddScanRanges(reader, ranges, true));
+  RETURN_IF_ERROR(range->GetNext(buffer));
+  DCHECK((*buffer) != nullptr);
+  DCHECK((*buffer)->eosr());
   return Status::OK();
- error:
-  DCHECK(!status.ok());
-  range->CleanUpBuffers(move(buffers));
-  return status;
 }
 
-vector<int64_t> DiskIoMgr::ChooseBufferSizes(int64_t scan_range_len, int64_t max_bytes) {
-  DCHECK_GE(max_bytes, min_buffer_size_);
-  vector<int64_t> buffer_sizes;
-  int64_t bytes_allocated = 0;
-  while (bytes_allocated < scan_range_len) {
-    int64_t bytes_remaining = scan_range_len - bytes_allocated;
-    // Either allocate a max-sized buffer or a smaller buffer to fit the rest of the
-    // range.
-    int64_t next_buffer_size;
-    if (bytes_remaining >= max_buffer_size_) {
-      next_buffer_size = max_buffer_size_;
+void DiskIoMgr::ReturnBuffer(unique_ptr<BufferDescriptor> buffer_desc) {
+  DCHECK(buffer_desc != nullptr);
+  if (!buffer_desc->status_.ok()) DCHECK(buffer_desc->buffer_ == nullptr);
+
+  RequestContext* reader = buffer_desc->reader_;
+  if (buffer_desc->buffer_ != nullptr) {
+    if (!buffer_desc->is_cached() && !buffer_desc->is_client_buffer()) {
+      // Buffers the were not allocated by DiskIoMgr don't need to be freed.
+      FreeBufferMemory(buffer_desc.get());
+    }
+    buffer_desc->buffer_ = nullptr;
+    num_buffers_in_readers_.Add(-1);
+    reader->num_buffers_in_reader_.Add(-1);
+  } else {
+    // A nullptr buffer means there was an error in which case there is no buffer
+    // to return.
+  }
+
+  if (buffer_desc->eosr_ || buffer_desc->scan_range_->is_cancelled_) {
+    // Need to close the scan range if returning the last buffer or the scan range
+    // has been cancelled (and the caller might never get the last buffer).
+    // Close() is idempotent so multiple cancelled buffers is okay.
+    buffer_desc->scan_range_->Close();
+  }
+}
+
+unique_ptr<BufferDescriptor> DiskIoMgr::GetFreeBuffer(
+    RequestContext* reader, ScanRange* range, int64_t buffer_size) {
+  DCHECK_LE(buffer_size, max_buffer_size_);
+  DCHECK_GT(buffer_size, 0);
+  buffer_size = min(static_cast<int64_t>(max_buffer_size_), buffer_size);
+  int idx = free_buffers_idx(buffer_size);
+  // Quantize buffer size to nearest power of 2 greater than the specified buffer size and
+  // convert to bytes
+  buffer_size = (1LL << idx) * min_buffer_size_;
+
+  // Track memory against the reader. This is checked the next time we start
+  // a read for the next reader in DiskIoMgr::GetNextScanRange().
+  DCHECK(reader->mem_tracker_ != nullptr);
+  reader->mem_tracker_->Consume(buffer_size);
+
+  uint8_t* buffer = nullptr;
+  {
+    unique_lock<mutex> lock(free_buffers_lock_);
+    if (free_buffers_[idx].empty()) {
+      num_allocated_buffers_.Add(1);
+      if (ImpaladMetrics::IO_MGR_NUM_BUFFERS != nullptr) {
+        ImpaladMetrics::IO_MGR_NUM_BUFFERS->Increment(1L);
+      }
+      if (ImpaladMetrics::IO_MGR_TOTAL_BYTES != nullptr) {
+        ImpaladMetrics::IO_MGR_TOTAL_BYTES->Increment(buffer_size);
+      }
+      // We already tracked this memory against the reader's MemTracker.
+      buffer = new uint8_t[buffer_size];
     } else {
-      next_buffer_size =
-          max(min_buffer_size_, BitUtil::RoundUpToPowerOfTwo(bytes_remaining));
+      if (ImpaladMetrics::IO_MGR_NUM_UNUSED_BUFFERS != nullptr) {
+        ImpaladMetrics::IO_MGR_NUM_UNUSED_BUFFERS->Increment(-1L);
+      }
+      buffer = free_buffers_[idx].front();
+      free_buffers_[idx].pop_front();
+      free_buffer_mem_tracker_->Release(buffer_size);
+      ASAN_UNPOISON_MEMORY_REGION(buffer, buffer_size);
     }
-    if (next_buffer_size + bytes_allocated > max_bytes) {
-      // Can't allocate the desired buffer size. Make sure to allocate at least one
-      // buffer.
-      if (bytes_allocated > 0) break;
-      next_buffer_size = BitUtil::RoundDownToPowerOfTwo(max_bytes);
+  }
+
+  // Validate more invariants.
+  DCHECK(range != nullptr);
+  DCHECK(reader != nullptr);
+  DCHECK(buffer != nullptr);
+  return unique_ptr<BufferDescriptor>(new BufferDescriptor(
+      this, reader, range, buffer, buffer_size, reader->mem_tracker_));
+}
+
+void DiskIoMgr::GcIoBuffers(int64_t bytes_to_free) {
+  unique_lock<mutex> lock(free_buffers_lock_);
+  int buffers_freed = 0;
+  int bytes_freed = 0;
+  // Free small-to-large to avoid retaining many small buffers and fragmenting memory.
+  for (int idx = 0; idx < free_buffers_.size(); ++idx) {
+    deque<uint8_t*>* free_buffers = &free_buffers_[idx];
+    while (
+        !free_buffers->empty() && (bytes_to_free == -1 || bytes_freed <= bytes_to_free)) {
+      uint8_t* buffer = free_buffers->front();
+      free_buffers->pop_front();
+      int64_t buffer_size = (1LL << idx) * min_buffer_size_;
+      ASAN_UNPOISON_MEMORY_REGION(buffer, buffer_size);
+      delete[] buffer;
+      free_buffer_mem_tracker_->Release(buffer_size);
+      num_allocated_buffers_.Add(-1);
+
+      ++buffers_freed;
+      bytes_freed += buffer_size;
+    }
+    if (bytes_to_free != -1 && bytes_freed >= bytes_to_free) break;
+  }
+
+  if (ImpaladMetrics::IO_MGR_NUM_BUFFERS != nullptr) {
+    ImpaladMetrics::IO_MGR_NUM_BUFFERS->Increment(-buffers_freed);
+  }
+  if (ImpaladMetrics::IO_MGR_TOTAL_BYTES != nullptr) {
+    ImpaladMetrics::IO_MGR_TOTAL_BYTES->Increment(-bytes_freed);
+  }
+  if (ImpaladMetrics::IO_MGR_NUM_UNUSED_BUFFERS != nullptr) {
+    ImpaladMetrics::IO_MGR_NUM_UNUSED_BUFFERS->Increment(-buffers_freed);
+  }
+}
+
+void DiskIoMgr::FreeBufferMemory(BufferDescriptor* desc) {
+  DCHECK(!desc->is_cached());
+  DCHECK(!desc->is_client_buffer());
+  uint8_t* buffer = desc->buffer_;
+  int64_t buffer_size = desc->buffer_len_;
+  int idx = free_buffers_idx(buffer_size);
+  DCHECK_EQ(BitUtil::Ceil(buffer_size, min_buffer_size_) & ~(1LL << idx), 0)
+      << "buffer_size_ / min_buffer_size_ should be power of 2, got buffer_size = "
+      << buffer_size << ", min_buffer_size_ = " << min_buffer_size_;
+
+  {
+    unique_lock<mutex> lock(free_buffers_lock_);
+    if (!FLAGS_disable_mem_pools &&
+        free_buffers_[idx].size() < FLAGS_max_free_io_buffers) {
+      // Poison buffers stored in cache.
+      ASAN_POISON_MEMORY_REGION(buffer, buffer_size);
+      free_buffers_[idx].push_back(buffer);
+      if (ImpaladMetrics::IO_MGR_NUM_UNUSED_BUFFERS != nullptr) {
+        ImpaladMetrics::IO_MGR_NUM_UNUSED_BUFFERS->Increment(1L);
+      }
+      // This consume call needs to be protected by 'free_buffers_lock_' to avoid a race
+      // with a Release() call for the same buffer that could make consumption negative.
+      // Note: we can't use TryConsume(), which can indirectly call GcIoBuffers().
+      // TODO: after IMPALA-3200 is completed, we should be able to leverage the buffer
+      // pool's free lists, and remove these free lists.
+      free_buffer_mem_tracker_->Consume(buffer_size);
+    } else {
+      num_allocated_buffers_.Add(-1);
+      delete[] buffer;
+      if (ImpaladMetrics::IO_MGR_NUM_BUFFERS != nullptr) {
+        ImpaladMetrics::IO_MGR_NUM_BUFFERS->Increment(-1L);
+      }
+      if (ImpaladMetrics::IO_MGR_TOTAL_BYTES != nullptr) {
+        ImpaladMetrics::IO_MGR_TOTAL_BYTES->Increment(-buffer_size);
+      }
     }
-    DCHECK(BitUtil::IsPowerOf2(next_buffer_size)) << next_buffer_size;
-    buffer_sizes.push_back(next_buffer_size);
-    bytes_allocated += next_buffer_size;
   }
-  return buffer_sizes;
+
+  // We transferred the buffer ownership from the BufferDescriptor to the DiskIoMgr.
+  desc->mem_tracker_->Release(buffer_size);
+  desc->buffer_ = nullptr;
 }
 
 // This function gets the next RequestRange to work on for this disk. It checks for
@@ -546,7 +787,7 @@ bool DiskIoMgr::GetNextRequestRange(DiskQueue* disk_queue, RequestRange** range,
       disk_queue->request_contexts.pop_front();
       DCHECK(*request_context != nullptr);
       request_disk_state = &((*request_context)->disk_states_[disk_id]);
-      request_disk_state->IncrementDiskThreadAndDequeue();
+      request_disk_state->IncrementRequestThreadAndDequeue();
     }
 
     // NOTE: no locks were taken in between.  We need to be careful about what state
@@ -554,13 +795,27 @@ bool DiskIoMgr::GetNextRequestRange(DiskQueue* disk_queue, RequestRange** range,
     // There are some invariants here.  Only one disk thread can have the
     // same reader here (the reader is removed from the queue).  There can be
     // other disk threads operating on this reader in other functions though.
+
+    // We just picked a reader. Before we may allocate a buffer on its behalf, check that
+    // it has not exceeded any memory limits (e.g. the query or process limit).
+    // TODO: once IMPALA-3200 is fixed, we should be able to remove the free lists and
+    // move these memory limit checks to GetFreeBuffer().
+    // Note that calling AnyLimitExceeded() can result in a call to GcIoBuffers().
+    // TODO: IMPALA-3209: we should not force a reader over its memory limit by
+    // pushing more buffers to it. Most readers can make progress and operate within
+    // a fixed memory limit.
+    if ((*request_context)->mem_tracker_ != nullptr
+        && (*request_context)->mem_tracker_->AnyLimitExceeded()) {
+      (*request_context)->Cancel(Status::MemLimitExceeded());
+    }
+
     unique_lock<mutex> request_lock((*request_context)->lock_);
     VLOG_FILE << "Disk (id=" << disk_id << ") reading for "
         << (*request_context)->DebugString();
 
     // Check if reader has been cancelled
     if ((*request_context)->state_ == RequestContext::Cancelled) {
-      request_disk_state->DecrementDiskThread(request_lock, *request_context);
+      request_disk_state->DecrementRequestThreadAndCheckDone(*request_context);
       continue;
     }
 
@@ -571,16 +826,16 @@ bool DiskIoMgr::GetNextRequestRange(DiskQueue* disk_queue, RequestRange** range,
         !request_disk_state->unstarted_scan_ranges()->empty()) {
       // We don't have a range queued for this disk for what the caller should
       // read next. Populate that.  We want to have one range waiting to minimize
-      // wait time in GetNextUnstartedRange().
+      // wait time in GetNextRange.
       ScanRange* new_range = request_disk_state->unstarted_scan_ranges()->Dequeue();
       (*request_context)->num_unstarted_scan_ranges_.Add(-1);
       (*request_context)->ready_to_start_ranges_.Enqueue(new_range);
       request_disk_state->set_next_scan_range_to_start(new_range);
 
       if ((*request_context)->num_unstarted_scan_ranges_.Load() == 0) {
-        // All the ranges have been started, notify everyone blocked on
-        // GetNextUnstartedRange(). Only one of them will get work so make sure to return
-        // nullptr to the other caller threads.
+        // All the ranges have been started, notify everyone blocked on GetNextRange.
+        // Only one of them will get work so make sure to return nullptr to the other
+        // caller threads.
         (*request_context)->ready_to_start_ranges_cv_.NotifyAll();
       } else {
         (*request_context)->ready_to_start_ranges_cv_.NotifyOne();
@@ -603,7 +858,7 @@ bool DiskIoMgr::GetNextRequestRange(DiskQueue* disk_queue, RequestRange** range,
 
     // There are no inflight ranges, nothing to do.
     if (request_disk_state->in_flight_ranges()->empty()) {
-      request_disk_state->DecrementDiskThread(request_lock, *request_context);
+      request_disk_state->DecrementRequestThread();
       continue;
     }
     DCHECK_GT(request_disk_state->num_remaining_ranges(), 0);
@@ -612,7 +867,7 @@ bool DiskIoMgr::GetNextRequestRange(DiskQueue* disk_queue, RequestRange** range,
 
     // Now that we've picked a request range, put the context back on the queue so
     // another thread can pick up another request range for this context.
-    request_disk_state->ScheduleContext(request_lock, *request_context, disk_id);
+    request_disk_state->ScheduleContext(*request_context, disk_id);
     DCHECK((*request_context)->Validate()) << endl << (*request_context)->DebugString();
     return true;
   }
@@ -626,62 +881,81 @@ void DiskIoMgr::HandleWriteFinished(
   // Copy disk_id before running callback: the callback may modify write_range.
   int disk_id = write_range->disk_id_;
 
-  // Execute the callback before decrementing the thread count. Otherwise
-  // RequestContext::Cancel() that waits for the disk ref count to be 0 will
-  // return, creating a race, e.g. see IMPALA-1890.
+  // Execute the callback before decrementing the thread count. Otherwise CancelContext()
+  // that waits for the disk ref count to be 0 will return, creating a race, e.g. see
+  // IMPALA-1890.
   // The status of the write does not affect the status of the writer context.
   write_range->callback_(write_status);
   {
     unique_lock<mutex> writer_lock(writer->lock_);
     DCHECK(writer->Validate()) << endl << writer->DebugString();
     RequestContext::PerDiskState& state = writer->disk_states_[disk_id];
-    state.DecrementDiskThread(writer_lock, writer);
+    if (writer->state_ == RequestContext::Cancelled) {
+      state.DecrementRequestThreadAndCheckDone(writer);
+    } else {
+      state.DecrementRequestThread();
+    }
     --state.num_remaining_ranges();
   }
 }
 
 void DiskIoMgr::HandleReadFinished(DiskQueue* disk_queue, RequestContext* reader,
-    Status read_status, unique_ptr<BufferDescriptor> buffer) {
+    unique_ptr<BufferDescriptor> buffer) {
   unique_lock<mutex> reader_lock(reader->lock_);
 
-  RequestContext::PerDiskState* disk_state = &reader->disk_states_[disk_queue->disk_id];
+  RequestContext::PerDiskState& state = reader->disk_states_[disk_queue->disk_id];
   DCHECK(reader->Validate()) << endl << reader->DebugString();
-  DCHECK_GT(disk_state->num_threads_in_op(), 0);
+  DCHECK_GT(state.num_threads_in_op(), 0);
   DCHECK(buffer->buffer_ != nullptr);
-  DCHECK(!buffer->is_cached()) << "HDFS cache reads don't go through this code path.";
 
-  // After calling EnqueueReadyBuffer() below, it is no longer valid to read from buffer.
-  // Store the state we need before calling EnqueueReadyBuffer().
-  bool eosr = buffer->eosr_;
+  if (reader->state_ == RequestContext::Cancelled) {
+    state.DecrementRequestThreadAndCheckDone(reader);
+    DCHECK(reader->Validate()) << endl << reader->DebugString();
+    if (!buffer->is_client_buffer()) FreeBufferMemory(buffer.get());
+    buffer->buffer_ = nullptr;
+    ScanRange* scan_range = buffer->scan_range_;
+    scan_range->Cancel(reader->status_);
+    // Enqueue the buffer to use the scan range's buffer cleanup path.
+    scan_range->EnqueueBuffer(reader_lock, move(buffer));
+    return;
+  }
+
+  DCHECK_EQ(reader->state_, RequestContext::Active);
+  DCHECK(buffer->buffer_ != nullptr);
 
-  // TODO: IMPALA-4249: it safe to touch 'scan_range' until DecrementDiskThread() is
-  // called because all clients of DiskIoMgr keep ScanRange objects alive until they
-  // unregister their RequestContext.
+  // Update the reader's scan ranges.  There are a three cases here:
+  //  1. Read error
+  //  2. End of scan range
+  //  3. Middle of scan range
+  if (!buffer->status_.ok()) {
+    // Error case
+    if (!buffer->is_client_buffer()) FreeBufferMemory(buffer.get());
+    buffer->buffer_ = nullptr;
+    buffer->eosr_ = true;
+    --state.num_remaining_ranges();
+    buffer->scan_range_->Cancel(buffer->status_);
+  } else if (buffer->eosr_) {
+    --state.num_remaining_ranges();
+  }
+
+  // After calling EnqueueBuffer(), it is no longer valid to read from buffer.
+  // Store the state we need before calling EnqueueBuffer().
+  bool eosr = buffer->eosr_;
   ScanRange* scan_range = buffer->scan_range_;
-  bool scan_range_done = eosr;
-  if (read_status.ok() && reader->state_ != RequestContext::Cancelled) {
-    DCHECK_EQ(reader->state_, RequestContext::Active);
-    // Read successfully - update the reader's scan ranges.  There are two cases here:
-    //  1. End of scan range or cancelled scan range - don't need to reschedule.
-    //  2. Middle of scan range - need to schedule to read next buffer.
-    bool enqueued = scan_range->EnqueueReadyBuffer(reader_lock, move(buffer));
-    if (!eosr && enqueued) reader->ScheduleScanRange(reader_lock, scan_range);
+  bool is_cached = buffer->is_cached();
+  bool queue_full = scan_range->EnqueueBuffer(reader_lock, move(buffer));
+  if (eosr) {
+    // For cached buffers, we can't close the range until the cached buffer is returned.
+    // Close() is called from DiskIoMgr::ReturnBuffer().
+    if (!is_cached) scan_range->Close();
   } else {
-    // The scan range will be cancelled, either because we hit an error or because the
-    // request context was cancelled.  The buffer is not needed - we must free it.
-    reader->FreeBuffer(buffer.get());
-    // Propagate 'read_status' to the scan range. If we are here because the context
-    // was cancelled, the scan range is already cancelled so we do not need to re-cancel
-    // it.
-    if (!read_status.ok()) scan_range->CancelFromReader(reader_lock, read_status);
-    scan_range_done = true;
-  }
-  if (scan_range_done) {
-    scan_range->Close();
-    --disk_state->num_remaining_ranges();
+    if (queue_full) {
+      reader->blocked_ranges_.Enqueue(scan_range);
+    } else {
+      reader->ScheduleScanRange(scan_range);
+    }
   }
-  DCHECK(reader->Validate()) << endl << reader->DebugString();
-  disk_state->DecrementDiskThread(reader_lock, reader);
+  state.DecrementRequestThread();
 }
 
 void DiskIoMgr::WorkLoop(DiskQueue* disk_queue) {
@@ -697,12 +971,14 @@ void DiskIoMgr::WorkLoop(DiskQueue* disk_queue) {
   //   3. Perform the read or write as specified.
   // Cancellation checking needs to happen in both steps 1 and 3.
   while (true) {
-    RequestContext* worker_context = nullptr;
+    RequestContext* worker_context = nullptr;;
     RequestRange* range = nullptr;
+
     if (!GetNextRequestRange(disk_queue, &range, &worker_context)) {
       DCHECK(shut_down_);
-      return;
+      break;
     }
+
     if (range->request_type() == RequestType::READ) {
       ReadRange(disk_queue, worker_context, static_cast<ScanRange*>(range));
     } else {
@@ -710,8 +986,12 @@ void DiskIoMgr::WorkLoop(DiskQueue* disk_queue) {
       Write(worker_context, static_cast<WriteRange*>(range));
     }
   }
+
+  DCHECK(shut_down_);
 }
 
+// This function reads the specified scan range associated with the
+// specified reader context and disk queue.
 void DiskIoMgr::ReadRange(
     DiskQueue* disk_queue, RequestContext* reader, ScanRange* range) {
   int64_t bytes_remaining = range->len_ - range->bytes_read_;
@@ -719,40 +999,86 @@ void DiskIoMgr::ReadRange(
   unique_ptr<BufferDescriptor> buffer_desc;
   if (range->external_buffer_tag_ == ScanRange::ExternalBufferTag::CLIENT_BUFFER) {
     buffer_desc = unique_ptr<BufferDescriptor>(new BufferDescriptor(this, reader, range,
-        range->client_buffer_.data, range->client_buffer_.len));
+        range->client_buffer_.data, range->client_buffer_.len, nullptr));
   } else {
-    DCHECK(range->external_buffer_tag_ == ScanRange::ExternalBufferTag::NO_BUFFER)
-        << "This code path does not handle other buffer types, i.e. HDFS cache"
-        << static_cast<int>(range->external_buffer_tag_);
-    buffer_desc = range->GetNextUnusedBufferForRange();
-    if (buffer_desc == nullptr) {
-      // No buffer available - the range will be rescheduled when a buffer is added.
-      unique_lock<mutex> reader_lock(reader->lock_);
-      reader->disk_states_[disk_queue->disk_id].DecrementDiskThread(reader_lock, reader);
-      DCHECK(reader->Validate()) << endl << reader->DebugString();
-      return;
-    }
+    // Need to allocate a buffer to read into.
+    int64_t buffer_size = ::min(bytes_remaining, static_cast<int64_t>(max_buffer_size_));
+    buffer_desc = TryAllocateNextBufferForRange(disk_queue, reader, range, buffer_size);
+    if (buffer_desc == nullptr) return;
   }
+  reader->num_used_buffers_.Add(1);
 
   // No locks in this section.  Only working on local vars.  We don't want to hold a
   // lock across the read call.
-  Status read_status = range->Open(detail::is_file_handle_caching_enabled());
-  if (read_status.ok()) {
+  buffer_desc->status_ = range->Open(detail::is_file_handle_caching_enabled());
+  if (buffer_desc->status_.ok()) {
     // Update counters.
-    COUNTER_ADD_IF_NOT_NULL(reader->active_read_thread_counter_, 1L);
-    COUNTER_BITOR_IF_NOT_NULL(reader->disks_accessed_bitmap_, 1LL << disk_queue->disk_id);
+    if (reader->active_read_thread_counter_) {
+      reader->active_read_thread_counter_->Add(1L);
+    }
+    if (reader->disks_accessed_bitmap_) {
+      int64_t disk_bit = 1LL << disk_queue->disk_id;
+      reader->disks_accessed_bitmap_->BitOr(disk_bit);
+    }
 
-    read_status = range->Read(buffer_desc->buffer_, buffer_desc->buffer_len_,
+    buffer_desc->status_ = range->Read(buffer_desc->buffer_, buffer_desc->buffer_len_,
         &buffer_desc->len_, &buffer_desc->eosr_);
     buffer_desc->scan_range_offset_ = range->bytes_read_ - buffer_desc->len_;
 
-    COUNTER_ADD_IF_NOT_NULL(reader->bytes_read_counter_, buffer_desc->len_);
+    if (reader->bytes_read_counter_ != nullptr) {
+      COUNTER_ADD(reader->bytes_read_counter_, buffer_desc->len_);
+    }
+
     COUNTER_ADD(&total_bytes_read_counter_, buffer_desc->len_);
-    COUNTER_ADD_IF_NOT_NULL(reader->active_read_thread_counter_, -1L);
+    if (reader->active_read_thread_counter_) {
+      reader->active_read_thread_counter_->Add(-1L);
+    }
   }
 
   // Finished read, update reader/disk based on the results
-  HandleReadFinished(disk_queue, reader, read_status, move(buffer_desc));
+  HandleReadFinished(disk_queue, reader, move(buffer_desc));
+}
+
+unique_ptr<BufferDescriptor> DiskIoMgr::TryAllocateNextBufferForRange(
+    DiskQueue* disk_queue, RequestContext* reader, ScanRange* range,
+    int64_t buffer_size) {
+  DCHECK(reader->mem_tracker_ != nullptr);
+  bool enough_memory = reader->mem_tracker_->SpareCapacity() > LOW_MEMORY;
+  if (!enough_memory) {
+    // Low memory, GC all the buffers and try again.
+    GcIoBuffers();
+    enough_memory = reader->mem_tracker_->SpareCapacity() > LOW_MEMORY;
+  }
+
+  if (!enough_memory) {
+    RequestContext::PerDiskState& state = reader->disk_states_[disk_queue->disk_id];
+    unique_lock<mutex> reader_lock(reader->lock_);
+
+    // Just grabbed the reader lock, check for cancellation.
+    if (reader->state_ == RequestContext::Cancelled) {
+      DCHECK(reader->Validate()) << endl << reader->DebugString();
+      state.DecrementRequestThreadAndCheckDone(reader);
+      range->Cancel(reader->status_);
+      DCHECK(reader->Validate()) << endl << reader->DebugString();
+      return nullptr;
+    }
+
+    if (!range->ready_buffers_.empty()) {
+      // We have memory pressure and this range doesn't need another buffer
+      // (it already has one queued). Skip this range and pick it up later.
+      range->blocked_on_queue_ = true;
+      reader->blocked_ranges_.Enqueue(range);
+      state.DecrementRequestThread();
+      return nullptr;
+    } else {
+      // We need to get a buffer anyway since there are none queued. The query
+      // is likely to fail due to mem limits but there's nothing we can do about that
+      // now.
+    }
+  }
+  unique_ptr<BufferDescriptor> buffer_desc = GetFreeBuffer(reader, range, buffer_size);
+  DCHECK(buffer_desc != nullptr);
+  return buffer_desc;
 }
 
 void DiskIoMgr::Write(RequestContext* writer_context, WriteRange* write_range) {
@@ -806,14 +1132,30 @@ Status DiskIoMgr::WriteRangeHelper(FILE* file_handle, WriteRange* write_range) {
         Substitute("fwrite(buffer, 1, $0, $1) failed with errno=$2 description=$3",
         write_range->len_, write_range->file_, errno, GetStrErrMsg())));
   }
-  ImpaladMetrics::IO_MGR_BYTES_WRITTEN->Increment(write_range->len_);
+  if (ImpaladMetrics::IO_MGR_BYTES_WRITTEN != nullptr) {
+    ImpaladMetrics::IO_MGR_BYTES_WRITTEN->Increment(write_range->len_);
+  }
+
   return Status::OK();
 }
 
+int DiskIoMgr::free_buffers_idx(int64_t buffer_size) {
+  int64_t buffer_size_scaled = BitUtil::Ceil(buffer_size, min_buffer_size_);
+  int idx = BitUtil::Log2Ceiling64(buffer_size_scaled);
+  DCHECK_GE(idx, 0);
+  DCHECK_LT(idx, free_buffers_.size());
+  return idx;
+}
+
 Status DiskIoMgr::AddWriteRange(RequestContext* writer, WriteRange* write_range) {
   unique_lock<mutex> writer_lock(writer->lock_);
-  if (writer->state_ == RequestContext::Cancelled) return Status::CANCELLED;
-  writer->AddRangeToDisk(writer_lock, write_range, ScheduleMode::IMMEDIATELY);
+
+  if (writer->state_ == RequestContext::Cancelled) {
+    DCHECK(!writer->status_.ok());
+    return writer->status_;
+  }
+
+  writer->AddRequestRange(write_range, false);
   return Status::OK();
 }
 

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr.h b/be/src/runtime/io/disk-io-mgr.h
index 2b6881b..e5d7eb4 100644
--- a/be/src/runtime/io/disk-io-mgr.h
+++ b/be/src/runtime/io/disk-io-mgr.h
@@ -30,7 +30,6 @@
 #include "common/hdfs.h"
 #include "common/object-pool.h"
 #include "common/status.h"
-#include "runtime/bufferpool/buffer-pool.h"
 #include "runtime/io/handle-cache.h"
 #include "runtime/io/request-ranges.h"
 #include "runtime/thread-resource-mgr.h"
@@ -43,19 +42,21 @@
 
 namespace impala {
 
+class MemTracker;
+
 namespace io {
 /// Manager object that schedules IO for all queries on all disks and remote filesystems
 /// (such as S3). Each query maps to one or more RequestContext objects, each of which
 /// has its own queue of scan ranges and/or write ranges.
-///
+//
 /// The API splits up requesting scan/write ranges (non-blocking) and reading the data
 /// (blocking). The DiskIoMgr has worker threads that will read from and write to
 /// disk/hdfs/remote-filesystems, allowing interleaving of IO and CPU. This allows us to
 /// keep all disks and all cores as busy as possible.
-///
+//
 /// All public APIs are thread-safe. It is not valid to call any of the APIs after
 /// UnregisterContext() returns.
-///
+//
 /// For Readers:
 /// We can model this problem as a multiple producer (threads for each disk), multiple
 /// consumer (scan ranges) problem. There are multiple queues that need to be
@@ -67,102 +68,84 @@ namespace io {
 /// Readers map to scan nodes. The reader then contains a queue of scan ranges. The caller
 /// asks the IoMgr for the next range to process. The IoMgr then selects the best range
 /// to read based on disk activity and begins reading and queuing buffers for that range.
-///
+/// TODO: We should map readers to queries. A reader is the unit of scheduling and queries
+/// that have multiple scan nodes shouldn't have more 'turns'.
+//
 /// For Writers:
 /// Data is written via AddWriteRange(). This is non-blocking and adds a WriteRange to a
 /// per-disk queue. After the write is complete, a callback in WriteRange is invoked.
 /// No memory is allocated within IoMgr for writes and no copies are made. It is the
 /// responsibility of the client to ensure that the data to be written is valid and that
 /// the file to be written to exists until the callback is invoked.
-///
-/// There are several key methods for scanning data with the IoMgr.
-///  1. StartScanRange(): adds range to the IoMgr to start immediately.
-///  2. AddScanRanges(): adds ranges to the IoMgr that the reader wants to scan, but does
-///     not start them until GetNextUnstartedRange() is called.
-///  3. GetNextUnstartedRange(): returns to the caller the next scan range it should
-///     process.
-///  4. ScanRange::GetNext(): returns the next buffer for this range, blocking until
-///     data is available.
-///
+//
+/// The IoMgr provides three key APIs.
+///  1. AddScanRanges: this is non-blocking and tells the IoMgr all the ranges that
+///     will eventually need to be read.
+///  2. GetNextRange: returns to the caller the next scan range it should process.
+///     This is based on disk load. This also begins reading the data in this scan
+///     range. This is blocking.
+///  3. ScanRange::GetNext: returns the next buffer for this range.  This is blocking.
+//
 /// The disk threads do not synchronize with each other. The readers and writers don't
 /// synchronize with each other. There is a lock and condition variable for each request
 /// context queue and each disk queue.
 /// IMPORTANT: whenever both locks are needed, the lock order is to grab the context lock
 /// before the disk lock.
-///
+//
 /// Scheduling: If there are multiple request contexts with work for a single disk, the
 /// request contexts are scheduled in round-robin order. Multiple disk threads can
 /// operate on the same request context. Exactly one request range is processed by a
-/// disk thread at a time. If there are multiple scan ranges scheduled for a single
-/// context, these are processed in round-robin order.
+/// disk thread at a time. If there are multiple scan ranges scheduled via
+/// GetNextRange() for a single context, these are processed in round-robin order.
 /// If there are multiple scan and write ranges for a disk, a read is always followed
 /// by a write, and a write is followed by a read, i.e. reads and writes alternate.
 /// If multiple write ranges are enqueued for a single disk, they will be processed
 /// by the disk threads in order, but may complete in any order. No guarantees are made
 /// on ordering of writes across disks.
-///
-/// Resource Management: the IoMgr is designed to share the available disk I/O capacity
-/// between many clients and to help use the available I/O capacity efficiently. The IoMgr
-/// interfaces are designed to let clients manage their own CPU and memory usage while the
-/// IoMgr manages the allocation of the I/O capacity of different I/O devices to scan
-/// ranges of different clients.
-///
-/// IoMgr clients may want to work on multiple scan ranges at a time to maximize CPU and
-/// I/O utilization. Clients can call GetNextUnstartedRange() to start as many concurrent
-/// scan ranges as required, e.g. from each parallel scanner thread. Once a scan range has
-/// been returned via GetNextUnstartedRange(), the caller must allocate any memory needed
-/// for buffering reads, after which the IoMgr wil start to fill the buffers with data
-/// while the caller concurrently consumes and processes the data. For example, the logic
-/// in a scanner thread might look like:
+//
+/// Resource Management: effective resource management in the IoMgr is key to good
+/// performance. The IoMgr helps coordinate two resources: CPU and disk. For CPU,
+/// spinning up too many threads causes thrashing.
+/// Memory usage in the IoMgr comes from queued read buffers.  If we queue the minimum
+/// (i.e. 1), then the disks are idle while we are processing the buffer. If we don't
+/// limit the queue, then it possible we end up queueing the entire data set (i.e. CPU
+/// is slower than disks) and run out of memory.
+/// For both CPU and memory, we want to model the machine as having a fixed amount of
+/// resources.  If a single query is running, it should saturate either CPU or Disk
+/// as well as using as little memory as possible. With multiple queries, each query
+/// should get less CPU. In that case each query will need fewer queued buffers and
+/// therefore have less memory usage.
+//
+/// The IoMgr defers CPU management to the caller. The IoMgr provides a GetNextRange
+/// API which will return the next scan range the caller should process. The caller
+/// can call this from the desired number of reading threads. Once a scan range
+/// has been returned via GetNextRange, the IoMgr will start to buffer reads for
+/// that range and it is expected the caller will pull those buffers promptly. For
+/// example, if the caller would like to have 1 scanner thread, the read loop
+/// would look like:
 ///   while (more_ranges)
-///     range = GetNextUnstartedRange()
+///     range = GetNextRange()
 ///     while (!range.eosr)
 ///       buffer = range.GetNext()
-///
-/// Note that the IoMgr rather than the client is responsible for choosing which scan
-/// range to process next, which allows optimizations like distributing load across disks.
-///
-/// Buffer Management:
-/// Buffers for reads are either a) allocated on behalf of the caller with
-/// AllocateBuffersForRange() ("IoMgr-allocated"), b) cached HDFS buffers if the scan
-/// range was read from the HDFS cache, or c) a client buffer, large enough to fit the
-/// whole scan range's data, that is provided by the caller when constructing the
+/// To have multiple reading threads, the caller would simply spin up the threads
+/// and each would process the loops above.
+//
+/// To control the number of IO buffers, each scan range has a limit of two queued
+/// buffers (SCAN_RANGE_READY_BUFFER_LIMIT). If the number of buffers is at capacity,
+/// the IoMgr will no longer read for that scan range until the caller has processed
+/// a buffer. Assuming the client returns each buffer before requesting the next one
+/// from the scan range, then this will consume up to 3 * 8MB = 24MB of I/O buffers per
 /// scan range.
+//
+/// Buffer Management:
+/// Buffers for reads are either a) allocated by the IoMgr and transferred to the caller,
+/// b) cached HDFS buffers if the scan range uses HDFS caching, or c) provided by the
+/// caller when constructing the scan range.
 ///
-/// All three kinds of buffers are wrapped in BufferDescriptors before returning to the
-/// caller. The caller must always call ReturnBuffer() on the buffer descriptor to allow
-/// recycling of the buffer memory and to release any resources associated with the buffer
-/// or scan range.
-///
-/// In case a), ReturnBuffer() may re-enqueue the buffer for GetNext() to return again if
-/// needed. E.g. if 24MB of buffers were allocated to read a 64MB scan range, each buffer
-/// must be returned multiple times. Callers must be careful to call ReturnBuffer() with
-/// the previous buffer returned from the range before calling before GetNext() so that
-/// at least one buffer is available for the I/O mgr to read data into. Calling GetNext()
-/// when the scan range has no buffers to read data into causes a resource deadlock.
-/// NB: if the scan range was allocated N buffers, then it's always ok for the caller
-/// to hold onto N - 1 buffers, but currently the IoMgr doesn't give the caller a way
-/// to determine the value of N.
-///
-/// If the caller wants to maximize I/O throughput, it can give the range enough memory
-/// for 3 max-sized buffers per scan range. Having two queued buffers (plus the buffer
-/// that is currently being processed by the client) gives good performance in most
-/// scenarios:
-/// 1. If the consumer is consuming data faster than we can read from disk, then the
-///    queue will be empty most of the time because the buffer will be immediately
-///    pulled off the queue as soon as it is added. There will always be an I/O request
-///    in the disk queue to maximize I/O throughput, which is the bottleneck in this
-///    case.
-/// 2. If we can read from disk faster than the consumer is consuming data, the queue
-///    will fill up and there will always be a buffer available for the consumer to
-///    read, so the consumer will not block and we maximize consumer throughput, which
-///    is the bottleneck in this case.
-/// 3. If the consumer is consuming data at approximately the same rate as we are
-///    reading from disk, then the steady state is that the consumer is processing one
-///    buffer and one buffer is in the disk queue. The additional buffer can absorb
-///    bursts where the producer runs faster than the consumer or the consumer runs
-///    faster than the producer without blocking either the producer or consumer.
-/// See IDEAL_MAX_SIZED_BUFFERS_PER_SCAN_RANGE.
+/// As a caller reads from a scan range, these buffers are wrapped in BufferDescriptors
+/// and returned to the caller. The caller must always call ReturnBuffer() on the buffer
+/// descriptor to allow recycling of the associated buffer (if there is an
+/// IoMgr-allocated or HDFS cached buffer).
 ///
 /// Caching support:
 /// Scan ranges contain metadata on whether or not it is cached on the DN. In that
@@ -178,13 +161,13 @@ namespace io {
 ///   - HDFS will time us out if we hold onto the mlock for too long
 ///   - Holding the lock prevents uncaching this file due to a caching policy change.
 /// Therefore, we only issue the cached read when the caller is ready to process the
-/// range (GetNextUnstartedRange()) instead of when the ranges are issued. This guarantees
-/// that there will be a CPU available to process the buffer and any throttling we do with
+/// range (GetNextRange()) instead of when the ranges are issued. This guarantees that
+/// there will be a CPU available to process the buffer and any throttling we do with
 /// the number of scanner threads properly controls the amount of files we mlock.
 /// With cached scan ranges, we cannot close the scan range until the cached buffer
 /// is returned (HDFS does not allow this). We therefore need to defer the close until
 /// the cached buffer is returned (ReturnBuffer()).
-///
+//
 /// Remote filesystem support (e.g. S3):
 /// Remote filesystems are modeled as "remote disks". That is, there is a seperate disk
 /// queue for each supported remote filesystem type. In order to maximize throughput,
@@ -193,13 +176,12 @@ namespace io {
 /// intensive than local disk/hdfs because of non-direct I/O and SSL processing, and can
 /// be CPU bottlenecked especially if not enough I/O threads for these queues are
 /// started.
-///
-/// TODO: We should implement more sophisticated resource management. Currently readers
-/// are the unit of scheduling and we attempt to distribute IOPS between them. Instead
-/// it would be better to have policies based on queries, resource pools, etc.
+//
 /// TODO: IoMgr should be able to request additional scan ranges from the coordinator
 /// to help deal with stragglers.
-///
+/// TODO: look into using a lock free queue
+/// TODO: simplify the common path (less locking, memory allocations).
+//
 /// Structure of the Implementation:
 ///  - All client APIs are defined in this file, request-ranges.h and request-context.h.
 ///    Clients can include only the files that they need.
@@ -222,12 +204,10 @@ class DiskIoMgr : public CacheLineAligned {
   ///    disk. This is also the max queue depth.
   ///  - threads_per_solid_state_disk: number of read threads to create per solid state
   ///    disk. This is also the max queue depth.
-  ///  - min_buffer_size: minimum io buffer size (in bytes). Will be rounded down to the
-  //     nearest power-of-two.
-  ///  - max_buffer_size: maximum io buffer size (in bytes). Will be rounded up to the
-  ///    nearest power-of-two. Also the max read size.
+  ///  - min_buffer_size: minimum io buffer size (in bytes)
+  ///  - max_buffer_size: maximum io buffer size (in bytes). Also the max read size.
   DiskIoMgr(int num_disks, int threads_per_rotational_disk,
-      int threads_per_solid_state_disk, int64_t min_buffer_size, int64_t max_buffer_size);
+      int threads_per_solid_state_disk, int min_buffer_size, int max_buffer_size);
 
   /// Create DiskIoMgr with default configs.
   DiskIoMgr();
@@ -237,13 +217,16 @@ class DiskIoMgr : public CacheLineAligned {
   ~DiskIoMgr();
 
   /// Initialize the IoMgr. Must be called once before any of the other APIs.
-  Status Init() WARN_UNUSED_RESULT;
-
+  Status Init(MemTracker* process_mem_tracker) WARN_UNUSED_RESULT;
 
   /// Allocates tracking structure for a request context.
   /// Register a new request context and return it to the caller. The caller must call
   /// UnregisterContext() for each context.
-  std::unique_ptr<RequestContext> RegisterContext();
+  /// reader_mem_tracker: Is non-null only for readers. IO buffers
+  ///    used for this reader will be tracked by this. If the limit is exceeded
+  ///    the reader will be cancelled and MEM_LIMIT_EXCEEDED will be returned via
+  ///    GetNext().
+  std::unique_ptr<RequestContext> RegisterContext(MemTracker* reader_mem_tracker);
 
   /// Unregisters context from the disk IoMgr by first cancelling it then blocking until
   /// all references to the context are removed from I/O manager internal data structures.
@@ -253,60 +236,50 @@ class DiskIoMgr : public CacheLineAligned {
   /// up.
   void UnregisterContext(RequestContext* context);
 
-  /// Adds the scan ranges to reader's queues, but does not start scheduling it. The range
-  /// can be scheduled by a thread calling GetNextUnstartedRange(). This call is
-  /// non-blocking. The caller must not deallocate the scan range pointers before
-  /// UnregisterContext(). 'ranges' must not be empty.
-  Status AddScanRanges(
-      RequestContext* reader, const std::vector<ScanRange*>& ranges) WARN_UNUSED_RESULT;
-
-  /// Adds the scan range to the queues, as with AddScanRanges(), but immediately
-  /// start scheduling the scan range. This can be used to do synchronous reads as well
-  /// as schedule dependent ranges, e.g. for columnar formats. This call is non-blocking.
-  /// The caller must not deallocate the scan range pointers before UnregisterContext().
-  ///
-  /// If this returns true in '*needs_buffers', the caller must then call
-  /// AllocateBuffersForRange() to add buffers for the data to be read into before the
-  /// range can be scheduled. Otherwise, the range is scheduled and the IoMgr will
-  /// asynchronously read the data for the range and the caller can call
-  /// ScanRange::GetNext() to read the data.
-  Status StartScanRange(
-      RequestContext* reader, ScanRange* range, bool* needs_buffers) WARN_UNUSED_RESULT;
+  /// This function cancels the context asychronously. All outstanding requests
+  /// are aborted and tracking structures cleaned up. This does not need to be
+  /// called if the context finishes normally.
+  /// This will also fail any outstanding GetNext()/Read requests.
+  void CancelContext(RequestContext* context);
+
+  /// Adds the scan ranges to the queues. This call is non-blocking. The caller must
+  /// not deallocate the scan range pointers before UnregisterContext().
+  /// If schedule_immediately, the ranges are immediately put on the read queue
+  /// (i.e. the caller should not/cannot call GetNextRange for these ranges).
+  /// This can be used to do synchronous reads as well as schedule dependent ranges,
+  /// as in the case for columnar formats.
+  Status AddScanRanges(RequestContext* reader,
+      const std::vector<ScanRange*>& ranges,
+      bool schedule_immediately = false) WARN_UNUSED_RESULT;
+  Status AddScanRange(RequestContext* reader, ScanRange* range,
+      bool schedule_immediately = false) WARN_UNUSED_RESULT;
 
   /// Add a WriteRange for the writer. This is non-blocking and schedules the context
   /// on the IoMgr disk queue. Does not create any files.
   Status AddWriteRange(
       RequestContext* writer, WriteRange* write_range) WARN_UNUSED_RESULT;
 
-  /// Tries to get an unstarted scan range that was added to 'reader' with
-  /// AddScanRanges(). On success, returns OK and returns the range in '*range'.
-  /// If 'reader' was cancelled, returns CANCELLED. If another error is encountered,
-  /// an error status is returned. Otherwise, if error or cancellation wasn't encountered
-  /// and there are no unstarted ranges for 'reader', returns OK and sets '*range' to
-  /// nullptr.
-  ///
-  /// If '*needs_buffers' is returned as true, the caller must call
-  /// AllocateBuffersForRange() to add buffers for the data to be read into before the
-  /// range can be scheduled. Otherwise, the range is scheduled and the IoMgr will
-  /// asynchronously read the data for the range and the caller can call
-  /// ScanRange::GetNext() to read the data.
-  Status GetNextUnstartedRange(RequestContext* reader, ScanRange** range,
-      bool* needs_buffers) WARN_UNUSED_RESULT;
-
-  /// Allocates up to 'max_bytes' buffers to read the data from 'range' into and schedules
-  /// the range. Called after StartScanRange() or GetNextUnstartedRange() returns
-  /// *needs_buffers=true.
-  ///
-  /// The buffer sizes are chosen based on range->len(). 'max_bytes' must be >=
-  /// min_read_buffer_size() so that at least one buffer can be allocated. The caller
-  /// must ensure that 'bp_client' has at least 'max_bytes' unused reservation. Returns ok
-  /// if the buffers were successfully allocated and the range was scheduled.
-  ///
-  /// Setting 'max_bytes' to IDEAL_MAX_SIZED_BUFFERS_PER_SCAN_RANGE * max_buffer_size()
-  /// will typically maximize I/O throughput. See the "Buffer Management" section of
-  /// the class comment for explanation.
-  Status AllocateBuffersForRange(RequestContext* reader,
-      BufferPool::ClientHandle* bp_client, ScanRange* range, int64_t max_bytes);
+  /// Returns the next unstarted scan range for this reader. When the range is returned,
+  /// the disk threads in the IoMgr will already have started reading from it. The
+  /// caller is expected to call ScanRange::GetNext on the returned range.
+  /// If there are no more unstarted ranges, nullptr is returned.
+  /// This call is blocking.
+  Status GetNextRange(RequestContext* reader, ScanRange** range) WARN_UNUSED_RESULT;
+
+  /// Reads the range and returns the result in buffer.
+  /// This behaves like the typical synchronous read() api, blocking until the data
+  /// is read. This can be called while there are outstanding ScanRanges and is
+  /// thread safe. Multiple threads can be calling Read() per reader at a time.
+  /// range *cannot* have already been added via AddScanRanges.
+  /// This can only be used if the scan range fits in a single IO buffer (i.e. is smaller
+  /// than max_read_buffer_size()) or if reading into a client-provided buffer.
+  Status Read(RequestContext* reader, ScanRange* range,
+      std::unique_ptr<BufferDescriptor>* buffer) WARN_UNUSED_RESULT;
+
+  /// Returns the buffer to the IoMgr. This must be called for every buffer
+  /// returned by GetNext()/Read() that did not return an error. This is non-blocking.
+  /// After calling this, the buffer descriptor is invalid and cannot be accessed.
+  void ReturnBuffer(std::unique_ptr<BufferDescriptor> buffer);
 
   /// Determine which disk queue this file should be assigned to.  Returns an index into
   /// disk_queues_.  The disk_id is the volume ID for the local disk that holds the
@@ -314,8 +287,32 @@ class DiskIoMgr : public CacheLineAligned {
   /// co-located with the datanode for this file.
   int AssignQueue(const char* file, int disk_id, bool expected_local);
 
-  int64_t min_buffer_size() const { return min_buffer_size_; }
-  int64_t max_buffer_size() const { return max_buffer_size_; }
+  /// TODO: The functions below can be moved to RequestContext.
+  /// Returns the current status of the context.
+  Status context_status(RequestContext* context) const WARN_UNUSED_RESULT;
+
+  void set_bytes_read_counter(RequestContext*, RuntimeProfile::Counter*);
+  void set_read_timer(RequestContext*, RuntimeProfile::Counter*);
+  void set_open_file_timer(RequestContext*, RuntimeProfile::Counter*);
+  void set_active_read_thread_counter(RequestContext*, RuntimeProfile::Counter*);
+  void set_disks_access_bitmap(RequestContext*, RuntimeProfile::Counter*);
+
+  int64_t queue_size(RequestContext* reader) const;
+  int64_t bytes_read_local(RequestContext* reader) const;
+  int64_t bytes_read_short_circuit(RequestContext* reader) const;
+  int64_t bytes_read_dn_cache(RequestContext* reader) const;
+  int num_remote_ranges(RequestContext* reader) const;
+  int64_t unexpected_remote_bytes(RequestContext* reader) const;
+  int cached_file_handles_hit_count(RequestContext* reader) const;
+  int cached_file_handles_miss_count(RequestContext* reader) const;
+
+  /// Returns the read throughput across all readers.
+  /// TODO: should this be a sliding window?  This should report metrics for the
+  /// last minute, hour and since the beginning.
+  int64_t GetReadThroughput();
+
+  /// Returns the maximum read buffer size
+  int max_read_buffer_size() const { return max_buffer_size_; }
 
   /// Returns the total number of disk queues (both local and remote).
   int num_total_disks() const { return disk_queues_.size(); }
@@ -368,6 +365,29 @@ class DiskIoMgr : public CacheLineAligned {
   Status ReopenCachedHdfsFileHandle(const hdfsFS& fs, std::string* fname, int64_t mtime,
       RequestContext* reader, CachedHdfsFileHandle** fid);
 
+  /// Garbage collect unused I/O buffers up to 'bytes_to_free', or all the buffers if
+  /// 'bytes_to_free' is -1.
+  void GcIoBuffers(int64_t bytes_to_free = -1);
+
+  /// The maximum number of ready buffers that can be queued in a scan range. Having two
+  /// queued buffers (plus the buffer that is returned to the client) gives good
+  /// performance in most scenarios:
+  /// 1. If the consumer is consuming data faster than we can read from disk, then the
+  ///    queue will be empty most of the time because the buffer will be immediately
+  ///    pulled off the queue as soon as it is added. There will always be an I/O request
+  ///    in the disk queue to maximize I/O throughput, which is the bottleneck in this
+  ///    case.
+  /// 2. If we can read from disk faster than the consumer is consuming data, the queue
+  ///    will fill up and there will always be a buffer available for the consumer to
+  ///    read, so the consumer will not block and we maximize consumer throughput, which
+  ///    is the bottleneck in this case.
+  /// 3. If the consumer is consuming data at approximately the same rate as we are
+  ///    reading from disk, then the steady state is that the consumer is processing one
+  ///    buffer and one buffer is in the disk queue. The additional buffer can absorb
+  ///    bursts where the producer runs faster than the consumer or the consumer runs
+  ///    faster than the producer without blocking either the producer or consumer.
+  static const int SCAN_RANGE_READY_BUFFER_LIMIT = 2;
+
   /// "Disk" queue offsets for remote accesses.  Offset 0 corresponds to
   /// disk ID (i.e. disk_queue_ index) of num_local_disks().
   enum {
@@ -377,10 +397,6 @@ class DiskIoMgr : public CacheLineAligned {
     REMOTE_NUM_DISKS
   };
 
-  /// The ideal number of max-sized buffers per scan range to maximise throughput.
-  /// See "Buffer Management" in the class comment for explanation.
-  static const int64_t IDEAL_MAX_SIZED_BUFFERS_PER_SCAN_RANGE = 3;
-
  private:
   friend class BufferDescriptor;
   friend class RequestContext;
@@ -389,9 +405,16 @@ class DiskIoMgr : public CacheLineAligned {
   struct DiskQueue;
 
   friend class DiskIoMgrTest_Buffers_Test;
-  friend class DiskIoMgrTest_BufferSizeSelection_Test;
   friend class DiskIoMgrTest_VerifyNumThreadsParameter_Test;
 
+  /// Memory tracker for unused I/O buffers owned by DiskIoMgr.
+  boost::scoped_ptr<MemTracker> free_buffer_mem_tracker_;
+
+  /// Memory tracker for I/O buffers where the RequestContext has no MemTracker.
+  /// TODO: once IMPALA-3200 is fixed, there should be no more cases where readers don't
+  /// provide a MemTracker.
+  boost::scoped_ptr<MemTracker> unowned_buffer_mem_tracker_;
+
   /// Number of worker(read) threads per rotational disk. Also the max depth of queued
   /// work to the disk.
   const int num_io_threads_per_rotational_disk_;
@@ -401,10 +424,10 @@ class DiskIoMgr : public CacheLineAligned {
   const int num_io_threads_per_solid_state_disk_;
 
   /// Maximum read size. This is also the maximum size of each allocated buffer.
-  const int64_t max_buffer_size_;
+  const int max_buffer_size_;
 
-  /// The minimum size of each read buffer. Must be >= BufferPool::min_buffer_len().
-  const int64_t min_buffer_size_;
+  /// The minimum size of each read buffer.
+  const int min_buffer_size_;
 
   /// Thread group containing all the worker threads.
   ThreadGroup disk_thread_group_;
@@ -422,6 +445,28 @@ class DiskIoMgr : public CacheLineAligned {
   /// Total time spent in hdfs reading
   RuntimeProfile::Counter read_timer_;
 
+  /// Protects free_buffers_
+  boost::mutex free_buffers_lock_;
+
+  /// Free buffers that can be handed out to clients. There is one list for each buffer
+  /// size, indexed by the Log2 of the buffer size in units of min_buffer_size_. The
+  /// maximum buffer size is max_buffer_size_, so the maximum index is
+  /// Log2(max_buffer_size_ / min_buffer_size_).
+  //
+  /// E.g. if min_buffer_size_ = 1024 bytes:
+  ///  free_buffers_[0]  => list of free buffers with size 1024 B
+  ///  free_buffers_[1]  => list of free buffers with size 2048 B
+  ///  free_buffers_[10] => list of free buffers with size 1 MB
+  ///  free_buffers_[13] => list of free buffers with size 8 MB
+  ///  free_buffers_[n]  => list of free buffers with size 2^n * 1024 B
+  std::vector<std::deque<uint8_t*>> free_buffers_;
+
+  /// Total number of allocated buffers, used for debugging.
+  AtomicInt32 num_allocated_buffers_;
+
+  /// Total number of buffers in readers
+  AtomicInt32 num_buffers_in_readers_;
+
   /// Per disk queues. This is static and created once at Init() time.  One queue is
   /// allocated for each local disk on the system and for each remote filesystem type.
   /// It is indexed by disk id.
@@ -437,6 +482,23 @@ class DiskIoMgr : public CacheLineAligned {
   // handles are closed.
   FileHandleCache file_handle_cache_;
 
+  /// Returns the index into free_buffers_ for a given buffer size
+  int free_buffers_idx(int64_t buffer_size);
+
+  /// Returns a buffer to read into with size between 'buffer_size' and
+  /// 'max_buffer_size_', If there is an appropriately-sized free buffer in the
+  /// 'free_buffers_', that is returned, otherwise a new one is allocated.
+  /// The returned *buffer_size must be between 0 and 'max_buffer_size_'.
+  /// The buffer memory is tracked against reader's mem tracker, or
+  /// 'unowned_buffer_mem_tracker_' if the reader does not have one.
+  std::unique_ptr<BufferDescriptor> GetFreeBuffer(
+      RequestContext* reader, ScanRange* range, int64_t buffer_size);
+
+  /// Disassociates the desc->buffer_ memory from 'desc' (which cannot be nullptr), either
+  /// freeing it or returning it to 'free_buffers_'. Memory tracking is updated to
+  /// reflect the transfer of ownership from desc->mem_tracker_ to the disk I/O mgr.
+  void FreeBufferMemory(BufferDescriptor* desc);
+
   /// Disk worker thread loop. This function retrieves the next range to process on
   /// the disk queue and invokes ReadRange() or Write() depending on the type of Range().
   /// There can be multiple threads per disk running this loop.
@@ -450,12 +512,10 @@ class DiskIoMgr : public CacheLineAligned {
   bool GetNextRequestRange(DiskQueue* disk_queue, RequestRange** range,
       RequestContext** request_context);
 
-  /// Updates disk queue and reader state after a read is complete. If the read
-  /// was successful, 'read_status' is ok and 'buffer' contains the result of the
-  /// read. If the read failed with an error, 'read_status' contains the error and
-  /// 'buffer' has the buffer that was meant to hold the result of the read.
+  /// Updates disk queue and reader state after a read is complete. The read result
+  /// is captured in the buffer descriptor.
   void HandleReadFinished(DiskQueue* disk_queue, RequestContext* reader,
-      Status read_status, std::unique_ptr<BufferDescriptor> buffer);
+      std::unique_ptr<BufferDescriptor> buffer);
 
   /// Invokes write_range->callback_  after the range has been written and
   /// updates per-disk state and handle state. The status of the write OK/RUNTIME_ERROR
@@ -478,14 +538,15 @@ class DiskIoMgr : public CacheLineAligned {
   /// Does not open or close the file that is written.
   Status WriteRangeHelper(FILE* file_handle, WriteRange* write_range) WARN_UNUSED_RESULT;
 
-  /// Reads the specified scan range and calls HandleReadFinished() when done. If no
-  /// buffer is available to read the range's data into, the read cannot proceed, the
-  /// range becomes blocked and this function returns without doing I/O.
+  /// Reads the specified scan range and calls HandleReadFinished when done.
   void ReadRange(DiskQueue* disk_queue, RequestContext* reader, ScanRange* range);
 
-  /// Helper for AllocateBuffersForRange() to compute the buffer sizes for a scan range
-  /// with length 'scan_range_len', given that 'max_bytes' of memory should be allocated.
-  std::vector<int64_t> ChooseBufferSizes(int64_t scan_range_len, int64_t max_bytes);
+  /// Try to allocate the next buffer for the scan range, returning the new buffer
+  /// if successful. If 'reader' is cancelled, cancels the range and returns nullptr.
+  /// If there is memory pressure and buffers are already queued, adds the range
+  /// to the blocked ranges and returns nullptr.
+  std::unique_ptr<BufferDescriptor> TryAllocateNextBufferForRange(DiskQueue* disk_queue,
+      RequestContext* reader, ScanRange* range, int64_t buffer_size);
 };
 }
 }


[4/9] impala git commit: Revert IMPALA-4835 and dependent changes

Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
index 0a945bd..23f2d88 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
@@ -153,6 +153,26 @@ public class SlotRef extends Expr {
     return "<slot " + Integer.toString(desc_.getId().asInt()) + ">";
   }
 
+  /**
+   * Checks if this slotRef refers to an array "pos" pseudo-column.
+   *
+   * Note: checking whether the column is null distinguishes between top-level columns
+   * and nested types. This check more specifically looks just for a reference to the
+   * "pos" field of an array type.
+   */
+  public boolean isArrayPosRef() {
+    TupleDescriptor parent = getDesc().getParent();
+    if (parent == null) return false;
+    Type parentType = parent.getType();
+    if (parentType instanceof CollectionStructType) {
+      if (((CollectionStructType)parentType).isArrayStruct() &&
+          getDesc().getLabel().equals(Path.ARRAY_POS_FIELD_NAME)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   @Override
   protected void toThrift(TExprNode msg) {
     msg.node_type = TExprNodeType.SLOT_REF;

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
index 87d1806..7735f98 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsScanNode.java
@@ -59,7 +59,6 @@ import org.apache.impala.common.ImpalaException;
 import org.apache.impala.common.ImpalaRuntimeException;
 import org.apache.impala.common.InternalException;
 import org.apache.impala.common.NotImplementedException;
-import org.apache.impala.common.Pair;
 import org.apache.impala.common.PrintUtils;
 import org.apache.impala.common.RuntimeEnv;
 import org.apache.impala.fb.FbFileBlock;
@@ -77,7 +76,6 @@ import org.apache.impala.thrift.TScanRange;
 import org.apache.impala.thrift.TScanRangeLocation;
 import org.apache.impala.thrift.TScanRangeLocationList;
 import org.apache.impala.thrift.TTableStats;
-import org.apache.impala.util.BitUtil;
 import org.apache.impala.util.MembershipSnapshot;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -145,7 +143,7 @@ public class HdfsScanNode extends ScanNode {
   // derived experimentally: running metadata-only Parquet count(*) scans on TPC-H
   // lineitem and TPC-DS store_sales of different sizes resulted in memory consumption
   // between 128kb and 1.1mb.
-  private static final long MIN_MEMORY_ESTIMATE = 1L * 1024L * 1024L;
+  private final static long MIN_MEMORY_ESTIMATE = 1 * 1024 * 1024;
 
   private final HdfsTable tbl_;
 
@@ -168,18 +166,6 @@ public class HdfsScanNode extends ScanNode {
   private long totalFiles_ = 0;
   private long totalBytes_ = 0;
 
-  // File formats scanned. Set in computeScanRangeLocations().
-  private Set<HdfsFileFormat> fileFormats_;
-
-  // Number of bytes in the largest scan range (i.e. hdfs split). Set in
-  // computeScanRangeLocations().
-  private long maxScanRangeBytes_ = 0;
-
-  // The ideal reservation to process a single scan range (i.e. hdfs split), >= the
-  // minimum reservation. Generally provides enough memory to overlap CPU and I/O and
-  // maximize throughput. Set in computeResourceProfile().
-  private long idealScanRangeReservation_ = -1;
-
   // Input cardinality based on the partition row counts or extrapolation. -1 if invalid.
   // Both values can be valid to report them in the explain plan, but only one of them is
   // used for determining the scan cardinality.
@@ -343,25 +329,25 @@ public class HdfsScanNode extends ScanNode {
     computeDictionaryFilterConjuncts(analyzer);
 
     // compute scan range locations with optional sampling
-    computeScanRangeLocations(analyzer);
+    Set<HdfsFileFormat> fileFormats = computeScanRangeLocations(analyzer);
 
     // Determine backend scan node implementation to use. The optimized MT implementation
     // is currently only supported for Parquet.
     if (analyzer.getQueryOptions().isSetMt_dop() &&
         analyzer.getQueryOptions().mt_dop > 0 &&
-        fileFormats_.size() == 1 &&
-        (fileFormats_.contains(HdfsFileFormat.PARQUET)
-          || fileFormats_.contains(HdfsFileFormat.TEXT))) {
+        fileFormats.size() == 1 &&
+        (fileFormats.contains(HdfsFileFormat.PARQUET)
+          || fileFormats.contains(HdfsFileFormat.TEXT))) {
       useMtScanNode_ = true;
     } else {
       useMtScanNode_ = false;
     }
 
-    if (fileFormats_.contains(HdfsFileFormat.PARQUET)) {
+    if (fileFormats.contains(HdfsFileFormat.PARQUET)) {
       computeMinMaxTupleAndConjuncts(analyzer);
     }
 
-    if (canApplyParquetCountStarOptimization(analyzer, fileFormats_)) {
+    if (canApplyParquetCountStarOptimization(analyzer, fileFormats)) {
       Preconditions.checkState(desc_.getPath().destTable() != null);
       Preconditions.checkState(collectionConjuncts_.isEmpty());
       countStarSlot_ = applyParquetCountStartOptimization(analyzer);
@@ -474,7 +460,7 @@ public class HdfsScanNode extends ScanNode {
     // This node is a table scan, so this must be a scanning slot.
     Preconditions.checkState(slotRef.getDesc().isScanSlot());
     // Skip the slot ref if it refers to an array's "pos" field.
-    if (slotRef.getDesc().isArrayPosRef()) return;
+    if (slotRef.isArrayPosRef()) return;
 
     Expr constExpr = binaryPred.getChild(1);
     // Only constant exprs can be evaluated against parquet::Statistics. This includes
@@ -502,7 +488,7 @@ public class HdfsScanNode extends ScanNode {
     // This node is a table scan, so this must be a scanning slot.
     Preconditions.checkState(slotRef.getDesc().isScanSlot());
     // Skip the slot ref if it refers to an array's "pos" field.
-    if (slotRef.getDesc().isArrayPosRef()) return;
+    if (slotRef.isArrayPosRef()) return;
     if (inPred.isNotIn()) return;
 
     ArrayList<Expr> children = inPred.getChildren();
@@ -709,15 +695,13 @@ public class HdfsScanNode extends ScanNode {
   }
 
   /**
-   * Computes scan ranges (i.e. hdfs splits) plus their storage locations, including
-   * volume ids, based on the given maximum number of bytes each scan range should scan.
+   * Computes scan ranges (hdfs splits) plus their storage locations, including volume
+   * ids, based on the given maximum number of bytes each scan range should scan.
    * If 'sampleParams_' is not null, generates a sample and computes the scan ranges
    * based on the sample.
-   *
-   * Initializes members with information about files and scan ranges, e.g. totalFiles_,
-   * fileFormats_, etc.
+   * Returns the set of file formats being scanned.
    */
-  private void computeScanRangeLocations(Analyzer analyzer)
+  private Set<HdfsFileFormat> computeScanRangeLocations(Analyzer analyzer)
       throws ImpalaRuntimeException {
     Map<Long, List<FileDescriptor>> sampledFiles = null;
     if (sampleParams_ != null) {
@@ -740,8 +724,7 @@ public class HdfsScanNode extends ScanNode {
     numPartitions_ = (sampledFiles != null) ? sampledFiles.size() : partitions_.size();
     totalFiles_ = 0;
     totalBytes_ = 0;
-    maxScanRangeBytes_ = 0;
-    fileFormats_ = Sets.newHashSet();
+    Set<HdfsFileFormat> fileFormats = Sets.newHashSet();
     for (HdfsPartition partition: partitions_) {
       List<FileDescriptor> fileDescs = partition.getFileDescriptors();
       if (sampledFiles != null) {
@@ -751,7 +734,7 @@ public class HdfsScanNode extends ScanNode {
       }
 
       analyzer.getDescTbl().addReferencedPartition(tbl_, partition.getId());
-      fileFormats_.add(partition.getFileFormat());
+      fileFormats.add(partition.getFileFormat());
       Preconditions.checkState(partition.getId() >= 0);
       // Missing disk id accounting is only done for file systems that support the notion
       // of disk/storage ids.
@@ -814,7 +797,6 @@ public class HdfsScanNode extends ScanNode {
             scanRangeLocations.scan_range = scanRange;
             scanRangeLocations.locations = locations;
             scanRanges_.add(scanRangeLocations);
-            maxScanRangeBytes_ = Math.max(maxScanRangeBytes_, currentLength);
             remainingLength -= currentLength;
             currentOffset += currentLength;
           }
@@ -829,58 +811,7 @@ public class HdfsScanNode extends ScanNode {
       }
       if (partitionMissingDiskIds) ++numPartitionsNoDiskIds_;
     }
-  }
-
-  /**
-   * Compute the number of columns that are read from the file, as opposed to
-   * materialised based on metadata. If there are nested collections, counts the
-   * number of leaf scalar slots per collection. This matches Parquet's "shredded"
-   * approach to nested collections, where each nested field is stored as a separate
-   * column. We may need to adjust this logic for non-shredded columnar formats if added.
-   */
-  private int computeNumColumnsReadFromFile() {
-    HdfsTable table = (HdfsTable) desc_.getTable();
-    int numColumns = 0;
-    boolean havePosSlot = false;
-    for (SlotDescriptor slot: desc_.getSlots()) {
-      if (!slot.isMaterialized() || slot == countStarSlot_) continue;
-      if (slot.getColumn() == null ||
-          slot.getColumn().getPosition() >= table.getNumClusteringCols()) {
-        if (slot.isArrayPosRef()) {
-          // Position virtual slots can be materialized by piggybacking on another slot.
-          havePosSlot = true;
-        } else if (slot.getType().isScalarType()) {
-          ++numColumns;
-        } else {
-          numColumns += computeNumColumnsReadForCollection(slot);
-        }
-      }
-    }
-    // Must scan something to materialize a position slot.
-    if (havePosSlot) numColumns = Math.max(numColumns, 1);
-    return numColumns;
-  }
-
-  /**
-   * Compute the number of columns read from disk for materialized scalar slots in
-   * the provided tuple.
-   */
-  private int computeNumColumnsReadForCollection(SlotDescriptor collectionSlot) {
-    Preconditions.checkState(collectionSlot.getType().isCollectionType());
-    int numColumns = 0;
-    for (SlotDescriptor nestedSlot: collectionSlot.getItemTupleDesc().getSlots()) {
-      // Position virtual slots can be materialized by piggybacking on another slot.
-      if (!nestedSlot.isMaterialized() || nestedSlot.isArrayPosRef()) continue;
-      if (nestedSlot.getType().isScalarType()) {
-        ++numColumns;
-      } else {
-        numColumns += computeNumColumnsReadForCollection(nestedSlot);
-      }
-    }
-    // Need to scan at least one column to materialize the pos virtual slot and/or
-    // determine the size of the nested array.
-    numColumns = Math.max(numColumns, 1);
-    return numColumns;
+    return fileFormats;
   }
 
   /**
@@ -1075,8 +1006,6 @@ public class HdfsScanNode extends ScanNode {
     }
     msg.hdfs_scan_node.setRandom_replica(randomReplica_);
     msg.node_type = TPlanNodeType.HDFS_SCAN_NODE;
-    Preconditions.checkState(idealScanRangeReservation_ >= 0, idealScanRangeReservation_);
-    msg.hdfs_scan_node.setIdeal_scan_range_reservation(idealScanRangeReservation_);
     if (!collectionConjuncts_.isEmpty()) {
       Map<Integer, List<TExpr>> tcollectionConjuncts = Maps.newLinkedHashMap();
       for (Map.Entry<TupleDescriptor, List<Expr>> entry:
@@ -1255,20 +1184,25 @@ public class HdfsScanNode extends ScanNode {
     Preconditions.checkNotNull(scanRanges_, "Cost estimation requires scan ranges.");
     if (scanRanges_.isEmpty()) {
       nodeResourceProfile_ = ResourceProfile.noReservation(0);
-      idealScanRangeReservation_ = 0;
       return;
     }
     Preconditions.checkState(0 < numNodes_ && numNodes_ <= scanRanges_.size());
     Preconditions.checkNotNull(desc_);
     Preconditions.checkNotNull(desc_.getTable() instanceof HdfsTable);
     HdfsTable table = (HdfsTable) desc_.getTable();
-    int numColumnsReadFromFile = computeNumColumnsReadFromFile();
     int perHostScanRanges;
     if (table.getMajorityFormat() == HdfsFileFormat.PARQUET) {
       // For the purpose of this estimation, the number of per-host scan ranges for
       // Parquet files are equal to the number of columns read from the file. I.e.
       // excluding partition columns and columns that are populated from file metadata.
-      perHostScanRanges = numColumnsReadFromFile;
+      perHostScanRanges = 0;
+      for (SlotDescriptor slot: desc_.getSlots()) {
+        if (!slot.isMaterialized() || slot == countStarSlot_) continue;
+        if (slot.getColumn() == null ||
+            slot.getColumn().getPosition() >= table.getNumClusteringCols()) {
+          ++perHostScanRanges;
+        }
+      }
     } else {
       perHostScanRanges = (int) Math.ceil((
           (double) scanRanges_.size() / (double) numNodes_) * SCAN_RANGE_SKEW_FACTOR);
@@ -1290,13 +1224,12 @@ public class HdfsScanNode extends ScanNode {
     long avgScanRangeBytes = (long) Math.ceil(totalBytes_ / (double) scanRanges_.size());
     // The +1 accounts for an extra I/O buffer to read past the scan range due to a
     // trailing record spanning Hdfs blocks.
-    long maxIoBufferSize =
-        BitUtil.roundUpToPowerOf2(BackendConfig.INSTANCE.getReadSize());
+    long readSize = BackendConfig.INSTANCE.getReadSize();
     long perThreadIoBuffers =
-        Math.min((long) Math.ceil(avgScanRangeBytes / (double) maxIoBufferSize),
+        Math.min((long) Math.ceil(avgScanRangeBytes / (double) readSize),
             MAX_IO_BUFFERS_PER_THREAD) + 1;
     long perInstanceMemEstimate = checkedMultiply(
-        checkedMultiply(maxScannerThreads, perThreadIoBuffers), maxIoBufferSize);
+        checkedMultiply(maxScannerThreads, perThreadIoBuffers), readSize);
 
     // Sanity check: the tighter estimation should not exceed the per-host maximum.
     long perHostUpperBound = getPerHostMemUpperBound();
@@ -1307,51 +1240,7 @@ public class HdfsScanNode extends ScanNode {
       perInstanceMemEstimate = perHostUpperBound;
     }
     perInstanceMemEstimate = Math.max(perInstanceMemEstimate, MIN_MEMORY_ESTIMATE);
-
-    Pair<Long, Long> reservation = computeReservation(numColumnsReadFromFile);
-    nodeResourceProfile_ = new ResourceProfileBuilder()
-        .setMemEstimateBytes(perInstanceMemEstimate)
-        .setMinReservationBytes(reservation.first).build();
-    idealScanRangeReservation_ = reservation.second;
-  }
-
-  /*
-   *  Compute the minimum and ideal memory reservation to process a single scan range
-   *  (i.e. hdfs split). Bound the reservation based on:
-   * - One minimum-sized buffer per IoMgr scan range, which is the absolute minimum
-   *   required to scan the data.
-   * - A maximum of either 1 or 3 max-sized I/O buffers per IoMgr scan range for
-   *   the minimum and ideal reservation respectively. 1 max-sized I/O buffer avoids
-   *   issuing small I/O unnecessarily while 3 max-sized I/O buffers guarantees higher
-   *   throughput by overlapping compute and I/O efficiently.
-   * - A maximum reservation of the hdfs split size, to avoid reserving excessive
-   *   memory for small files or ranges, e.g. small dimension tables with very few
-   *   rows.
-   */
-  private Pair<Long, Long> computeReservation(int numColumnsReadFromFile) {
-    Preconditions.checkState(maxScanRangeBytes_ >= 0);
-    long maxIoBufferSize =
-        BitUtil.roundUpToPowerOf2(BackendConfig.INSTANCE.getReadSize());
-    // Scanners for columnar formats issue one IoMgr scan range for metadata, followed by
-    // one IoMgr scan range per column in parallel. Scanners for row-oriented formats
-    // issue only one IoMgr scan range at a time.
-    int iomgrScanRangesPerSplit = fileFormats_.contains(HdfsFileFormat.PARQUET) ?
-        Math.max(1, numColumnsReadFromFile) : 1;
-    // Need one buffer per IoMgr scan range to execute the scan.
-    long minReservationToExecute =
-        iomgrScanRangesPerSplit * BackendConfig.INSTANCE.getMinBufferSize();
-
-    // Quantize the max scan range (i.e. hdfs split) size to an I/O buffer size.
-    long quantizedMaxScanRangeBytes = maxScanRangeBytes_ < maxIoBufferSize ?
-        BitUtil.roundUpToPowerOf2(maxScanRangeBytes_) :
-        BitUtil.roundUpToPowerOf2Factor(maxScanRangeBytes_, maxIoBufferSize);
-    long minReservationBytes = Math.max(minReservationToExecute,
-        Math.min(iomgrScanRangesPerSplit * maxIoBufferSize,
-            quantizedMaxScanRangeBytes));
-    long idealReservationBytes = Math.max(minReservationToExecute,
-        Math.min(iomgrScanRangesPerSplit * maxIoBufferSize * 3,
-            quantizedMaxScanRangeBytes));
-    return Pair.create(minReservationBytes, idealReservationBytes);
+    nodeResourceProfile_ = ResourceProfile.noReservation(perInstanceMemEstimate);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/fe/src/main/java/org/apache/impala/util/BitUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/BitUtil.java b/fe/src/main/java/org/apache/impala/util/BitUtil.java
index 6bb670d..839dd6e 100644
--- a/fe/src/main/java/org/apache/impala/util/BitUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/BitUtil.java
@@ -29,10 +29,4 @@ public class BitUtil {
   public static long roundUpToPowerOf2(long val) {
     return 1L << log2Ceiling(val);
   }
-
-  // Round up 'val' to the nearest multiple of a power-of-two 'factor'.
-  // 'val' must be > 0.
-  public static long roundUpToPowerOf2Factor(long val, long factor) {
-    return (val + (factor - 1)) & ~(factor - 1);
-  }
 }

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/fe/src/test/java/org/apache/impala/util/BitUtilTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/util/BitUtilTest.java b/fe/src/test/java/org/apache/impala/util/BitUtilTest.java
index a6da80c..a134b6a 100644
--- a/fe/src/test/java/org/apache/impala/util/BitUtilTest.java
+++ b/fe/src/test/java/org/apache/impala/util/BitUtilTest.java
@@ -46,10 +46,4 @@ public class BitUtilTest {
     assertEquals(0x8000000000000000L, BitUtil.roundUpToPowerOf2(0x8000000000000000L - 1));
   }
 
-  @Test
-  public void testPowerOf2Factor() {
-    assertEquals(BitUtil.roundUpToPowerOf2Factor(7, 8), 8);
-    assertEquals(BitUtil.roundUpToPowerOf2Factor(8, 8), 8);
-    assertEquals(BitUtil.roundUpToPowerOf2Factor(9, 8), 16);
-  }
 }

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test b/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test
index 533ac42..f25ad0a 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/constant-folding.test
@@ -5,7 +5,7 @@ where 5 + 5 < c_custkey and o_orderkey = (2 + 2)
   and (coalesce(2, 3, 4) * 10) + l_linenumber < (0 * 1)
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=264.00MB mem-reservation=24.00MB
+|  Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -59,7 +59,7 @@ PLAN-ROOT SINK
    parquet dictionary predicates: c_custkey > 10
    parquet dictionary predicates on o: o_orderkey = 4
    parquet dictionary predicates on o_lineitems: 20 + l_linenumber < 0
-   mem-estimate=264.00MB mem-reservation=24.00MB
+   mem-estimate=176.00MB mem-reservation=0B
    tuple-ids=0 row-size=24B cardinality=15000
 ====
 # Test HBase scan node.
@@ -107,7 +107,7 @@ having 1024 * 1024 * count(*) % 2 = 0
   and (sm between 5 and 10)
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=138.00MB mem-reservation=1.97MB
+|  Per-Host Resources: mem-estimate=138.00MB mem-reservation=1.94MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -125,7 +125,7 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=20B cardinality=7300
 ====
 # Test hash join.
@@ -136,7 +136,7 @@ left outer join functional.alltypes b
 where round(1.11 + 2.22 + 3.33 + 4.44, 1) < cast(b.double_col as decimal(3, 2))
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=257.94MB mem-reservation=2.00MB
+|  Per-Host Resources: mem-estimate=257.94MB mem-reservation=1.94MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -157,7 +157,7 @@ PLAN-ROOT SINK
 |       columns: all
 |     extrapolated-rows=disabled
 |     parquet dictionary predicates: CAST(b.double_col AS DECIMAL(3,2)) > 11.1
-|     mem-estimate=128.00MB mem-reservation=32.00KB
+|     mem-estimate=128.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=20B cardinality=730
 |
 00:SCAN HDFS [functional.alltypes a]
@@ -167,7 +167,7 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=7300
 ====
 # Test nested-loop join. Same as above but and with a disjunction in the On clause.
@@ -179,7 +179,7 @@ left outer join functional.alltypes b
 where cast(b.double_col as decimal(3, 2)) > round(1.11 + 2.22 + 3.33 + 4.44, 1)
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=256.01MB mem-reservation=64.00KB
+|  Per-Host Resources: mem-estimate=256.01MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -198,7 +198,7 @@ PLAN-ROOT SINK
 |       columns: all
 |     extrapolated-rows=disabled
 |     parquet dictionary predicates: CAST(b.double_col AS DECIMAL(3,2)) > 11.1
-|     mem-estimate=128.00MB mem-reservation=32.00KB
+|     mem-estimate=128.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=20B cardinality=730
 |
 00:SCAN HDFS [functional.alltypes a]
@@ -208,7 +208,7 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=7300
 ====
 # Test distinct aggregation with grouping.
@@ -242,7 +242,7 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=20B cardinality=7300
 ====
 # Test non-grouping distinct aggregation.
@@ -251,7 +251,7 @@ from functional.alltypes
 having 1024 * 1024 * count(*) % 2 = 0
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=138.00MB mem-reservation=1.97MB
+|  Per-Host Resources: mem-estimate=138.00MB mem-reservation=1.94MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -274,7 +274,7 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=4B cardinality=7300
 ====
 # Test analytic eval node.
@@ -284,7 +284,7 @@ select first_value(1 + 1 + int_col - (1 - 1)) over
 from functional.alltypes
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.03MB
+|  Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -309,7 +309,7 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=29B cardinality=7300
 ====
 # Test sort node.
@@ -317,7 +317,7 @@ select int_col from functional.alltypes
 order by id * abs((factorial(5) / power(2, 4)))
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=134.00MB mem-reservation=6.03MB
+|  Per-Host Resources: mem-estimate=134.00MB mem-reservation=6.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -333,7 +333,7 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=7300
 ====
 # Test HDFS table sink.
@@ -342,7 +342,7 @@ select id, int_col, cast(1 + 1 + 1 + year as int), cast(month - (1 - 1 - 1) as i
 from functional.alltypessmall
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=8.00KB
+|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
 WRITE TO HDFS [functional.alltypes, OVERWRITE=false, PARTITION-KEYS=(CAST(3 + year AS INT),CAST(month - -1 AS INT))]
 |  partitions=4
 |  mem-estimate=1.56KB mem-reservation=0B
@@ -354,7 +354,7 @@ WRITE TO HDFS [functional.alltypes, OVERWRITE=false, PARTITION-KEYS=(CAST(3 + ye
      partitions: 4/4 rows=100
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=32.00MB mem-reservation=8.00KB
+   mem-estimate=32.00MB mem-reservation=0B
    tuple-ids=0 row-size=16B cardinality=100
 ====
 # Constant folding does not work across query blocks.
@@ -366,7 +366,7 @@ select sum(id + c3) from
   ) v3
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=138.00MB mem-reservation=32.00KB
+|  Per-Host Resources: mem-estimate=138.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -383,6 +383,6 @@ PLAN-ROOT SINK
      columns: all
    extrapolated-rows=disabled
    limit: 2
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=4B cardinality=2
 ====

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test b/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test
index 3d234d1..c5081a7 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/disable-codegen.test
@@ -1,7 +1,7 @@
 # Rows per node is < 3000: codegen should be disabled.
 select count(*) from functional.alltypes
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=148.00MB
 Codegen disabled by planner
 
@@ -21,7 +21,7 @@ PLAN-ROOT SINK
 # Rows per node is > 3000: codegen should be enabled.
 select count(*) from functional.alltypesagg
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=128.00KB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=100.00MB
 
 PLAN-ROOT SINK
@@ -40,7 +40,7 @@ PLAN-ROOT SINK
 # No stats on functional_parquet: codegen should be disabled.
 select count(*) from functional_parquet.alltypes
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00KB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=21.00MB
 WARNING: The following tables are missing relevant table and/or column statistics.
 functional_parquet.alltypes
@@ -61,7 +61,7 @@ PLAN-ROOT SINK
 # > 3000 rows returned to coordinator: codegen should be enabled
 select * from functional_parquet.alltypes
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=88.00KB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=128.00MB
 WARNING: The following tables are missing relevant table and/or column statistics.
 functional_parquet.alltypes
@@ -78,7 +78,7 @@ select count(*)
 from functional.alltypes t1
 join functional.alltypestiny t2 on t1.id = t2.id
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=2.98MB
+Max Per-Host Resource Reservation: Memory=2.94MB
 Per-Host Resource Estimates: Memory=182.94MB
 Codegen disabled by planner
 
@@ -108,7 +108,7 @@ PLAN-ROOT SINK
 # Optimisation is disabled by cross join producing > 3000 rows
 select count(*) from functional.alltypes t1, functional.alltypes t2
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=64.00KB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=276.00MB
 
 PLAN-ROOT SINK
@@ -137,7 +137,7 @@ select count(*) from (
   union all
   select * from functional.alltypestiny) v
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=148.00MB
 Codegen disabled by planner
 
@@ -166,7 +166,7 @@ select count(*) from (
   union all
   select * from functional.alltypes) v
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=148.00MB
 
 PLAN-ROOT SINK
@@ -193,7 +193,7 @@ PLAN-ROOT SINK
 select sum(l_discount)
 from (select * from tpch.lineitem limit 1000) v
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=274.00MB
 Codegen disabled by planner
 
@@ -214,7 +214,7 @@ PLAN-ROOT SINK
 select sum(l_discount)
 from (select * from tpch.lineitem where l_orderkey > 100 limit 1000) v
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=274.00MB
 
 PLAN-ROOT SINK

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test b/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test
index 55439d6..bb12bca 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/fk-pk-join-detection.test
@@ -5,7 +5,7 @@ on ss_customer_sk = c_customer_sk
 where c_salutation = 'Mrs.'
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=185.50MB mem-reservation=25.50MB runtime-filters-memory=1.00MB
+|  Per-Host Resources: mem-estimate=185.50MB mem-reservation=9.50MB runtime-filters-memory=1.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -24,7 +24,7 @@ PLAN-ROOT SINK
 |       columns: all
 |     extrapolated-rows=disabled
 |     parquet dictionary predicates: c_salutation = 'Mrs.'
-|     mem-estimate=48.00MB mem-reservation=8.00MB
+|     mem-estimate=48.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=255B cardinality=16667
 |
 00:SCAN HDFS [tpcds.store_sales]
@@ -35,7 +35,7 @@ PLAN-ROOT SINK
      partitions: 1824/1824 rows=2880404
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=8.00MB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=100B cardinality=2880404
 ====
 # Single-column FK/PK join detection on left outer join. The join cardinality
@@ -46,7 +46,7 @@ on ss_customer_sk = c_customer_sk
 where c_salutation = 'Mrs.'
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=184.50MB mem-reservation=24.50MB
+|  Per-Host Resources: mem-estimate=184.50MB mem-reservation=8.50MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -65,7 +65,7 @@ PLAN-ROOT SINK
 |       columns: all
 |     extrapolated-rows=disabled
 |     parquet dictionary predicates: c_salutation = 'Mrs.'
-|     mem-estimate=48.00MB mem-reservation=8.00MB
+|     mem-estimate=48.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=255B cardinality=16667
 |
 00:SCAN HDFS [tpcds.store_sales]
@@ -75,7 +75,7 @@ PLAN-ROOT SINK
      partitions: 1824/1824 rows=2880404
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=8.00MB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=100B cardinality=2880404
 ====
 # Single-column FK/PK join detection on right outer join. The join cardinality
@@ -86,7 +86,7 @@ on ss_customer_sk = c_customer_sk
 where c_salutation = 'Mrs.'
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=185.50MB mem-reservation=25.50MB runtime-filters-memory=1.00MB
+|  Per-Host Resources: mem-estimate=185.50MB mem-reservation=9.50MB runtime-filters-memory=1.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -105,7 +105,7 @@ PLAN-ROOT SINK
 |       columns: all
 |     extrapolated-rows=disabled
 |     parquet dictionary predicates: c_salutation = 'Mrs.'
-|     mem-estimate=48.00MB mem-reservation=8.00MB
+|     mem-estimate=48.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=255B cardinality=16667
 |
 00:SCAN HDFS [tpcds.store_sales]
@@ -116,7 +116,7 @@ PLAN-ROOT SINK
      partitions: 1824/1824 rows=2880404
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=8.00MB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=100B cardinality=2880404
 ====
 # Multi-column FK/PK join detection
@@ -126,7 +126,7 @@ on ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number
 where sr_return_quantity < 10
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=214.75MB mem-reservation=22.75MB runtime-filters-memory=2.00MB
+|  Per-Host Resources: mem-estimate=214.75MB mem-reservation=6.75MB runtime-filters-memory=2.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -145,7 +145,7 @@ PLAN-ROOT SINK
 |       columns: all
 |     extrapolated-rows=disabled
 |     parquet dictionary predicates: sr_return_quantity < 10
-|     mem-estimate=80.00MB mem-reservation=8.00MB
+|     mem-estimate=80.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=88B cardinality=28751
 |
 00:SCAN HDFS [tpcds.store_sales]
@@ -156,7 +156,7 @@ PLAN-ROOT SINK
      partitions: 1824/1824 rows=2880404
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=8.00MB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=100B cardinality=2880404
 ====
 # Many-to-many join detection.
@@ -165,7 +165,7 @@ tpcds.store_sales inner join tpcds.web_sales
 on ss_sold_time_sk = ws_sold_time_sk
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=397.67MB mem-reservation=51.00MB runtime-filters-memory=1.00MB
+|  Per-Host Resources: mem-estimate=397.67MB mem-reservation=35.00MB runtime-filters-memory=1.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -182,7 +182,7 @@ PLAN-ROOT SINK
 |       table: rows=719384 size=140.07MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=160.00MB mem-reservation=8.00MB
+|     mem-estimate=160.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=144B cardinality=719384
 |
 00:SCAN HDFS [tpcds.store_sales]
@@ -193,7 +193,7 @@ PLAN-ROOT SINK
      partitions: 1824/1824 rows=2880404
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=8.00MB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=100B cardinality=2880404
 ====
 # PK/PK join is detected as FK/PK.
@@ -203,7 +203,7 @@ on a.d_date_sk = b.d_date_sk
 where a.d_holiday = "Y"
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=114.00MB mem-reservation=34.00MB runtime-filters-memory=1.00MB
+|  Per-Host Resources: mem-estimate=114.00MB mem-reservation=18.00MB runtime-filters-memory=1.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -222,7 +222,7 @@ PLAN-ROOT SINK
 |       columns: all
 |     extrapolated-rows=disabled
 |     parquet dictionary predicates: a.d_holiday = 'Y'
-|     mem-estimate=48.00MB mem-reservation=8.00MB
+|     mem-estimate=48.00MB mem-reservation=0B
 |     tuple-ids=0 row-size=303B cardinality=36525
 |
 01:SCAN HDFS [tpcds.date_dim b]
@@ -232,7 +232,7 @@ PLAN-ROOT SINK
      table: rows=73049 size=9.84MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=48.00MB mem-reservation=8.00MB
+   mem-estimate=48.00MB mem-reservation=0B
    tuple-ids=1 row-size=303B cardinality=73049
 ====
 # Single query with various join types combined.
@@ -246,7 +246,7 @@ where ss_item_sk = sr_item_sk and ss_ticket_number = sr_ticket_number
   and d1.d_fy_week_seq = 1000
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=362.81MB mem-reservation=50.81MB runtime-filters-memory=5.00MB
+|  Per-Host Resources: mem-estimate=362.81MB mem-reservation=12.75MB runtime-filters-memory=5.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -263,7 +263,7 @@ PLAN-ROOT SINK
 |       table: rows=100000 size=12.60MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=48.00MB mem-reservation=8.00MB
+|     mem-estimate=48.00MB mem-reservation=0B
 |     tuple-ids=2 row-size=4B cardinality=100000
 |
 07:HASH JOIN [INNER JOIN]
@@ -279,7 +279,7 @@ PLAN-ROOT SINK
 |       table: rows=73049 size=9.84MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=48.00MB mem-reservation=8.00MB
+|     mem-estimate=48.00MB mem-reservation=0B
 |     tuple-ids=4 row-size=4B cardinality=73049
 |
 06:HASH JOIN [INNER JOIN]
@@ -304,7 +304,7 @@ PLAN-ROOT SINK
 |  |       columns: all
 |  |     extrapolated-rows=disabled
 |  |     parquet dictionary predicates: d1.d_fy_week_seq = 1000
-|  |     mem-estimate=48.00MB mem-reservation=8.00MB
+|  |     mem-estimate=48.00MB mem-reservation=0B
 |  |     tuple-ids=3 row-size=8B cardinality=7
 |  |
 |  00:SCAN HDFS [tpcds.store_sales]
@@ -315,7 +315,7 @@ PLAN-ROOT SINK
 |       partitions: 1824/1824 rows=2880404
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=128.00MB mem-reservation=8.00MB
+|     mem-estimate=128.00MB mem-reservation=0B
 |     tuple-ids=0 row-size=24B cardinality=2880404
 |
 01:SCAN HDFS [tpcds.store_returns]
@@ -325,7 +325,7 @@ PLAN-ROOT SINK
      table: rows=287514 size=31.19MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=1 row-size=20B cardinality=287514
 ====
 # Assumed FK/PK join becasue of non-trivial equi-join exprs.
@@ -334,7 +334,7 @@ tpcds.store_sales inner join tpcds.customer
 on ss_customer_sk % 10 = c_customer_sk / 100
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=211.00MB mem-reservation=51.00MB runtime-filters-memory=1.00MB
+|  Per-Host Resources: mem-estimate=211.00MB mem-reservation=35.00MB runtime-filters-memory=1.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -351,7 +351,7 @@ PLAN-ROOT SINK
 |       table: rows=100000 size=12.60MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=48.00MB mem-reservation=8.00MB
+|     mem-estimate=48.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=255B cardinality=100000
 |
 00:SCAN HDFS [tpcds.store_sales]
@@ -362,7 +362,7 @@ PLAN-ROOT SINK
      partitions: 1824/1824 rows=2880404
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=8.00MB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=100B cardinality=2880404
 ====
 # Assumed FK/PK join due to missing stats on the rhs. Join cardinality is equal to
@@ -372,7 +372,7 @@ tpcds.store_sales inner join tpcds_seq_snap.customer
 on ss_customer_sk = c_customer_sk
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=2.17GB mem-reservation=51.00MB runtime-filters-memory=1.00MB
+|  Per-Host Resources: mem-estimate=2.17GB mem-reservation=35.00MB runtime-filters-memory=1.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -384,12 +384,12 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1 row-size=8B cardinality=2880404
 |
 |--01:SCAN HDFS [tpcds_seq_snap.customer]
-|     partitions=1/1 files=1 size=8.58MB
+|     partitions=1/1 files=1 size=8.59MB
 |     stored statistics:
-|       table: rows=unavailable size=8.58MB
+|       table: rows=unavailable size=8.59MB
 |       columns: unavailable
 |     extrapolated-rows=disabled
-|     mem-estimate=48.00MB mem-reservation=8.00MB
+|     mem-estimate=48.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=4B cardinality=unavailable
 |
 00:SCAN HDFS [tpcds.store_sales]
@@ -400,7 +400,7 @@ PLAN-ROOT SINK
      partitions: 1824/1824 rows=2880404
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=8.00MB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=4B cardinality=2880404
 ====
 # Assumed FK/PK join due to missing stats on the lhs. Join cardinality is unknown.
@@ -409,7 +409,7 @@ tpcds_seq_snap.store_sales inner join tpcds.customer
 on ss_customer_sk = c_customer_sk
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=178.94MB mem-reservation=18.94MB runtime-filters-memory=1.00MB
+|  Per-Host Resources: mem-estimate=178.94MB mem-reservation=2.94MB runtime-filters-memory=1.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -426,18 +426,18 @@ PLAN-ROOT SINK
 |       table: rows=100000 size=12.60MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=48.00MB mem-reservation=8.00MB
+|     mem-estimate=48.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=4B cardinality=100000
 |
 00:SCAN HDFS [tpcds_seq_snap.store_sales]
-   partitions=1824/1824 files=1824 size=207.85MB
+   partitions=1824/1824 files=1824 size=207.90MB
    runtime filters: RF000[bloom] -> ss_customer_sk
    stored statistics:
      table: rows=unavailable size=unavailable
      partitions: 0/1824 rows=unavailable
      columns: unavailable
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=8.00MB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=4B cardinality=unavailable
 ====
 # Join is detected as many-to-many even though the rhs join columns
@@ -448,7 +448,7 @@ tpcds.store_sales inner join
 on ss_sold_time_sk = ws_sold_time_sk
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=299.00MB mem-reservation=18.94MB runtime-filters-memory=1.00MB
+|  Per-Host Resources: mem-estimate=299.00MB mem-reservation=4.88MB runtime-filters-memory=1.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -470,7 +470,7 @@ PLAN-ROOT SINK
 |       table: rows=719384 size=140.07MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=160.00MB mem-reservation=8.00MB
+|     mem-estimate=160.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=4B cardinality=719384
 |
 00:SCAN HDFS [tpcds.store_sales]
@@ -481,6 +481,6 @@ PLAN-ROOT SINK
      partitions: 1824/1824 rows=2880404
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=8.00MB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=100B cardinality=2880404
 ====

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test b/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
index 1df9270..96015e0 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/max-row-size.test
@@ -4,7 +4,7 @@ select straight_join *
 from tpch_parquet.customer
     inner join tpch_parquet.nation on c_nationkey = n_nationkey
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=33.97MB
+Max Per-Host Resource Reservation: Memory=17.94MB
 Per-Host Resource Estimates: Memory=57.94MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -17,7 +17,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1 row-size=355B cardinality=150000
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-Per-Host Resources: mem-estimate=41.94MB mem-reservation=33.94MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=41.94MB mem-reservation=17.94MB runtime-filters-memory=1.00MB
 02:HASH JOIN [INNER JOIN, BROADCAST]
 |  hash predicates: c_nationkey = n_nationkey
 |  fk/pk conjuncts: c_nationkey = n_nationkey
@@ -30,14 +30,14 @@ Per-Host Resources: mem-estimate=41.94MB mem-reservation=33.94MB runtime-filters
 |  |  tuple-ids=1 row-size=117B cardinality=25
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
+|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.nation, RANDOM]
 |     partitions=1/1 files=1 size=2.74KB
 |     stored statistics:
 |       table: rows=25 size=2.74KB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=16.00MB mem-reservation=32.00KB
+|     mem-estimate=16.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=117B cardinality=25
 |
 00:SCAN HDFS [tpch_parquet.customer, RANDOM]
@@ -47,7 +47,7 @@ Per-Host Resources: mem-estimate=41.94MB mem-reservation=33.94MB runtime-filters
      table: rows=150000 size=12.31MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=24.00MB mem-reservation=16.00MB
+   mem-estimate=24.00MB mem-reservation=0B
    tuple-ids=0 row-size=238B cardinality=150000
 ====
 # Join with large build side.
@@ -56,8 +56,8 @@ select straight_join *
 from tpch_parquet.lineitem
     left join tpch_parquet.orders on l_orderkey = o_orderkey
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=166.00MB
-Per-Host Resource Estimates: Memory=428.41MB
+Max Per-Host Resource Reservation: Memory=46.00MB
+Per-Host Resource Estimates: Memory=420.41MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
 |  Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -69,7 +69,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1N row-size=454B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=380.41MB mem-reservation=118.00MB
+Per-Host Resources: mem-estimate=380.41MB mem-reservation=46.00MB
 02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
 |  hash predicates: l_orderkey = o_orderkey
 |  fk/pk conjuncts: l_orderkey = o_orderkey
@@ -81,23 +81,23 @@ Per-Host Resources: mem-estimate=380.41MB mem-reservation=118.00MB
 |  |  tuple-ids=1 row-size=191B cardinality=1500000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-|  Per-Host Resources: mem-estimate=48.00MB mem-reservation=48.00MB
+|  Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.orders, RANDOM]
 |     partitions=1/1 files=2 size=54.07MB
 |     stored statistics:
 |       table: rows=1500000 size=54.07MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=48.00MB mem-reservation=48.00MB
+|     mem-estimate=40.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=191B cardinality=1500000
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.72MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.72MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ====
 # Null-aware anti-join with medium build side.
@@ -105,7 +105,7 @@ Per-Host Resources: mem-estimate=380.41MB mem-reservation=118.00MB
 select * from tpch_parquet.lineitem
 where l_orderkey not in (select o_orderkey from tpch_parquet.orders)
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=114.00MB
+Max Per-Host Resource Reservation: Memory=34.00MB
 Per-Host Resource Estimates: Memory=154.00MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -118,7 +118,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0 row-size=263B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=114.00MB mem-reservation=106.00MB
+Per-Host Resources: mem-estimate=114.00MB mem-reservation=34.00MB
 02:HASH JOIN [NULL AWARE LEFT ANTI JOIN, BROADCAST]
 |  hash predicates: l_orderkey = o_orderkey
 |  mem-estimate=34.00MB mem-reservation=34.00MB spill-buffer=1.00MB
@@ -129,23 +129,23 @@ Per-Host Resources: mem-estimate=114.00MB mem-reservation=106.00MB
 |  |  tuple-ids=1 row-size=8B cardinality=1500000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-|  Per-Host Resources: mem-estimate=40.00MB mem-reservation=8.00MB
+|  Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.orders, RANDOM]
 |     partitions=1/1 files=2 size=54.07MB
 |     stored statistics:
 |       table: rows=1500000 size=54.07MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=40.00MB mem-reservation=8.00MB
+|     mem-estimate=40.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=8B cardinality=1500000
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.72MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.72MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ====
 # Mid NDV aggregation.
@@ -156,7 +156,7 @@ from tpch_parquet.lineitem
 group by 1, 2
 having count(*) = 1
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=137.00MB
+Max Per-Host Resource Reservation: Memory=113.00MB
 Per-Host Resource Estimates: Memory=253.12MB
 
 F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -201,14 +201,14 @@ Per-Host Resources: mem-estimate=86.12MB mem-reservation=66.00MB runtime-filters
 |  |  tuple-ids=1 row-size=25B cardinality=1500000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-|  Per-Host Resources: mem-estimate=40.00MB mem-reservation=16.00MB
+|  Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.orders, RANDOM]
 |     partitions=1/1 files=2 size=54.07MB
 |     stored statistics:
 |       table: rows=1500000 size=54.07MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=40.00MB mem-reservation=16.00MB
+|     mem-estimate=40.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=25B cardinality=1500000
 |
 04:EXCHANGE [HASH(l_orderkey)]
@@ -216,15 +216,15 @@ Per-Host Resources: mem-estimate=86.12MB mem-reservation=66.00MB runtime-filters
 |  tuple-ids=0 row-size=8B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=81.00MB mem-reservation=9.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=81.00MB mem-reservation=1.00MB runtime-filters-memory=1.00MB
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.72MB
+   partitions=1/1 files=3 size=193.73MB
    runtime filters: RF000[bloom] -> l_orderkey
    stored statistics:
-     table: rows=6001215 size=193.72MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=6001215
 ====
 # High NDV aggregation.
@@ -232,7 +232,7 @@ Per-Host Resources: mem-estimate=81.00MB mem-reservation=9.00MB runtime-filters-
 select distinct *
 from tpch_parquet.lineitem
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=152.00MB
+Max Per-Host Resource Reservation: Memory=80.00MB
 Per-Host Resource Estimates: Memory=3.31GB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -256,19 +256,19 @@ Per-Host Resources: mem-estimate=1.62GB mem-reservation=46.00MB
 |  tuple-ids=1 row-size=263B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=1.69GB mem-reservation=106.00MB
+Per-Host Resources: mem-estimate=1.69GB mem-reservation=34.00MB
 01:AGGREGATE [STREAMING]
 |  group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
 |  mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
 |  tuple-ids=1 row-size=263B cardinality=6001215
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.72MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.72MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ====
 # High NDV aggregation with string aggregation function.
@@ -277,7 +277,7 @@ select l_orderkey, l_partkey, group_concat(l_linestatus, ",")
 from tpch_parquet.lineitem
 group by 1, 2
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=106.00MB
+Max Per-Host Resource Reservation: Memory=82.00MB
 Per-Host Resource Estimates: Memory=482.91MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -302,7 +302,7 @@ Per-Host Resources: mem-estimate=201.46MB mem-reservation=48.00MB
 |  tuple-ids=1 row-size=32B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=281.46MB mem-reservation=58.00MB
+Per-Host Resources: mem-estimate=281.46MB mem-reservation=34.00MB
 01:AGGREGATE [STREAMING]
 |  output: group_concat(l_linestatus, ',')
 |  group by: l_orderkey, l_partkey
@@ -310,12 +310,12 @@ Per-Host Resources: mem-estimate=281.46MB mem-reservation=58.00MB
 |  tuple-ids=1 row-size=32B cardinality=6001215
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.72MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.72MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=24.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=33B cardinality=6001215
 ====
 # Sort + Analytic.
@@ -323,7 +323,7 @@ Per-Host Resources: mem-estimate=281.46MB mem-reservation=58.00MB
 select max(tinyint_col) over(partition by int_col)
 from functional.alltypes
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=40.03MB
+Max Per-Host Resource Reservation: Memory=40.00MB
 Per-Host Resource Estimates: Memory=56.00MB
 Codegen disabled by planner
 
@@ -354,7 +354,7 @@ Per-Host Resources: mem-estimate=40.00MB mem-reservation=40.00MB
 |  tuple-ids=0 row-size=5B cardinality=7300
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
+Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
 00:SCAN HDFS [functional.alltypes, RANDOM]
    partitions=24/24 files=24 size=478.45KB
    stored statistics:
@@ -362,6 +362,6 @@ Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=32.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=5B cardinality=7300
 ====

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test b/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test
index 8dd2593..7f6d96b 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/min-max-runtime-filters.test
@@ -99,7 +99,7 @@ select count(*) from functional_kudu.alltypes a, functional_parquet.alltypes b,
 where a.int_col = b.int_col and a.int_col = c.int_col
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=2.02GB mem-reservation=36.95MB runtime-filters-memory=1.00MB
+|  Per-Host Resources: mem-estimate=2.02GB mem-reservation=36.94MB runtime-filters-memory=1.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -127,14 +127,14 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1 row-size=8B cardinality=7300
 |
 |--01:SCAN HDFS [functional_parquet.alltypes b]
-|     partitions=24/24 files=24 size=174.62KB
+|     partitions=24/24 files=24 size=174.39KB
 |     runtime filters: RF000[bloom] -> b.int_col
 |     stored statistics:
 |       table: rows=unavailable size=unavailable
 |       partitions: 0/24 rows=unavailable
 |       columns: unavailable
 |     extrapolated-rows=disabled
-|     mem-estimate=16.00MB mem-reservation=8.00KB
+|     mem-estimate=16.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=4B cardinality=unavailable
 |
 00:SCAN KUDU [functional_kudu.alltypes a]

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test b/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
index 31129ff..61d646b 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/mt-dop-validation.test
@@ -40,7 +40,7 @@ order by cnt, bigint_col
 limit 10
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.02MB
+|  Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -56,7 +56,7 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=16B cardinality=unavailable
 |
 00:SCAN HDFS [functional_parquet.alltypes]
-   partitions=24/24 files=24 size=174.39KB
+   partitions=24/24 files=24 size=179.19KB
    predicates: id < 10
    stored statistics:
      table: rows=unavailable size=unavailable
@@ -65,7 +65,7 @@ PLAN-ROOT SINK
    extrapolated-rows=disabled
    parquet statistics predicates: id < 10
    parquet dictionary predicates: id < 10
-   mem-estimate=16.00MB mem-reservation=24.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=16B cardinality=unavailable
 ---- PARALLELPLANS
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -97,7 +97,7 @@ Per-Host Resources: mem-estimate=384.00MB mem-reservation=102.00MB
 |  tuple-ids=1 row-size=16B cardinality=unavailable
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=9
-Per-Host Resources: mem-estimate=432.00MB mem-reservation=102.07MB
+Per-Host Resources: mem-estimate=432.00MB mem-reservation=102.00MB
 01:AGGREGATE [STREAMING]
 |  output: count(int_col)
 |  group by: bigint_col
@@ -105,7 +105,7 @@ Per-Host Resources: mem-estimate=432.00MB mem-reservation=102.07MB
 |  tuple-ids=1 row-size=16B cardinality=unavailable
 |
 00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
-   partitions=24/24 files=24 size=174.39KB
+   partitions=24/24 files=24 size=179.19KB
    predicates: id < 10
    stored statistics:
      table: rows=unavailable size=unavailable
@@ -114,7 +114,7 @@ Per-Host Resources: mem-estimate=432.00MB mem-reservation=102.07MB
    extrapolated-rows=disabled
    parquet statistics predicates: id < 10
    parquet dictionary predicates: id < 10
-   mem-estimate=16.00MB mem-reservation=24.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=16B cardinality=unavailable
 ====
 # Single-table scan/filter/analytic should work.
@@ -123,7 +123,7 @@ from functional_parquet.alltypes
 where id < 10
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=26.00MB mem-reservation=10.02MB
+|  Per-Host Resources: mem-estimate=26.00MB mem-reservation=10.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -141,7 +141,7 @@ PLAN-ROOT SINK
 |  tuple-ids=4 row-size=8B cardinality=unavailable
 |
 00:SCAN HDFS [functional_parquet.alltypes]
-   partitions=24/24 files=24 size=174.39KB
+   partitions=24/24 files=24 size=179.19KB
    predicates: id < 10
    stored statistics:
      table: rows=unavailable size=unavailable
@@ -150,7 +150,7 @@ PLAN-ROOT SINK
    extrapolated-rows=disabled
    parquet statistics predicates: id < 10
    parquet dictionary predicates: id < 10
-   mem-estimate=16.00MB mem-reservation=16.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=unavailable
 ---- PARALLELPLANS
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -182,9 +182,9 @@ Per-Host Resources: mem-estimate=30.00MB mem-reservation=30.00MB
 |  tuple-ids=0 row-size=8B cardinality=unavailable
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=9
-Per-Host Resources: mem-estimate=48.00MB mem-reservation=48.00KB
+Per-Host Resources: mem-estimate=48.00MB mem-reservation=0B
 00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
-   partitions=24/24 files=24 size=174.39KB
+   partitions=24/24 files=24 size=179.19KB
    predicates: id < 10
    stored statistics:
      table: rows=unavailable size=unavailable
@@ -193,7 +193,7 @@ Per-Host Resources: mem-estimate=48.00MB mem-reservation=48.00KB
    extrapolated-rows=disabled
    parquet statistics predicates: id < 10
    parquet dictionary predicates: id < 10
-   mem-estimate=16.00MB mem-reservation=16.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=unavailable
 ====
 # Nested-loop join in a subplan should work.
@@ -202,7 +202,7 @@ from tpch_nested_parquet.customer c, c.c_orders o, o.o_lineitems
 where c_custkey < 10 and o_orderkey < 5 and l_linenumber < 3
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=88.00MB
+|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -257,7 +257,7 @@ PLAN-ROOT SINK
    parquet dictionary predicates: c_custkey < 10
    parquet dictionary predicates on o: o_orderkey < 5
    parquet dictionary predicates on o_lineitems: l_linenumber < 3
-   mem-estimate=88.00MB mem-reservation=88.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=254B cardinality=15000
 ---- PARALLELPLANS
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -270,7 +270,7 @@ PLAN-ROOT SINK
 |  tuple-ids=2,1,0 row-size=562B cardinality=1500000
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=9
-Per-Host Resources: mem-estimate=264.00MB mem-reservation=264.00MB
+Per-Host Resources: mem-estimate=264.00MB mem-reservation=0B
 01:SUBPLAN
 |  mem-estimate=0B mem-reservation=0B
 |  tuple-ids=2,1,0 row-size=562B cardinality=1500000
@@ -322,7 +322,7 @@ Per-Host Resources: mem-estimate=264.00MB mem-reservation=264.00MB
    parquet dictionary predicates: c_custkey < 10
    parquet dictionary predicates on o: o_orderkey < 5
    parquet dictionary predicates on o_lineitems: l_linenumber < 3
-   mem-estimate=88.00MB mem-reservation=88.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=254B cardinality=15000
 ====
 # Hash-join in a subplan should work.
@@ -331,7 +331,7 @@ from tpch_nested_parquet.customer c, c.c_orders o1, c.c_orders o2
 where o1.o_orderkey = o2.o_orderkey + 2 and o1.o_orderkey < 5
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=89.94MB mem-reservation=81.94MB
+|  Per-Host Resources: mem-estimate=89.94MB mem-reservation=1.94MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -374,7 +374,7 @@ PLAN-ROOT SINK
    extrapolated-rows=disabled
    parquet statistics predicates on o1: o1.o_orderkey < 5
    parquet dictionary predicates on o1: o1.o_orderkey < 5
-   mem-estimate=88.00MB mem-reservation=80.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=270B cardinality=150000
 ---- PARALLELPLANS
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -387,7 +387,7 @@ PLAN-ROOT SINK
 |  tuple-ids=1,0,2 row-size=286B cardinality=1500000
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=9
-Per-Host Resources: mem-estimate=269.81MB mem-reservation=245.81MB
+Per-Host Resources: mem-estimate=269.81MB mem-reservation=5.81MB
 01:SUBPLAN
 |  mem-estimate=0B mem-reservation=0B
 |  tuple-ids=1,0,2 row-size=286B cardinality=1500000
@@ -427,6 +427,6 @@ Per-Host Resources: mem-estimate=269.81MB mem-reservation=245.81MB
    extrapolated-rows=disabled
    parquet statistics predicates on o1: o1.o_orderkey < 5
    parquet dictionary predicates on o1: o1.o_orderkey < 5
-   mem-estimate=88.00MB mem-reservation=80.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=270B cardinality=150000
 ====

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test b/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
index 3e812c6..2b602c9 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/parquet-filtering.test
@@ -8,7 +8,7 @@ where int_col > 1 and int_col * rand() > 50 and int_col is null
 and int_col > tinyint_col;
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=42.00MB mem-reservation=16.00KB
+|  Per-Host Resources: mem-estimate=42.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -18,7 +18,7 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=8B cardinality=1
 |
 00:SCAN HDFS [functional_parquet.alltypes]
-   partitions=24/24 files=24 size=174.39KB
+   partitions=24/24 files=24 size=179.19KB
    predicates: int_col IS NULL, int_col > 1, int_col > tinyint_col, int_col * rand() > 50
    stored statistics:
      table: rows=unavailable size=unavailable
@@ -27,7 +27,7 @@ PLAN-ROOT SINK
    extrapolated-rows=disabled
    parquet statistics predicates: int_col > 1
    parquet dictionary predicates: int_col > 1
-   mem-estimate=32.00MB mem-reservation=16.00KB
+   mem-estimate=32.00MB mem-reservation=0B
    tuple-ids=0 row-size=5B cardinality=unavailable
 ====
 # Test a variety of types
@@ -40,7 +40,7 @@ and timestamp_cmp(timestamp_col, '2016-11-20 00:00:00') = 1
 and year > 2000 and month < 12;
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=138.00MB mem-reservation=88.00KB
+|  Per-Host Resources: mem-estimate=138.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -50,7 +50,7 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=8B cardinality=1
 |
 00:SCAN HDFS [functional_parquet.alltypes]
-   partitions=22/24 files=22 size=159.69KB
+   partitions=22/24 files=22 size=164.09KB
    predicates: bool_col, bigint_col < 5000, double_col > 100.00, float_col > 50.00, id = 1, tinyint_col < 50, string_col IN ('aaaa', 'bbbb', 'cccc'), smallint_col IN (1, 2, 3, 4, 5), mod(int_col, 2) = 1, timestamp_cmp(timestamp_col, TIMESTAMP '2016-11-20 00:00:00') = 1, date_string_col > '1993-10-01'
    stored statistics:
      table: rows=unavailable size=unavailable
@@ -59,7 +59,7 @@ PLAN-ROOT SINK
    extrapolated-rows=disabled
    parquet statistics predicates: bigint_col < 5000, double_col > 100.00, float_col > 50.00, id = 1, tinyint_col < 50, string_col IN ('aaaa', 'bbbb', 'cccc'), smallint_col IN (1, 2, 3, 4, 5), date_string_col > '1993-10-01'
    parquet dictionary predicates: bool_col, bigint_col < 5000, double_col > 100.00, float_col > 50.00, id = 1, tinyint_col < 50, string_col IN ('aaaa', 'bbbb', 'cccc'), smallint_col IN (1, 2, 3, 4, 5), mod(int_col, 2) = 1, timestamp_cmp(timestamp_col, TIMESTAMP '2016-11-20 00:00:00') = 1, date_string_col > '1993-10-01'
-   mem-estimate=128.00MB mem-reservation=88.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=80B cardinality=unavailable
 ====
 # Test negative cases for IN predicate min/max filtering
@@ -73,7 +73,7 @@ and mod(int_col,50) IN (0,1)
 and id IN (int_col);
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=58.00MB mem-reservation=24.00KB
+|  Per-Host Resources: mem-estimate=58.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -83,7 +83,7 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=8B cardinality=1
 |
 00:SCAN HDFS [functional_parquet.alltypes]
-   partitions=24/24 files=24 size=174.39KB
+   partitions=24/24 files=24 size=179.19KB
    predicates: id IN (int_col), id NOT IN (0, 1, 2), string_col IN ('aaaa', 'bbbb', 'cccc', NULL), mod(int_col, 50) IN (0, 1)
    stored statistics:
      table: rows=unavailable size=unavailable
@@ -91,7 +91,7 @@ PLAN-ROOT SINK
      columns: unavailable
    extrapolated-rows=disabled
    parquet dictionary predicates: id NOT IN (0, 1, 2), string_col IN ('aaaa', 'bbbb', 'cccc', NULL), mod(int_col, 50) IN (0, 1)
-   mem-estimate=48.00MB mem-reservation=24.00KB
+   mem-estimate=48.00MB mem-reservation=0B
    tuple-ids=0 row-size=24B cardinality=unavailable
 ====
 # Test collection types where all collections on the path are required (inner
@@ -101,7 +101,7 @@ select id from functional_parquet.complextypestbl c, c.nested_struct.c.d cn, cn.
 where a.item.e < -10;
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=16.00KB
+|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -152,7 +152,7 @@ PLAN-ROOT SINK
    extrapolated-rows=disabled
    parquet statistics predicates on a: a.item.e < -10
    parquet dictionary predicates on a: a.item.e < -10
-   mem-estimate=32.00MB mem-reservation=16.00KB
+   mem-estimate=32.00MB mem-reservation=0B
    tuple-ids=0 row-size=24B cardinality=unavailable
 ====
 # Test collection types where the lower collection in the path is optional
@@ -164,7 +164,7 @@ left outer join cn.item a
 where a.item.e < -10;
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=16.00KB
+|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -213,7 +213,7 @@ PLAN-ROOT SINK
      table: rows=unavailable size=unavailable
      columns missing stats: id
    extrapolated-rows=disabled
-   mem-estimate=32.00MB mem-reservation=16.00KB
+   mem-estimate=32.00MB mem-reservation=0B
    tuple-ids=0 row-size=24B cardinality=unavailable
 ====
 # Tests collection types where the outer is optional (outer join descent)
@@ -223,7 +223,7 @@ select id from functional_parquet.complextypestbl c
 left outer join c.nested_struct.c.d cn, cn.item a where a.item.e < -10;
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=16.00KB
+|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -270,7 +270,7 @@ PLAN-ROOT SINK
      table: rows=unavailable size=unavailable
      columns missing stats: id
    extrapolated-rows=disabled
-   mem-estimate=32.00MB mem-reservation=16.00KB
+   mem-estimate=32.00MB mem-reservation=0B
    tuple-ids=0 row-size=24B cardinality=unavailable
 ====
 # Test collections so that each level has a filter applied.
@@ -278,7 +278,7 @@ select c_custkey from tpch_nested_parquet.customer c, c.c_orders o,
 o.o_lineitems l where c_custkey > 0 and o.o_orderkey > 0 and l.l_partkey > 0;
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=264.00MB mem-reservation=24.00MB
+|  Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -333,7 +333,7 @@ PLAN-ROOT SINK
    parquet dictionary predicates: c_custkey > 0
    parquet dictionary predicates on o: o.o_orderkey > 0
    parquet dictionary predicates on l: l.l_partkey > 0
-   mem-estimate=264.00MB mem-reservation=24.00MB
+   mem-estimate=176.00MB mem-reservation=0B
    tuple-ids=0 row-size=24B cardinality=15000
 ====
 # Test collections in a way that would incorrectly apply a min-max
@@ -342,7 +342,7 @@ select count(*) from functional_parquet.complextypestbl c left outer join
 (select * from c.int_array where item > 10) v;
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=26.00MB mem-reservation=8.00KB
+|  Per-Host Resources: mem-estimate=26.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -376,7 +376,7 @@ PLAN-ROOT SINK
      table: rows=unavailable size=unavailable
      columns: unavailable
    extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=8.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=16B cardinality=unavailable
 ====
 # Multiple nested collection values (at the same nesting level) where dictionary
@@ -388,7 +388,7 @@ l.l_receiptdate = '1994-08-24' and l.l_shipmode = 'RAIL' and l.l_returnflag = 'R
 l.l_comment is null;
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=616.00MB mem-reservation=56.00MB
+|  Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -439,6 +439,6 @@ PLAN-ROOT SINK
    extrapolated-rows=disabled
    parquet statistics predicates on l: l.l_shipdate = '1994-08-19', l.l_receiptdate = '1994-08-24', l.l_shipmode = 'RAIL', l.l_returnflag = 'R'
    parquet dictionary predicates on l: l.l_shipdate = '1994-08-19', l.l_receiptdate = '1994-08-24', l.l_shipmode = 'RAIL', l.l_returnflag = 'R'
-   mem-estimate=616.00MB mem-reservation=56.00MB
+   mem-estimate=176.00MB mem-reservation=0B
    tuple-ids=0 row-size=50B cardinality=150000
 ====

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/partition-pruning.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/partition-pruning.test b/testdata/workloads/functional-planner/queries/PlannerTest/partition-pruning.test
index 5cbba7b..610136d 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/partition-pruning.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/partition-pruning.test
@@ -4,7 +4,7 @@ select * from functional.stringpartitionkey
 where string_col=cast("2009-01-01 00:00:00" as timestamp);
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=8.00KB
+|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -15,6 +15,6 @@ PLAN-ROOT SINK
      partitions: 1/1 rows=1
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=32.00MB mem-reservation=8.00KB
+   mem-estimate=32.00MB mem-reservation=0B
    tuple-ids=0 row-size=20B cardinality=1
 ====


[5/9] impala git commit: Revert IMPALA-4835 and dependent changes

Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/request-context.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/request-context.cc b/be/src/runtime/io/request-context.cc
index dec6aa6..287f53a 100644
--- a/be/src/runtime/io/request-context.cc
+++ b/be/src/runtime/io/request-context.cc
@@ -17,122 +17,74 @@
 
 #include "runtime/io/disk-io-mgr-internal.h"
 
-#include "runtime/exec-env.h"
-
 #include "common/names.h"
 
 using namespace impala;
 using namespace impala::io;
 
-BufferDescriptor::BufferDescriptor(DiskIoMgr* io_mgr,
-    RequestContext* reader, ScanRange* scan_range, uint8_t* buffer,
-    int64_t buffer_len)
-  : io_mgr_(io_mgr),
-    reader_(reader),
-    scan_range_(scan_range),
-    buffer_(buffer),
-    buffer_len_(buffer_len) {
-  DCHECK(io_mgr != nullptr);
-  DCHECK(scan_range != nullptr);
-  DCHECK(buffer != nullptr);
-  DCHECK_GE(buffer_len, 0);
-}
+void RequestContext::Cancel(const Status& status) {
+  DCHECK(!status.ok());
 
-BufferDescriptor::BufferDescriptor(DiskIoMgr* io_mgr, RequestContext* reader,
-    ScanRange* scan_range, BufferPool::ClientHandle* bp_client,
-    BufferPool::BufferHandle handle) :
-  io_mgr_(io_mgr),
-  reader_(reader),
-  scan_range_(scan_range),
-  buffer_(handle.data()),
-  buffer_len_(handle.len()),
-  bp_client_(bp_client),
-  handle_(move(handle)) {
-  DCHECK(io_mgr != nullptr);
-  DCHECK(scan_range != nullptr);
-  DCHECK(bp_client_->is_registered());
-  DCHECK(handle_.is_open());
-}
-
-void RequestContext::FreeBuffer(BufferDescriptor* buffer) {
-  DCHECK(buffer->buffer_ != nullptr);
-  if (!buffer->is_cached() && !buffer->is_client_buffer()) {
-    // Only buffers that were allocated by DiskIoMgr need to be freed.
-    ExecEnv::GetInstance()->buffer_pool()->FreeBuffer(
-        buffer->bp_client_, &buffer->handle_);
-  }
-  buffer->buffer_ = nullptr;
-}
-
-// Cancellation of a RequestContext requires coordination from multiple threads that may
-// hold references to the context:
-//  1. Disk threads that are currently processing a range for this context.
-//  2. Caller threads that are waiting in GetNext().
-//
-// Each thread that currently has a reference to the request context must notice the
-// cancel, cancel any pending operations involving the context and remove the contxt from
-// tracking structures. Once no more operations are pending on the context and no more
-// I/O mgr threads hold references to the context, the context can be marked inactive
-// (see CancelAndMarkInactive()), after which the owner of the context object can free
-// it.
-//
-// The steps are:
-// 1. Cancel() will immediately set the context in the Cancelled state. This prevents any
-// other thread from adding more ready buffers to the context (they all take a lock and
-// check the state before doing so), or any write ranges to the context.
-// 2. Cancel() will call Cancel() on each ScanRange that is not yet complete, unblocking
-// any threads in GetNext(). If there was no prior error for a scan range, any reads from
-// that scan range will return a CANCELLED Status. Cancel() also invokes callbacks for
-// all WriteRanges with a CANCELLED Status.
-// 3. Disk threads notice the context is cancelled either when picking the next context
-// to process or when they try to enqueue a ready buffer. Upon noticing the cancelled
-// state, removes the context from the disk queue. The last thread per disk then calls
-// DecrementDiskRefCount(). After the last disk thread has called DecrementDiskRefCount(),
-// cancellation is done and it is safe to unregister the context.
-void RequestContext::Cancel() {
   // Callbacks are collected in this vector and invoked while no lock is held.
   vector<WriteRange::WriteDoneCallback> write_callbacks;
   {
-    unique_lock<mutex> lock(lock_);
+    lock_guard<mutex> lock(lock_);
     DCHECK(Validate()) << endl << DebugString();
 
     // Already being cancelled
     if (state_ == RequestContext::Cancelled) return;
 
+    DCHECK(status_.ok());
+    status_ = status;
+
     // The reader will be put into a cancelled state until call cleanup is complete.
     state_ = RequestContext::Cancelled;
 
-    // Clear out all request ranges from queues for this reader. Cancel the scan
-    // ranges and invoke the write range callbacks to propagate the cancellation.
-    for (ScanRange* range : active_scan_ranges_) range->CancelInternal(Status::CANCELLED);
-    active_scan_ranges_.clear();
-    for (PerDiskState& disk_state : disk_states_) {
-      RequestRange* range;
-      while ((range = disk_state.in_flight_ranges()->Dequeue()) != nullptr) {
-        if (range->request_type() == RequestType::WRITE) {
+    // Cancel all scan ranges for this reader. Each range could be one one of
+    // four queues.
+    for (int i = 0; i < disk_states_.size(); ++i) {
+      RequestContext::PerDiskState& state = disk_states_[i];
+      RequestRange* range = NULL;
+      while ((range = state.in_flight_ranges()->Dequeue()) != NULL) {
+        if (range->request_type() == RequestType::READ) {
+          static_cast<ScanRange*>(range)->Cancel(status);
+        } else {
+          DCHECK(range->request_type() == RequestType::WRITE);
           write_callbacks.push_back(static_cast<WriteRange*>(range)->callback_);
         }
       }
-      while (disk_state.unstarted_scan_ranges()->Dequeue() != nullptr);
+
+      ScanRange* scan_range;
+      while ((scan_range = state.unstarted_scan_ranges()->Dequeue()) != NULL) {
+        scan_range->Cancel(status);
+      }
       WriteRange* write_range;
-      while ((write_range = disk_state.unstarted_write_ranges()->Dequeue()) != nullptr) {
+      while ((write_range = state.unstarted_write_ranges()->Dequeue()) != NULL) {
         write_callbacks.push_back(write_range->callback_);
       }
     }
-    // Clear out the lists of scan ranges.
-    while (ready_to_start_ranges_.Dequeue() != nullptr);
-    while (cached_ranges_.Dequeue() != nullptr);
 
-    // Ensure that the reader is scheduled on all disks (it may already be scheduled on
-    // some). The disk threads will notice that the context is cancelled and do any
-    // required cleanup for the disk state.
+    ScanRange* range = NULL;
+    while ((range = ready_to_start_ranges_.Dequeue()) != NULL) {
+      range->Cancel(status);
+    }
+    while ((range = blocked_ranges_.Dequeue()) != NULL) {
+      range->Cancel(status);
+    }
+    while ((range = cached_ranges_.Dequeue()) != NULL) {
+      range->Cancel(status);
+    }
+
+    // Schedule reader on all disks. The disks will notice it is cancelled and do any
+    // required cleanup
     for (int i = 0; i < disk_states_.size(); ++i) {
-      disk_states_[i].ScheduleContext(lock, this, i);
+      RequestContext::PerDiskState& state = disk_states_[i];
+      state.ScheduleContext(this, i);
     }
   }
 
   for (const WriteRange::WriteDoneCallback& write_callback: write_callbacks) {
-    write_callback(Status::CANCELLED);
+    write_callback(status_);
   }
 
   // Signal reader and unblock the GetNext/Read thread.  That read will fail with
@@ -141,7 +93,7 @@ void RequestContext::Cancel() {
 }
 
 void RequestContext::CancelAndMarkInactive() {
-  Cancel();
+  Cancel(Status::CANCELLED);
 
   boost::unique_lock<boost::mutex> l(lock_);
   DCHECK_NE(state_, Inactive);
@@ -150,76 +102,54 @@ void RequestContext::CancelAndMarkInactive() {
   // Wait until the ranges finish up.
   while (num_disks_with_ranges_ > 0) disks_complete_cond_var_.Wait(l);
 
-  // Validate that no ranges are active.
-  DCHECK_EQ(0, active_scan_ranges_.size()) << endl << DebugString();
-
-  // Validate that no threads are active and the context is not queued.
-  for (const PerDiskState& disk_state : disk_states_) {
-    DCHECK_EQ(0, disk_state.in_flight_ranges()->size()) << endl << DebugString();
-    DCHECK_EQ(0, disk_state.unstarted_scan_ranges()->size()) << endl << DebugString();
-    DCHECK_EQ(0, disk_state.num_threads_in_op()) << endl << DebugString();
-    DCHECK(!disk_state.is_on_queue()) << endl << DebugString();
-  }
+  // Validate that no buffers were leaked from this context.
+  DCHECK_EQ(num_buffers_in_reader_.Load(), 0) << endl << DebugString();
+  DCHECK_EQ(num_used_buffers_.Load(), 0) << endl << DebugString();
   DCHECK(Validate()) << endl << DebugString();
   state_ = Inactive;
 }
 
-void RequestContext::AddRangeToDisk(const unique_lock<mutex>& lock,
-    RequestRange* range, ScheduleMode schedule_mode) {
-  DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
-  DCHECK_EQ(state_, Active) << DebugString();
-  PerDiskState* disk_state = &disk_states_[range->disk_id()];
-  if (disk_state->done()) {
-    DCHECK_EQ(disk_state->num_remaining_ranges(), 0);
-    disk_state->set_done(false);
+void RequestContext::AddRequestRange(
+    RequestRange* range, bool schedule_immediately) {
+  // DCHECK(lock_.is_locked()); // TODO: boost should have this API
+  RequestContext::PerDiskState& state = disk_states_[range->disk_id()];
+  if (state.done()) {
+    DCHECK_EQ(state.num_remaining_ranges(), 0);
+    state.set_done(false);
     ++num_disks_with_ranges_;
   }
+
+  bool schedule_context;
   if (range->request_type() == RequestType::READ) {
     ScanRange* scan_range = static_cast<ScanRange*>(range);
-    if (schedule_mode == ScheduleMode::IMMEDIATELY) {
-      ScheduleScanRange(lock, scan_range);
-    } else if (schedule_mode == ScheduleMode::UPON_GETNEXT) {
-      disk_state->unstarted_scan_ranges()->Enqueue(scan_range);
+    if (schedule_immediately) {
+      ScheduleScanRange(scan_range);
+    } else {
+      state.unstarted_scan_ranges()->Enqueue(scan_range);
       num_unstarted_scan_ranges_.Add(1);
-      // If there's no 'next_scan_range_to_start', schedule this RequestContext so that
-      // one of the 'unstarted_scan_ranges' will become the 'next_scan_range_to_start'.
-      if (disk_state->next_scan_range_to_start() == nullptr) {
-        disk_state->ScheduleContext(lock, this, range->disk_id());
-      }
     }
+    // If next_scan_range_to_start is NULL, schedule this RequestContext so that it will
+    // be set. If it's not NULL, this context will be scheduled when GetNextRange() is
+    // invoked.
+    schedule_context = state.next_scan_range_to_start() == NULL;
   } else {
     DCHECK(range->request_type() == RequestType::WRITE);
-    DCHECK(schedule_mode == ScheduleMode::IMMEDIATELY) << static_cast<int>(schedule_mode);
+    DCHECK(!schedule_immediately);
     WriteRange* write_range = static_cast<WriteRange*>(range);
-    disk_state->unstarted_write_ranges()->Enqueue(write_range);
+    state.unstarted_write_ranges()->Enqueue(write_range);
 
-    // Ensure that the context is scheduled so that the write range gets picked up.
-    // ScheduleContext() has no effect if already scheduled, so this is safe to do always.
-    disk_state->ScheduleContext(lock, this, range->disk_id());
+    // ScheduleContext() has no effect if the context is already scheduled,
+    // so this is safe.
+    schedule_context = true;
   }
-  ++disk_state->num_remaining_ranges();
-}
-
-void RequestContext::AddActiveScanRangeLocked(
-    const unique_lock<mutex>& lock, ScanRange* range) {
-  DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
-  DCHECK(state_ == Active);
-  active_scan_ranges_.insert(range);
-}
 
-void RequestContext::RemoveActiveScanRange(ScanRange* range) {
-  unique_lock<mutex> lock(lock_);
-  RemoveActiveScanRangeLocked(lock, range);
+  if (schedule_context) state.ScheduleContext(this, range->disk_id());
+  ++state.num_remaining_ranges();
 }
 
-void RequestContext::RemoveActiveScanRangeLocked(
-    const unique_lock<mutex>& lock, ScanRange* range) {
-  DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
-  active_scan_ranges_.erase(range);
-}
-
-RequestContext::RequestContext(DiskIoMgr* parent, int num_disks)
-  : parent_(parent), disk_states_(num_disks) {}
+RequestContext::RequestContext(
+    DiskIoMgr* parent, int num_disks, MemTracker* tracker)
+  : parent_(parent), mem_tracker_(tracker), disk_states_(num_disks) {}
 
 // Dumps out request context information. Lock should be taken by caller
 string RequestContext::DebugString() const {
@@ -229,9 +159,13 @@ string RequestContext::DebugString() const {
   if (state_ == RequestContext::Cancelled) ss << "Cancelled";
   if (state_ == RequestContext::Active) ss << "Active";
   if (state_ != RequestContext::Inactive) {
-    ss << " #disk_with_ranges=" << num_disks_with_ranges_
-       << " #disks=" << num_disks_with_ranges_
-       << " #active scan ranges=" << active_scan_ranges_.size();
+    ss << " status_=" << (status_.ok() ? "OK" : status_.GetDetail())
+       << " #ready_buffers=" << num_ready_buffers_.Load()
+       << " #used_buffers=" << num_used_buffers_.Load()
+       << " #num_buffers_in_reader=" << num_buffers_in_reader_.Load()
+       << " #finished_scan_ranges=" << num_finished_ranges_.Load()
+       << " #disk_with_ranges=" << num_disks_with_ranges_
+       << " #disks=" << num_disks_with_ranges_;
     for (int i = 0; i < disk_states_.size(); ++i) {
       ss << endl << "   " << i << ": "
          << "is_on_queue=" << disk_states_[i].is_on_queue()
@@ -254,6 +188,16 @@ bool RequestContext::Validate() const {
     return false;
   }
 
+  if (num_used_buffers_.Load() < 0) {
+    LOG(WARNING) << "num_used_buffers_ < 0: #used=" << num_used_buffers_.Load();
+    return false;
+  }
+
+  if (num_ready_buffers_.Load() < 0) {
+    LOG(WARNING) << "num_ready_buffers_ < 0: #used=" << num_ready_buffers_.Load();
+    return false;
+  }
+
   int total_unstarted_ranges = 0;
   for (int i = 0; i < disk_states_.size(); ++i) {
     const PerDiskState& state = disk_states_[i];
@@ -331,8 +275,8 @@ bool RequestContext::Validate() const {
       LOG(WARNING) << "Reader cancelled but has ready to start ranges.";
       return false;
     }
-    if (!active_scan_ranges_.empty()) {
-      LOG(WARNING) << "Reader cancelled but has active ranges.";
+    if (!blocked_ranges_.empty()) {
+      LOG(WARNING) << "Reader cancelled but has blocked ranges.";
       return false;
     }
   }
@@ -340,11 +284,10 @@ bool RequestContext::Validate() const {
   return true;
 }
 
-void RequestContext::PerDiskState::ScheduleContext(const unique_lock<mutex>& context_lock,
+void RequestContext::PerDiskState::ScheduleContext(
     RequestContext* context, int disk_id) {
-  DCHECK(context_lock.mutex() == &context->lock_ && context_lock.owns_lock());
-  if (is_on_queue_.Load() == 0 && !done_) {
-    is_on_queue_.Store(1);
+  if (!is_on_queue_ && !done_) {
+    is_on_queue_ = true;
     context->parent_->disk_queues_[disk_id]->EnqueueContext(context);
   }
 }

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/request-context.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/request-context.h b/be/src/runtime/io/request-context.h
index b028596..fd68669 100644
--- a/be/src/runtime/io/request-context.h
+++ b/be/src/runtime/io/request-context.h
@@ -23,13 +23,10 @@
 
 namespace impala {
 namespace io {
-
-// Mode argument for AddRangeToDisk().
-enum class ScheduleMode {
-  IMMEDIATELY, UPON_GETNEXT, BY_CALLER
-};
 /// A request context is used to group together I/O requests belonging to a client of the
-/// I/O manager for management and scheduling.
+/// I/O manager for management and scheduling. For most I/O manager clients it is an
+/// opaque pointer, but some clients may need to include this header, e.g. to make the
+/// unique_ptr<DiskIoRequestContext> destructor work correctly.
 ///
 /// Implementation Details
 /// ======================
@@ -37,109 +34,56 @@ enum class ScheduleMode {
 /// maintains state across all disks as well as per disk state.
 /// The unit for an IO request is a RequestRange, which may be a ScanRange or a
 /// WriteRange.
-/// A scan range for the reader is on one of six states:
-/// 1) PerDiskState's 'unstarted_scan_ranges_': This range has only been queued
+/// A scan range for the reader is on one of five states:
+/// 1) PerDiskState's unstarted_ranges: This range has only been queued
 ///    and nothing has been read from it.
-/// 2) RequestContext's 'ready_to_start_ranges_': This range is about to be started.
-///    As soon as the reader picks it up, it will move to the 'in_flight_ranges_'
+/// 2) RequestContext's ready_to_start_ranges_: This range is about to be started.
+///    As soon as the reader picks it up, it will move to the in_flight_ranges
 ///    queue.
-/// 3) PerDiskState's 'in_flight_ranges_': This range is being processed and will
+/// 3) PerDiskState's in_flight_ranges: This range is being processed and will
 ///    be read from the next time a disk thread picks it up in GetNextRequestRange()
-/// 4) The ScanRange is blocked waiting for buffers because it does not have any unused
-///    buffers to read data into. It is unblocked when a client adds new buffers via
-///    AllocateBuffersForRange() or returns existing buffers via ReturnBuffer().
-///    ScanRanges in this state are identified by 'blocked_on_buffer_' == true.
-/// 5) ScanRange is cached and in the 'cached_ranges_' queue.
-/// 6) Inactive - either all the data for the range was returned or the range was
-///    cancelled. I.e. ScanRange::eosr_ is true or ScanRange::cancel_status_ != OK.
-///
-/// If the scan range is read and does not get blocked waiting for buffers, the
+/// 4) ScanRange's outgoing ready buffers is full. We can't read for this range
+///    anymore. We need the caller to pull a buffer off which will put this in
+///    the in_flight_ranges queue. These ranges are in the RequestContext's
+///    blocked_ranges_ queue.
+/// 5) ScanRange is cached and in the cached_ranges_ queue.
+//
+/// If the scan range is read and does not get blocked on the outgoing queue, the
 /// transitions are: 1 -> 2 -> 3.
 /// If the scan range does get blocked, the transitions are
 /// 1 -> 2 -> 3 -> (4 -> 3)*
-///
-/// In the case of a cached scan range, the range is immediately put in 'cached_ranges_'.
+//
+/// In the case of a cached scan range, the range is immediately put in cached_ranges_.
 /// When the caller asks for the next range to process, we first pull ranges from
-/// the 'cache_ranges_' queue. If the range was cached, the range is removed and
+/// the cache_ranges_ queue. If the range was cached, the range is removed and
 /// done (ranges are either entirely cached or not at all). If the cached read attempt
 /// fails, we put the range in state 1.
-///
-/// All scan ranges in states 1-5 are tracked in 'active_scan_ranges_' so that they can be
-/// cancelled when the RequestContext is cancelled. Scan ranges are removed from
-/// 'active_scan_ranges_' during their transition to state 6.
-///
-/// A write range for a context may be in one of two queues:
-/// 1) 'unstarted_write_ranges_': Ranges that have been queued but not processed.
-/// 2) 'in_flight_ranges_': The write range is ready to be processed by the next disk
-///    thread that picks it up in GetNextRequestRange().
-///
+//
+/// A write range for a context may be in one of two lists:
+/// 1) unstarted_write_ranges_ : Ranges that have been queued but not processed.
+/// 2) in_flight_ranges_: The write range is ready to be processed by the next disk thread
+///    that picks it up in GetNextRequestRange().
+//
 /// AddWriteRange() adds WriteRanges for a disk.
 /// It is the responsibility of the client to pin the data to be written via a WriteRange
 /// in memory. After a WriteRange has been written, a callback is invoked to inform the
 /// client that the write has completed.
-///
+//
 /// An important assumption is that write does not exceed the maximum read size and that
 /// the entire range is written when the write request is handled. (In other words, writes
 /// are not broken up.)
-///
+//
 /// When a RequestContext is processed by a disk thread in GetNextRequestRange(),
 /// a write range is always removed from the list of unstarted write ranges and appended
 /// to the in_flight_ranges_ queue. This is done to alternate reads and writes - a read
-/// that is scheduled (by calling GetNextUnstartedRange()) is always followed by a write
-/// (if one exists). And since at most one WriteRange can be present in in_flight_ranges_
-/// at any time (once a write range is returned from GetNetxRequestRange() it is completed
-/// and not re-enqueued), a scan range scheduled via a call to GetNextUnstartedRange() can
-/// be queued up behind at most one write range.
+/// that is scheduled (by calling GetNextRange()) is always followed by a write (if one
+/// exists).  And since at most one WriteRange can be present in in_flight_ranges_ at any
+/// time (once a write range is returned from GetNetxRequestRange() it is completed an
+/// not re-enqueued), a scan range scheduled via a call to GetNextRange() can be queued up
+/// behind at most one write range.
 class RequestContext {
  public:
-  ~RequestContext() {
-    DCHECK_EQ(state_, Inactive) << "Must be unregistered. " << DebugString();
-  }
-
-  /// Cancel the context asynchronously. All outstanding requests are cancelled
-  /// asynchronously. This does not need to be called if the context finishes normally.
-  /// Calling GetNext() on any scan ranges belonging to this RequestContext will return
-  /// CANCELLED (or another error, if an error was encountered for that scan range before
-  /// it is cancelled).
-  void Cancel();
-
-  bool IsCancelled() {
-    boost::unique_lock<boost::mutex> lock(lock_);
-    return state_ == Cancelled;
-  }
-
-  int64_t bytes_read_local() const { return bytes_read_local_.Load(); }
-  int64_t bytes_read_short_circuit() const { return bytes_read_short_circuit_.Load(); }
-  int64_t bytes_read_dn_cache() const { return bytes_read_dn_cache_.Load(); }
-  int num_remote_ranges() const { return num_remote_ranges_.Load(); }
-  int64_t unexpected_remote_bytes() const { return unexpected_remote_bytes_.Load(); }
-
-  int cached_file_handles_hit_count() const {
-    return cached_file_handles_hit_count_.Load();
-  }
-
-  int cached_file_handles_miss_count() const {
-    return cached_file_handles_miss_count_.Load();
-  }
-
-  void set_bytes_read_counter(RuntimeProfile::Counter* bytes_read_counter) {
-    bytes_read_counter_ = bytes_read_counter;
-  }
-
-  void set_read_timer(RuntimeProfile::Counter* read_timer) { read_timer_ = read_timer; }
-
-  void set_open_file_timer(RuntimeProfile::Counter* open_file_timer) {
-    open_file_timer_ = open_file_timer;
-  }
-
-  void set_active_read_thread_counter(
-      RuntimeProfile::Counter* active_read_thread_counter) {
-   active_read_thread_counter_ = active_read_thread_counter;
-  }
-
-  void set_disks_accessed_bitmap(RuntimeProfile::Counter* disks_accessed_bitmap) {
-    disks_accessed_bitmap_ = disks_accessed_bitmap;
-  }
+  ~RequestContext() { DCHECK_EQ(state_, Inactive) << "Must be unregistered."; }
 
  private:
   DISALLOW_COPY_AND_ASSIGN(RequestContext);
@@ -162,19 +106,13 @@ class RequestContext {
     Inactive,
   };
 
-  RequestContext(DiskIoMgr* parent, int num_disks);
-
-  /// Cleans up a buffer. If the buffer was allocated with AllocateBuffersForRange(),
-  /// frees the buffer. Otherwise (e.g. a client or HDFS cache buffer), just prepares the
-  /// descriptor to be destroyed. After this is called, buffer->buffer() is NULL.
-  /// Does not acquire 'lock_'.
-  void FreeBuffer(BufferDescriptor* buffer);
+  RequestContext(DiskIoMgr* parent, int num_disks, MemTracker* tracker);
 
   /// Decrements the number of active disks for this reader.  If the disk count
   /// goes to 0, the disk complete condition variable is signaled.
-  /// 'lock_' must be held via 'lock'.
-  void DecrementDiskRefCount(const boost::unique_lock<boost::mutex>& lock) {
-    DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
+  /// Reader lock must be taken before this call.
+  void DecrementDiskRefCount() {
+    // boost doesn't let us dcheck that the reader lock is taken
     DCHECK_GT(num_disks_with_ranges_, 0);
     if (--num_disks_with_ranges_ == 0) {
       disks_complete_cond_var_.NotifyAll();
@@ -191,48 +129,25 @@ class RequestContext {
 
   /// Adds range to in_flight_ranges, scheduling this reader on the disk threads
   /// if necessary.
-  /// 'lock_' must be held via 'lock'. Only valid to call if this context is active.
-  void ScheduleScanRange(const boost::unique_lock<boost::mutex>& lock, ScanRange* range) {
-    DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
+  /// Reader lock must be taken before this.
+  void ScheduleScanRange(ScanRange* range) {
     DCHECK_EQ(state_, Active);
-    DCHECK(range != nullptr);
+    DCHECK(range != NULL);
     RequestContext::PerDiskState& state = disk_states_[range->disk_id()];
     state.in_flight_ranges()->Enqueue(range);
-    state.ScheduleContext(lock, this, range->disk_id());
+    state.ScheduleContext(this, range->disk_id());
   }
 
+  /// Cancels the context with status code 'status'
+  void Cancel(const Status& status);
+
   /// Cancel the context if not already cancelled, wait for all scan ranges to finish
   /// and mark the context as inactive, after which it cannot be used.
   void CancelAndMarkInactive();
 
-  /// Adds a request range to the appropriate disk state. 'schedule_mode' controls which
-  /// queue the range is placed in. This RequestContext is scheduled on the disk state
-  /// if required by 'schedule_mode'.
-  ///
-  /// Write ranges must always have 'schedule_mode' IMMEDIATELY and are added to the
-  /// 'unstarted_write_ranges_' queue, from which they will be asynchronously moved to the
-  /// 'in_flight_ranges_' queue.
-  ///
-  /// Scan ranges can have different 'schedule_mode' values. If IMMEDIATELY, the range is
-  /// immediately added to the 'in_flight_ranges_' queue where it will be processed
-  /// asynchronously by disk threads. If UPON_GETNEXT, the range is added to the
-  /// 'unstarted_ranges_' queue, from which it can be returned to a client by
-  /// DiskIoMgr::GetNextUnstartedRange(). If BY_CALLER, the scan range is not added to
-  /// any queues. The range will be scheduled later as a separate step, e.g. when it is
-  /// unblocked by adding buffers to it. Caller must hold 'lock_' via 'lock'.
-  void AddRangeToDisk(const boost::unique_lock<boost::mutex>& lock, RequestRange* range,
-      ScheduleMode schedule_mode);
-
-  /// Adds an active range to 'active_scan_ranges_'
-  void AddActiveScanRangeLocked(
-      const boost::unique_lock<boost::mutex>& lock, ScanRange* range);
-
-  /// Removes the range from 'active_scan_ranges_'. Called by ScanRange after eos or
-  /// cancellation. If calling the Locked version, the caller must hold
-  /// 'lock_'. Otherwise the function will acquire 'lock_'.
-  void RemoveActiveScanRange(ScanRange* range);
-  void RemoveActiveScanRangeLocked(
-      const boost::unique_lock<boost::mutex>& lock, ScanRange* range);
+  /// Adds request range to disk queue for this request context. Currently,
+  /// schedule_immediately must be false is RequestRange is a write range.
+  void AddRequestRange(RequestRange* range, bool schedule_immediately);
 
   /// Validates invariants of reader.  Reader lock must be taken beforehand.
   bool Validate() const;
@@ -243,6 +158,9 @@ class RequestContext {
   /// Parent object
   DiskIoMgr* const parent_;
 
+  /// Memory used for this reader.  This is unowned by this object.
+  MemTracker* const mem_tracker_;
+
   /// Total bytes read for this reader
   RuntimeProfile::Counter* bytes_read_counter_ = nullptr;
 
@@ -272,6 +190,13 @@ class RequestContext {
   /// Total number of bytes from remote reads that were expected to be local.
   AtomicInt64 unexpected_remote_bytes_{0};
 
+  /// The number of buffers that have been returned to the reader (via GetNext) that the
+  /// reader has not returned. Only included for debugging and diagnostics.
+  AtomicInt32 num_buffers_in_reader_{0};
+
+  /// The number of scan ranges that have been completed for this reader.
+  AtomicInt32 num_finished_ranges_{0};
+
   /// The number of scan ranges that required a remote read, updated at the end of each
   /// range scan. Only used for diagnostics.
   AtomicInt32 num_remote_ranges_{0};
@@ -286,6 +211,17 @@ class RequestContext {
   /// Total number of file handle opens where the file handle was not in the cache
   AtomicInt32 cached_file_handles_miss_count_{0};
 
+  /// The number of buffers that are being used for this reader. This is the sum
+  /// of all buffers in ScanRange queues and buffers currently being read into (i.e. about
+  /// to be queued). This includes both IOMgr-allocated buffers and client-provided
+  /// buffers.
+  AtomicInt32 num_used_buffers_{0};
+
+  /// The total number of ready buffers across all ranges.  Ready buffers are buffers
+  /// that have been read from disk but not retrieved by the caller.
+  /// This is the sum of all queued buffers in all ranges for this reader context.
+  AtomicInt32 num_ready_buffers_{0};
+
   /// All fields below are accessed by multiple threads and the lock needs to be
   /// taken before accessing them. Must be acquired before ScanRange::lock_ if both
   /// are held simultaneously.
@@ -294,16 +230,8 @@ class RequestContext {
   /// Current state of the reader
   State state_ = Active;
 
-  /// Scan ranges that have been added to the IO mgr for this context. Ranges can only
-  /// be added when 'state_' is Active. When this context is cancelled, Cancel() is
-  /// called for all the active ranges. If a client attempts to add a range while
-  /// 'state_' is Cancelled, the range is not added to this list and Status::CANCELLED
-  /// is returned to the client. This ensures that all active ranges are cancelled as a
-  /// result of RequestContext cancellation.
-  /// Ranges can be cancelled or hit eos non-atomically with their removal from this set,
-  /// so eos or cancelled ranges may be temporarily present here. Cancelling these ranges
-  /// a second time or cancelling after eos is safe and has no effect.
-  boost::unordered_set<ScanRange*> active_scan_ranges_;
+  /// Status of this reader.  Set to non-ok if cancelled.
+  Status status_;
 
   /// The number of disks with scan ranges remaining (always equal to the sum of
   /// disks with ranges).
@@ -312,18 +240,21 @@ class RequestContext {
   /// This is the list of ranges that are expected to be cached on the DN.
   /// When the reader asks for a new range (GetNextScanRange()), we first
   /// return ranges from this list.
-  InternalList<ScanRange> cached_ranges_;
+  InternalQueue<ScanRange> cached_ranges_;
 
   /// A list of ranges that should be returned in subsequent calls to
-  /// GetNextUnstartedRange().
+  /// GetNextRange.
   /// There is a trade-off with when to populate this list.  Populating it on
-  /// demand means consumers need to wait (happens in DiskIoMgr::GetNextUnstartedRange()).
+  /// demand means consumers need to wait (happens in DiskIoMgr::GetNextRange()).
   /// Populating it preemptively means we make worse scheduling decisions.
   /// We currently populate one range per disk.
   /// TODO: think about this some more.
-  InternalList<ScanRange> ready_to_start_ranges_;
+  InternalQueue<ScanRange> ready_to_start_ranges_;
   ConditionVariable ready_to_start_ranges_cv_; // used with lock_
 
+  /// Ranges that are blocked due to back pressure on outgoing buffers.
+  InternalQueue<ScanRange> blocked_ranges_;
+
   /// Condition variable for UnregisterContext() to wait for all disks to complete
   ConditionVariable disks_complete_cond_var_;
 
@@ -342,9 +273,21 @@ class RequestContext {
       next_scan_range_to_start_ = range;
     }
 
-    bool is_on_queue() const { return is_on_queue_.Load() != 0; }
+    /// We need to have a memory barrier to prevent this load from being reordered
+    /// with num_threads_in_op(), since these variables are set without the reader
+    /// lock taken
+    bool is_on_queue() const {
+      bool b = is_on_queue_;
+      __sync_synchronize();
+      return b;
+    }
 
-    int num_threads_in_op() const { return num_threads_in_op_.Load(); }
+    int num_threads_in_op() const {
+      int v = num_threads_in_op_.Load();
+      // TODO: determine whether this barrier is necessary for any callsites.
+      AtomicUtil::MemoryBarrier();
+      return v;
+    }
 
     const InternalQueue<ScanRange>* unstarted_scan_ranges() const {
       return &unstarted_scan_ranges_;
@@ -363,41 +306,26 @@ class RequestContext {
     InternalQueue<RequestRange>* in_flight_ranges() { return &in_flight_ranges_; }
 
     /// Schedules the request context on this disk if it's not already on the queue.
-    /// context->lock_ must be held by the caller via 'context_lock'.
-    void ScheduleContext(const boost::unique_lock<boost::mutex>& context_lock,
-        RequestContext* context, int disk_id);
-
-    /// Increment the count of disk threads that have a reference to this context. These
-    /// threads do not hold any locks while reading from HDFS, so we need to prevent the
-    /// RequestContext from being destroyed underneath them.
-    ///
-    /// The caller does not need to hold 'lock_', so this can execute concurrently with
-    /// itself and DecrementDiskThread().
-    void IncrementDiskThreadAndDequeue() {
-      /// Incrementing 'num_threads_in_op_' first so that there is no window when other
-      /// threads see 'is_on_queue_ == num_threads_in_op_ == 0' and think there are no
-      /// references left to this context.
+    /// Context lock must be taken before this.
+    void ScheduleContext(RequestContext* context, int disk_id);
+
+    /// Increment the ref count on reader.  We need to track the number of threads per
+    /// reader per disk that are in the unlocked hdfs read code section. This is updated
+    /// by multiple threads without a lock so we need to use an atomic int.
+    void IncrementRequestThreadAndDequeue() {
       num_threads_in_op_.Add(1);
-      is_on_queue_.Store(0);
+      is_on_queue_ = false;
     }
 
-    /// Decrement the count of disks threads with a reference to this context. Does final
-    /// cleanup if the context is cancelled and this is the last thread for the disk.
-    /// context->lock_ must be held by the caller via 'context_lock'.
-    void DecrementDiskThread(const boost::unique_lock<boost::mutex>& context_lock,
-        RequestContext* context) {
-      DCHECK(context_lock.mutex() == &context->lock_ && context_lock.owns_lock());
-      num_threads_in_op_.Add(-1);
-
-      if (context->state_ != Cancelled) {
-        DCHECK_EQ(context->state_, Active);
-        return;
-      }
-      // The state is cancelled, check to see if we're the last thread to touch the
-      // context on this disk. We need to load 'is_on_queue_' and 'num_threads_in_op_'
-      // in this order to avoid a race with IncrementDiskThreadAndDequeue().
-      if (is_on_queue_.Load() == 0 && num_threads_in_op_.Load() == 0 && !done_) {
-        context->DecrementDiskRefCount(context_lock);
+    void DecrementRequestThread() { num_threads_in_op_.Add(-1); }
+
+    /// Decrement request thread count and do final cleanup if this is the last
+    /// thread. RequestContext lock must be taken before this.
+    void DecrementRequestThreadAndCheckDone(RequestContext* context) {
+      num_threads_in_op_.Add(-1); // Also acts as a barrier.
+      if (!is_on_queue_ && num_threads_in_op_.Load() == 0 && !done_) {
+        // This thread is the last one for this reader on this disk, do final cleanup
+        context->DecrementDiskRefCount();
         done_ = true;
       }
     }
@@ -410,12 +338,7 @@ class RequestContext {
     bool done_ = true;
 
     /// For each disk, keeps track if the context is on this disk's queue, indicating
-    /// the disk must do some work for this context. 1 means that the context is on the
-    /// disk queue, 0 means that it's not on the queue (either because it has on ranges
-    /// active for the disk or because a disk thread dequeued the context and is
-    /// currently processing a request).
-    ///
-    /// The disk needs to do work in 4 cases:
+    /// the disk must do some work for this context. The disk needs to do work in 4 cases:
     ///  1) in_flight_ranges is not empty, the disk needs to read for this reader.
     ///  2) next_range_to_start is NULL, the disk needs to prepare a scan range to be
     ///     read next.
@@ -426,15 +349,7 @@ class RequestContext {
     /// useful that can be done. If there's nothing useful, the disk queue will wake up
     /// and then remove the reader from the queue. Doing this causes thrashing of the
     /// threads.
-    ///
-    /// This variable is important during context cancellation because it indicates
-    /// whether a queue has a reference to the context that must be released before
-    /// the context is considered unregistered. Atomically set to false after
-    /// incrementing 'num_threads_in_op_' when dequeueing so that there is no window
-    /// when other threads see 'is_on_queue_ == num_threads_in_op_ == 0' and think there
-    /// are no references left to this context.
-    /// TODO: this could be combined with 'num_threads_in_op_' to be a single refcount.
-    AtomicInt32 is_on_queue_{0};
+    bool is_on_queue_ = false;
 
     /// For each disks, the number of request ranges that have not been fully read.
     /// In the non-cancellation path, this will hit 0, and done will be set to true
@@ -448,7 +363,7 @@ class RequestContext {
 
     /// Queue of pending IO requests for this disk in the order that they will be
     /// processed. A ScanRange is added to this queue when it is returned in
-    /// GetNextUnstartedRange(), or when it is added with schedule_mode == IMMEDIATELY.
+    /// GetNextRange(), or when it is added with schedule_immediately = true.
     /// A WriteRange is added to this queue from unstarted_write_ranges_ for each
     /// invocation of GetNextRequestRange() in WorkLoop().
     /// The size of this queue is always less than or equal to num_remaining_ranges.
@@ -464,11 +379,11 @@ class RequestContext {
     /// range to ready_to_start_ranges_.
     ScanRange* next_scan_range_to_start_ = nullptr;
 
-    /// For each disk, the number of disk threads issuing the underlying read/write on
-    /// behalf of this context. There are a few places where we release the context lock,
-    /// do some work, and then grab the lock again.  Because we don't hold the lock for
-    /// the entire operation, we need this ref count to keep track of which thread should
-    /// do final resource cleanup during cancellation.
+    /// For each disk, the number of threads issuing the underlying read/write on behalf
+    /// of this context. There are a few places where we release the context lock, do some
+    /// work, and then grab the lock again.  Because we don't hold the lock for the
+    /// entire operation, we need this ref count to keep track of which thread should do
+    /// final resource cleanup during cancellation.
     /// Only the thread that sees the count at 0 should do the final cleanup.
     AtomicInt32 num_threads_in_op_{0};
 
@@ -477,8 +392,7 @@ class RequestContext {
     /// unstarted_read_ranges_ and unstarted_write_ranges_ to alternate between reads
     /// and writes. (Otherwise, since next_scan_range_to_start is set
     /// in GetNextRequestRange() whenever it is null, repeated calls to
-    /// GetNextRequestRange() and GetNextUnstartedRange() may result in only reads being
-    /// processed)
+    /// GetNextRequestRange() and GetNextRange() may result in only reads being processed)
     InternalQueue<WriteRange> unstarted_write_ranges_;
   };
 

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/request-ranges.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/request-ranges.h b/be/src/runtime/io/request-ranges.h
index 0b234ac..222f847 100644
--- a/be/src/runtime/io/request-ranges.h
+++ b/be/src/runtime/io/request-ranges.h
@@ -23,15 +23,14 @@
 
 #include <boost/thread/mutex.hpp>
 
-#include "common/atomic.h"
 #include "common/hdfs.h"
 #include "common/status.h"
-#include "runtime/bufferpool/buffer-pool.h"
 #include "util/condition-variable.h"
 #include "util/internal-queue.h"
-#include "util/mem-range.h"
 
 namespace impala {
+class MemTracker;
+
 namespace io {
 class DiskIoMgr;
 class RequestContext;
@@ -56,19 +55,24 @@ class BufferDescriptor {
   /// Returns the offset within the scan range that this buffer starts at
   int64_t scan_range_offset() const { return scan_range_offset_; }
 
+  /// Transfer ownership of buffer memory from 'mem_tracker_' to 'dst' and set
+  /// 'mem_tracker_' to 'dst'. 'mem_tracker_' and 'dst' must be non-NULL. Does not
+  /// check memory limits on 'dst': the caller should check the memory limit if a
+  /// different memory limit may apply to 'dst'. If the buffer was a client-provided
+  /// buffer, transferring is not allowed.
+  /// TODO: IMPALA-3209: revisit this as part of scanner memory usage revamp.
+  void TransferOwnership(MemTracker* dst);
+
  private:
   friend class DiskIoMgr;
   friend class ScanRange;
   friend class RequestContext;
 
-  /// Create a buffer descriptor for a new reader, range and data buffer.
-  BufferDescriptor(DiskIoMgr* io_mgr, RequestContext* reader,
-      ScanRange* scan_range, uint8_t* buffer, int64_t buffer_len);
-
-  /// Create a buffer descriptor allocated from the buffer pool.
+  /// Create a buffer descriptor for a new reader, range and data buffer. The buffer
+  /// memory should already be accounted against 'mem_tracker'.
   BufferDescriptor(DiskIoMgr* io_mgr, RequestContext* reader,
-      ScanRange* scan_range, BufferPool::ClientHandle* bp_client,
-      BufferPool::BufferHandle handle);
+      ScanRange* scan_range, uint8_t* buffer, int64_t buffer_len,
+      MemTracker* mem_tracker);
 
   /// Return true if this is a cached buffer owned by HDFS.
   bool is_cached() const;
@@ -82,11 +86,14 @@ class BufferDescriptor {
   /// Reader that this buffer is for.
   RequestContext* const reader_;
 
+  /// The current tracker this buffer is associated with. After initialisation,
+  /// NULL for cached buffers and non-NULL for all other buffers.
+  MemTracker* mem_tracker_;
+
   /// Scan range that this buffer is for. Non-NULL when initialised.
   ScanRange* const scan_range_;
 
-  /// Buffer for the read contents. Must be set to NULL in RequestContext::FreeBuffer()
-  /// before destruction of the descriptor.
+  /// buffer with the read contents
   uint8_t* buffer_;
 
   /// length of buffer_. For buffers from cached reads, the length is 0.
@@ -98,12 +105,10 @@ class BufferDescriptor {
   /// true if the current scan range is complete
   bool eosr_ = false;
 
-  int64_t scan_range_offset_ = 0;
+  /// Status of the read to this buffer. if status is not ok, 'buffer' is nullptr
+  Status status_;
 
-  // Handle to an allocated buffer and the client used to allocate it buffer. Only used
-  // for non-external buffers.
-  BufferPool::ClientHandle* bp_client_ = nullptr;
-  BufferPool::BufferHandle handle_;
+  int64_t scan_range_offset_ = 0;
 };
 
 /// The request type, read or write associated with a request range.
@@ -211,11 +216,11 @@ class ScanRange : public RequestRange {
 
   /// Resets this scan range object with the scan range description. The scan range
   /// is for bytes [offset, offset + len) in 'file' on 'fs' (which is nullptr for the
-  /// local filesystem). The scan range must be non-empty and fall within the file bounds
-  /// (len > 0 and offset >= 0 and offset + len <= file_length). 'disk_id' is the disk
-  /// queue to add the range to. If 'expected_local' is true, a warning is generated if
-  /// the read did not come from a local disk. 'buffer_opts' specifies buffer management
-  /// options - see the DiskIoMgr class comment and the BufferOpts comments for details.
+  /// local filesystem). The scan range must fall within the file bounds (offset >= 0
+  /// and offset + len <= file_length). 'disk_id' is the disk queue to add the range
+  /// to. If 'expected_local' is true, a warning is generated if the read did not
+  /// come from a local disk. 'buffer_opts' specifies buffer management options -
+  /// see the DiskIoMgr class comment and the BufferOpts comments for details.
   /// 'meta_data' is an arbitrary client-provided pointer for any auxiliary data.
   void Reset(hdfsFS fs, const char* file, int64_t len, int64_t offset, int disk_id,
       bool expected_local, const BufferOpts& buffer_opts, void* meta_data = nullptr);
@@ -231,17 +236,10 @@ class ScanRange : public RequestRange {
   /// Only one thread can be in GetNext() at any time.
   Status GetNext(std::unique_ptr<BufferDescriptor>* buffer) WARN_UNUSED_RESULT;
 
-  /// Returns the buffer to the scan range. This must be called for every buffer
-  /// returned by GetNext(). After calling this, the buffer descriptor is invalid
-  /// and cannot be accessed.
-  void ReturnBuffer(std::unique_ptr<BufferDescriptor> buffer);
-
-  /// Cancel this scan range. This cleans up all queued buffers and wakes up any threads
-  /// blocked on GetNext(). Status is a non-ok status with the reason the range was
-  /// cancelled, e.g. CANCELLED if the range was cancelled because it was not needed, or
-  /// another error if an error was encountered while scanning the range. Status is
-  /// returned to the any callers of GetNext(). If a thread is currently blocked in
-  /// GetNext(), it is woken up.
+  /// Cancel this scan range. This cleans up all queued buffers and
+  /// wakes up any threads blocked on GetNext().
+  /// Status is the reason the range was cancelled. Must not be ok().
+  /// Status is returned to the user in GetNext().
   void Cancel(const Status& status);
 
   /// return a descriptive string for debug.
@@ -257,13 +255,18 @@ class ScanRange : public RequestRange {
   /// Initialize internal fields
   void InitInternal(DiskIoMgr* io_mgr, RequestContext* reader);
 
-  /// Enqueues a ready buffer with valid data for this range. This does not block.
+  /// Enqueues a buffer for this range. This does not block.
+  /// Returns true if this scan range has hit the queue capacity, false otherwise.
   /// The caller passes ownership of buffer to the scan range and it is not
   /// valid to access buffer after this call. The reader lock must be held by the
-  /// caller. Returns false if the scan range was cancelled.
-  bool EnqueueReadyBuffer(const boost::unique_lock<boost::mutex>& reader_lock,
+  /// caller.
+  bool EnqueueBuffer(const boost::unique_lock<boost::mutex>& reader_lock,
       std::unique_ptr<BufferDescriptor> buffer);
 
+  /// Cleanup any queued buffers (i.e. due to cancellation). This cannot
+  /// be called with any locks taken.
+  void CleanupQueuedBuffers();
+
   /// Validates the internal state of this range. lock_ must be taken
   /// before calling this.
   bool Validate();
@@ -280,10 +283,6 @@ class ScanRange : public RequestRange {
   /// exclusive use by this scan range. The scan range is the exclusive owner of the
   /// file handle, and the file handle is destroyed in Close().
   /// All local OS files are opened using normal OS file APIs.
-  ///
-  /// If an error is encountered during opening, returns a status describing the error.
-  /// If the scan range was cancelled, returns the reason for cancellation. Otherwise, on
-  /// success, returns OK.
   Status Open(bool use_file_handle_cache) WARN_UNUSED_RESULT;
 
   /// Closes the file for this range. This function only modifies state in this range.
@@ -291,10 +290,6 @@ class ScanRange : public RequestRange {
 
   /// Reads from this range into 'buffer', which has length 'buffer_len' bytes. Returns
   /// the number of bytes read. The read position in this scan range is updated.
-  ///
-  /// If an error is encountered during reading, returns a status describing the error.
-  /// If the scan range was cancelled, returns the reason for cancellation. Otherwise, on
-  /// success, returns OK.
   Status Read(uint8_t* buffer, int64_t buffer_len, int64_t* bytes_read,
       bool* eosr) WARN_UNUSED_RESULT;
 
@@ -312,58 +307,6 @@ class ScanRange : public RequestRange {
   Status ReadFromCache(const boost::unique_lock<boost::mutex>& reader_lock,
       bool* read_succeeded) WARN_UNUSED_RESULT;
 
-  /// Add buffers for the range to read data into and schedule the range if blocked.
-  /// If 'returned' is true, the buffers returned from GetNext() that are being recycled
-  /// via ReturnBuffer(). Otherwise the buffers are newly allocated buffers to be added.
-  void AddUnusedBuffers(
-      std::vector<std::unique_ptr<BufferDescriptor>>&& buffers, bool returned);
-
-  /// Remove a buffer from 'unused_iomgr_buffers_' and update
-  /// 'unused_iomgr_buffer_bytes_'. If 'unused_iomgr_buffers_' is empty, return NULL.
-  /// 'lock_' must be held by the caller via 'scan_range_lock'.
-  std::unique_ptr<BufferDescriptor> GetUnusedBuffer(
-      const boost::unique_lock<boost::mutex>& scan_range_lock);
-
-  /// Get the next buffer for this scan range for a disk thread to read into. Returns
-  /// the new buffer if successful.  If no buffers are available, marks the range
-  /// as blocked and returns nullptr. Called must not hold 'lock_'.
-  std::unique_ptr<BufferDescriptor> GetNextUnusedBufferForRange();
-
-  /// Cleans up a buffer that was not returned to the client.
-  /// Either ReturnBuffer() or CleanUpBuffer() is called for every BufferDescriptor.
-  /// The caller must hold 'lock_' via 'scan_range_lock'.
-  /// This function may acquire 'hdfs_lock_'
-  void CleanUpBuffer(const boost::unique_lock<boost::mutex>& scan_range_lock,
-      std::unique_ptr<BufferDescriptor> buffer);
-
-  /// Same as CleanUpBuffer() except cleans up multiple buffers and caller must not
-  /// hold 'lock_'.
-  void CleanUpBuffers(std::vector<std::unique_ptr<BufferDescriptor>>&& buffers);
-
-  /// Clean up all buffers in 'unused_iomgr_buffers_'. Only valid to call when the scan
-  /// range is cancelled or at eos. The caller must hold 'lock_' via 'scan_range_lock'.
-  void CleanUpUnusedBuffers(const boost::unique_lock<boost::mutex>& scan_range_lock);
-
-  /// Same as Cancel() except reader_->lock must be held by the caller.
-  void CancelFromReader(const boost::unique_lock<boost::mutex>& reader_lock,
-      const Status& status);
-
-  /// Same as Cancel() except doesn't remove the scan range from
-  /// reader_->active_scan_ranges_. This is invoked by RequestContext::Cancel(),
-  /// which removes the range itself to avoid invalidating its active_scan_ranges_
-  /// iterator.
-  void CancelInternal(const Status& status);
-
-  /// Marks the scan range as blocked waiting for a buffer. Caller must not hold 'lock_'.
-  void SetBlockedOnBuffer();
-
-  /// Returns true if no more buffers will be returned to clients in the future,
-  /// either because of hitting eosr or cancellation.
-  bool all_buffers_returned(const boost::unique_lock<boost::mutex>& lock) const {
-    DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
-    return !cancel_status_.ok() || (eosr_queued_ && ready_buffers_.empty());
-  }
-
   /// Pointer to caller specified metadata. This is untouched by the io manager
   /// and the caller can put whatever auxiliary data in here.
   void* meta_data_ = nullptr;
@@ -380,9 +323,6 @@ class ScanRange : public RequestRange {
   /// TODO: we can do more with this
   bool expected_local_ = false;
 
-  /// Last modified time of the file associated with the scan range. Set in Reset().
-  int64_t mtime_;
-
   /// Total number of bytes read remotely. This is necessary to maintain a count of
   /// the number of remote scan ranges. Since IO statistics can be collected multiple
   /// times for a scan range, it is necessary to keep some state about whether this
@@ -429,10 +369,6 @@ class ScanRange : public RequestRange {
     struct hadoopRzBuffer* cached_buffer_ = nullptr;
   };
 
-  /// The number of buffers that have been returned to a client via GetNext() that have
-  /// not yet been returned with ReturnBuffer().
-  AtomicInt32 num_buffers_in_reader_{0};
-
   /// Lock protecting fields below.
   /// This lock should not be taken during Open()/Read()/Close().
   /// If RequestContext::lock_ and this lock need to be held simultaneously,
@@ -442,40 +378,25 @@ class ScanRange : public RequestRange {
   /// Number of bytes read so far for this scan range
   int bytes_read_;
 
-  /// Buffers to read into, used if the 'external_buffer_tag_' is NO_BUFFER. These are
-  /// initially populated when the client calls AllocateBuffersForRange() and
-  /// and are used to read scanned data into. Buffers are taken from this vector for
-  /// every read and added back, if needed, when the client calls ReturnBuffer().
-  std::vector<std::unique_ptr<BufferDescriptor>> unused_iomgr_buffers_;
-
-  /// Total number of bytes of buffers in 'unused_iomgr_buffers_'.
-  int64_t unused_iomgr_buffer_bytes_ = 0;
-
-  /// Number of bytes of buffers returned from GetNextUnusedBufferForRange(). Used to
-  /// infer how many bytes of buffers need to be held onto to read the rest of the scan
-  /// range.
-  int64_t iomgr_buffer_bytes_returned_ = 0;
+  /// Status for this range. This is non-ok if is_cancelled_ is true.
+  /// Note: an individual range can fail without the RequestContext being
+  /// cancelled. This allows us to skip individual ranges.
+  Status status_;
 
   /// If true, the last buffer for this scan range has been queued.
-  /// If this is true and 'ready_buffers_' is empty, then no more buffers will be
-  /// returned to the caller by this scan range.
   bool eosr_queued_ = false;
 
-  /// If true, this scan range is not scheduled because a buffer is not available for
-  /// the next I/O in the range. This can happen when the scan range is initially created
-  /// or if the buffers added to the range have all been filled with data an not yet
-  /// returned.
-  bool blocked_on_buffer_ = false;
+  /// If true, the last buffer for this scan range has been returned.
+  bool eosr_returned_ = false;
 
-  /// IO buffers that are queued for this scan range. When Cancel() is called
-  /// this is drained by the cancelling thread. I.e. this is always empty if
-  /// 'cancel_status_' is not OK.
-  std::deque<std::unique_ptr<BufferDescriptor>> ready_buffers_;
+  /// If true, this scan range has been removed from the reader's in_flight_ranges
+  /// queue because the ready_buffers_ queue is full.
+  bool blocked_on_queue_ = false;
 
-  /// Condition variable for threads in GetNext() that are waiting for the next buffer.
-  /// Signalled when a buffer is enqueued in 'ready_buffers_' or the scan range is
-  /// cancelled.
+  /// IO buffers that are queued for this scan range.
+  /// Condition variable for GetNext
   ConditionVariable buffer_ready_cv_;
+  std::deque<std::unique_ptr<BufferDescriptor>> ready_buffers_;
 
   /// Lock that should be taken during hdfs calls. Only one thread (the disk reading
   /// thread) calls into hdfs at a time so this lock does not have performance impact.
@@ -485,16 +406,11 @@ class ScanRange : public RequestRange {
   /// If this lock and lock_ need to be taken, lock_ must be taken first.
   boost::mutex hdfs_lock_;
 
-  /// If non-OK, this scan range has been cancelled. This status is the reason for
-  /// cancellation - CANCELLED if cancelled without error, or another status if an
-  /// error caused cancellation. Note that a range can be cancelled without cancelling
-  /// the owning context. This means that ranges can be cancelled or hit errors without
-  /// aborting all scan ranges.
-  //
-  /// Writers must hold both 'lock_' and 'hdfs_lock_'. Readers must hold either 'lock_'
-  /// or 'hdfs_lock_'. This prevents the range from being cancelled while any thread
-  /// is inside a critical section.
-  Status cancel_status_;
+  /// If true, this scan range has been cancelled.
+  bool is_cancelled_ = false;
+
+  /// Last modified time of the file associated with the scan range
+  int64_t mtime_;
 };
 
 /// Used to specify data to be written to a file and offset.

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/scan-range.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/scan-range.cc b/be/src/runtime/io/scan-range.cc
index 4f7c38b..bd89846 100644
--- a/be/src/runtime/io/scan-range.cc
+++ b/be/src/runtime/io/scan-range.cc
@@ -42,27 +42,35 @@ DEFINE_int64(adls_read_chunk_size, 128 * 1024, "The maximum read chunk size to u
 // any time and only one thread will remove from the queue. This is to guarantee
 // that buffers are queued and read in file order.
 
-bool ScanRange::EnqueueReadyBuffer(
+bool ScanRange::EnqueueBuffer(
     const unique_lock<mutex>& reader_lock, unique_ptr<BufferDescriptor> buffer) {
   DCHECK(reader_lock.mutex() == &reader_->lock_ && reader_lock.owns_lock());
-  DCHECK(buffer->buffer_ != nullptr) << "Cannot enqueue freed buffer";
   {
     unique_lock<mutex> scan_range_lock(lock_);
     DCHECK(Validate()) << DebugString();
+    DCHECK(!eosr_returned_);
     DCHECK(!eosr_queued_);
-    if (!cancel_status_.ok()) {
-      // This range has been cancelled, no need to enqueue the buffer.
-      CleanUpBuffer(scan_range_lock, move(buffer));
+    if (is_cancelled_) {
+      // Return the buffer, this range has been cancelled
+      if (buffer->buffer_ != nullptr) {
+        io_mgr_->num_buffers_in_readers_.Add(1);
+        reader_->num_buffers_in_reader_.Add(1);
+      }
+      reader_->num_used_buffers_.Add(-1);
+      io_mgr_->ReturnBuffer(move(buffer));
       return false;
     }
-    // Clean up any surplus buffers. E.g. we may have allocated too many if the file was
-    // shorter than expected.
-    if (buffer->eosr()) CleanUpUnusedBuffers(scan_range_lock);
+    reader_->num_ready_buffers_.Add(1);
     eosr_queued_ = buffer->eosr();
     ready_buffers_.emplace_back(move(buffer));
+
+    DCHECK_LE(ready_buffers_.size(), DiskIoMgr::SCAN_RANGE_READY_BUFFER_LIMIT);
+    blocked_on_queue_ = ready_buffers_.size() == DiskIoMgr::SCAN_RANGE_READY_BUFFER_LIMIT;
   }
+
   buffer_ready_cv_.NotifyOne();
-  return true;
+
+  return blocked_on_queue_;
 }
 
 Status ScanRange::GetNext(unique_ptr<BufferDescriptor>* buffer) {
@@ -70,225 +78,123 @@ Status ScanRange::GetNext(unique_ptr<BufferDescriptor>* buffer) {
   bool eosr;
   {
     unique_lock<mutex> scan_range_lock(lock_);
+    if (eosr_returned_) return Status::OK();
     DCHECK(Validate()) << DebugString();
-    // No more buffers to return - return the cancel status or OK if not cancelled.
-    if (all_buffers_returned(scan_range_lock)) return cancel_status_;
 
-    while (ready_buffers_.empty() && cancel_status_.ok()) {
+    while (ready_buffers_.empty() && !is_cancelled_) {
       buffer_ready_cv_.Wait(scan_range_lock);
     }
-    /// Propagate cancellation to the client if it happened while we were waiting.
-    RETURN_IF_ERROR(cancel_status_);
+
+    if (is_cancelled_) {
+      DCHECK(!status_.ok());
+      return status_;
+    }
 
     // Remove the first ready buffer from the queue and return it
     DCHECK(!ready_buffers_.empty());
+    DCHECK_LE(ready_buffers_.size(), DiskIoMgr::SCAN_RANGE_READY_BUFFER_LIMIT);
     *buffer = move(ready_buffers_.front());
     ready_buffers_.pop_front();
+    eosr_returned_ = (*buffer)->eosr();
     eosr = (*buffer)->eosr();
-    DCHECK(!eosr || unused_iomgr_buffers_.empty()) << DebugString();
-  }
-
-  // Update tracking counters. The buffer has now moved from the IoMgr to the caller.
-  if (eosr) reader_->RemoveActiveScanRange(this);
-  num_buffers_in_reader_.Add(1);
-  return Status::OK();
-}
-
-void ScanRange::ReturnBuffer(unique_ptr<BufferDescriptor> buffer_desc) {
-  vector<unique_ptr<BufferDescriptor>> buffers;
-  buffers.emplace_back(move(buffer_desc));
-  AddUnusedBuffers(move(buffers), true);
-}
-
-void ScanRange::AddUnusedBuffers(vector<unique_ptr<BufferDescriptor>>&& buffers,
-      bool returned) {
-  DCHECK_GT(buffers.size(), 0);
-  /// Keep track of whether the range was unblocked in this function. If so, we need
-  /// to schedule it so it resumes progress.
-  bool unblocked = false;
-  {
-    unique_lock<mutex> scan_range_lock(lock_);
-    if (returned) {
-      // Buffers were in reader but now aren't.
-      num_buffers_in_reader_.Add(-buffers.size());
-    }
-
-    for (unique_ptr<BufferDescriptor>& buffer : buffers) {
-      // We should not hold onto the buffers in the following cases:
-      // 1. the scan range is using external buffers, e.g. cached buffers.
-      // 2. the scan range is cancelled
-      // 3. the scan range already hit eosr
-      // 4. we already have enough buffers to read the remainder of the scan range.
-      if (external_buffer_tag_ != ExternalBufferTag::NO_BUFFER
-          || !cancel_status_.ok()
-          || eosr_queued_
-          || unused_iomgr_buffer_bytes_ >= len_ - iomgr_buffer_bytes_returned_) {
-        CleanUpBuffer(scan_range_lock, move(buffer));
-      } else {
-        unused_iomgr_buffer_bytes_ += buffer->buffer_len();
-        unused_iomgr_buffers_.emplace_back(move(buffer));
-        if (blocked_on_buffer_) {
-          blocked_on_buffer_ = false;
-          unblocked = true;
-        }
-      }
-    }
-  }
-  // Must drop the ScanRange lock before acquiring the RequestContext lock.
-  if (unblocked) {
-    unique_lock<mutex> reader_lock(reader_->lock_);
-    // Reader may have been cancelled after we dropped 'scan_range_lock' above.
-    if (reader_->state_ == RequestContext::Cancelled) {
-      DCHECK(!cancel_status_.ok());
-    } else {
-      reader_->ScheduleScanRange(reader_lock, this);
-    }
   }
-}
-
-unique_ptr<BufferDescriptor> ScanRange::GetUnusedBuffer(
-    const unique_lock<mutex>& scan_range_lock) {
-  DCHECK(scan_range_lock.mutex() == &lock_ && scan_range_lock.owns_lock());
-  if (unused_iomgr_buffers_.empty()) return nullptr;
-  unique_ptr<BufferDescriptor> result = move(unused_iomgr_buffers_.back());
-  unused_iomgr_buffers_.pop_back();
-  unused_iomgr_buffer_bytes_ -= result->buffer_len();
-  return result;
-}
 
-unique_ptr<BufferDescriptor> ScanRange::GetNextUnusedBufferForRange() {
-  unique_lock<mutex> lock(lock_);
-  unique_ptr<BufferDescriptor> buffer_desc = GetUnusedBuffer(lock);
-  if (buffer_desc == nullptr) {
-    blocked_on_buffer_ = true;
-  } else {
-    iomgr_buffer_bytes_returned_ += buffer_desc->buffer_len();
+  // Update tracking counters. The buffer has now moved from the IoMgr to the
+  // caller.
+  io_mgr_->num_buffers_in_readers_.Add(1);
+  reader_->num_buffers_in_reader_.Add(1);
+  reader_->num_ready_buffers_.Add(-1);
+  reader_->num_used_buffers_.Add(-1);
+  if (eosr) reader_->num_finished_ranges_.Add(1);
+
+  Status status = (*buffer)->status_;
+  if (!status.ok()) {
+    io_mgr_->ReturnBuffer(move(*buffer));
+    return status;
   }
-  return buffer_desc;
-}
 
-void ScanRange::SetBlockedOnBuffer() {
-  unique_lock<mutex> lock(lock_);
-  blocked_on_buffer_ = true;
-}
+  unique_lock<mutex> reader_lock(reader_->lock_);
 
-void ScanRange::CleanUpBuffer(
-    const boost::unique_lock<boost::mutex>& scan_range_lock,
-    unique_ptr<BufferDescriptor> buffer_desc) {
-  DCHECK(scan_range_lock.mutex() == &lock_ && scan_range_lock.owns_lock());
-  DCHECK(buffer_desc != nullptr);
-  DCHECK_EQ(this, buffer_desc->scan_range_);
-  buffer_desc->reader_->FreeBuffer(buffer_desc.get());
-
-  if (all_buffers_returned(scan_range_lock) && num_buffers_in_reader_.Load() == 0) {
-    // Close the scan range if there are no more buffers in the reader and no more buffers
-    // will be returned to readers in future. Close() is idempotent so it is ok to call
-    // multiple times during cleanup so long as the range is actually finished.
-    Close();
+  DCHECK(reader_->Validate()) << endl << reader_->DebugString();
+  if (reader_->state_ == RequestContext::Cancelled) {
+    reader_->blocked_ranges_.Remove(this);
+    Cancel(reader_->status_);
+    io_mgr_->ReturnBuffer(move(*buffer));
+    return status_;
   }
-}
-
-void ScanRange::CleanUpBuffers(vector<unique_ptr<BufferDescriptor>>&& buffers) {
-  unique_lock<mutex> lock(lock_);
-  for (unique_ptr<BufferDescriptor>& buffer : buffers) CleanUpBuffer(lock, move(buffer));
-}
 
-void ScanRange::CleanUpUnusedBuffers(const unique_lock<mutex>& scan_range_lock) {
-  while (!unused_iomgr_buffers_.empty()) {
-    CleanUpBuffer(scan_range_lock, GetUnusedBuffer(scan_range_lock));
+  {
+    // Check to see if we can re-schedule a blocked range. Note that EnqueueBuffer()
+    // may have been called after we released 'lock_' above so we need to re-check
+    // whether the queue is full.
+    unique_lock<mutex> scan_range_lock(lock_);
+    if (blocked_on_queue_
+        && ready_buffers_.size() < DiskIoMgr::SCAN_RANGE_READY_BUFFER_LIMIT
+        && !eosr_queued_) {
+      blocked_on_queue_ = false;
+      // This scan range was blocked and is no longer, add it to the reader
+      // queue again.
+      reader_->blocked_ranges_.Remove(this);
+      reader_->ScheduleScanRange(this);
+    }
   }
+  return Status::OK();
 }
 
 void ScanRange::Cancel(const Status& status) {
   // Cancelling a range that was never started, ignore.
   if (io_mgr_ == nullptr) return;
-  CancelInternal(status);
-  reader_->RemoveActiveScanRange(this);
-}
 
-void ScanRange::CancelFromReader(const boost::unique_lock<boost::mutex>& reader_lock,
-    const Status& status) {
-  DCHECK(reader_lock.mutex() == &reader_->lock_ && reader_lock.owns_lock());
-  CancelInternal(status);
-  reader_->RemoveActiveScanRangeLocked(reader_lock, this);
-}
-
-void ScanRange::CancelInternal(const Status& status) {
-  DCHECK(io_mgr_ != nullptr);
   DCHECK(!status.ok());
   {
-    // Grab both locks to make sure that we don't change 'cancel_status_' while other
-    // threads are in critical sections.
+    // Grab both locks to make sure that all working threads see is_cancelled_.
     unique_lock<mutex> scan_range_lock(lock_);
-    {
-      unique_lock<mutex> hdfs_lock(hdfs_lock_);
-      DCHECK(Validate()) << DebugString();
-      // If already cancelled, preserve the original reason for cancellation. The first
-      // thread to set 'cancel_status_' does the cleanup below.
-      RETURN_VOID_IF_ERROR(cancel_status_);
-      cancel_status_ = status;
-    }
-
-    /// Clean up 'ready_buffers_' while still holding 'lock_' to prevent other threads
-    /// from seeing inconsistent state.
-    while (!ready_buffers_.empty()) {
-      CleanUpBuffer(scan_range_lock, move(ready_buffers_.front()));
-      ready_buffers_.pop_front();
-    }
-
-    /// Clean up buffers that we don't need any more because we won't read any more data.
-    CleanUpUnusedBuffers(scan_range_lock);
+    unique_lock<mutex> hdfs_lock(hdfs_lock_);
+    DCHECK(Validate()) << DebugString();
+    if (is_cancelled_) return;
+    is_cancelled_ = true;
+    status_ = status;
   }
   buffer_ready_cv_.NotifyAll();
+  CleanupQueuedBuffers();
 
   // For cached buffers, we can't close the range until the cached buffer is returned.
-  // Close() is called from ScanRange::CleanUpBufferLocked().
+  // Close() is called from DiskIoMgr::ReturnBuffer().
   if (external_buffer_tag_ != ExternalBufferTag::CACHED_BUFFER) Close();
 }
 
+void ScanRange::CleanupQueuedBuffers() {
+  DCHECK(is_cancelled_);
+  io_mgr_->num_buffers_in_readers_.Add(ready_buffers_.size());
+  reader_->num_buffers_in_reader_.Add(ready_buffers_.size());
+  reader_->num_used_buffers_.Add(-ready_buffers_.size());
+  reader_->num_ready_buffers_.Add(-ready_buffers_.size());
+
+  while (!ready_buffers_.empty()) {
+    io_mgr_->ReturnBuffer(move(ready_buffers_.front()));
+    ready_buffers_.pop_front();
+  }
+}
+
 string ScanRange::DebugString() const {
   stringstream ss;
   ss << "file=" << file_ << " disk_id=" << disk_id_ << " offset=" << offset_
      << " len=" << len_ << " bytes_read=" << bytes_read_
-     << " cancel_status=" << cancel_status_.GetDetail()
      << " buffer_queue=" << ready_buffers_.size()
-     << " num_buffers_in_readers=" << num_buffers_in_reader_.Load()
-     << " unused_iomgr_buffers=" << unused_iomgr_buffers_.size()
-     << " unused_iomgr_buffer_bytes=" << unused_iomgr_buffer_bytes_
-     << " blocked_on_buffer=" << blocked_on_buffer_
      << " hdfs_file=" << exclusive_hdfs_fh_;
   return ss.str();
 }
 
 bool ScanRange::Validate() {
   if (bytes_read_ > len_) {
-    LOG(ERROR) << "Bytes read tracking is wrong. Shouldn't read past the scan range."
+    LOG(WARNING) << "Bytes read tracking is wrong. Shouldn't read past the scan range."
                  << " bytes_read_=" << bytes_read_ << " len_=" << len_;
     return false;
   }
-  if (!cancel_status_.ok() && !ready_buffers_.empty()) {
-    LOG(ERROR) << "Cancelled range should not have queued buffers " << DebugString();
-    return false;
-  }
-  int64_t unused_iomgr_buffer_bytes = 0;
-  for (auto& buffer : unused_iomgr_buffers_)
-    unused_iomgr_buffer_bytes += buffer->buffer_len();
-  if (unused_iomgr_buffer_bytes != unused_iomgr_buffer_bytes_) {
-    LOG(ERROR) << "unused_iomgr_buffer_bytes_ incorrect actual: "
-               << unused_iomgr_buffer_bytes_
-               << " vs. expected: " << unused_iomgr_buffer_bytes;
-    return false;
-  }
-  bool is_finished = !cancel_status_.ok() || eosr_queued_;
-  if (is_finished && !unused_iomgr_buffers_.empty()) {
-    LOG(ERROR) << "Held onto too many buffers " << unused_iomgr_buffers_.size()
-               << " bytes: " << unused_iomgr_buffer_bytes_
-               << " cancel_status: " << cancel_status_.GetDetail()
-               << " eosr_queued: " << eosr_queued_;
-    return false;
-  }
-  if (!is_finished && blocked_on_buffer_ && !unused_iomgr_buffers_.empty()) {
-    LOG(ERROR) << "Blocked despite having buffers: " << DebugString();
+  if (eosr_returned_ && !eosr_queued_) {
+    LOG(WARNING) << "Returned eosr to reader before finishing reading the scan range"
+                 << " eosr_returned_=" << eosr_returned_
+                 << " eosr_queued_=" << eosr_queued_;
     return false;
   }
   return true;
@@ -297,14 +203,13 @@ bool ScanRange::Validate() {
 ScanRange::ScanRange()
   : RequestRange(RequestType::READ),
     num_remote_bytes_(0),
-    external_buffer_tag_(ExternalBufferTag::NO_BUFFER) {}
+    external_buffer_tag_(ExternalBufferTag::NO_BUFFER),
+    mtime_(-1) {}
 
 ScanRange::~ScanRange() {
   DCHECK(exclusive_hdfs_fh_ == nullptr) << "File was not closed.";
   DCHECK(external_buffer_tag_ != ExternalBufferTag::CACHED_BUFFER)
       << "Cached buffer was not released.";
-  DCHECK_EQ(0, ready_buffers_.size());
-  DCHECK_EQ(0, num_buffers_in_reader_.Load());
 }
 
 void ScanRange::Reset(hdfsFS fs, const char* file, int64_t len, int64_t offset,
@@ -340,22 +245,24 @@ void ScanRange::Reset(hdfsFS fs, const char* file, int64_t len, int64_t offset,
 void ScanRange::InitInternal(DiskIoMgr* io_mgr, RequestContext* reader) {
   DCHECK(exclusive_hdfs_fh_ == nullptr);
   DCHECK(local_file_ == nullptr);
+  // Reader must provide MemTracker or a buffer.
+  DCHECK(external_buffer_tag_ == ExternalBufferTag::CLIENT_BUFFER
+      || reader->mem_tracker_ != nullptr);
   io_mgr_ = io_mgr;
   reader_ = reader;
   local_file_ = nullptr;
   exclusive_hdfs_fh_ = nullptr;
   bytes_read_ = 0;
-  unused_iomgr_buffer_bytes_ = 0;
-  iomgr_buffer_bytes_returned_ = 0;
-  cancel_status_ = Status::OK();
-  eosr_queued_ = false;
-  blocked_on_buffer_ = false;
+  is_cancelled_ = false;
+  eosr_queued_= false;
+  eosr_returned_= false;
+  blocked_on_queue_ = false;
   DCHECK(Validate()) << DebugString();
 }
 
 Status ScanRange::Open(bool use_file_handle_cache) {
   unique_lock<mutex> hdfs_lock(hdfs_lock_);
-  RETURN_IF_ERROR(cancel_status_);
+  if (is_cancelled_) return Status::CANCELLED;
 
   if (fs_ != nullptr) {
     if (exclusive_hdfs_fh_ != nullptr) return Status::OK();
@@ -395,7 +302,9 @@ Status ScanRange::Open(bool use_file_handle_cache) {
           "for file: $1: $2", offset_, file_, GetStrErrMsg()));
     }
   }
-  ImpaladMetrics::IO_MGR_NUM_OPEN_FILES->Increment(1L);
+  if (ImpaladMetrics::IO_MGR_NUM_OPEN_FILES != nullptr) {
+    ImpaladMetrics::IO_MGR_NUM_OPEN_FILES->Increment(1L);
+  }
   return Status::OK();
 }
 
@@ -447,7 +356,9 @@ void ScanRange::Close() {
     local_file_ = nullptr;
     closed_file = true;
   }
-  if (closed_file) ImpaladMetrics::IO_MGR_NUM_OPEN_FILES->Increment(-1L);
+  if (closed_file && ImpaladMetrics::IO_MGR_NUM_OPEN_FILES != nullptr) {
+    ImpaladMetrics::IO_MGR_NUM_OPEN_FILES->Increment(-1L);
+  }
 }
 
 int64_t ScanRange::MaxReadChunkSize() const {
@@ -475,7 +386,7 @@ int64_t ScanRange::MaxReadChunkSize() const {
 Status ScanRange::Read(
     uint8_t* buffer, int64_t buffer_len, int64_t* bytes_read, bool* eosr) {
   unique_lock<mutex> hdfs_lock(hdfs_lock_);
-  RETURN_IF_ERROR(cancel_status_);
+  if (is_cancelled_) return Status::CANCELLED;
 
   *eosr = false;
   *bytes_read = 0;
@@ -622,7 +533,7 @@ Status ScanRange::ReadFromCache(
 
   {
     unique_lock<mutex> hdfs_lock(hdfs_lock_);
-    RETURN_IF_ERROR(cancel_status_);
+    if (is_cancelled_) return Status::CANCELLED;
 
     DCHECK(exclusive_hdfs_fh_ != nullptr);
     DCHECK(external_buffer_tag_ == ExternalBufferTag::NO_BUFFER);
@@ -657,16 +568,20 @@ Status ScanRange::ReadFromCache(
   }
 
   // Create a single buffer desc for the entire scan range and enqueue that.
-  // The memory is owned by the HDFS java client, not the Impala backend.
+  // 'mem_tracker' is nullptr because the memory is owned by the HDFS java client,
+  // not the Impala backend.
   unique_ptr<BufferDescriptor> desc = unique_ptr<BufferDescriptor>(new BufferDescriptor(
-      io_mgr_, reader_, this, reinterpret_cast<uint8_t*>(buffer), 0));
+      io_mgr_, reader_, this, reinterpret_cast<uint8_t*>(buffer), 0, nullptr));
   desc->len_ = bytes_read;
   desc->scan_range_offset_ = 0;
   desc->eosr_ = true;
   bytes_read_ = bytes_read;
-  EnqueueReadyBuffer(reader_lock, move(desc));
-  COUNTER_ADD_IF_NOT_NULL(reader_->bytes_read_counter_, bytes_read);
+  EnqueueBuffer(reader_lock, move(desc));
+  if (reader_->bytes_read_counter_ != nullptr) {
+    COUNTER_ADD(reader_->bytes_read_counter_, bytes_read);
+  }
   *read_succeeded = true;
+  reader_->num_used_buffers_.Add(1);
   return Status::OK();
 }
 

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/mem-tracker.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/mem-tracker.h b/be/src/runtime/mem-tracker.h
index 8d28f8f..10a3424 100644
--- a/be/src/runtime/mem-tracker.h
+++ b/be/src/runtime/mem-tracker.h
@@ -107,6 +107,7 @@ class MemTracker {
   /// destruction to prevent other threads from getting a reference to the MemTracker
   /// via its parent. Only used to deregister the query-level MemTracker from the
   /// global hierarchy.
+  /// TODO: IMPALA-3200: this is also used by BufferedBlockMgr, which will be deleted.
   void CloseAndUnregisterFromParent();
 
   /// Include counters from a ReservationTracker in logs and other diagnostics.

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/test-env.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/test-env.cc b/be/src/runtime/test-env.cc
index 0d6c6f0..770eaba 100644
--- a/be/src/runtime/test-env.cc
+++ b/be/src/runtime/test-env.cc
@@ -49,7 +49,7 @@ Status TestEnv::Init() {
   exec_env_.reset(new ExecEnv);
   // Populate the ExecEnv state that the backend tests need.
   exec_env_->mem_tracker_.reset(new MemTracker(-1, "Process"));
-  RETURN_IF_ERROR(exec_env_->disk_io_mgr()->Init());
+  RETURN_IF_ERROR(exec_env_->disk_io_mgr()->Init(exec_env_->process_mem_tracker()));
   exec_env_->metrics_.reset(new MetricGroup("test-env-metrics"));
   exec_env_->tmp_file_mgr_.reset(new TmpFileMgr);
   if (have_tmp_file_mgr_args_) {

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/tmp-file-mgr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/tmp-file-mgr-test.cc b/be/src/runtime/tmp-file-mgr-test.cc
index 9161b63..3091c58 100644
--- a/be/src/runtime/tmp-file-mgr-test.cc
+++ b/be/src/runtime/tmp-file-mgr-test.cc
@@ -25,7 +25,6 @@
 #include <gtest/gtest.h>
 
 #include "common/init.h"
-#include "runtime/io/request-context.h"
 #include "runtime/test-env.h"
 #include "runtime/tmp-file-mgr-internal.h"
 #include "runtime/tmp-file-mgr.h"
@@ -135,7 +134,7 @@ class TmpFileMgrTest : public ::testing::Test {
 
   /// Helper to cancel the FileGroup RequestContext.
   static void CancelIoContext(TmpFileMgr::FileGroup* group) {
-    group->io_ctx_->Cancel();
+    group->io_mgr_->CancelContext(group->io_ctx_.get());
   }
 
   /// Helper to get the # of bytes allocated by the group. Validates that the sum across

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/tmp-file-mgr.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/tmp-file-mgr.cc b/be/src/runtime/tmp-file-mgr.cc
index e0c58d4..3807670 100644
--- a/be/src/runtime/tmp-file-mgr.cc
+++ b/be/src/runtime/tmp-file-mgr.cc
@@ -243,7 +243,7 @@ TmpFileMgr::FileGroup::FileGroup(TmpFileMgr* tmp_file_mgr, DiskIoMgr* io_mgr,
     next_allocation_index_(0),
     free_ranges_(64) {
   DCHECK(tmp_file_mgr != nullptr);
-  io_ctx_ = io_mgr_->RegisterContext();
+  io_ctx_ = io_mgr_->RegisterContext(nullptr);
 }
 
 TmpFileMgr::FileGroup::~FileGroup() {
@@ -396,10 +396,7 @@ Status TmpFileMgr::FileGroup::ReadAsync(WriteHandle* handle, MemRange buffer) {
       BufferOpts::ReadInto(buffer.data(), buffer.len()));
   read_counter_->Add(1);
   bytes_read_counter_->Add(buffer.len());
-  bool needs_buffers;
-  RETURN_IF_ERROR(io_mgr_->StartScanRange(
-      io_ctx_.get(), handle->read_range_, &needs_buffers));
-  DCHECK(!needs_buffers) << "Already provided a buffer";
+  RETURN_IF_ERROR(io_mgr_->AddScanRange(io_ctx_.get(), handle->read_range_, true));
   return Status::OK();
 }
 
@@ -429,7 +426,7 @@ Status TmpFileMgr::FileGroup::WaitForAsyncRead(WriteHandle* handle, MemRange buf
   }
 exit:
   // Always return the buffer before exiting to avoid leaking it.
-  if (io_mgr_buffer != nullptr) handle->read_range_->ReturnBuffer(move(io_mgr_buffer));
+  if (io_mgr_buffer != nullptr) io_mgr_->ReturnBuffer(move(io_mgr_buffer));
   handle->read_range_ = nullptr;
   return status;
 }
@@ -525,20 +522,11 @@ TmpFileMgr::WriteHandle::WriteHandle(
     is_cancelled_(false),
     write_in_flight_(false) {}
 
-TmpFileMgr::WriteHandle::~WriteHandle() {
-  DCHECK(!write_in_flight_);
-  DCHECK(read_range_ == nullptr);
-}
-
 string TmpFileMgr::WriteHandle::TmpFilePath() const {
   if (file_ == nullptr) return "";
   return file_->path();
 }
 
-int64_t TmpFileMgr::WriteHandle::len() const {
-  return write_range_->len();
-}
-
 Status TmpFileMgr::WriteHandle::Write(DiskIoMgr* io_mgr, RequestContext* io_ctx,
     File* file, int64_t offset, MemRange buffer,
     WriteRange::WriteDoneCallback callback) {

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/tmp-file-mgr.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/tmp-file-mgr.h b/be/src/runtime/tmp-file-mgr.h
index 55901f4..95072ae 100644
--- a/be/src/runtime/tmp-file-mgr.h
+++ b/be/src/runtime/tmp-file-mgr.h
@@ -28,6 +28,7 @@
 #include "common/object-pool.h"
 #include "common/status.h"
 #include "gen-cpp/Types_types.h" // for TUniqueId
+#include "runtime/io/request-ranges.h"
 #include "util/collection-metrics.h"
 #include "util/condition-variable.h"
 #include "util/mem-range.h"
@@ -36,12 +37,6 @@
 #include "util/spinlock.h"
 
 namespace impala {
-namespace io {
-  class DiskIoMgr;
-  class RequestContext;
-  class ScanRange;
-  class WriteRange;
-}
 
 /// TmpFileMgr provides an abstraction for management of temporary (a.k.a. scratch) files
 /// on the filesystem and I/O to and from them. TmpFileMgr manages multiple scratch
@@ -89,7 +84,6 @@ class TmpFileMgr {
   /// Needs to be public for TmpFileMgrTest.
   typedef int DeviceId;
 
-  /// Same typedef as io::WriteRange::WriteDoneCallback.
   typedef std::function<void(const Status&)> WriteDoneCallback;
 
   /// Represents a group of temporary files - one per disk with a scratch directory. The
@@ -283,7 +277,10 @@ class TmpFileMgr {
    public:
     /// The write must be destroyed by passing it to FileGroup - destroying it before
     /// the write completes is an error.
-    ~WriteHandle();
+    ~WriteHandle() {
+      DCHECK(!write_in_flight_);
+      DCHECK(read_range_ == nullptr);
+    }
 
     /// Cancel any in-flight read synchronously.
     void CancelRead();
@@ -293,7 +290,7 @@ class TmpFileMgr {
     std::string TmpFilePath() const;
 
     /// The length of the write range in bytes.
-    int64_t len() const;
+    int64_t len() const { return write_range_->len(); }
 
     std::string DebugString();
 
@@ -308,7 +305,7 @@ class TmpFileMgr {
     /// failure and 'is_cancelled_' is set to true on failure.
     Status Write(io::DiskIoMgr* io_mgr, io::RequestContext* io_ctx, File* file,
         int64_t offset, MemRange buffer,
-        WriteDoneCallback callback) WARN_UNUSED_RESULT;
+        io::WriteRange::WriteDoneCallback callback) WARN_UNUSED_RESULT;
 
     /// Retry the write after the initial write failed with an error, instead writing to
     /// 'offset' of 'file'. 'write_in_flight_' must be true before calling.

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/util/bit-util-test.cc
----------------------------------------------------------------------
diff --git a/be/src/util/bit-util-test.cc b/be/src/util/bit-util-test.cc
index 6f70727..5f8d443 100644
--- a/be/src/util/bit-util-test.cc
+++ b/be/src/util/bit-util-test.cc
@@ -258,17 +258,6 @@ TEST(BitUtil, Log2) {
   EXPECT_EQ(BitUtil::Log2CeilingNonZero64(ULLONG_MAX), 64);
 }
 
-TEST(BitUtil, RoundToPowerOfTwo) {
-  EXPECT_EQ(16, BitUtil::RoundUpToPowerOfTwo(9));
-  EXPECT_EQ(16, BitUtil::RoundUpToPowerOfTwo(15));
-  EXPECT_EQ(16, BitUtil::RoundUpToPowerOfTwo(16));
-  EXPECT_EQ(32, BitUtil::RoundUpToPowerOfTwo(17));
-  EXPECT_EQ(8, BitUtil::RoundDownToPowerOfTwo(9));
-  EXPECT_EQ(8, BitUtil::RoundDownToPowerOfTwo(15));
-  EXPECT_EQ(16, BitUtil::RoundDownToPowerOfTwo(16));
-  EXPECT_EQ(16, BitUtil::RoundDownToPowerOfTwo(17));
-}
-
 TEST(BitUtil, RoundUpToPowerOf2) {
   EXPECT_EQ(BitUtil::RoundUpToPowerOf2(7, 8), 8);
   EXPECT_EQ(BitUtil::RoundUpToPowerOf2(8, 8), 8);

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/util/bit-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/bit-util.h b/be/src/util/bit-util.h
index 8a65509..5c9a29b 100644
--- a/be/src/util/bit-util.h
+++ b/be/src/util/bit-util.h
@@ -98,12 +98,6 @@ class BitUtil {
     return v;
   }
 
-  /// Returns the largest power of two <= v.
-  static inline int64_t RoundDownToPowerOfTwo(int64_t v) {
-    int64_t v_rounded_up = RoundUpToPowerOfTwo(v);
-    return v_rounded_up == v ? v : v_rounded_up / 2;
-  }
-
   /// Returns 'value' rounded up to the nearest multiple of 'factor' when factor is
   /// a power of two
   static inline int64_t RoundUpToPowerOf2(int64_t value, int64_t factor) {
@@ -111,7 +105,7 @@ class BitUtil {
     return (value + (factor - 1)) & ~(factor - 1);
   }
 
-  static inline int64_t RoundDownToPowerOf2(int64_t value, int64_t factor) {
+  static inline int RoundDownToPowerOf2(int value, int factor) {
     DCHECK((factor > 0) && ((factor & (factor - 1)) == 0));
     return value & ~(factor - 1);
   }

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/util/impalad-metrics.cc
----------------------------------------------------------------------
diff --git a/be/src/util/impalad-metrics.cc b/be/src/util/impalad-metrics.cc
index 815e4af..c2a2644 100644
--- a/be/src/util/impalad-metrics.cc
+++ b/be/src/util/impalad-metrics.cc
@@ -46,6 +46,12 @@ const char* ImpaladMetricKeys::HASH_TABLE_TOTAL_BYTES =
     "impala-server.hash-table.total-bytes";
 const char* ImpaladMetricKeys::IO_MGR_NUM_OPEN_FILES =
     "impala-server.io-mgr.num-open-files";
+const char* ImpaladMetricKeys::IO_MGR_NUM_BUFFERS =
+    "impala-server.io-mgr.num-buffers";
+const char* ImpaladMetricKeys::IO_MGR_TOTAL_BYTES =
+    "impala-server.io-mgr.total-bytes";
+const char* ImpaladMetricKeys::IO_MGR_NUM_UNUSED_BUFFERS =
+    "impala-server.io-mgr.num-unused-buffers";
 const char* ImpaladMetricKeys::IO_MGR_BYTES_READ =
     "impala-server.io-mgr.bytes-read";
 const char* ImpaladMetricKeys::IO_MGR_LOCAL_BYTES_READ =
@@ -205,8 +211,11 @@ void ImpaladMetrics::CreateMetrics(MetricGroup* m) {
       ImpaladMetricKeys::NUM_FILES_OPEN_FOR_INSERT, 0);
 
   // Initialize IO mgr metrics
-  IO_MGR_NUM_OPEN_FILES = m->AddGauge(
-      ImpaladMetricKeys::IO_MGR_NUM_OPEN_FILES, 0);
+  IO_MGR_NUM_OPEN_FILES = m->AddGauge(ImpaladMetricKeys::IO_MGR_NUM_OPEN_FILES, 0);
+  IO_MGR_NUM_BUFFERS = m->AddGauge(ImpaladMetricKeys::IO_MGR_NUM_BUFFERS, 0);
+  IO_MGR_TOTAL_BYTES = m->AddGauge(ImpaladMetricKeys::IO_MGR_TOTAL_BYTES, 0);
+  IO_MGR_NUM_UNUSED_BUFFERS = m->AddGauge(
+      ImpaladMetricKeys::IO_MGR_NUM_UNUSED_BUFFERS, 0);
   IO_MGR_NUM_CACHED_FILE_HANDLES = m->AddGauge(
       ImpaladMetricKeys::IO_MGR_NUM_CACHED_FILE_HANDLES, 0);
   IO_MGR_NUM_FILE_HANDLES_OUTSTANDING = m->AddGauge(

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/util/impalad-metrics.h
----------------------------------------------------------------------
diff --git a/be/src/util/impalad-metrics.h b/be/src/util/impalad-metrics.h
index 7de7aa8..b49caf9 100644
--- a/be/src/util/impalad-metrics.h
+++ b/be/src/util/impalad-metrics.h
@@ -67,6 +67,15 @@ class ImpaladMetricKeys {
   /// Number of files currently opened by the io mgr
   static const char* IO_MGR_NUM_OPEN_FILES;
 
+  /// Number of IO buffers allocated by the io mgr
+  static const char* IO_MGR_NUM_BUFFERS;
+
+  /// Number of bytes used by IO buffers (used and unused).
+  static const char* IO_MGR_TOTAL_BYTES;
+
+  /// Number of IO buffers that are currently unused (and can be GC'ed)
+  static const char* IO_MGR_NUM_UNUSED_BUFFERS;
+
   /// Total number of bytes read by the io mgr
   static const char* IO_MGR_BYTES_READ;
 

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/common/thrift/PlanNodes.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/PlanNodes.thrift b/common/thrift/PlanNodes.thrift
index 1ab05a0..c5df1cd 100644
--- a/common/thrift/PlanNodes.thrift
+++ b/common/thrift/PlanNodes.thrift
@@ -239,9 +239,6 @@ struct THdfsScanNode {
   // The byte offset of the slot for Parquet metadata if Parquet count star optimization
   // is enabled.
   10: optional i32 parquet_count_star_slot_offset
-
-  // The ideal memory reservation in bytes to process an input split.
-  11: optional i64 ideal_scan_range_reservation
 }
 
 struct TDataSourceScanNode {

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java b/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
index aae3863..4f0a0e1 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SlotDescriptor.java
@@ -163,25 +163,6 @@ public class SlotDescriptor {
   }
 
   /**
-   * Checks if this descriptor describes  an array "pos" pseudo-column.
-   *
-   * Note: checking whether the column is null distinguishes between top-level columns
-   * and nested types. This check more specifically looks just for a reference to the
-   * "pos" field of an array type.
-   */
-  public boolean isArrayPosRef() {
-    if (parent_ == null) return false;
-    Type parentType = parent_.getType();
-    if (parentType instanceof CollectionStructType) {
-      if (((CollectionStructType)parentType).isArrayStruct() &&
-          label_.equals(Path.ARRAY_POS_FIELD_NAME)) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /**
    * Assembles the absolute materialized path to this slot starting from the schema
    * root. The materialized path points to the first non-struct schema element along the
    * path starting from the parent's tuple path to this slot's path.


[9/9] impala git commit: Revert IMPALA-4835 and dependent changes

Posted by lv...@apache.org.
Revert IMPALA-4835 and dependent changes

Revert "IMPALA-6585: increase test_low_mem_limit_q21 limit"

This reverts commit 25bcb258dfd712f1514cf188206667a5e6be0e26.

Revert "IMPALA-6588: don't add empty list of ranges in text scan"

This reverts commit d57fbec6f67b83227b4c6117476da8f7d75fc4f6.

Revert "IMPALA-4835: Part 3: switch I/O buffers to buffer pool"

This reverts commit 24b4ed0b29a44090350e630d625291c01b753a36.

Revert "IMPALA-4835: Part 2: Allocate scan range buffers upfront"

This reverts commit 5699b59d0c5cbe37e888a367adb42fa12dfb0916.

Revert "IMPALA-4835: Part 1: simplify I/O mgr mem mgmt and cancellation"

This reverts commit 65680dc42107db4ff2273c635cedf83d20f0ea94.

Change-Id: Ie5ca451cd96602886b0a8ecaa846957df0269cbb
Reviewed-on: http://gerrit.cloudera.org:8080/9480
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Tested-by: Impala Public Jenkins
Reviewed-on: http://gerrit.cloudera.org:8080/9485
Reviewed-by: Tim Armstrong <ta...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/e5689fb5
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/e5689fb5
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/e5689fb5

Branch: refs/heads/2.x
Commit: e5689fb5c6a011af9e526effa9aa5399b194e977
Parents: 42c604e
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Fri Mar 2 16:09:25 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Sat Mar 3 22:07:15 2018 +0000

----------------------------------------------------------------------
 be/src/exec/CMakeLists.txt                      |    1 -
 be/src/exec/base-sequence-scanner.cc            |    1 -
 be/src/exec/base-sequence-scanner.h             |    3 +-
 be/src/exec/hdfs-lzo-text-scanner.cc            |    1 -
 be/src/exec/hdfs-parquet-scanner-test.cc        |   96 -
 be/src/exec/hdfs-parquet-scanner.cc             |  234 +--
 be/src/exec/hdfs-parquet-scanner.h              |   44 +-
 be/src/exec/hdfs-scan-node-base.cc              |   95 +-
 be/src/exec/hdfs-scan-node-base.h               |    3 -
 be/src/exec/hdfs-scan-node-mt.cc                |   20 +-
 be/src/exec/hdfs-scan-node.cc                   |  172 +-
 be/src/exec/hdfs-scan-node.h                    |   66 +-
 be/src/exec/hdfs-text-scanner.cc                |    6 +-
 be/src/exec/parquet-column-readers.cc           |  108 +-
 be/src/exec/parquet-column-readers.h            |   88 +-
 be/src/exec/scanner-context.cc                  |   42 +-
 be/src/exec/scanner-context.h                   |   54 +-
 be/src/runtime/bufferpool/buffer-pool.h         |    1 -
 .../bufferpool/reservation-tracker-test.cc      |    8 +-
 be/src/runtime/bufferpool/reservation-util.cc   |    2 +-
 be/src/runtime/exec-env.cc                      |    7 +-
 be/src/runtime/io/disk-io-mgr-internal.h        |   16 -
 be/src/runtime/io/disk-io-mgr-stress-test.cc    |   43 +-
 be/src/runtime/io/disk-io-mgr-stress.cc         |   89 +-
 be/src/runtime/io/disk-io-mgr-stress.h          |   26 +-
 be/src/runtime/io/disk-io-mgr-test.cc           |  849 ++++----
 be/src/runtime/io/disk-io-mgr.cc                |  716 +++++--
 be/src/runtime/io/disk-io-mgr.h                 |  383 ++--
 be/src/runtime/io/request-context.cc            |  239 +--
 be/src/runtime/io/request-context.h             |  318 ++-
 be/src/runtime/io/request-ranges.h              |  196 +-
 be/src/runtime/io/scan-range.cc                 |  309 ++-
 be/src/runtime/mem-tracker.h                    |    1 +
 be/src/runtime/test-env.cc                      |    2 +-
 be/src/runtime/tmp-file-mgr-test.cc             |    3 +-
 be/src/runtime/tmp-file-mgr.cc                  |   18 +-
 be/src/runtime/tmp-file-mgr.h                   |   17 +-
 be/src/util/bit-util-test.cc                    |   11 -
 be/src/util/bit-util.h                          |    8 +-
 be/src/util/impalad-metrics.cc                  |   13 +-
 be/src/util/impalad-metrics.h                   |    9 +
 common/thrift/PlanNodes.thrift                  |    3 -
 .../apache/impala/analysis/SlotDescriptor.java  |   19 -
 .../org/apache/impala/analysis/SlotRef.java     |   20 +
 .../org/apache/impala/planner/HdfsScanNode.java |  167 +-
 .../java/org/apache/impala/util/BitUtil.java    |    6 -
 .../org/apache/impala/util/BitUtilTest.java     |    6 -
 .../queries/PlannerTest/constant-folding.test   |   42 +-
 .../queries/PlannerTest/disable-codegen.test    |   20 +-
 .../PlannerTest/fk-pk-join-detection.test       |   78 +-
 .../queries/PlannerTest/max-row-size.test       |   80 +-
 .../PlannerTest/min-max-runtime-filters.test    |    6 +-
 .../queries/PlannerTest/mt-dop-validation.test  |   40 +-
 .../queries/PlannerTest/parquet-filtering.test  |   42 +-
 .../queries/PlannerTest/partition-pruning.test  |    4 +-
 .../PlannerTest/resource-requirements.test      | 1814 ++++--------------
 .../PlannerTest/sort-expr-materialization.test  |   32 +-
 .../PlannerTest/spillable-buffer-sizing.test    |  192 +-
 .../queries/PlannerTest/tablesample.test        |   44 +-
 .../queries/PlannerTest/union.test              |    8 +-
 .../admission-reject-min-reservation.test       |   12 +-
 .../queries/QueryTest/analytic-fns.test         |    5 +-
 .../queries/QueryTest/codegen-mem-limit.test    |    5 +-
 .../QueryTest/disk-spill-encryption.test        |    2 +-
 .../queries/QueryTest/explain-level0.test       |    4 +-
 .../queries/QueryTest/explain-level1.test       |    4 +-
 .../queries/QueryTest/explain-level2.test       |   14 +-
 .../queries/QueryTest/explain-level3.test       |   14 +-
 .../queries/QueryTest/nested-types-tpch.test    |    6 +-
 .../queries/QueryTest/runtime_row_filters.test  |   11 +-
 .../queries/QueryTest/scanners.test             |    7 -
 .../functional-query/queries/QueryTest/set.test |    2 +-
 .../queries/QueryTest/spilling-aggs.test        |   19 +-
 .../spilling-naaj-no-deny-reservation.test      |    7 +-
 .../queries/QueryTest/spilling-naaj.test        |    8 +-
 .../QueryTest/spilling-no-debug-action.test     |   66 -
 .../QueryTest/spilling-sorts-exhaustive.test    |   10 +-
 .../queries/QueryTest/spilling.test             |   76 +-
 .../queries/QueryTest/stats-extrapolation.test  |   52 +-
 .../tpch/queries/sort-reservation-usage.test    |    9 +-
 tests/common/test_dimensions.py                 |   16 +-
 tests/custom_cluster/test_scratch_disk.py       |    2 +-
 tests/query_test/test_mem_usage_scaling.py      |   11 +-
 tests/query_test/test_query_mem_limit.py        |    4 +-
 tests/query_test/test_scanners.py               |   25 +-
 tests/query_test/test_scanners_fuzz.py          |    9 +-
 tests/query_test/test_sort.py                   |   16 +-
 tests/query_test/test_spilling.py               |    5 -
 88 files changed, 2799 insertions(+), 4563 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt
index c1f91d6..aab1383 100644
--- a/be/src/exec/CMakeLists.txt
+++ b/be/src/exec/CMakeLists.txt
@@ -108,4 +108,3 @@ ADD_BE_TEST(parquet-version-test)
 ADD_BE_TEST(row-batch-list-test)
 ADD_BE_TEST(incr-stats-util-test)
 ADD_BE_TEST(hdfs-avro-scanner-test)
-ADD_BE_TEST(hdfs-parquet-scanner-test)

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/base-sequence-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/base-sequence-scanner.cc b/be/src/exec/base-sequence-scanner.cc
index 9d95b0b..9cb6330 100644
--- a/be/src/exec/base-sequence-scanner.cc
+++ b/be/src/exec/base-sequence-scanner.cc
@@ -46,7 +46,6 @@ static const int MIN_SYNC_READ_SIZE = 64 * 1024; // bytes
 
 Status BaseSequenceScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
     const vector<HdfsFileDesc*>& files) {
-  DCHECK(!files.empty());
   // Issue just the header range for each file.  When the header is complete,
   // we'll issue the splits for that file.  Splits cannot be processed until the
   // header is parsed (the header object is then shared across splits for that file).

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/base-sequence-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/base-sequence-scanner.h b/be/src/exec/base-sequence-scanner.h
index 3c2326e..887ff6f 100644
--- a/be/src/exec/base-sequence-scanner.h
+++ b/be/src/exec/base-sequence-scanner.h
@@ -47,8 +47,7 @@ class ScannerContext;
 /// situation, causing the block to be incorrectly skipped.
 class BaseSequenceScanner : public HdfsScanner {
  public:
-  /// Issue the initial ranges for all sequence container files. 'files' must not be
-  /// empty.
+  /// Issue the initial ranges for all sequence container files.
   static Status IssueInitialRanges(HdfsScanNodeBase* scan_node,
                                    const std::vector<HdfsFileDesc*>& files)
                                    WARN_UNUSED_RESULT;

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-lzo-text-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-lzo-text-scanner.cc b/be/src/exec/hdfs-lzo-text-scanner.cc
index 8af89f2..88ae295 100644
--- a/be/src/exec/hdfs-lzo-text-scanner.cc
+++ b/be/src/exec/hdfs-lzo-text-scanner.cc
@@ -62,7 +62,6 @@ HdfsScanner* HdfsLzoTextScanner::GetHdfsLzoTextScanner(
 
 Status HdfsLzoTextScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
     const vector<HdfsFileDesc*>& files) {
-  DCHECK(!files.empty());
   if (LzoIssueInitialRanges == NULL) {
     lock_guard<SpinLock> l(lzo_load_lock_);
     if (library_load_status_.ok()) {

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-parquet-scanner-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-scanner-test.cc b/be/src/exec/hdfs-parquet-scanner-test.cc
deleted file mode 100644
index cbc6e76..0000000
--- a/be/src/exec/hdfs-parquet-scanner-test.cc
+++ /dev/null
@@ -1,96 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "exec/hdfs-parquet-scanner.h"
-#include "testutil/gtest-util.h"
-
-#include "common/names.h"
-
-static const int64_t MIN_BUFFER_SIZE = 64 * 1024;
-static const int64_t MAX_BUFFER_SIZE = 8 * 1024 * 1024;
-
-namespace impala {
-
-class HdfsParquetScannerTest : public testing::Test {
- protected:
-  void TestDivideReservation(const vector<int64_t>& col_range_lengths,
-      int64_t total_col_reservation, const vector<int64_t>& expected_reservations);
-};
-
-/// Test that DivideReservationBetweenColumns() returns 'expected_reservations' for
-/// inputs 'col_range_lengths' and 'total_col_reservation'.
-void HdfsParquetScannerTest::TestDivideReservation(const vector<int64_t>& col_range_lengths,
-      int64_t total_col_reservation, const vector<int64_t>& expected_reservations) {
-  vector<pair<int, int64_t>> reservations =
-      HdfsParquetScanner::DivideReservationBetweenColumnsHelper(
-      MIN_BUFFER_SIZE, MAX_BUFFER_SIZE, col_range_lengths, total_col_reservation);
-  for (int i = 0; i < reservations.size(); ++i) {
-    LOG(INFO) << i << " " << reservations[i].first << " " << reservations[i].second;
-  }
-  EXPECT_EQ(reservations.size(), expected_reservations.size());
-  vector<bool> present(expected_reservations.size(), false);
-  for (auto& reservation: reservations) {
-    // Ensure that each appears exactly once.
-    EXPECT_FALSE(present[reservation.first]);
-    present[reservation.first] = true;
-    EXPECT_EQ(expected_reservations[reservation.first], reservation.second)
-        << reservation.first;
-  }
-}
-
-TEST_F(HdfsParquetScannerTest, DivideReservation) {
-  // Test a long scan ranges with lots of memory - should allocate 3 max-size
-  // buffers per range.
-  TestDivideReservation({100 * 1024 * 1024}, 50 * 1024 * 1024, {3 * MAX_BUFFER_SIZE});
-  TestDivideReservation({100 * 1024 * 1024, 50 * 1024 * 1024}, 100 * 1024 * 1024,
-        {3 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE});
-
-  // Long scan ranges, not enough memory for 3 buffers each. Should only allocate
-  // max-sized buffers, preferring the longer scan range.
-  TestDivideReservation({50 * 1024 * 1024, 100 * 1024 * 1024}, 5 * MAX_BUFFER_SIZE,
-        {2 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE});
-  TestDivideReservation({50 * 1024 * 1024, 100 * 1024 * 1024},
-        5 * MAX_BUFFER_SIZE + MIN_BUFFER_SIZE,
-        {2 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE});
-  TestDivideReservation({50 * 1024 * 1024, 100 * 1024 * 1024}, 6 * MAX_BUFFER_SIZE - 1,
-        {2 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE});
-
-  // Test a short range with lots of memory - should round up buffer size.
-  TestDivideReservation({100 * 1024}, 50 * 1024 * 1024, {128 * 1024});
-
-  // Test a range << MIN_BUFFER_SIZE - should round up to buffer size.
-  TestDivideReservation({13}, 50 * 1024 * 1024, {MIN_BUFFER_SIZE});
-
-  // Test long ranges with limited memory.
-  TestDivideReservation({100 * 1024 * 1024}, 100 * 1024, {MIN_BUFFER_SIZE});
-  TestDivideReservation({100 * 1024 * 1024}, MIN_BUFFER_SIZE, {MIN_BUFFER_SIZE});
-  TestDivideReservation({100 * 1024 * 1024}, 2 * MIN_BUFFER_SIZE, {2 * MIN_BUFFER_SIZE});
-  TestDivideReservation({100 * 1024 * 1024}, MAX_BUFFER_SIZE - 1, {MAX_BUFFER_SIZE / 2});
-  TestDivideReservation({100 * 1024 * 1024, 1024 * 1024, MIN_BUFFER_SIZE},
-      3 * MIN_BUFFER_SIZE, {MIN_BUFFER_SIZE, MIN_BUFFER_SIZE, MIN_BUFFER_SIZE});
-
-  // Test a mix of scan range lengths larger than and smaller than the max I/O buffer
-  // size. Long ranges get allocated most memory.
-  TestDivideReservation(
-      {15145047, 5019635, 5019263, 15145047, 15145047, 5019635, 5019263, 317304},
-      25165824,
-      {8388608, 2097152, 524288, 8388608, 4194304, 1048576, 262144, 262144});
-}
-
-}
-
-IMPALA_TEST_MAIN();

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-parquet-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-scanner.cc b/be/src/exec/hdfs-parquet-scanner.cc
index 574ddb0..e279369 100644
--- a/be/src/exec/hdfs-parquet-scanner.cc
+++ b/be/src/exec/hdfs-parquet-scanner.cc
@@ -17,7 +17,6 @@
 
 #include "exec/hdfs-parquet-scanner.h"
 
-#include <algorithm>
 #include <queue>
 
 #include <gutil/strings/substitute.h>
@@ -28,7 +27,6 @@
 #include "exec/parquet-column-stats.h"
 #include "exec/scanner-context.inline.h"
 #include "runtime/collection-value-builder.h"
-#include "runtime/exec-env.h"
 #include "runtime/io/disk-io-mgr.h"
 #include "runtime/runtime-state.h"
 #include "runtime/runtime-filter.inline.h"
@@ -37,7 +35,6 @@
 #include "common/names.h"
 
 using std::move;
-using std::sort;
 using namespace impala;
 using namespace impala::io;
 
@@ -50,6 +47,10 @@ constexpr int BATCHES_PER_FILTER_SELECTIVITY_CHECK = 16;
 static_assert(BitUtil::IsPowerOf2(BATCHES_PER_FILTER_SELECTIVITY_CHECK),
     "BATCHES_PER_FILTER_SELECTIVITY_CHECK must be a power of two");
 
+// Max dictionary page header size in bytes. This is an estimate and only needs to be an
+// upper bound.
+const int MAX_DICT_HEADER_SIZE = 100;
+
 // Max entries in the dictionary before switching to PLAIN encoding. If a dictionary
 // has fewer entries, then the entire column is dictionary encoded. This threshold
 // is guaranteed to be true for Impala versions 2.9 or below.
@@ -68,7 +69,6 @@ const string PARQUET_MEM_LIMIT_EXCEEDED =
 
 Status HdfsParquetScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
     const vector<HdfsFileDesc*>& files) {
-  DCHECK(!files.empty());
   vector<ScanRange*> footer_ranges;
   for (int i = 0; i < files.size(); ++i) {
     // If the file size is less than 12 bytes, it is an invalid Parquet file.
@@ -98,7 +98,7 @@ Status HdfsParquetScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
             static_cast<ScanRangeMetadata*>(split->meta_data());
         // Each split is processed by first issuing a scan range for the file footer, which
         // is done here, followed by scan ranges for the columns of each row group within
-        // the actual split (see InitScalarColumns()). The original split is stored in the
+        // the actual split (in InitColumns()). The original split is stored in the
         // metadata associated with the footer range.
         ScanRange* footer_range;
         if (footer_split != nullptr) {
@@ -121,13 +121,9 @@ Status HdfsParquetScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
       }
     }
   }
-  // We may not have any scan ranges if this scan node does not have the footer split for
-  // any Parquet file.
-  if (footer_ranges.size() > 0) {
-    // The threads that process the footer will also do the scan, so we mark all the files
-    // as complete here.
-    RETURN_IF_ERROR(scan_node->AddDiskIoRanges(footer_ranges, files.size()));
-  }
+  // The threads that process the footer will also do the scan, so we mark all the files
+  // as complete here.
+  RETURN_IF_ERROR(scan_node->AddDiskIoRanges(footer_ranges, files.size()));
   return Status::OK();
 }
 
@@ -232,14 +228,9 @@ Status HdfsParquetScanner::Open(ScannerContext* context) {
 
   // Release I/O buffers immediately to make sure they are cleaned up
   // in case we return a non-OK status anywhere below.
-  int64_t stream_reservation = stream_->reservation();
-  stream_ = nullptr;
   context_->ReleaseCompletedResources(true);
   context_->ClearStreams();
   RETURN_IF_ERROR(footer_status);
-  // The scanner-wide stream was used only to read the file footer.  Each column has added
-  // its own stream. We can use the reservation from 'stream_' for the columns now.
-  total_col_reservation_ = stream_reservation;
 
   // Parse the file schema into an internal representation for schema resolution.
   schema_resolver_.reset(new ParquetSchemaResolver(*scan_node_->hdfs_table(),
@@ -256,6 +247,10 @@ Status HdfsParquetScanner::Open(ScannerContext* context) {
   template_tuple_ = template_tuple_map_[scan_node_->tuple_desc()];
 
   RETURN_IF_ERROR(InitDictFilterStructures());
+
+  // The scanner-wide stream was used only to read the file footer.  Each column has added
+  // its own stream.
+  stream_ = nullptr;
   return Status::OK();
 }
 
@@ -679,13 +674,15 @@ Status HdfsParquetScanner::NextRowGroup() {
     }
 
     InitCollectionColumns();
-    RETURN_IF_ERROR(InitScalarColumns());
 
-    // Start scanning dictionary filtering column readers, so we can read the dictionary
-    // pages in EvalDictionaryFilters().
-    RETURN_IF_ERROR(BaseScalarColumnReader::StartScans(dict_filterable_readers_));
+    // Prepare dictionary filtering columns for first read
+    // This must be done before dictionary filtering, because this code initializes
+    // the column offsets and streams needed to read the dictionaries.
+    // TODO: Restructure the code so that the dictionary can be read without the rest
+    // of the column.
+    RETURN_IF_ERROR(InitScalarColumns(row_group_idx_, dict_filterable_readers_));
 
-    // StartScans() may have allocated resources to scan columns. If we skip this row
+    // InitColumns() may have allocated resources to scan columns. If we skip this row
     // group below, we must call ReleaseSkippedRowGroupResources() before continuing.
 
     // If there is a dictionary-encoded column where every value is eliminated
@@ -706,10 +703,10 @@ Status HdfsParquetScanner::NextRowGroup() {
     }
 
     // At this point, the row group has passed any filtering criteria
-    // Start scanning non-dictionary filtering column readers and initialize their
-    // dictionaries.
-    RETURN_IF_ERROR(BaseScalarColumnReader::StartScans(non_dict_filterable_readers_));
-    status = BaseScalarColumnReader::InitDictionaries(non_dict_filterable_readers_);
+    // Prepare non-dictionary filtering column readers for first read and
+    // initialize their dictionaries.
+    RETURN_IF_ERROR(InitScalarColumns(row_group_idx_, non_dict_filterable_readers_));
+    status = InitDictionaries(non_dict_filterable_readers_);
     if (!status.ok()) {
       // Either return an error or skip this row group if it is ok to ignore errors
       RETURN_IF_ERROR(state_->LogOrReturnError(status.msg()));
@@ -744,6 +741,7 @@ Status HdfsParquetScanner::NextRowGroup() {
       break;
     }
   }
+
   DCHECK(parse_status_.ok());
   return Status::OK();
 }
@@ -801,7 +799,6 @@ void HdfsParquetScanner::PartitionReaders(
     } else {
       BaseScalarColumnReader* scalar_reader =
           static_cast<BaseScalarColumnReader*>(reader);
-      scalar_readers_.push_back(scalar_reader);
       if (can_eval_dict_filters && IsDictFilterable(scalar_reader)) {
         dict_filterable_readers_.push_back(scalar_reader);
       } else {
@@ -993,7 +990,7 @@ Status HdfsParquetScanner::EvalDictionaryFilters(const parquet::RowGroup& row_gr
 
   // Any columns that were not 100% dictionary encoded need to initialize
   // their dictionaries here.
-  RETURN_IF_ERROR(BaseScalarColumnReader::InitDictionaries(deferred_dict_init_list));
+  RETURN_IF_ERROR(InitDictionaries(deferred_dict_init_list));
 
   return Status::OK();
 }
@@ -1442,15 +1439,12 @@ Status HdfsParquetScanner::ProcessFooter() {
         BufferOpts::ReadInto(metadata_buffer.buffer(), metadata_size));
 
     unique_ptr<BufferDescriptor> io_buffer;
-    bool needs_buffers;
-    RETURN_IF_ERROR(io_mgr->StartScanRange(
-          scan_node_->reader_context(), metadata_range, &needs_buffers));
-    DCHECK(!needs_buffers) << "Already provided a buffer";
-    RETURN_IF_ERROR(metadata_range->GetNext(&io_buffer));
+    RETURN_IF_ERROR(
+        io_mgr->Read(scan_node_->reader_context(), metadata_range, &io_buffer));
     DCHECK_EQ(io_buffer->buffer(), metadata_buffer.buffer());
     DCHECK_EQ(io_buffer->len(), metadata_size);
     DCHECK(io_buffer->eosr());
-    metadata_range->ReturnBuffer(move(io_buffer));
+    io_mgr->ReturnBuffer(move(io_buffer));
   }
 
   // Deserialize file header
@@ -1650,16 +1644,23 @@ void HdfsParquetScanner::InitCollectionColumns() {
   }
 }
 
-Status HdfsParquetScanner::InitScalarColumns() {
+Status HdfsParquetScanner::InitScalarColumns(
+    int row_group_idx, const vector<BaseScalarColumnReader*>& column_readers) {
   int64_t partition_id = context_->partition_descriptor()->id();
   const HdfsFileDesc* file_desc = scan_node_->GetFileDesc(partition_id, filename());
   DCHECK(file_desc != nullptr);
-  parquet::RowGroup& row_group = file_metadata_.row_groups[row_group_idx_];
+  parquet::RowGroup& row_group = file_metadata_.row_groups[row_group_idx];
 
+  // All the scan ranges (one for each column).
+  vector<ScanRange*> col_ranges;
   // Used to validate that the number of values in each reader in column_readers_ at the
   // same SchemaElement is the same.
   unordered_map<const parquet::SchemaElement*, int> num_values_map;
-  for (BaseScalarColumnReader* scalar_reader : scalar_readers_) {
+  // Used to validate we issued the right number of scan ranges
+  int num_scalar_readers = 0;
+
+  for (BaseScalarColumnReader* scalar_reader: column_readers) {
+    ++num_scalar_readers;
     const parquet::ColumnChunk& col_chunk = row_group.columns[scalar_reader->col_idx()];
     auto num_values_it = num_values_map.find(&scalar_reader->schema_element());
     int num_values = -1;
@@ -1668,115 +1669,78 @@ Status HdfsParquetScanner::InitScalarColumns() {
     } else {
       num_values_map[&scalar_reader->schema_element()] = col_chunk.meta_data.num_values;
     }
+    int64_t col_start = col_chunk.meta_data.data_page_offset;
+
     if (num_values != -1 && col_chunk.meta_data.num_values != num_values) {
       // TODO: improve this error message by saying which columns are different,
       // and also specify column in other error messages as appropriate
       return Status(TErrorCode::PARQUET_NUM_COL_VALS_ERROR, scalar_reader->col_idx(),
           col_chunk.meta_data.num_values, num_values, filename());
     }
-    RETURN_IF_ERROR(scalar_reader->Reset(*file_desc, col_chunk, row_group_idx_));
-  }
-  RETURN_IF_ERROR(
-      DivideReservationBetweenColumns(scalar_readers_, total_col_reservation_));
-  return Status::OK();
-}
 
-Status HdfsParquetScanner::DivideReservationBetweenColumns(
-    const vector<BaseScalarColumnReader*>& column_readers,
-    int64_t reservation_to_distribute) {
-  DiskIoMgr* io_mgr = ExecEnv::GetInstance()->disk_io_mgr();
-  const int64_t min_buffer_size = io_mgr->min_buffer_size();
-  const int64_t max_buffer_size = io_mgr->max_buffer_size();
-  // The HdfsScanNode reservation calculation in the planner ensures that we have
-  // reservation for at least one buffer per column.
-  if (reservation_to_distribute < min_buffer_size * column_readers.size()) {
-    return Status(TErrorCode::INTERNAL_ERROR,
-        Substitute("Not enough reservation in Parquet scanner for file '$0'. Need at "
-                   "least $1 bytes per column for $2 columns but had $3 bytes",
-            filename(), min_buffer_size, column_readers.size(),
-            reservation_to_distribute));
-  }
+    RETURN_IF_ERROR(ParquetMetadataUtils::ValidateRowGroupColumn(file_metadata_,
+        filename(), row_group_idx, scalar_reader->col_idx(),
+        scalar_reader->schema_element(), state_));
 
-  vector<int64_t> col_range_lengths(column_readers.size());
-  for (int i = 0; i < column_readers.size(); ++i) {
-    col_range_lengths[i] = column_readers[i]->scan_range()->len();
-  }
-  vector<pair<int, int64_t>> tmp_reservations = DivideReservationBetweenColumnsHelper(
-      min_buffer_size, max_buffer_size, col_range_lengths, reservation_to_distribute);
-  for (auto& tmp_reservation : tmp_reservations) {
-    column_readers[tmp_reservation.first]->set_io_reservation(tmp_reservation.second);
-  }
-  return Status::OK();
-}
+    if (col_chunk.meta_data.__isset.dictionary_page_offset) {
+      // Already validated in ValidateColumnOffsets()
+      DCHECK_LT(col_chunk.meta_data.dictionary_page_offset, col_start);
+      col_start = col_chunk.meta_data.dictionary_page_offset;
+    }
+    int64_t col_len = col_chunk.meta_data.total_compressed_size;
+    if (col_len <= 0) {
+      return Status(Substitute("File '$0' contains invalid column chunk size: $1",
+          filename(), col_len));
+    }
+    int64_t col_end = col_start + col_len;
+
+    // Already validated in ValidateColumnOffsets()
+    DCHECK_GT(col_end, 0);
+    DCHECK_LT(col_end, file_desc->file_length);
+    if (file_version_.application == "parquet-mr" && file_version_.VersionLt(1, 2, 9)) {
+      // The Parquet MR writer had a bug in 1.2.8 and below where it didn't include the
+      // dictionary page header size in total_compressed_size and total_uncompressed_size
+      // (see IMPALA-694). We pad col_len to compensate.
+      int64_t bytes_remaining = file_desc->file_length - col_end;
+      int64_t pad = min<int64_t>(MAX_DICT_HEADER_SIZE, bytes_remaining);
+      col_len += pad;
+    }
 
-vector<pair<int, int64_t>> HdfsParquetScanner::DivideReservationBetweenColumnsHelper(
-    int64_t min_buffer_size, int64_t max_buffer_size,
-    const vector<int64_t>& col_range_lengths, int64_t reservation_to_distribute) {
-  // Pair of (column index, reservation allocated).
-  vector<pair<int, int64_t>> tmp_reservations;
-  for (int i = 0; i < col_range_lengths.size(); ++i) tmp_reservations.emplace_back(i, 0);
-
-  // Sort in descending order of length, breaking ties by index so that larger columns
-  // get allocated reservation first. It is common to have dramatically different column
-  // sizes in a single file because of different value sizes and compressibility. E.g.
-  // consider a large STRING "comment" field versus a highly compressible
-  // dictionary-encoded column with only a few distinct values. We want to give max-sized
-  // buffers to large columns first to maximize the size of I/Os that we do while reading
-  // this row group.
-  sort(tmp_reservations.begin(), tmp_reservations.end(),
-      [&col_range_lengths](const pair<int, int64_t>& left, const pair<int, int64_t>& right) {
-        int64_t left_len = col_range_lengths[left.first];
-        int64_t right_len = col_range_lengths[right.first];
-        return left_len != right_len ? left_len > right_len : left.first < right.first;
-      });
-
-  // Set aside the minimum reservation per column.
-  reservation_to_distribute -= min_buffer_size * col_range_lengths.size();
-
-  // Allocate reservations to columns by repeatedly allocating either a max-sized buffer
-  // or a large enough buffer to fit the remaining data for each column. Do this
-  // round-robin up to the ideal number of I/O buffers.
-  for (int i = 0; i < DiskIoMgr::IDEAL_MAX_SIZED_BUFFERS_PER_SCAN_RANGE; ++i) {
-    for (auto& tmp_reservation : tmp_reservations) {
-      // Add back the reservation we set aside above.
-      if (i == 0) reservation_to_distribute += min_buffer_size;
-
-      int64_t bytes_left_in_range =
-          col_range_lengths[tmp_reservation.first] - tmp_reservation.second;
-      int64_t bytes_to_add;
-      if (bytes_left_in_range >= max_buffer_size) {
-        if (reservation_to_distribute >= max_buffer_size) {
-          bytes_to_add = max_buffer_size;
-        } else if (i == 0) {
-          DCHECK_EQ(0, tmp_reservation.second);
-          // Ensure this range gets at least one buffer on the first iteration.
-          bytes_to_add = BitUtil::RoundDownToPowerOfTwo(reservation_to_distribute);
-        } else {
-          DCHECK_GT(tmp_reservation.second, 0);
-          // We need to read more than the max buffer size, but can't allocate a
-          // max-sized buffer. Stop adding buffers to this column: we prefer to use
-          // the existing max-sized buffers without small buffers mixed in so that
-          // we will alway do max-sized I/Os, which make efficient use of I/O devices.
-          bytes_to_add = 0;
-        }
-      } else if (bytes_left_in_range > 0 &&
-          reservation_to_distribute >= min_buffer_size) {
-        // Choose a buffer size that will fit the rest of the bytes left in the range.
-        bytes_to_add = max(min_buffer_size, BitUtil::RoundUpToPowerOfTwo(bytes_left_in_range));
-        // But don't add more reservation than is available.
-        bytes_to_add =
-            min(bytes_to_add, BitUtil::RoundDownToPowerOfTwo(reservation_to_distribute));
-      } else {
-        bytes_to_add = 0;
-      }
-      DCHECK(bytes_to_add == 0 || bytes_to_add >= min_buffer_size) << bytes_to_add;
-      reservation_to_distribute -= bytes_to_add;
-      tmp_reservation.second += bytes_to_add;
-      DCHECK_GE(reservation_to_distribute, 0);
-      DCHECK_GT(tmp_reservation.second, 0);
+    // TODO: this will need to change when we have co-located files and the columns
+    // are different files.
+    if (!col_chunk.file_path.empty() && col_chunk.file_path != filename()) {
+      return Status(Substitute("Expected parquet column file path '$0' to match "
+          "filename '$1'", col_chunk.file_path, filename()));
     }
+
+    const ScanRange* split_range =
+        static_cast<ScanRangeMetadata*>(metadata_range_->meta_data())->original_split;
+
+    // Determine if the column is completely contained within a local split.
+    bool col_range_local = split_range->expected_local()
+        && col_start >= split_range->offset()
+        && col_end <= split_range->offset() + split_range->len();
+    ScanRange* col_range = scan_node_->AllocateScanRange(metadata_range_->fs(),
+        filename(), col_len, col_start, partition_id, split_range->disk_id(),
+        col_range_local,
+        BufferOpts(split_range->try_cache(), file_desc->mtime));
+    col_ranges.push_back(col_range);
+
+    // Get the stream that will be used for this column
+    ScannerContext::Stream* stream = context_->AddStream(col_range);
+    DCHECK(stream != nullptr);
+
+    RETURN_IF_ERROR(scalar_reader->Reset(&col_chunk.meta_data, stream));
   }
-  return tmp_reservations;
+  DCHECK_EQ(col_ranges.size(), num_scalar_readers);
+
+  // Issue all the column chunks to the io mgr and have them scheduled immediately.
+  // This means these ranges aren't returned via DiskIoMgr::GetNextRange and
+  // instead are scheduled to be read immediately.
+  RETURN_IF_ERROR(scan_node_->runtime_state()->io_mgr()->AddScanRanges(
+      scan_node_->reader_context(), col_ranges, true));
+
+  return Status::OK();
 }
 
 Status HdfsParquetScanner::InitDictionaries(

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-parquet-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-scanner.h b/be/src/exec/hdfs-parquet-scanner.h
index 92f2550..f0043b5 100644
--- a/be/src/exec/hdfs-parquet-scanner.h
+++ b/be/src/exec/hdfs-parquet-scanner.h
@@ -34,7 +34,7 @@ class CollectionValueBuilder;
 struct HdfsFileDesc;
 
 /// Internal schema representation and resolution.
-struct SchemaNode;
+class SchemaNode;
 
 /// Class that implements Parquet definition and repetition level decoding.
 class ParquetLevelDecoder;
@@ -69,8 +69,8 @@ class BoolColumnReader;
 /// the split size, the mid point guarantees that we have at least 50% of the row group in
 /// the current split. ProcessSplit() then computes the column ranges for these row groups
 /// and submits them to the IoMgr for immediate scheduling (so they don't surface in
-/// DiskIoMgr::GetNextUnstartedRange()). Scheduling them immediately also guarantees they
-/// are all read at once.
+/// DiskIoMgr::GetNextRange()). Scheduling them immediately also guarantees they are all
+/// read at once.
 ///
 /// Like the other scanners, each parquet scanner object is one to one with a
 /// ScannerContext. Unlike the other scanners though, the context will have multiple
@@ -328,7 +328,7 @@ class HdfsParquetScanner : public HdfsScanner {
   virtual ~HdfsParquetScanner() {}
 
   /// Issue just the footer range for each file.  We'll then parse the footer and pick
-  /// out the columns we want. 'files' must not be empty.
+  /// out the columns we want.
   static Status IssueInitialRanges(HdfsScanNodeBase* scan_node,
                                    const std::vector<HdfsFileDesc*>& files)
                                    WARN_UNUSED_RESULT;
@@ -361,7 +361,6 @@ class HdfsParquetScanner : public HdfsScanner {
   template<typename InternalType, parquet::Type::type PARQUET_TYPE, bool MATERIALIZED>
   friend class ScalarColumnReader;
   friend class BoolColumnReader;
-  friend class HdfsParquetScannerTest;
 
   /// Size of the file footer.  This is a guess.  If this value is too little, we will
   /// need to issue another read.
@@ -430,7 +429,7 @@ class HdfsParquetScanner : public HdfsScanner {
   /// Number of scratch batches processed so far.
   int64_t row_batches_produced_;
 
-  /// Column reader for each top-level materialized slot in the output tuple.
+  /// Column reader for each materialized columns for this file.
   std::vector<ParquetColumnReader*> column_readers_;
 
   /// Column readers will write slot values into this scratch batch for
@@ -446,9 +445,6 @@ class HdfsParquetScanner : public HdfsScanner {
   /// Scan range for the metadata.
   const io::ScanRange* metadata_range_;
 
-  /// Reservation available for scanning columns, in bytes.
-  int64_t total_col_reservation_ = 0;
-
   /// Pool to copy dictionary page buffer into. This pool is shared across all the
   /// pages in a column chunk.
   boost::scoped_ptr<MemPool> dictionary_pool_;
@@ -465,9 +461,6 @@ class HdfsParquetScanner : public HdfsScanner {
   /// or nested within a collection.
   std::vector<BaseScalarColumnReader*> non_dict_filterable_readers_;
 
-  /// Flattened list of all scalar column readers in column_readers_.
-  std::vector<BaseScalarColumnReader*> scalar_readers_;
-
   /// Flattened collection column readers that point to readers in column_readers_.
   std::vector<CollectionColumnReader*> collection_readers_;
 
@@ -634,24 +627,12 @@ class HdfsParquetScanner : public HdfsScanner {
       WARN_UNUSED_RESULT;
 
   /// Walks file_metadata_ and initiates reading the materialized columns.  This
-  /// initializes 'scalar_readers_' and divides reservation between the columns but
-  /// does not start any scan ranges.
-  Status InitScalarColumns() WARN_UNUSED_RESULT;
-
-  /// Decides how to divide 'reservation_to_distribute' bytes of reservation between the
-  /// columns. Sets the reservation on each corresponding reader in 'column_readers'.
-  Status DivideReservationBetweenColumns(
-      const std::vector<BaseScalarColumnReader*>& column_readers,
-      int64_t reservation_to_distribute);
-
-  /// Helper for DivideReservationBetweenColumns. Implements the core algorithm for
-  /// dividing a reservation of 'reservation_to_distribute' bytes between columns with
-  /// scan range lengths 'col_range_lengths' given a min and max buffer size. Returns
-  /// a vector with an entry per column with the index into 'col_range_lengths' and the
-  /// amount of reservation in bytes to give to that column.
-  static std::vector<std::pair<int, int64_t>> DivideReservationBetweenColumnsHelper(
-      int64_t min_buffer_size, int64_t max_buffer_size,
-      const std::vector<int64_t>& col_range_lengths, int64_t reservation_to_distribute);
+  /// initializes 'column_readers' and issues the reads for the columns. 'column_readers'
+  /// includes a mix of scalar readers from multiple schema nodes (i.e., readers of
+  /// top-level scalar columns and readers of scalar columns within a collection node).
+  Status InitScalarColumns(
+      int row_group_idx, const std::vector<BaseScalarColumnReader*>& column_readers)
+      WARN_UNUSED_RESULT;
 
   /// Initializes the column readers in collection_readers_.
   void InitCollectionColumns();
@@ -687,8 +668,7 @@ class HdfsParquetScanner : public HdfsScanner {
   /// Partitions the readers into scalar and collection readers. The collection readers
   /// are flattened into collection_readers_. The scalar readers are partitioned into
   /// dict_filterable_readers_ and non_dict_filterable_readers_ depending on whether
-  /// dictionary filtering is enabled and the reader can be dictionary filtered. All
-  /// scalar readers are also flattened into scalar_readers_.
+  /// dictionary filtering is enabled and the reader can be dictionary filtered.
   void PartitionReaders(const vector<ParquetColumnReader*>& readers,
                         bool can_eval_dict_filters);
 

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-scan-node-base.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-base.cc b/be/src/exec/hdfs-scan-node-base.cc
index 0dfd6f0..80cb6c5 100644
--- a/be/src/exec/hdfs-scan-node-base.cc
+++ b/be/src/exec/hdfs-scan-node-base.cc
@@ -35,7 +35,6 @@
 #include "exprs/scalar-expr-evaluator.h"
 #include "exprs/scalar-expr.h"
 #include "runtime/descriptors.h"
-#include "runtime/exec-env.h"
 #include "runtime/hdfs-fs-cache.h"
 #include "runtime/io/disk-io-mgr.h"
 #include "runtime/io/request-context.h"
@@ -68,7 +67,6 @@ const int UNEXPECTED_REMOTE_BYTES_WARN_THRESHOLD = 64 * 1024 * 1024;
 HdfsScanNodeBase::HdfsScanNodeBase(ObjectPool* pool, const TPlanNode& tnode,
     const DescriptorTbl& descs)
     : ScanNode(pool, tnode, descs),
-      ideal_scan_range_reservation_(tnode.hdfs_scan_node.ideal_scan_range_reservation),
       min_max_tuple_id_(tnode.hdfs_scan_node.__isset.min_max_tuple_id ?
           tnode.hdfs_scan_node.min_max_tuple_id : -1),
       skip_header_line_count_(tnode.hdfs_scan_node.__isset.skip_header_line_count ?
@@ -85,7 +83,6 @@ HdfsScanNodeBase::HdfsScanNodeBase(ObjectPool* pool, const TPlanNode& tnode,
           &tnode.hdfs_scan_node.dictionary_filter_conjuncts : nullptr),
       disks_accessed_bitmap_(TUnit::UNIT, 0),
       active_hdfs_read_thread_counter_(TUnit::UNIT, 0) {
-  DCHECK_GE(ideal_scan_range_reservation_, resource_profile_.min_reservation);
 }
 
 HdfsScanNodeBase::~HdfsScanNodeBase() {
@@ -115,6 +112,7 @@ Status HdfsScanNodeBase::Init(const TPlanNode& tnode, RuntimeState* state) {
     RETURN_IF_ERROR(ScalarExpr::Create(tnode.hdfs_scan_node.min_max_conjuncts,
         *min_max_row_desc, state, &min_max_conjuncts_));
   }
+
   return Status::OK();
 }
 
@@ -248,16 +246,6 @@ Status HdfsScanNodeBase::Prepare(RuntimeState* state) {
   ImpaladMetrics::NUM_RANGES_PROCESSED->Increment(scan_range_params_->size());
   ImpaladMetrics::NUM_RANGES_MISSING_VOLUME_ID->Increment(num_ranges_missing_volume_id);
 
-  // Check if reservation was enough to allocate at least one buffer. The
-  // reservation calculation in HdfsScanNode.java should guarantee this.
-  // Hitting this error indicates a misconfiguration or bug.
-  int64_t min_buffer_size = ExecEnv::GetInstance()->disk_io_mgr()->min_buffer_size();
-  if (scan_range_params_->size() > 0
-      && resource_profile_.min_reservation < min_buffer_size) {
-    return Status(TErrorCode::INTERNAL_ERROR,
-      Substitute("HDFS scan min reservation $0 must be >= min buffer size $1",
-       resource_profile_.min_reservation, min_buffer_size));
-  }
   // Add per volume stats to the runtime profile
   PerVolumeStats per_volume_stats;
   stringstream str;
@@ -344,18 +332,7 @@ Status HdfsScanNodeBase::Open(RuntimeState* state) {
         partition_desc->partition_key_value_evals(), scan_node_pool_.get(), state);
   }
 
-  RETURN_IF_ERROR(ClaimBufferReservation(state));
-  // We got the minimum reservation. Now try to get ideal reservation.
-  if (resource_profile_.min_reservation != ideal_scan_range_reservation_) {
-    bool increased = buffer_pool_client_.IncreaseReservation(
-        ideal_scan_range_reservation_ - resource_profile_.min_reservation);
-    VLOG_FILE << "Increasing reservation from minimum "
-              << resource_profile_.min_reservation << "B to ideal "
-              << ideal_scan_range_reservation_ << "B "
-              << (increased ? "succeeded" : "failed");
-  }
-
-  reader_context_ = runtime_state_->io_mgr()->RegisterContext();
+  reader_context_ = runtime_state_->io_mgr()->RegisterContext(mem_tracker());
 
   // Initialize HdfsScanNode specific counters
   // TODO: Revisit counters and move the counters specific to multi-threaded scans
@@ -376,11 +353,14 @@ Status HdfsScanNodeBase::Open(RuntimeState* state) {
   num_scanner_threads_started_counter_ =
       ADD_COUNTER(runtime_profile(), NUM_SCANNER_THREADS_STARTED, TUnit::UNIT);
 
-  reader_context_->set_bytes_read_counter(bytes_read_counter());
-  reader_context_->set_read_timer(read_timer());
-  reader_context_->set_open_file_timer(open_file_timer());
-  reader_context_->set_active_read_thread_counter(&active_hdfs_read_thread_counter_);
-  reader_context_->set_disks_accessed_bitmap(&disks_accessed_bitmap_);
+  runtime_state_->io_mgr()->set_bytes_read_counter(
+      reader_context_.get(), bytes_read_counter());
+  runtime_state_->io_mgr()->set_read_timer(reader_context_.get(), read_timer());
+  runtime_state_->io_mgr()->set_open_file_timer(reader_context_.get(), open_file_timer());
+  runtime_state_->io_mgr()->set_active_read_thread_counter(
+      reader_context_.get(), &active_hdfs_read_thread_counter_);
+  runtime_state_->io_mgr()->set_disks_access_bitmap(
+      reader_context_.get(), &disks_accessed_bitmap_);
 
   average_scanner_thread_concurrency_ = runtime_profile()->AddSamplingCounter(
       AVERAGE_SCANNER_THREAD_CONCURRENCY, &active_scanner_thread_counter_);
@@ -474,27 +454,18 @@ Status HdfsScanNodeBase::IssueInitialScanRanges(RuntimeState* state) {
     }
   }
 
-  // Issue initial ranges for all file types. Only call functions for file types that
-  // actually exist - trying to add empty lists of ranges can result in spurious
-  // CANCELLED errors - see IMPALA-6564.
-  for (const auto& entry : matching_per_type_files) {
-    if (entry.second.empty()) continue;
-    switch (entry.first) {
-      case THdfsFileFormat::PARQUET:
-        RETURN_IF_ERROR(HdfsParquetScanner::IssueInitialRanges(this, entry.second));
-        break;
-      case THdfsFileFormat::TEXT:
-        RETURN_IF_ERROR(HdfsTextScanner::IssueInitialRanges(this, entry.second));
-        break;
-      case THdfsFileFormat::SEQUENCE_FILE:
-      case THdfsFileFormat::RC_FILE:
-      case THdfsFileFormat::AVRO:
-        RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this, entry.second));
-        break;
-      default:
-        DCHECK(false) << "Unexpected file type " << entry.first;
-    }
-  }
+  // Issue initial ranges for all file types.
+  RETURN_IF_ERROR(HdfsParquetScanner::IssueInitialRanges(this,
+      matching_per_type_files[THdfsFileFormat::PARQUET]));
+  RETURN_IF_ERROR(HdfsTextScanner::IssueInitialRanges(this,
+      matching_per_type_files[THdfsFileFormat::TEXT]));
+  RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this,
+      matching_per_type_files[THdfsFileFormat::SEQUENCE_FILE]));
+  RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this,
+      matching_per_type_files[THdfsFileFormat::RC_FILE]));
+  RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this,
+      matching_per_type_files[THdfsFileFormat::AVRO]));
+
   return Status::OK();
 }
 
@@ -553,8 +524,6 @@ ScanRange* HdfsScanNodeBase::AllocateScanRange(hdfsFS fs, const char* file,
 
 Status HdfsScanNodeBase::AddDiskIoRanges(
     const vector<ScanRange*>& ranges, int num_files_queued) {
-  DCHECK(!progress_.done()) << "Don't call AddScanRanges() after all ranges finished.";
-  DCHECK_GT(ranges.size(), 0);
   RETURN_IF_ERROR(runtime_state_->io_mgr()->AddScanRanges(reader_context_.get(), ranges));
   num_unqueued_files_.Add(-num_files_queued);
   DCHECK_GE(num_unqueued_files_.Load(), 0);
@@ -857,14 +826,20 @@ void HdfsScanNodeBase::StopAndFinalizeCounters() {
       Substitute("Codegen enabled: $0 out of $1", num_enabled, total));
 
   if (reader_context_ != nullptr) {
-    bytes_read_local_->Set(reader_context_->bytes_read_local());
-    bytes_read_short_circuit_->Set(reader_context_->bytes_read_short_circuit());
-    bytes_read_dn_cache_->Set(reader_context_->bytes_read_dn_cache());
-    num_remote_ranges_->Set(reader_context_->num_remote_ranges());
-    unexpected_remote_bytes_->Set(reader_context_->unexpected_remote_bytes());
-    cached_file_handles_hit_count_->Set(reader_context_->cached_file_handles_hit_count());
+    bytes_read_local_->Set(
+        runtime_state_->io_mgr()->bytes_read_local(reader_context_.get()));
+    bytes_read_short_circuit_->Set(
+        runtime_state_->io_mgr()->bytes_read_short_circuit(reader_context_.get()));
+    bytes_read_dn_cache_->Set(
+        runtime_state_->io_mgr()->bytes_read_dn_cache(reader_context_.get()));
+    num_remote_ranges_->Set(static_cast<int64_t>(
+        runtime_state_->io_mgr()->num_remote_ranges(reader_context_.get())));
+    unexpected_remote_bytes_->Set(
+        runtime_state_->io_mgr()->unexpected_remote_bytes(reader_context_.get()));
+    cached_file_handles_hit_count_->Set(
+        runtime_state_->io_mgr()->cached_file_handles_hit_count(reader_context_.get()));
     cached_file_handles_miss_count_->Set(
-        reader_context_->cached_file_handles_miss_count());
+        runtime_state_->io_mgr()->cached_file_handles_miss_count(reader_context_.get()));
 
     if (unexpected_remote_bytes_->value() >= UNEXPECTED_REMOTE_BYTES_WARN_THRESHOLD) {
       runtime_state_->LogError(ErrorMsg(TErrorCode::GENERAL, Substitute(

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-scan-node-base.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-base.h b/be/src/exec/hdfs-scan-node-base.h
index 3a9c37f..70fbac2 100644
--- a/be/src/exec/hdfs-scan-node-base.h
+++ b/be/src/exec/hdfs-scan-node-base.h
@@ -322,9 +322,6 @@ class HdfsScanNodeBase : public ScanNode {
   friend class ScannerContext;
   friend class HdfsScanner;
 
-  /// Ideal reservation to process each input split, computed by the planner.
-  const int64_t ideal_scan_range_reservation_;
-
   /// Tuple id of the tuple used to evaluate conjuncts on parquet::Statistics.
   const int min_max_tuple_id_;
 

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-scan-node-mt.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-mt.cc b/be/src/exec/hdfs-scan-node-mt.cc
index f4948d9..7ea4d80 100644
--- a/be/src/exec/hdfs-scan-node-mt.cc
+++ b/be/src/exec/hdfs-scan-node-mt.cc
@@ -26,7 +26,6 @@
 
 #include "gen-cpp/PlanNodes_types.h"
 
-using namespace impala::io;
 using std::stringstream;
 
 namespace impala {
@@ -77,27 +76,20 @@ Status HdfsScanNodeMt::GetNext(RuntimeState* state, RowBatch* row_batch, bool* e
       scanner_->Close(row_batch);
       scanner_.reset();
     }
-    DiskIoMgr* io_mgr = runtime_state_->io_mgr();
-    bool needs_buffers;
-    RETURN_IF_ERROR(io_mgr->GetNextUnstartedRange(
-        reader_context_.get(), &scan_range_, &needs_buffers));
-    if (scan_range_ == nullptr) {
+    RETURN_IF_ERROR(
+        runtime_state_->io_mgr()->GetNextRange(reader_context_.get(), &scan_range_));
+    if (scan_range_ == NULL) {
       *eos = true;
       StopAndFinalizeCounters();
       return Status::OK();
     }
-    int64_t scanner_reservation = buffer_pool_client_.GetReservation();
-    if (needs_buffers) {
-      RETURN_IF_ERROR(io_mgr->AllocateBuffersForRange(reader_context_.get(),
-          &buffer_pool_client_, scan_range_, scanner_reservation));
-    }
     ScanRangeMetadata* metadata =
         static_cast<ScanRangeMetadata*>(scan_range_->meta_data());
     int64_t partition_id = metadata->partition_id;
     HdfsPartitionDescriptor* partition = hdfs_table_->GetPartition(partition_id);
-    scanner_ctx_.reset(new ScannerContext(runtime_state_, this, &buffer_pool_client_,
-        partition, filter_ctxs(), expr_results_pool()));
-    scanner_ctx_->AddStream(scan_range_, scanner_reservation);
+    scanner_ctx_.reset(new ScannerContext(
+        runtime_state_, this, partition, scan_range_, filter_ctxs(),
+        expr_results_pool()));
     Status status = CreateAndOpenScanner(partition, scanner_ctx_.get(), &scanner_);
     if (!status.ok()) {
       DCHECK(scanner_ == NULL);

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-scan-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node.cc b/be/src/exec/hdfs-scan-node.cc
index a95e47a..710a8af 100644
--- a/be/src/exec/hdfs-scan-node.cc
+++ b/be/src/exec/hdfs-scan-node.cc
@@ -26,7 +26,6 @@
 #include "exec/scanner-context.h"
 #include "runtime/descriptors.h"
 #include "runtime/fragment-instance-state.h"
-#include "runtime/io/request-context.h"
 #include "runtime/runtime-filter.inline.h"
 #include "runtime/runtime-state.h"
 #include "runtime/mem-tracker.h"
@@ -46,6 +45,16 @@ DECLARE_bool(skip_file_runtime_filtering);
 using namespace impala;
 using namespace impala::io;
 
+// Amount of memory that we approximate a scanner thread will use not including IoBuffers.
+// The memory used does not vary considerably between file formats (just a couple of MBs).
+// This value is conservative and taken from running against the tpch lineitem table.
+// TODO: revisit how we do this.
+const int SCANNER_THREAD_MEM_USAGE = 32 * 1024 * 1024;
+
+// Estimated upper bound on the compression ratio of compressed text files. Used to
+// estimate scanner thread memory usage.
+const int COMPRESSED_TEXT_COMPRESSION_RATIO = 11;
+
 // Amount of time to block waiting for GetNext() to release scanner threads between
 // checking if a scanner thread should yield itself back to the global thread pool.
 const int SCANNER_THREAD_WAIT_TIME_MS = 20;
@@ -53,6 +62,11 @@ const int SCANNER_THREAD_WAIT_TIME_MS = 20;
 HdfsScanNode::HdfsScanNode(ObjectPool* pool, const TPlanNode& tnode,
                            const DescriptorTbl& descs)
     : HdfsScanNodeBase(pool, tnode, descs),
+      ranges_issued_barrier_(1),
+      scanner_thread_bytes_required_(0),
+      done_(false),
+      all_ranges_started_(false),
+      thread_avail_cb_id_(-1),
       max_num_scanner_threads_(CpuInfo::num_cores()) {
 }
 
@@ -153,6 +167,36 @@ Status HdfsScanNode::Init(const TPlanNode& tnode, RuntimeState* state) {
 Status HdfsScanNode::Prepare(RuntimeState* state) {
   SCOPED_TIMER(runtime_profile_->total_time_counter());
   RETURN_IF_ERROR(HdfsScanNodeBase::Prepare(state));
+
+  // Compute the minimum bytes required to start a new thread. This is based on the
+  // file format.
+  // The higher the estimate, the less likely it is the query will fail but more likely
+  // the query will be throttled when it does not need to be.
+  // TODO: how many buffers should we estimate per range. The IoMgr will throttle down to
+  // one but if there are already buffers queued before memory pressure was hit, we can't
+  // reclaim that memory.
+  if (per_type_files_[THdfsFileFormat::PARQUET].size() > 0) {
+    // Parquet files require buffers per column
+    scanner_thread_bytes_required_ =
+        materialized_slots_.size() * 3 * runtime_state_->io_mgr()->max_read_buffer_size();
+  } else {
+    scanner_thread_bytes_required_ =
+        3 * runtime_state_->io_mgr()->max_read_buffer_size();
+  }
+  // scanner_thread_bytes_required_ now contains the IoBuffer requirement.
+  // Next we add in the other memory the scanner thread will use.
+  // e.g. decompression buffers, tuple buffers, etc.
+  // For compressed text, we estimate this based on the file size (since the whole file
+  // will need to be decompressed at once). For all other formats, we use a constant.
+  // TODO: can we do something better?
+  int64_t scanner_thread_mem_usage = SCANNER_THREAD_MEM_USAGE;
+  for (HdfsFileDesc* file: per_type_files_[THdfsFileFormat::TEXT]) {
+    if (file->file_compression != THdfsCompression::NONE) {
+      int64_t bytes_required = file->file_length * COMPRESSED_TEXT_COMPRESSION_RATIO;
+      scanner_thread_mem_usage = ::max(bytes_required, scanner_thread_mem_usage);
+    }
+  }
+  scanner_thread_bytes_required_ += scanner_thread_mem_usage;
   row_batches_get_timer_ = ADD_TIMER(runtime_profile(), "RowBatchQueueGetWaitTime");
   row_batches_put_timer_ = ADD_TIMER(runtime_profile(), "RowBatchQueuePutWaitTime");
   return Status::OK();
@@ -174,9 +218,10 @@ Status HdfsScanNode::Open(RuntimeState* state) {
     max_num_scanner_threads_ = runtime_state_->query_options().num_scanner_threads;
   }
   DCHECK_GT(max_num_scanner_threads_, 0);
-  spare_reservation_.Store(buffer_pool_client_.GetReservation());
+
   thread_avail_cb_id_ = runtime_state_->resource_pool()->AddThreadAvailableCb(
       bind<void>(mem_fn(&HdfsScanNode::ThreadTokenAvailableCb), this, _1));
+
   return Status::OK();
 }
 
@@ -217,28 +262,37 @@ Status HdfsScanNode::AddDiskIoRanges(const vector<ScanRange*>& ranges,
   return Status::OK();
 }
 
-bool HdfsScanNode::EnoughReservationForExtraThread(const unique_lock<mutex>& lock) {
-  DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
-  if (spare_reservation_.Load() >= ideal_scan_range_reservation_) return true;
-  int64_t increase = ideal_scan_range_reservation_ - spare_reservation_.Load();
-  if (!buffer_pool_client_.IncreaseReservation(increase)) return false;
-  spare_reservation_.Add(increase);
-  return true;
-}
-
-int64_t HdfsScanNode::DeductReservationForScannerThread(const unique_lock<mutex>& lock,
-    bool first_thread) {
-  DCHECK(lock.mutex() == &lock_ && lock.owns_lock());
-  int64_t amount;
-  if (first_thread) {
-    amount = spare_reservation_.Load() >= ideal_scan_range_reservation_ ?
-        ideal_scan_range_reservation_ : resource_profile_.min_reservation;
-  } else {
-    amount = ideal_scan_range_reservation_;
+// For controlling the amount of memory used for scanners, we approximate the
+// scanner mem usage based on scanner_thread_bytes_required_, rather than the
+// consumption in the scan node's mem tracker. The problem with the scan node
+// trackers is that it does not account for the memory the scanner will use.
+// For example, if there is 110 MB of memory left (based on the mem tracker)
+// and we estimate that a scanner will use 100MB, we want to make sure to only
+// start up one additional thread. However, after starting the first thread, the
+// mem tracker value will not change immediately (it takes some time before the
+// scanner is running and starts using memory). Therefore we just use the estimate
+// based on the number of running scanner threads.
+bool HdfsScanNode::EnoughMemoryForScannerThread(bool new_thread) {
+  int64_t committed_scanner_mem =
+      active_scanner_thread_counter_.value() * scanner_thread_bytes_required_;
+  int64_t tracker_consumption = mem_tracker()->consumption();
+  int64_t est_additional_scanner_mem = committed_scanner_mem - tracker_consumption;
+  if (est_additional_scanner_mem < 0) {
+    // This is the case where our estimate was too low. Expand the estimate based
+    // on the usage.
+    int64_t avg_consumption =
+        tracker_consumption / active_scanner_thread_counter_.value();
+    // Take the average and expand it by 50%. Some scanners will not have hit their
+    // steady state mem usage yet.
+    // TODO: how can we scale down if we've overestimated.
+    // TODO: better heuristic?
+    scanner_thread_bytes_required_ = static_cast<int64_t>(avg_consumption * 1.5);
+    est_additional_scanner_mem = 0;
   }
-  int64_t remainder = spare_reservation_.Add(-amount);
-  DCHECK_GE(remainder, 0);
-  return amount;
+
+  // If we are starting a new thread, take that into account now.
+  if (new_thread) est_additional_scanner_mem += scanner_thread_bytes_required_;
+  return est_additional_scanner_mem < mem_tracker()->SpareCapacity();
 }
 
 void HdfsScanNode::ThreadTokenAvailableCb(ThreadResourceMgr::ResourcePool* pool) {
@@ -268,45 +322,36 @@ void HdfsScanNode::ThreadTokenAvailableCb(ThreadResourceMgr::ResourcePool* pool)
     // TODO: This still leans heavily on starvation-free locks, come up with a more
     // correct way to communicate between this method and ScannerThreadHelper
     unique_lock<mutex> lock(lock_);
-
-    const int64_t num_active_scanner_threads = active_scanner_thread_counter_.value();
-    const bool first_thread = num_active_scanner_threads == 0;
     // Cases 1, 2, 3.
     if (done_ || all_ranges_started_ ||
-        num_active_scanner_threads >= progress_.remaining()) {
+        active_scanner_thread_counter_.value() >= progress_.remaining()) {
       break;
     }
 
     // Cases 5 and 6.
-    if (!first_thread &&
+    if (active_scanner_thread_counter_.value() > 0 &&
         (materialized_row_batches_->Size() >= max_materialized_row_batches_ ||
-         !EnoughReservationForExtraThread(lock))) {
+         !EnoughMemoryForScannerThread(true))) {
       break;
     }
 
     // Case 7 and 8.
-    if (num_active_scanner_threads >= max_num_scanner_threads_ ||
-        !pool->TryAcquireThreadToken()) {
+    bool is_reserved = false;
+    if (active_scanner_thread_counter_.value() >= max_num_scanner_threads_ ||
+        !pool->TryAcquireThreadToken(&is_reserved)) {
       break;
     }
 
-    // Deduct the reservation. We haven't dropped the lock since the
-    // first_thread/EnoughReservationForExtraThread() checks so spare reservation
-    // must be available.
-    int64_t scanner_thread_reservation =
-        DeductReservationForScannerThread(lock, first_thread);
     COUNTER_ADD(&active_scanner_thread_counter_, 1);
     string name = Substitute("scanner-thread (finst:$0, plan-node-id:$1, thread-idx:$2)",
         PrintId(runtime_state_->fragment_instance_id()), id(),
         num_scanner_threads_started_counter_->value());
-    auto fn = [this, scanner_thread_reservation]() {
-      this->ScannerThread(scanner_thread_reservation);
-    };
+
+    auto fn = [this]() { this->ScannerThread(); };
     std::unique_ptr<Thread> t;
     status =
       Thread::Create(FragmentInstanceState::FINST_THREAD_GROUP_NAME, name, fn, &t, true);
     if (!status.ok()) {
-      ReturnReservationFromScannerThread(scanner_thread_reservation);
       COUNTER_ADD(&active_scanner_thread_counter_, -1);
       // Release the token and skip running callbacks to find a replacement. Skipping
       // serves two purposes. First, it prevents a mutual recursion between this function
@@ -327,10 +372,9 @@ void HdfsScanNode::ThreadTokenAvailableCb(ThreadResourceMgr::ResourcePool* pool)
   }
 }
 
-void HdfsScanNode::ScannerThread(int64_t scanner_thread_reservation) {
+void HdfsScanNode::ScannerThread() {
   SCOPED_THREAD_COUNTER_MEASUREMENT(scanner_thread_counters());
   SCOPED_THREAD_COUNTER_MEASUREMENT(runtime_state_->total_thread_statistics());
-  DiskIoMgr* io_mgr = runtime_state_->io_mgr();
 
   // Make thread-local copy of filter contexts to prune scan ranges, and to pass to the
   // scanner for finer-grained filtering. Use a thread-local MemPool for the filter
@@ -356,7 +400,8 @@ void HdfsScanNode::ScannerThread(int64_t scanner_thread_reservation) {
       // this thread.
       unique_lock<mutex> l(lock_);
       if (active_scanner_thread_counter_.value() > 1) {
-        if (runtime_state_->resource_pool()->optional_exceeded()) {
+        if (runtime_state_->resource_pool()->optional_exceeded() ||
+            !EnoughMemoryForScannerThread(false)) {
           // We can't break here. We need to update the counter with the lock held or else
           // all threads might see active_scanner_thread_counter_.value > 1
           COUNTER_ADD(&active_scanner_thread_counter_, -1);
@@ -376,29 +421,21 @@ void HdfsScanNode::ScannerThread(int64_t scanner_thread_reservation) {
     // to return if there's an error.
     ranges_issued_barrier_.Wait(SCANNER_THREAD_WAIT_TIME_MS, &unused);
 
-    // Take a snapshot of num_unqueued_files_ before calling GetNextUnstartedRange().
+    ScanRange* scan_range;
+    // Take a snapshot of num_unqueued_files_ before calling GetNextRange().
     // We don't want num_unqueued_files_ to go to zero between the return from
-    // GetNextUnstartedRange() and the check for when all ranges are complete.
+    // GetNextRange() and the check for when all ranges are complete.
     int num_unqueued_files = num_unqueued_files_.Load();
     // TODO: the Load() acts as an acquire barrier.  Is this needed? (i.e. any earlier
     // stores that need to complete?)
     AtomicUtil::MemoryBarrier();
-    ScanRange* scan_range;
-    bool needs_buffers;
     Status status =
-        io_mgr->GetNextUnstartedRange(reader_context_.get(), &scan_range, &needs_buffers);
+        runtime_state_->io_mgr()->GetNextRange(reader_context_.get(), &scan_range);
 
-    if (status.ok() && scan_range != nullptr) {
-      if (needs_buffers) {
-        status = io_mgr->AllocateBuffersForRange(
-            reader_context_.get(), &buffer_pool_client_, scan_range,
-            scanner_thread_reservation);
-      }
-      if (status.ok()) {
-        // Got a scan range. Process the range end to end (in this thread).
-        status = ProcessSplit(filter_status.ok() ? filter_ctxs : vector<FilterContext>(),
-            &expr_results_pool, scan_range, scanner_thread_reservation);
-      }
+    if (status.ok() && scan_range != NULL) {
+      // Got a scan range. Process the range end to end (in this thread).
+      status = ProcessSplit(filter_status.ok() ? filter_ctxs : vector<FilterContext>(),
+          &expr_results_pool, scan_range);
     }
 
     if (!status.ok()) {
@@ -428,8 +465,8 @@ void HdfsScanNode::ScannerThread(int64_t scanner_thread_reservation) {
       // TODO: Based on the usage pattern of all_ranges_started_, it looks like it is not
       // needed to acquire the lock in x86.
       unique_lock<mutex> l(lock_);
-      // All ranges have been queued and GetNextUnstartedRange() returned NULL. This means
-      // that every range is either done or being processed by another thread.
+      // All ranges have been queued and GetNextRange() returned NULL. This means that
+      // every range is either done or being processed by another thread.
       all_ranges_started_ = true;
       break;
     }
@@ -437,7 +474,6 @@ void HdfsScanNode::ScannerThread(int64_t scanner_thread_reservation) {
   COUNTER_ADD(&active_scanner_thread_counter_, -1);
 
 exit:
-  ReturnReservationFromScannerThread(scanner_thread_reservation);
   runtime_state_->resource_pool()->ReleaseThreadToken(false);
   for (auto& ctx: filter_ctxs) ctx.expr_eval->Close(runtime_state_);
   filter_mem_pool.FreeAll();
@@ -445,8 +481,7 @@ exit:
 }
 
 Status HdfsScanNode::ProcessSplit(const vector<FilterContext>& filter_ctxs,
-    MemPool* expr_results_pool, ScanRange* scan_range,
-    int64_t scanner_thread_reservation) {
+    MemPool* expr_results_pool, ScanRange* scan_range) {
   DCHECK(scan_range != NULL);
 
   ScanRangeMetadata* metadata = static_cast<ScanRangeMetadata*>(scan_range->meta_data());
@@ -471,9 +506,8 @@ Status HdfsScanNode::ProcessSplit(const vector<FilterContext>& filter_ctxs,
     return Status::OK();
   }
 
-  ScannerContext context(runtime_state_, this, &buffer_pool_client_, partition,
-      filter_ctxs, expr_results_pool);
-  context.AddStream(scan_range, scanner_thread_reservation);
+  ScannerContext context(
+      runtime_state_, this, partition, scan_range, filter_ctxs, expr_results_pool);
   scoped_ptr<HdfsScanner> scanner;
   Status status = CreateAndOpenScanner(partition, &context, &scanner);
   if (!status.ok()) {
@@ -515,7 +549,9 @@ Status HdfsScanNode::ProcessSplit(const vector<FilterContext>& filter_ctxs,
 void HdfsScanNode::SetDoneInternal() {
   if (done_) return;
   done_ = true;
-  if (reader_context_ != nullptr) reader_context_->Cancel();
+  if (reader_context_ != nullptr) {
+    runtime_state_->io_mgr()->CancelContext(reader_context_.get());
+  }
   materialized_row_batches_->Shutdown();
 }
 

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-scan-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node.h b/be/src/exec/hdfs-scan-node.h
index 81e826e..a1c97cf 100644
--- a/be/src/exec/hdfs-scan-node.h
+++ b/be/src/exec/hdfs-scan-node.h
@@ -27,7 +27,6 @@
 #include <boost/scoped_ptr.hpp>
 #include <boost/thread/mutex.hpp>
 
-#include "common/atomic.h"
 #include "exec/filter-context.h"
 #include "exec/hdfs-scan-node-base.h"
 #include "runtime/io/disk-io-mgr.h"
@@ -59,14 +58,8 @@ class TPlanNode;
 /// 5. The scanner finishes the scan range and informs the scan node so it can track
 ///    end of stream.
 ///
-/// Buffer management:
-/// ------------------
-/// The different scanner threads all allocate I/O buffers from the node's Buffer Pool
-/// client. The scan node ensures that enough reservation is available to start a
-/// scanner thread before launching each one with (see
-/// EnoughReservationForExtraThread()), after which the scanner thread is responsible
-/// for staying within the reservation handed off to it.
-///
+/// TODO: This class allocates a bunch of small utility objects that should be
+/// recycled.
 /// TODO: Remove this class once the fragment-based multi-threaded execution is
 /// fully functional.
 class HdfsScanNode : public HdfsScanNodeBase {
@@ -107,7 +100,12 @@ class HdfsScanNode : public HdfsScanNodeBase {
 
  private:
   /// Released when initial ranges are issued in the first call to GetNext().
-  CountingBarrier ranges_issued_barrier_{1};
+  CountingBarrier ranges_issued_barrier_;
+
+  /// The estimated memory required to start up a new scanner thread. If the memory
+  /// left (due to limits) is less than this value, we won't start up optional
+  /// scanner threads.
+  int64_t scanner_thread_bytes_required_;
 
   /// Thread group for all scanner worker threads
   ThreadGroup scanner_threads_;
@@ -132,16 +130,16 @@ class HdfsScanNode : public HdfsScanNodeBase {
   /// are finished, an error/cancellation occurred, or the limit was reached.
   /// Setting this to true triggers the scanner threads to clean up.
   /// This should not be explicitly set. Instead, call SetDone().
-  bool done_ = false;
+  bool done_;
 
   /// Set to true if all ranges have started. Some of the ranges may still be in flight
   /// being processed by scanner threads, but no new ScannerThreads should be started.
-  bool all_ranges_started_ = false;
+  bool all_ranges_started_;
 
   /// The id of the callback added to the thread resource manager when thread token
   /// is available. Used to remove the callback before this scan node is destroyed.
   /// -1 if no callback is registered.
-  int thread_avail_cb_id_ = -1;
+  int thread_avail_cb_id_;
 
   /// Maximum number of scanner threads. Set to 'NUM_SCANNER_THREADS' if that query
   /// option is set. Otherwise, it's set to the number of cpu cores. Scanner threads
@@ -149,14 +147,6 @@ class HdfsScanNode : public HdfsScanNodeBase {
   /// the number of cores.
   int max_num_scanner_threads_;
 
-  /// Amount of the 'buffer_pool_client_' reservation that is not allocated to scanner
-  /// threads. Doled out to scanner threads when they are started and returned when
-  /// those threads no longer need it. Can be atomically incremented without holding
-  /// 'lock_' but 'lock_' is held when decrementing to ensure that the check for
-  /// reservation and the actual deduction are atomic with respect to other threads
-  /// trying to claim reservation.
-  AtomicInt64 spare_reservation_{0};
-
   /// The wait time for fetching a row batch from the row batch queue.
   RuntimeProfile::Counter* row_batches_get_timer_;
 
@@ -170,35 +160,21 @@ class HdfsScanNode : public HdfsScanNodeBase {
   /// Main function for scanner thread. This thread pulls the next range to be
   /// processed from the IoMgr and then processes the entire range end to end.
   /// This thread terminates when all scan ranges are complete or an error occurred.
-  /// The caller must have reserved 'scanner_thread_reservation' bytes of memory for
-  /// this thread with DeductReservationForScannerThread().
-  void ScannerThread(int64_t scanner_thread_reservation);
+  void ScannerThread();
 
   /// Process the entire scan range with a new scanner object. Executed in scanner
   /// thread. 'filter_ctxs' is a clone of the class-wide filter_ctxs_, used to filter rows
   /// in this split.
   Status ProcessSplit(const std::vector<FilterContext>& filter_ctxs,
-      MemPool* expr_results_pool, io::ScanRange* scan_range,
-      int64_t scanner_thread_reservation) WARN_UNUSED_RESULT;
-
-  /// Return true if there is enough reservation to start an extra scanner thread.
-  /// Tries to increase reservation if enough is not already available in
-  /// 'spare_reservation_'. 'lock_' must be held via 'lock'
-  bool EnoughReservationForExtraThread(const boost::unique_lock<boost::mutex>& lock);
-
-  /// Deduct reservation to start a new scanner thread from 'spare_reservation_'. If
-  /// 'first_thread' is true, this is the first thread to be started and only the
-  /// minimum reservation is required to be available. Otherwise
-  /// EnoughReservationForExtra() thread must have returned true in the current
-  /// critical section so that 'ideal_scan_range_bytes_' is available for the extra
-  /// thread. Returns the amount deducted. 'lock_' must be held via 'lock'.
-  int64_t DeductReservationForScannerThread(const boost::unique_lock<boost::mutex>& lock,
-      bool first_thread);
-
-  /// Called by scanner thread to return or all of its reservation that is not needed.
-  void ReturnReservationFromScannerThread(int64_t bytes) {
-    spare_reservation_.Add(bytes);
-  }
+      MemPool* expr_results_pool, io::ScanRange* scan_range) WARN_UNUSED_RESULT;
+
+  /// Returns true if there is enough memory (against the mem tracker limits) to
+  /// have a scanner thread.
+  /// If new_thread is true, the calculation is for starting a new scanner thread.
+  /// If false, it determines whether there's adequate memory for the existing
+  /// set of scanner threads.
+  /// lock_ must be taken before calling this.
+  bool EnoughMemoryForScannerThread(bool new_thread);
 
   /// Checks for eos conditions and returns batches from materialized_row_batches_.
   Status GetNextInternal(RuntimeState* state, RowBatch* row_batch, bool* eos)

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/hdfs-text-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-text-scanner.cc b/be/src/exec/hdfs-text-scanner.cc
index 2a6a912..253bcc8 100644
--- a/be/src/exec/hdfs-text-scanner.cc
+++ b/be/src/exec/hdfs-text-scanner.cc
@@ -144,10 +144,8 @@ Status HdfsTextScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
         DCHECK(false);
     }
   }
-  if (compressed_text_scan_ranges.size() > 0) {
-    RETURN_IF_ERROR(scan_node->AddDiskIoRanges(compressed_text_scan_ranges,
-        compressed_text_files));
-  }
+  RETURN_IF_ERROR(scan_node->AddDiskIoRanges(compressed_text_scan_ranges,
+          compressed_text_files));
   if (lzo_text_files.size() > 0) {
     // This will dlopen the lzo binary and can fail if the lzo binary is not present.
     RETURN_IF_ERROR(HdfsLzoTextScanner::IssueInitialRanges(scan_node, lzo_text_files));

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/parquet-column-readers.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/parquet-column-readers.cc b/be/src/exec/parquet-column-readers.cc
index 153b24a..2939691 100644
--- a/be/src/exec/parquet-column-readers.cc
+++ b/be/src/exec/parquet-column-readers.cc
@@ -30,7 +30,6 @@
 #include "exec/scanner-context.inline.h"
 #include "rpc/thrift-util.h"
 #include "runtime/collection-value-builder.h"
-#include "runtime/exec-env.h"
 #include "runtime/tuple-row.h"
 #include "runtime/tuple.h"
 #include "runtime/runtime-state.h"
@@ -50,10 +49,6 @@ DEFINE_bool(convert_legacy_hive_parquet_utc_timestamps, false,
 
 static const int BITPACKED_DEPRECATION_WARNING_FREQUENCY = 100;
 
-// Max dictionary page header size in bytes. This is an estimate and only needs to be an
-// upper bound.
-static const int MAX_DICT_HEADER_SIZE = 100;
-
 // Max data page header size in bytes. This is an estimate and only needs to be an upper
 // bound. It is theoretically possible to have a page header of any size due to string
 // value statistics, but in practice we'll have trouble reading string values this large.
@@ -73,8 +68,6 @@ static int debug_count = 0;
 #define SHOULD_TRIGGER_DEBUG_ACTION(x) (false)
 #endif
 
-using namespace impala::io;
-
 namespace impala {
 
 const string PARQUET_COL_MEM_LIMIT_EXCEEDED =
@@ -854,99 +847,8 @@ static bool RequiresSkippedDictionaryHeaderCheck(
   return v.VersionEq(1,1,0) || (v.VersionEq(1,2,0) && v.is_impala_internal);
 }
 
-Status BaseScalarColumnReader::Reset(const HdfsFileDesc& file_desc,
-    const parquet::ColumnChunk& col_chunk, int row_group_idx) {
-  num_buffered_values_ = 0;
-  data_ = nullptr;
-  data_end_ = nullptr;
-  stream_ = nullptr;
-  io_reservation_ = 0;
-  metadata_ = &col_chunk.meta_data;
-  num_values_read_ = 0;
-  def_level_ = -1;
-  // See ColumnReader constructor.
-  rep_level_ = max_rep_level() == 0 ? 0 : -1;
-  pos_current_value_ = -1;
-
-  if (metadata_->codec != parquet::CompressionCodec::UNCOMPRESSED) {
-    RETURN_IF_ERROR(Codec::CreateDecompressor(
-        nullptr, false, PARQUET_TO_IMPALA_CODEC[metadata_->codec], &decompressor_));
-  }
-  int64_t col_start = col_chunk.meta_data.data_page_offset;
-
-  RETURN_IF_ERROR(ParquetMetadataUtils::ValidateRowGroupColumn(parent_->file_metadata_,
-      parent_->filename(), row_group_idx, col_idx(), schema_element(),
-      parent_->state_));
-
-  if (col_chunk.meta_data.__isset.dictionary_page_offset) {
-    // Already validated in ValidateColumnOffsets()
-    DCHECK_LT(col_chunk.meta_data.dictionary_page_offset, col_start);
-    col_start = col_chunk.meta_data.dictionary_page_offset;
-  }
-  int64_t col_len = col_chunk.meta_data.total_compressed_size;
-  if (col_len <= 0) {
-    return Status(Substitute("File '$0' contains invalid column chunk size: $1",
-        filename(), col_len));
-  }
-  int64_t col_end = col_start + col_len;
-
-  // Already validated in ValidateColumnOffsets()
-  DCHECK_GT(col_end, 0);
-  DCHECK_LT(col_end, file_desc.file_length);
-  const ParquetFileVersion& file_version = parent_->file_version_;
-  if (file_version.application == "parquet-mr" && file_version.VersionLt(1, 2, 9)) {
-    // The Parquet MR writer had a bug in 1.2.8 and below where it didn't include the
-    // dictionary page header size in total_compressed_size and total_uncompressed_size
-    // (see IMPALA-694). We pad col_len to compensate.
-    int64_t bytes_remaining = file_desc.file_length - col_end;
-    int64_t pad = min<int64_t>(MAX_DICT_HEADER_SIZE, bytes_remaining);
-    col_len += pad;
-  }
-
-  // TODO: this will need to change when we have co-located files and the columns
-  // are different files.
-  if (!col_chunk.file_path.empty() && col_chunk.file_path != filename()) {
-    return Status(Substitute("Expected parquet column file path '$0' to match "
-        "filename '$1'", col_chunk.file_path, filename()));
-  }
-
-  const ScanRange* metadata_range = parent_->metadata_range_;
-  int64_t partition_id = parent_->context_->partition_descriptor()->id();
-  const ScanRange* split_range =
-      static_cast<ScanRangeMetadata*>(metadata_range->meta_data())->original_split;
-  // Determine if the column is completely contained within a local split.
-  bool col_range_local = split_range->expected_local()
-      && col_start >= split_range->offset()
-      && col_end <= split_range->offset() + split_range->len();
-  scan_range_ = parent_->scan_node_->AllocateScanRange(metadata_range->fs(),
-      filename(), col_len, col_start, partition_id, split_range->disk_id(),
-      col_range_local,
-      BufferOpts(split_range->try_cache(), file_desc.mtime));
-  ClearDictionaryDecoder();
-  return Status::OK();
-}
-
-Status BaseScalarColumnReader::StartScan() {
-  DCHECK(scan_range_ != nullptr) << "Must Reset() before starting scan.";
-  DiskIoMgr* io_mgr = ExecEnv::GetInstance()->disk_io_mgr();
-  ScannerContext* context = parent_->context_;
-  DCHECK_GT(io_reservation_, 0);
-  bool needs_buffers;
-  RETURN_IF_ERROR(io_mgr->StartScanRange(
-      parent_->scan_node_->reader_context(), scan_range_, &needs_buffers));
-  if (needs_buffers) {
-    RETURN_IF_ERROR(io_mgr->AllocateBuffersForRange(
-        parent_->scan_node_->reader_context(), context->bp_client(),
-        scan_range_, io_reservation_));
-  }
-  stream_ = parent_->context_->AddStream(scan_range_, io_reservation_);
-  DCHECK(stream_ != nullptr);
-  return Status::OK();
-}
-
 Status BaseScalarColumnReader::ReadPageHeader(bool peek,
     parquet::PageHeader* next_page_header, uint32_t* next_header_size, bool* eos) {
-  DCHECK(stream_ != nullptr);
   *eos = false;
 
   uint8_t* buffer;
@@ -1030,7 +932,7 @@ Status BaseScalarColumnReader::InitDictionary() {
   bool eos;
   parquet::PageHeader next_page_header;
   uint32_t next_header_size;
-  DCHECK(stream_ != nullptr);
+
   DCHECK(!HasDictionaryDecoder());
 
   RETURN_IF_ERROR(ReadPageHeader(true /* peek */, &next_page_header,
@@ -1142,14 +1044,6 @@ Status BaseScalarColumnReader::InitDictionary() {
   return Status::OK();
 }
 
-Status BaseScalarColumnReader::InitDictionaries(
-    const vector<BaseScalarColumnReader*> readers) {
-  for (BaseScalarColumnReader* reader : readers) {
-    RETURN_IF_ERROR(reader->InitDictionary());
-  }
-  return Status::OK();
-}
-
 Status BaseScalarColumnReader::ReadDataPage() {
   // We're about to move to the next data page.  The previous data page is
   // now complete, free up any memory allocated for it. If the data page contained

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/parquet-column-readers.h
----------------------------------------------------------------------
diff --git a/be/src/exec/parquet-column-readers.h b/be/src/exec/parquet-column-readers.h
index bfb20ec..3a8ad70 100644
--- a/be/src/exec/parquet-column-readers.h
+++ b/be/src/exec/parquet-column-readers.h
@@ -326,29 +326,42 @@ class BaseScalarColumnReader : public ParquetColumnReader {
   BaseScalarColumnReader(HdfsParquetScanner* parent, const SchemaNode& node,
       const SlotDescriptor* slot_desc)
     : ParquetColumnReader(parent, node, slot_desc),
+      data_(NULL),
+      data_end_(NULL),
+      def_levels_(true),
+      rep_levels_(false),
+      page_encoding_(parquet::Encoding::PLAIN_DICTIONARY),
+      num_buffered_values_(0),
+      num_values_read_(0),
+      metadata_(NULL),
+      stream_(NULL),
       data_page_pool_(new MemPool(parent->scan_node_->mem_tracker())) {
     DCHECK_GE(node_.col_idx, 0) << node_.DebugString();
   }
 
   virtual ~BaseScalarColumnReader() { }
 
-  /// Resets the reader for each row group in the file and creates the scan
-  /// range for the column, but does not start it. To start scanning,
-  /// set_io_reservation() must be called to assign reservation to this
-  /// column, followed by StartScan().
-  Status Reset(const HdfsFileDesc& file_desc, const parquet::ColumnChunk& col_chunk,
-    int row_group_idx);
-
-  /// Starts the column scan range. The reader must be Reset() and have a
-  /// reservation assigned via set_io_reservation(). This must be called
-  /// before any of the column data can be read (including dictionary and
-  /// data pages). Returns an error status if there was an error starting the
-  /// scan or allocating buffers for it.
-  Status StartScan();
-
-  /// Helper to start scans for multiple columns at once.
-  static Status StartScans(const std::vector<BaseScalarColumnReader*> readers) {
-    for (BaseScalarColumnReader* reader : readers) RETURN_IF_ERROR(reader->StartScan());
+  /// This is called once for each row group in the file.
+  Status Reset(const parquet::ColumnMetaData* metadata, ScannerContext::Stream* stream) {
+    DCHECK(stream != NULL);
+    DCHECK(metadata != NULL);
+
+    num_buffered_values_ = 0;
+    data_ = NULL;
+    data_end_ = NULL;
+    stream_ = stream;
+    metadata_ = metadata;
+    num_values_read_ = 0;
+    def_level_ = -1;
+    // See ColumnReader constructor.
+    rep_level_ = max_rep_level() == 0 ? 0 : -1;
+    pos_current_value_ = -1;
+
+    if (metadata_->codec != parquet::CompressionCodec::UNCOMPRESSED) {
+      RETURN_IF_ERROR(Codec::CreateDecompressor(
+          NULL, false, PARQUET_TO_IMPALA_CODEC[metadata_->codec], &decompressor_));
+    }
+    ClearDictionaryDecoder();
     return Status::OK();
   }
 
@@ -363,27 +376,22 @@ class BaseScalarColumnReader : public ParquetColumnReader {
     if (dict_decoder != nullptr) dict_decoder->Close();
   }
 
-  io::ScanRange* scan_range() const { return scan_range_; }
   int64_t total_len() const { return metadata_->total_compressed_size; }
   int col_idx() const { return node_.col_idx; }
   THdfsCompression::type codec() const {
     if (metadata_ == NULL) return THdfsCompression::NONE;
     return PARQUET_TO_IMPALA_CODEC[metadata_->codec];
   }
-  void set_io_reservation(int bytes) { io_reservation_ = bytes; }
 
   /// Reads the next definition and repetition levels for this column. Initializes the
   /// next data page if necessary.
   virtual bool NextLevels() { return NextLevels<true>(); }
 
-  /// Check the data stream to see if there is a dictionary page. If there is,
-  /// use that page to initialize dict_decoder_ and advance the data stream
-  /// past the dictionary page.
+  // Check the data stream to see if there is a dictionary page. If there is,
+  // use that page to initialize dict_decoder_ and advance the data stream
+  // past the dictionary page.
   Status InitDictionary();
 
-  /// Convenience function to initialize multiple dictionaries.
-  static Status InitDictionaries(const std::vector<BaseScalarColumnReader*> readers);
-
   // Returns the dictionary or NULL if the dictionary doesn't exist
   virtual DictDecoderBase* GetDictionaryDecoder() { return nullptr; }
 
@@ -409,45 +417,33 @@ class BaseScalarColumnReader : public ParquetColumnReader {
   // fit in as few cache lines as possible.
 
   /// Pointer to start of next value in data page
-  uint8_t* data_ = nullptr;
+  uint8_t* data_;
 
   /// End of the data page.
-  const uint8_t* data_end_ = nullptr;
+  const uint8_t* data_end_;
 
   /// Decoder for definition levels.
-  ParquetLevelDecoder def_levels_{true};
+  ParquetLevelDecoder def_levels_;
 
   /// Decoder for repetition levels.
-  ParquetLevelDecoder rep_levels_{false};
+  ParquetLevelDecoder rep_levels_;
 
   /// Page encoding for values of the current data page. Cached here for perf. Set in
   /// InitDataPage().
-  parquet::Encoding::type page_encoding_ = parquet::Encoding::PLAIN_DICTIONARY;
+  parquet::Encoding::type page_encoding_;
 
   /// Num values remaining in the current data page
-  int num_buffered_values_ = 0;
+  int num_buffered_values_;
 
   // Less frequently used members that are not accessed in inner loop should go below
   // here so they do not occupy precious cache line space.
 
   /// The number of values seen so far. Updated per data page.
-  int64_t num_values_read_ = 0;
-
-  /// Metadata for the column for the current row group.
-  const parquet::ColumnMetaData* metadata_ = nullptr;
+  int64_t num_values_read_;
 
+  const parquet::ColumnMetaData* metadata_;
   boost::scoped_ptr<Codec> decompressor_;
-
-  /// The scan range for the column's data. Initialized for each row group by Reset().
-  io::ScanRange* scan_range_ = nullptr;
-
-  // Stream used to read data from 'scan_range_'. Initialized by StartScan().
-  ScannerContext::Stream* stream_ = nullptr;
-
-  /// Reservation in bytes to use for I/O buffers in 'scan_range_'/'stream_'. Must be set
-  /// with set_io_reservation() before 'stream_' is initialized. Reset for each row group
-  /// by Reset().
-  int64_t io_reservation_ = 0;
+  ScannerContext::Stream* stream_;
 
   /// Pool to allocate storage for data pages from - either decompression buffers for
   /// compressed data pages or copies of the data page with var-len data to attach to


[7/9] impala git commit: Revert IMPALA-4835 and dependent changes

Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr-test.cc b/be/src/runtime/io/disk-io-mgr-test.cc
index 3a0f727..b03ec31 100644
--- a/be/src/runtime/io/disk-io-mgr-test.cc
+++ b/be/src/runtime/io/disk-io-mgr-test.cc
@@ -22,16 +22,12 @@
 
 #include "codegen/llvm-codegen.h"
 #include "common/init.h"
-#include "runtime/bufferpool/buffer-pool.h"
-#include "runtime/bufferpool/reservation-tracker.h"
+#include "runtime/io/request-context.h"
 #include "runtime/io/disk-io-mgr-stress.h"
 #include "runtime/io/disk-io-mgr.h"
-#include "runtime/io/request-context.h"
-#include "runtime/test-env.h"
+#include "runtime/mem-tracker.h"
 #include "runtime/thread-resource-mgr.h"
-#include "service/fe-support.h"
 #include "testutil/gtest-util.h"
-#include "testutil/rand-util.h"
 #include "util/condition-variable.h"
 #include "util/cpu-info.h"
 #include "util/disk-info.h"
@@ -39,20 +35,13 @@
 
 #include "common/names.h"
 
-using std::mt19937;
-using std::uniform_int_distribution;
-using std::uniform_real_distribution;
-
-DECLARE_int64(min_buffer_size);
 DECLARE_int32(num_remote_hdfs_io_threads);
 DECLARE_int32(num_s3_io_threads);
 DECLARE_int32(num_adls_io_threads);
 
-const int MIN_BUFFER_SIZE = 128;
+const int MIN_BUFFER_SIZE = 512;
 const int MAX_BUFFER_SIZE = 1024;
-const int64_t LARGE_RESERVATION_LIMIT = 4L * 1024L * 1024L * 1024L;
-const int64_t LARGE_INITIAL_RESERVATION = 128L * 1024L * 1024L;
-const int64_t BUFFER_POOL_CAPACITY = LARGE_RESERVATION_LIMIT;
+const int LARGE_MEM_LIMIT = 1024 * 1024 * 1024;
 
 namespace impala {
 namespace io {
@@ -60,41 +49,14 @@ namespace io {
 class DiskIoMgrTest : public testing::Test {
  public:
 
-  virtual void SetUp() {
-    test_env_.reset(new TestEnv);
-    // Tests try to allocate arbitrarily small buffers. Ensure Buffer Pool allows it.
-    test_env_->SetBufferPoolArgs(1, BUFFER_POOL_CAPACITY);
-    ASSERT_OK(test_env_->Init());
-    RandTestUtil::SeedRng("DISK_IO_MGR_TEST_SEED", &rng_);
-  }
+  virtual void SetUp() {}
 
   virtual void TearDown() {
-    root_reservation_.Close();
     pool_.Clear();
-    test_env_.reset();
-  }
-
-  /// Initialises 'root_reservation_'. The reservation is automatically closed in
-  /// TearDown().
-  void InitRootReservation(int64_t reservation_limit) {
-    root_reservation_.InitRootTracker(
-        RuntimeProfile::Create(&pool_, "root"), reservation_limit);
   }
-
-  /// Initialise 'client' with the given reservation limit. The client reservation is a
-  /// child of 'root_reservation_'.
-  void RegisterBufferPoolClient(int64_t reservation_limit, int64_t initial_reservation,
-      BufferPool::ClientHandle* client) {
-    ASSERT_OK(buffer_pool()->RegisterClient("", nullptr, &root_reservation_, nullptr,
-        reservation_limit, RuntimeProfile::Create(&pool_, ""), client));
-    if (initial_reservation > 0) {
-      ASSERT_TRUE(client->IncreaseReservation(initial_reservation));
-    }
-  }
-
   void WriteValidateCallback(int num_writes, WriteRange** written_range,
-      DiskIoMgr* io_mgr, RequestContext* reader, BufferPool::ClientHandle* client,
-      int32_t* data, Status expected_status, const Status& status) {
+      DiskIoMgr* io_mgr, RequestContext* reader, int32_t* data,
+      Status expected_status, const Status& status) {
     if (expected_status.code() == TErrorCode::CANCELLED) {
       EXPECT_TRUE(status.ok() || status.IsCancelled()) << "Error: " << status.GetDetail();
     } else {
@@ -104,8 +66,8 @@ class DiskIoMgrTest : public testing::Test {
       ScanRange* scan_range = pool_.Add(new ScanRange());
       scan_range->Reset(nullptr, (*written_range)->file(), (*written_range)->len(),
           (*written_range)->offset(), 0, false, BufferOpts::Uncached());
-      ValidateSyncRead(io_mgr, reader, client, scan_range,
-          reinterpret_cast<const char*>(data), sizeof(int32_t));
+      ValidateSyncRead(io_mgr, reader, scan_range, reinterpret_cast<const char*>(data),
+          sizeof(int32_t));
     }
 
     {
@@ -126,13 +88,9 @@ class DiskIoMgrTest : public testing::Test {
 
  protected:
   void CreateTempFile(const char* filename, const char* data) {
-    CreateTempFile(filename, data, strlen(data));
-  }
-
-  void CreateTempFile(const char* filename, const char* data, int64_t data_bytes) {
     FILE* file = fopen(filename, "w");
     EXPECT_TRUE(file != nullptr);
-    fwrite(data, 1, data_bytes, file);
+    fwrite(data, 1, strlen(data), file);
     fclose(file);
   }
 
@@ -157,22 +115,15 @@ class DiskIoMgrTest : public testing::Test {
   }
 
   static void ValidateSyncRead(DiskIoMgr* io_mgr, RequestContext* reader,
-      BufferPool::ClientHandle* client, ScanRange* range, const char* expected,
-      int expected_len = -1) {
+      ScanRange* range, const char* expected, int expected_len = -1) {
     unique_ptr<BufferDescriptor> buffer;
-    bool needs_buffers;
-    ASSERT_OK(io_mgr->StartScanRange(reader, range, &needs_buffers));
-    if (needs_buffers) {
-      ASSERT_OK(io_mgr->AllocateBuffersForRange(
-          reader, client, range, io_mgr->max_buffer_size()));
-    }
-    ASSERT_OK(range->GetNext(&buffer));
+    ASSERT_OK(io_mgr->Read(reader, range, &buffer));
     ASSERT_TRUE(buffer != nullptr);
     EXPECT_EQ(buffer->len(), range->len());
     if (expected_len < 0) expected_len = strlen(expected);
     int cmp = memcmp(buffer->buffer(), expected, expected_len);
     EXPECT_TRUE(cmp == 0);
-    range->ReturnBuffer(move(buffer));
+    io_mgr->ReturnBuffer(move(buffer));
   }
 
   static void ValidateScanRange(DiskIoMgr* io_mgr, ScanRange* range,
@@ -185,13 +136,13 @@ class DiskIoMgrTest : public testing::Test {
       Status status = range->GetNext(&buffer);
       ASSERT_TRUE(status.ok() || status.code() == expected_status.code());
       if (buffer == nullptr || !status.ok()) {
-        if (buffer != nullptr) range->ReturnBuffer(move(buffer));
+        if (buffer != nullptr) io_mgr->ReturnBuffer(move(buffer));
         break;
       }
       ASSERT_LE(buffer->len(), expected_len);
       memcpy(result + range->offset() + buffer->scan_range_offset(),
           buffer->buffer(), buffer->len());
-      range->ReturnBuffer(move(buffer));
+      io_mgr->ReturnBuffer(move(buffer));
     }
     ValidateEmptyOrCorrect(expected, result, expected_len);
   }
@@ -199,46 +150,35 @@ class DiskIoMgrTest : public testing::Test {
   // Continues pulling scan ranges from the io mgr until they are all done.
   // Updates num_ranges_processed with the number of ranges seen by this thread.
   static void ScanRangeThread(DiskIoMgr* io_mgr, RequestContext* reader,
-      BufferPool::ClientHandle* client, const char* expected_result, int expected_len,
-      const Status& expected_status, int max_ranges, AtomicInt32* num_ranges_processed) {
+      const char* expected_result, int expected_len, const Status& expected_status,
+      int max_ranges, AtomicInt32* num_ranges_processed) {
     int num_ranges = 0;
     while (max_ranges == 0 || num_ranges < max_ranges) {
       ScanRange* range;
-      bool needs_buffers;
-      Status status = io_mgr->GetNextUnstartedRange(reader, &range, &needs_buffers);
+      Status status = io_mgr->GetNextRange(reader, &range);
       ASSERT_TRUE(status.ok() || status.code() == expected_status.code());
       if (range == nullptr) break;
-      if (needs_buffers) {
-        ASSERT_OK(io_mgr->AllocateBuffersForRange(
-            reader, client, range, io_mgr->max_buffer_size() * 3));
-      }
       ValidateScanRange(io_mgr, range, expected_result, expected_len, expected_status);
       num_ranges_processed->Add(1);
       ++num_ranges;
     }
   }
 
-  ScanRange* InitRange(ObjectPool* pool, const char* file_path, int offset, int len,
+  ScanRange* AllocateRange() {
+    return pool_.Add(new ScanRange);
+  }
+
+  ScanRange* InitRange(const char* file_path, int offset, int len,
       int disk_id, int64_t mtime, void* meta_data = nullptr, bool is_cached = false) {
-    ScanRange* range = pool->Add(new ScanRange);
+    ScanRange* range = AllocateRange();
     range->Reset(nullptr, file_path, len, offset, disk_id, true,
         BufferOpts(is_cached, mtime), meta_data);
     EXPECT_EQ(mtime, range->mtime());
     return range;
   }
 
-  /// Convenience function to get a reference to the buffer pool.
-  BufferPool* buffer_pool() const { return ExecEnv::GetInstance()->buffer_pool(); }
-
-  boost::scoped_ptr<TestEnv> test_env_;
-
-  /// Per-test random number generator. Seeded before every test.
-  mt19937 rng_;
-
   ObjectPool pool_;
 
-  ReservationTracker root_reservation_;
-
   mutex written_mutex_;
   ConditionVariable writes_done_;
   int num_ranges_written_;
@@ -249,7 +189,7 @@ class DiskIoMgrTest : public testing::Test {
 // by reading the data back via a separate IoMgr instance. All writes are expected to
 // complete successfully.
 TEST_F(DiskIoMgrTest, SingleWriter) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   num_ranges_written_ = 0;
   string tmp_file = "/tmp/disk_io_mgr_test.txt";
   int num_ranges = 100;
@@ -263,27 +203,25 @@ TEST_F(DiskIoMgrTest, SingleWriter) {
   }
 
   scoped_ptr<DiskIoMgr> read_io_mgr(new DiskIoMgr(1, 1, 1, 1, 10));
-  BufferPool::ClientHandle read_client;
-  RegisterBufferPoolClient(
-      LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
-  ASSERT_OK(read_io_mgr->Init());
-  unique_ptr<RequestContext> reader = read_io_mgr->RegisterContext();
+  MemTracker reader_mem_tracker(LARGE_MEM_LIMIT);
+  ASSERT_OK(read_io_mgr->Init(&reader_mem_tracker));
+  unique_ptr<RequestContext> reader =
+      read_io_mgr->RegisterContext(&reader_mem_tracker);
   for (int num_threads_per_disk = 1; num_threads_per_disk <= 5; ++num_threads_per_disk) {
     for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
-      // Pool for temporary objects from this iteration only.
-      ObjectPool tmp_pool;
+      pool_.Clear(); // Destroy scan ranges from previous iterations.
       DiskIoMgr io_mgr(num_disks, num_threads_per_disk, num_threads_per_disk, 1, 10);
-      ASSERT_OK(io_mgr.Init());
-      unique_ptr<RequestContext> writer = io_mgr.RegisterContext();
+      ASSERT_OK(io_mgr.Init(&mem_tracker));
+      unique_ptr<RequestContext> writer = io_mgr.RegisterContext(&mem_tracker);
       for (int i = 0; i < num_ranges; ++i) {
-        int32_t* data = tmp_pool.Add(new int32_t);
+        int32_t* data = pool_.Add(new int32_t);
         *data = rand();
-        WriteRange** new_range = tmp_pool.Add(new WriteRange*);
-        WriteRange::WriteDoneCallback callback = bind(
-            mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, num_ranges, new_range,
-            read_io_mgr.get(), reader.get(), &read_client, data, Status::OK(), _1);
-        *new_range = tmp_pool.Add(
-            new WriteRange(tmp_file, cur_offset, num_ranges % num_disks, callback));
+        WriteRange** new_range = pool_.Add(new WriteRange*);
+        WriteRange::WriteDoneCallback callback =
+            bind(mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, num_ranges,
+                new_range, read_io_mgr.get(), reader.get(), data, Status::OK(), _1);
+        *new_range = pool_.Add(new WriteRange(
+            tmp_file, cur_offset, num_ranges % num_disks, callback));
         (*new_range)->SetData(reinterpret_cast<uint8_t*>(data), sizeof(int32_t));
         EXPECT_OK(io_mgr.AddWriteRange(writer.get(), *new_range));
         cur_offset += sizeof(int32_t);
@@ -299,26 +237,27 @@ TEST_F(DiskIoMgrTest, SingleWriter) {
   }
 
   read_io_mgr->UnregisterContext(reader.get());
-  buffer_pool()->DeregisterClient(&read_client);
+  read_io_mgr.reset();
 }
 
 // Perform invalid writes (e.g. file in non-existent directory, negative offset) and
 // validate that an error status is returned via the write callback.
 TEST_F(DiskIoMgrTest, InvalidWrite) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   num_ranges_written_ = 0;
   string tmp_file = "/non-existent/file.txt";
   DiskIoMgr io_mgr(1, 1, 1, 1, 10);
-  ASSERT_OK(io_mgr.Init());
-  unique_ptr<RequestContext> writer = io_mgr.RegisterContext();
+  ASSERT_OK(io_mgr.Init(&mem_tracker));
+  unique_ptr<RequestContext> writer = io_mgr.RegisterContext(nullptr);
   int32_t* data = pool_.Add(new int32_t);
   *data = rand();
 
   // Write to file in non-existent directory.
   WriteRange** new_range = pool_.Add(new WriteRange*);
   WriteRange::WriteDoneCallback callback =
-      bind(mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, 2, new_range, nullptr,
-          nullptr, nullptr, data, Status(TErrorCode::DISK_IO_ERROR, "Test Failure"), _1);
+      bind(mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, 2, new_range,
+          (DiskIoMgr*)nullptr, (RequestContext*)nullptr, data,
+          Status(TErrorCode::DISK_IO_ERROR, "Test Failure"), _1);
   *new_range = pool_.Add(new WriteRange(tmp_file, rand(), 0, callback));
 
   (*new_range)->SetData(reinterpret_cast<uint8_t*>(data), sizeof(int32_t));
@@ -333,9 +272,9 @@ TEST_F(DiskIoMgrTest, InvalidWrite) {
   }
 
   new_range = pool_.Add(new WriteRange*);
-  callback = bind(mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, 2, new_range,
-      nullptr, nullptr, nullptr, data,
-      Status(TErrorCode::DISK_IO_ERROR, "Test Failure"), _1);
+  callback = bind(mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, 2,
+      new_range, (DiskIoMgr*)nullptr, (RequestContext*)nullptr,
+      data, Status(TErrorCode::DISK_IO_ERROR, "Test Failure"), _1);
 
   *new_range = pool_.Add(new WriteRange(tmp_file, -1, 0, callback));
   (*new_range)->SetData(reinterpret_cast<uint8_t*>(data), sizeof(int32_t));
@@ -353,7 +292,7 @@ TEST_F(DiskIoMgrTest, InvalidWrite) {
 // AddWriteRange() is expected to succeed before the cancel and fail after it.
 // The writes themselves may finish with status cancelled or ok.
 TEST_F(DiskIoMgrTest, SingleWriterCancel) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   num_ranges_written_ = 0;
   string tmp_file = "/tmp/disk_io_mgr_test.txt";
   int num_ranges = 100;
@@ -368,33 +307,30 @@ TEST_F(DiskIoMgrTest, SingleWriterCancel) {
   }
 
   scoped_ptr<DiskIoMgr> read_io_mgr(new DiskIoMgr(1, 1, 1, 1, 10));
-  BufferPool::ClientHandle read_client;
-  RegisterBufferPoolClient(
-      LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
-  ASSERT_OK(read_io_mgr->Init());
-  unique_ptr<RequestContext> reader = read_io_mgr->RegisterContext();
+  MemTracker reader_mem_tracker(LARGE_MEM_LIMIT);
+  ASSERT_OK(read_io_mgr->Init(&reader_mem_tracker));
+  unique_ptr<RequestContext> reader =
+      read_io_mgr->RegisterContext(&reader_mem_tracker);
   for (int num_threads_per_disk = 1; num_threads_per_disk <= 5; ++num_threads_per_disk) {
     for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
-      // Pool for temporary objects from this iteration only.
-      ObjectPool tmp_pool;
+      pool_.Clear(); // Destroy scan ranges from previous iterations.
       DiskIoMgr io_mgr(num_disks, num_threads_per_disk, num_threads_per_disk, 1, 10);
-      ASSERT_OK(io_mgr.Init());
-      unique_ptr<RequestContext> writer = io_mgr.RegisterContext();
+      ASSERT_OK(io_mgr.Init(&mem_tracker));
+      unique_ptr<RequestContext> writer = io_mgr.RegisterContext(&mem_tracker);
       Status validate_status = Status::OK();
       for (int i = 0; i < num_ranges; ++i) {
         if (i == num_ranges_before_cancel) {
-          writer->Cancel();
+          io_mgr.CancelContext(writer.get());
           validate_status = Status::CANCELLED;
         }
-        int32_t* data = tmp_pool.Add(new int32_t);
+        int32_t* data = pool_.Add(new int32_t);
         *data = rand();
-        WriteRange** new_range = tmp_pool.Add(new WriteRange*);
-        WriteRange::WriteDoneCallback callback =
-            bind(mem_fn(&DiskIoMgrTest::WriteValidateCallback), this,
-                num_ranges_before_cancel, new_range, read_io_mgr.get(), reader.get(),
-                &read_client, data, Status::CANCELLED, _1);
-        *new_range = tmp_pool.Add(
-            new WriteRange(tmp_file, cur_offset, num_ranges % num_disks, callback));
+        WriteRange** new_range = pool_.Add(new WriteRange*);
+        WriteRange::WriteDoneCallback callback = bind(
+            mem_fn(&DiskIoMgrTest::WriteValidateCallback), this, num_ranges_before_cancel,
+            new_range, read_io_mgr.get(), reader.get(), data, Status::CANCELLED, _1);
+        *new_range = pool_.Add(new WriteRange(
+            tmp_file, cur_offset, num_ranges % num_disks, callback));
         (*new_range)->SetData(reinterpret_cast<uint8_t*>(data), sizeof(int32_t));
         cur_offset += sizeof(int32_t);
         Status add_status = io_mgr.AddWriteRange(writer.get(), *new_range);
@@ -411,13 +347,13 @@ TEST_F(DiskIoMgrTest, SingleWriterCancel) {
   }
 
   read_io_mgr->UnregisterContext(reader.get());
-  buffer_pool()->DeregisterClient(&read_client);
+  read_io_mgr.reset();
 }
 
 // Basic test with a single reader, testing multiple threads, disks and a different
 // number of buffers.
 TEST_F(DiskIoMgrTest, SingleReader) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
   const char* data = "abcdefghijklm";
   int len = strlen(data);
@@ -431,7 +367,7 @@ TEST_F(DiskIoMgrTest, SingleReader) {
   for (int num_threads_per_disk = 1; num_threads_per_disk <= 5; ++num_threads_per_disk) {
     for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
       for (int num_read_threads = 1; num_read_threads <= 5; ++num_read_threads) {
-        ObjectPool tmp_pool;
+        ObjectPool pool;
         LOG(INFO) << "Starting test with num_threads_per_disk=" << num_threads_per_disk
                   << " num_disk=" << num_disks
                   << " num_read_threads=" << num_read_threads;
@@ -439,40 +375,38 @@ TEST_F(DiskIoMgrTest, SingleReader) {
         if (++iters % 5000 == 0) LOG(ERROR) << "Starting iteration " << iters;
         DiskIoMgr io_mgr(num_disks, num_threads_per_disk, num_threads_per_disk, 1, 1);
 
-        ASSERT_OK(io_mgr.Init());
-        BufferPool::ClientHandle read_client;
-        RegisterBufferPoolClient(
-            LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
-        unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+        ASSERT_OK(io_mgr.Init(&mem_tracker));
+        MemTracker reader_mem_tracker;
+        unique_ptr<RequestContext> reader =
+            io_mgr.RegisterContext(&reader_mem_tracker);
 
         vector<ScanRange*> ranges;
         for (int i = 0; i < len; ++i) {
           int disk_id = i % num_disks;
-          ranges.push_back(InitRange(&tmp_pool, tmp_file, 0, len, disk_id, stat_val.st_mtime));
+          ranges.push_back(InitRange(tmp_file, 0, len, disk_id, stat_val.st_mtime));
         }
         ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges));
 
         AtomicInt32 num_ranges_processed;
         thread_group threads;
         for (int i = 0; i < num_read_threads; ++i) {
-          threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(),
-              &read_client, data, len, Status::OK(), 0, &num_ranges_processed));
+          threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(), data, len,
+              Status::OK(), 0, &num_ranges_processed));
         }
         threads.join_all();
 
         EXPECT_EQ(num_ranges_processed.Load(), ranges.size());
         io_mgr.UnregisterContext(reader.get());
-        EXPECT_EQ(read_client.GetUsedReservation(), 0);
-        buffer_pool()->DeregisterClient(&read_client);
+        EXPECT_EQ(reader_mem_tracker.consumption(), 0);
       }
     }
   }
-  EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+  EXPECT_EQ(mem_tracker.consumption(), 0);
 }
 
 // This test issues adding additional scan ranges while there are some still in flight.
 TEST_F(DiskIoMgrTest, AddScanRangeTest) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
   const char* data = "abcdefghijklm";
   int len = strlen(data);
@@ -485,19 +419,17 @@ TEST_F(DiskIoMgrTest, AddScanRangeTest) {
   int64_t iters = 0;
   for (int num_threads_per_disk = 1; num_threads_per_disk <= 5; ++num_threads_per_disk) {
     for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
-      // Pool for temporary objects from this iteration only.
-      ObjectPool tmp_pool;
+      pool_.Clear(); // Destroy scan ranges from previous iterations.
       LOG(INFO) << "Starting test with num_threads_per_disk=" << num_threads_per_disk
                 << " num_disk=" << num_disks;
 
       if (++iters % 5000 == 0) LOG(ERROR) << "Starting iteration " << iters;
       DiskIoMgr io_mgr(num_disks, num_threads_per_disk, num_threads_per_disk, 1, 1);
 
-      ASSERT_OK(io_mgr.Init());
-      BufferPool::ClientHandle read_client;
-      RegisterBufferPoolClient(
-          LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
-      unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+      ASSERT_OK(io_mgr.Init(&mem_tracker));
+      MemTracker reader_mem_tracker;
+      unique_ptr<RequestContext> reader =
+          io_mgr.RegisterContext(&reader_mem_tracker);
 
       vector<ScanRange*> ranges_first_half;
       vector<ScanRange*> ranges_second_half;
@@ -505,10 +437,10 @@ TEST_F(DiskIoMgrTest, AddScanRangeTest) {
         int disk_id = i % num_disks;
         if (i > len / 2) {
           ranges_second_half.push_back(
-              InitRange(&tmp_pool, tmp_file, i, 1, disk_id, stat_val.st_mtime));
+              InitRange(tmp_file, i, 1, disk_id, stat_val.st_mtime));
         } else {
           ranges_first_half.push_back(
-              InitRange(&tmp_pool, tmp_file, i, 1, disk_id, stat_val.st_mtime));
+              InitRange(tmp_file, i, 1, disk_id, stat_val.st_mtime));
         }
       }
       AtomicInt32 num_ranges_processed;
@@ -517,8 +449,8 @@ TEST_F(DiskIoMgrTest, AddScanRangeTest) {
       ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges_first_half));
 
       // Read a couple of them
-      ScanRangeThread(&io_mgr, reader.get(), &read_client, data, strlen(data),
-          Status::OK(), 2, &num_ranges_processed);
+      ScanRangeThread(&io_mgr, reader.get(), data, strlen(data), Status::OK(), 2,
+          &num_ranges_processed);
 
       // Issue second half
       ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges_second_half));
@@ -526,26 +458,24 @@ TEST_F(DiskIoMgrTest, AddScanRangeTest) {
       // Start up some threads and then cancel
       thread_group threads;
       for (int i = 0; i < 3; ++i) {
-        threads.add_thread(
-            new thread(ScanRangeThread, &io_mgr, reader.get(), &read_client, data,
-                strlen(data), Status::CANCELLED, 0, &num_ranges_processed));
+        threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(), data,
+            strlen(data), Status::CANCELLED, 0, &num_ranges_processed));
       }
 
       threads.join_all();
       EXPECT_EQ(num_ranges_processed.Load(), len);
       io_mgr.UnregisterContext(reader.get());
-      EXPECT_EQ(read_client.GetUsedReservation(), 0);
-      buffer_pool()->DeregisterClient(&read_client);
+      EXPECT_EQ(reader_mem_tracker.consumption(), 0);
     }
   }
-  EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+  EXPECT_EQ(mem_tracker.consumption(), 0);
 }
 
 // Test to make sure that sync reads and async reads work together
 // Note: this test is constructed so the number of buffers is greater than the
 // number of scan ranges.
 TEST_F(DiskIoMgrTest, SyncReadTest) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
   const char* data = "abcdefghijklm";
   int len = strlen(data);
@@ -558,8 +488,7 @@ TEST_F(DiskIoMgrTest, SyncReadTest) {
   int64_t iters = 0;
   for (int num_threads_per_disk = 1; num_threads_per_disk <= 5; ++num_threads_per_disk) {
     for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
-      // Pool for temporary objects from this iteration only.
-      ObjectPool tmp_pool;
+      pool_.Clear(); // Destroy scan ranges from previous iterations.
       LOG(INFO) << "Starting test with num_threads_per_disk=" << num_threads_per_disk
                 << " num_disk=" << num_disks;
 
@@ -567,56 +496,54 @@ TEST_F(DiskIoMgrTest, SyncReadTest) {
       DiskIoMgr io_mgr(num_disks, num_threads_per_disk, num_threads_per_disk,
           MIN_BUFFER_SIZE, MAX_BUFFER_SIZE);
 
-      ASSERT_OK(io_mgr.Init());
-      BufferPool::ClientHandle read_client;
-      RegisterBufferPoolClient(
-          LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
-      unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+      ASSERT_OK(io_mgr.Init(&mem_tracker));
+      MemTracker reader_mem_tracker;
+      unique_ptr<RequestContext> reader =
+          io_mgr.RegisterContext(&reader_mem_tracker);
 
       ScanRange* complete_range =
-          InitRange(&tmp_pool, tmp_file, 0, strlen(data), 0, stat_val.st_mtime);
+          InitRange(tmp_file, 0, strlen(data), 0, stat_val.st_mtime);
 
       // Issue some reads before the async ones are issued
-      ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
-      ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
+      ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
+      ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
+
       vector<ScanRange*> ranges;
       for (int i = 0; i < len; ++i) {
         int disk_id = i % num_disks;
-        ranges.push_back(
-            InitRange(&tmp_pool, tmp_file, 0, len, disk_id, stat_val.st_mtime));
+        ranges.push_back(InitRange(tmp_file, 0, len, disk_id, stat_val.st_mtime));
       }
       ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges));
 
       AtomicInt32 num_ranges_processed;
       thread_group threads;
       for (int i = 0; i < 5; ++i) {
-        threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(),
-            &read_client, data, strlen(data), Status::OK(), 0, &num_ranges_processed));
+        threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(), data,
+            strlen(data), Status::OK(), 0, &num_ranges_processed));
       }
 
       // Issue some more sync ranges
       for (int i = 0; i < 5; ++i) {
         sched_yield();
-        ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
+        ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
       }
 
       threads.join_all();
 
-      ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
-      ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
+      ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
+      ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
 
       EXPECT_EQ(num_ranges_processed.Load(), ranges.size());
       io_mgr.UnregisterContext(reader.get());
-      EXPECT_EQ(read_client.GetUsedReservation(), 0);
-      buffer_pool()->DeregisterClient(&read_client);
+      EXPECT_EQ(reader_mem_tracker.consumption(), 0);
     }
   }
-  EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+  EXPECT_EQ(mem_tracker.consumption(), 0);
 }
 
 // Tests a single reader cancelling half way through scan ranges.
 TEST_F(DiskIoMgrTest, SingleReaderCancel) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
   const char* data = "abcdefghijklm";
   int len = strlen(data);
@@ -629,25 +556,22 @@ TEST_F(DiskIoMgrTest, SingleReaderCancel) {
   int64_t iters = 0;
   for (int num_threads_per_disk = 1; num_threads_per_disk <= 5; ++num_threads_per_disk) {
     for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
-      // Pool for temporary objects from this iteration only.
-      ObjectPool tmp_pool;
+      pool_.Clear(); // Destroy scan ranges from previous iterations.
       LOG(INFO) << "Starting test with num_threads_per_disk=" << num_threads_per_disk
                 << " num_disk=" << num_disks;
 
       if (++iters % 5000 == 0) LOG(ERROR) << "Starting iteration " << iters;
       DiskIoMgr io_mgr(num_disks, num_threads_per_disk, num_threads_per_disk, 1, 1);
 
-      ASSERT_OK(io_mgr.Init());
-      BufferPool::ClientHandle read_client;
-      RegisterBufferPoolClient(
-          LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
-      unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+      ASSERT_OK(io_mgr.Init(&mem_tracker));
+      MemTracker reader_mem_tracker;
+      unique_ptr<RequestContext> reader =
+          io_mgr.RegisterContext(&reader_mem_tracker);
 
       vector<ScanRange*> ranges;
       for (int i = 0; i < len; ++i) {
         int disk_id = i % num_disks;
-        ranges.push_back(
-            InitRange(&tmp_pool, tmp_file, 0, len, disk_id, stat_val.st_mtime));
+        ranges.push_back(InitRange(tmp_file, 0, len, disk_id, stat_val.st_mtime));
       }
       ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges));
 
@@ -655,112 +579,94 @@ TEST_F(DiskIoMgrTest, SingleReaderCancel) {
       int num_succesful_ranges = ranges.size() / 2;
       // Read half the ranges
       for (int i = 0; i < num_succesful_ranges; ++i) {
-        ScanRangeThread(&io_mgr, reader.get(), &read_client, data, strlen(data),
-            Status::OK(), 1, &num_ranges_processed);
+        ScanRangeThread(&io_mgr, reader.get(), data, strlen(data), Status::OK(), 1,
+            &num_ranges_processed);
       }
       EXPECT_EQ(num_ranges_processed.Load(), num_succesful_ranges);
 
       // Start up some threads and then cancel
       thread_group threads;
       for (int i = 0; i < 3; ++i) {
-        threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(),
-            &read_client, data, strlen(data), Status::CANCELLED, 0,
-            &num_ranges_processed));
+        threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(), data,
+            strlen(data), Status::CANCELLED, 0, &num_ranges_processed));
       }
 
-      reader->Cancel();
+      io_mgr.CancelContext(reader.get());
       sched_yield();
 
       threads.join_all();
-      EXPECT_TRUE(reader->IsCancelled());
+      EXPECT_TRUE(io_mgr.context_status(reader.get()).IsCancelled());
       io_mgr.UnregisterContext(reader.get());
-      EXPECT_EQ(read_client.GetUsedReservation(), 0);
-      buffer_pool()->DeregisterClient(&read_client);
+      EXPECT_EQ(reader_mem_tracker.consumption(), 0);
     }
   }
-  EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+  EXPECT_EQ(mem_tracker.consumption(), 0);
 }
 
-// Test readers running with different amounts of memory and getting blocked on scan
-// ranges that have run out of buffers.
-TEST_F(DiskIoMgrTest, MemScarcity) {
+// Test when the reader goes over the mem limit
+TEST_F(DiskIoMgrTest, MemLimits) {
   const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
-  // File is 2.5 max buffers so that we can scan file without returning buffers
-  // when we get the max reservation below.
-  const int64_t DATA_BYTES = MAX_BUFFER_SIZE * 5 / 2;
-  char data[DATA_BYTES];
-  for (int i = 0; i < DATA_BYTES; ++i) {
-    data[i] = uniform_int_distribution<uint8_t>(0, 255)(rng_);
-  }
-  CreateTempFile(tmp_file, data, DATA_BYTES);
+  const char* data = "abcdefghijklm";
+  int len = strlen(data);
+  CreateTempFile(tmp_file, data);
 
   // Get mtime for file
   struct stat stat_val;
   stat(tmp_file, &stat_val);
 
-  const int RESERVATION_LIMIT_NUM_BUFFERS = 20;
-  const int64_t RESERVATION_LIMIT = RESERVATION_LIMIT_NUM_BUFFERS * MAX_BUFFER_SIZE;
-  InitRootReservation(RESERVATION_LIMIT);
-
-  thread_group threads;
-  // Allocate enough ranges so that the total buffers exceeds the limit.
+  const int mem_limit_num_buffers = 2;
+  // Allocate enough ranges so that the total buffers exceeds the mem limit.
   const int num_ranges = 25;
   {
+    MemTracker root_mem_tracker(mem_limit_num_buffers * MAX_BUFFER_SIZE);
     DiskIoMgr io_mgr(1, 1, 1, MIN_BUFFER_SIZE, MAX_BUFFER_SIZE);
 
-    ASSERT_OK(io_mgr.Init());
-    BufferPool::ClientHandle read_client;
-    RegisterBufferPoolClient(RESERVATION_LIMIT, RESERVATION_LIMIT, &read_client);
-    unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+    ASSERT_OK(io_mgr.Init(&root_mem_tracker));
+    MemTracker reader_mem_tracker(-1, "Reader", &root_mem_tracker);
+    unique_ptr<RequestContext> reader = io_mgr.RegisterContext(&reader_mem_tracker);
 
     vector<ScanRange*> ranges;
     for (int i = 0; i < num_ranges; ++i) {
-      ranges.push_back(InitRange(&pool_, tmp_file, 0, DATA_BYTES, 0, stat_val.st_mtime));
+      ranges.push_back(InitRange(tmp_file, 0, len, 0, stat_val.st_mtime));
     }
     ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges));
-    // Keep starting new ranges without returning buffers until we run out of
-    // reservation.
-    while (read_client.GetUnusedReservation() >= MIN_BUFFER_SIZE) {
+
+    // Don't return buffers to force memory pressure
+    vector<unique_ptr<BufferDescriptor>> buffers;
+
+    AtomicInt32 num_ranges_processed;
+    ScanRangeThread(&io_mgr, reader.get(), data, strlen(data), Status::MemLimitExceeded(),
+        1, &num_ranges_processed);
+
+    char result[strlen(data) + 1];
+    // Keep reading new ranges without returning buffers. This forces us
+    // to go over the limit eventually.
+    while (true) {
+      memset(result, 0, strlen(data) + 1);
       ScanRange* range = nullptr;
-      bool needs_buffers;
-      ASSERT_OK(io_mgr.GetNextUnstartedRange(reader.get(), &range, &needs_buffers));
+      Status status = io_mgr.GetNextRange(reader.get(), &range);
+      ASSERT_TRUE(status.ok() || status.IsMemLimitExceeded());
       if (range == nullptr) break;
-      ASSERT_TRUE(needs_buffers);
-      // Pick a random amount of memory to reserve.
-      int64_t max_bytes_to_alloc = uniform_int_distribution<int64_t>(MIN_BUFFER_SIZE,
-          min<int64_t>(read_client.GetUnusedReservation(), MAX_BUFFER_SIZE * 3))(rng_);
-      ASSERT_OK(io_mgr.AllocateBuffersForRange(
-          reader.get(), &read_client, range, max_bytes_to_alloc));
-      // Start a thread fetching from the range. The thread will either finish the
-      // range or be cancelled.
-      threads.add_thread(new thread([&data, DATA_BYTES, range] {
-        // Don't return buffers to force memory pressure.
-        vector<unique_ptr<BufferDescriptor>> buffers;
-        int64_t data_offset = 0;
-        Status status;
-        while (true) {
-          unique_ptr<BufferDescriptor> buffer;
-          status = range->GetNext(&buffer);
-          ASSERT_TRUE(status.ok() || status.IsCancelled()) << status.GetDetail();
-          if (status.IsCancelled() || buffer == nullptr) break;
-          EXPECT_EQ(0, memcmp(data + data_offset, buffer->buffer(), buffer->len()));
-          data_offset += buffer->len();
-          buffers.emplace_back(move(buffer));
-        }
-        if (status.ok()) ASSERT_EQ(DATA_BYTES, data_offset);
-        for (auto& buffer : buffers) range->ReturnBuffer(move(buffer));
-      }));
-      // Let the thread start running before starting the next.
-      SleepForMs(10);
+
+      while (true) {
+        unique_ptr<BufferDescriptor> buffer;
+        Status status = range->GetNext(&buffer);
+        ASSERT_TRUE(status.ok() || status.IsMemLimitExceeded());
+        if (buffer == nullptr) break;
+        memcpy(result + range->offset() + buffer->scan_range_offset(),
+            buffer->buffer(), buffer->len());
+        buffers.push_back(move(buffer));
+      }
+      ValidateEmptyOrCorrect(data, result, strlen(data));
     }
-    // Let the threads run for a bit then cancel everything.
-    SleepForMs(500);
-    reader->Cancel();
-    // Wait until the threads have returned their buffers before unregistering.
-    threads.join_all();
+
+    for (int i = 0; i < buffers.size(); ++i) {
+      io_mgr.ReturnBuffer(move(buffers[i]));
+    }
+
+    EXPECT_TRUE(io_mgr.context_status(reader.get()).IsMemLimitExceeded());
     io_mgr.UnregisterContext(reader.get());
-    EXPECT_EQ(read_client.GetUsedReservation(), 0);
-    buffer_pool()->DeregisterClient(&read_client);
+    EXPECT_EQ(reader_mem_tracker.consumption(), 0);
   }
 }
 
@@ -769,7 +675,7 @@ TEST_F(DiskIoMgrTest, MemScarcity) {
 // only tests the fallback mechanism.
 // TODO: we can fake the cached read path without HDFS
 TEST_F(DiskIoMgrTest, CachedReads) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
   const char* data = "abcdefghijklm";
   int len = strlen(data);
@@ -783,55 +689,52 @@ TEST_F(DiskIoMgrTest, CachedReads) {
   {
     DiskIoMgr io_mgr(num_disks, 1, 1, MIN_BUFFER_SIZE, MAX_BUFFER_SIZE);
 
-    ASSERT_OK(io_mgr.Init());
-    BufferPool::ClientHandle read_client;
-    RegisterBufferPoolClient(
-        LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
-    unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+    ASSERT_OK(io_mgr.Init(&mem_tracker));
+    MemTracker reader_mem_tracker;
+    unique_ptr<RequestContext> reader = io_mgr.RegisterContext(&reader_mem_tracker);
 
     ScanRange* complete_range =
-        InitRange(&pool_, tmp_file, 0, strlen(data), 0, stat_val.st_mtime, nullptr, true);
+        InitRange(tmp_file, 0, strlen(data), 0, stat_val.st_mtime, nullptr, true);
 
     // Issue some reads before the async ones are issued
-    ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
-    ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
+    ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
+    ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
 
     vector<ScanRange*> ranges;
     for (int i = 0; i < len; ++i) {
       int disk_id = i % num_disks;
       ranges.push_back(
-          InitRange(&pool_, tmp_file, 0, len, disk_id, stat_val.st_mtime, nullptr, true));
+          InitRange(tmp_file, 0, len, disk_id, stat_val.st_mtime, nullptr, true));
     }
     ASSERT_OK(io_mgr.AddScanRanges(reader.get(), ranges));
 
     AtomicInt32 num_ranges_processed;
     thread_group threads;
     for (int i = 0; i < 5; ++i) {
-      threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(), &read_client,
-          data, strlen(data), Status::OK(), 0, &num_ranges_processed));
+      threads.add_thread(new thread(ScanRangeThread, &io_mgr, reader.get(), data,
+          strlen(data), Status::OK(), 0, &num_ranges_processed));
     }
 
     // Issue some more sync ranges
     for (int i = 0; i < 5; ++i) {
       sched_yield();
-      ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
+      ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
     }
 
     threads.join_all();
 
-    ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
-    ValidateSyncRead(&io_mgr, reader.get(), &read_client, complete_range, data);
+    ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
+    ValidateSyncRead(&io_mgr, reader.get(), complete_range, data);
 
     EXPECT_EQ(num_ranges_processed.Load(), ranges.size());
     io_mgr.UnregisterContext(reader.get());
-    EXPECT_EQ(read_client.GetUsedReservation(), 0);
-    buffer_pool()->DeregisterClient(&read_client);
+    EXPECT_EQ(reader_mem_tracker.consumption(), 0);
   }
-  EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+  EXPECT_EQ(mem_tracker.consumption(), 0);
 }
 
 TEST_F(DiskIoMgrTest, MultipleReaderWriter) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   const int ITERATIONS = 1;
   const char* data = "abcdefghijklmnopqrstuvwxyz";
   const int num_contexts = 5;
@@ -853,22 +756,17 @@ TEST_F(DiskIoMgrTest, MultipleReaderWriter) {
 
   int64_t iters = 0;
   vector<unique_ptr<RequestContext>> contexts(num_contexts);
-  unique_ptr<BufferPool::ClientHandle[]> clients(
-      new BufferPool::ClientHandle[num_contexts]);
   Status status;
   for (int iteration = 0; iteration < ITERATIONS; ++iteration) {
     for (int threads_per_disk = 1; threads_per_disk <= 5; ++threads_per_disk) {
       for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
-        // Pool for temporary objects from this iteration only.
-        ObjectPool tmp_pool;
         DiskIoMgr io_mgr(num_disks, threads_per_disk, threads_per_disk, MIN_BUFFER_SIZE,
             MAX_BUFFER_SIZE);
-        ASSERT_OK(io_mgr.Init());
+        ASSERT_OK(io_mgr.Init(&mem_tracker));
         for (int file_index = 0; file_index < num_contexts; ++file_index) {
-          RegisterBufferPoolClient(
-              LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &clients[file_index]);
-          contexts[file_index] = io_mgr.RegisterContext();
+          contexts[file_index] = io_mgr.RegisterContext(&mem_tracker);
         }
+        pool_.Clear();
         int read_offset = 0;
         int write_offset = 0;
         while (read_offset < file_size) {
@@ -880,11 +778,11 @@ TEST_F(DiskIoMgrTest, MultipleReaderWriter) {
             int num_scan_ranges = min<int>(num_reads_queued, write_offset - read_offset);
             for (int i = 0; i < num_scan_ranges; ++i) {
               ranges.push_back(InitRange(
-                  &tmp_pool, file_name.c_str(), read_offset, 1, i % num_disks, stat_val.st_mtime));
-              threads.add_thread(new thread(ScanRangeThread, &io_mgr,
-                  contexts[context_index].get(), &clients[context_index],
-                  reinterpret_cast<const char*>(data + (read_offset % strlen(data))),
-                  1, Status::OK(), num_scan_ranges, &num_ranges_processed));
+                  file_name.c_str(), read_offset, 1, i % num_disks, stat_val.st_mtime));
+              threads.add_thread(
+                  new thread(ScanRangeThread, &io_mgr, contexts[context_index].get(),
+                      reinterpret_cast<const char*>(data + (read_offset % strlen(data))),
+                      1, Status::OK(), num_scan_ranges, &num_ranges_processed));
               ++read_offset;
             }
 
@@ -894,7 +792,7 @@ TEST_F(DiskIoMgrTest, MultipleReaderWriter) {
               WriteRange::WriteDoneCallback callback =
                   bind(mem_fn(&DiskIoMgrTest::WriteCompleteCallback),
                       this, num_write_ranges, _1);
-              WriteRange* new_range = tmp_pool.Add(new WriteRange(
+              WriteRange* new_range = pool_.Add(new WriteRange(
                   file_name, write_offset, i % num_disks, callback));
               new_range->SetData(
                   reinterpret_cast<const uint8_t*>(data + (write_offset % strlen(data))),
@@ -911,9 +809,9 @@ TEST_F(DiskIoMgrTest, MultipleReaderWriter) {
             threads.join_all();
           } // for (int context_index
         } // while (read_offset < file_size)
+
         for (int file_index = 0; file_index < num_contexts; ++file_index) {
           io_mgr.UnregisterContext(contexts[file_index].get());
-          buffer_pool()->DeregisterClient(&clients[file_index]);
         }
       } // for (int num_disks
     } // for (int threads_per_disk
@@ -922,19 +820,23 @@ TEST_F(DiskIoMgrTest, MultipleReaderWriter) {
 
 // This test will test multiple concurrent reads each reading a different file.
 TEST_F(DiskIoMgrTest, MultipleReader) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   const int NUM_READERS = 5;
   const int DATA_LEN = 50;
   const int ITERATIONS = 25;
   const int NUM_THREADS_PER_READER = 3;
 
-  vector<string> file_names(NUM_READERS);
-  vector<int64_t> mtimes(NUM_READERS);
-  vector<string> data(NUM_READERS);
-  unique_ptr<BufferPool::ClientHandle[]> clients(
-      new BufferPool::ClientHandle[NUM_READERS]);
-  vector<unique_ptr<RequestContext>> readers(NUM_READERS);
-  vector<char*> results(NUM_READERS);
+  vector<string> file_names;
+  vector<int64_t> mtimes;
+  vector<string> data;
+  vector<unique_ptr<RequestContext>> readers;
+  vector<char*> results;
+
+  file_names.resize(NUM_READERS);
+  readers.resize(NUM_READERS);
+  mtimes.resize(NUM_READERS);
+  data.resize(NUM_READERS);
+  results.resize(NUM_READERS);
 
   // Initialize data for each reader.  The data will be
   // 'abcd...' for reader one, 'bcde...' for reader two (wrapping around at 'z')
@@ -965,25 +867,22 @@ TEST_F(DiskIoMgrTest, MultipleReader) {
   for (int iteration = 0; iteration < ITERATIONS; ++iteration) {
     for (int threads_per_disk = 1; threads_per_disk <= 5; ++threads_per_disk) {
       for (int num_disks = 1; num_disks <= 5; num_disks += 2) {
-        // Pool for temporary objects from this iteration only.
-        ObjectPool tmp_pool;
+        pool_.Clear(); // Destroy scan ranges from previous iterations.
         LOG(INFO) << "Starting test with num_threads_per_disk=" << threads_per_disk
                   << " num_disk=" << num_disks;
         if (++iters % 2500 == 0) LOG(ERROR) << "Starting iteration " << iters;
 
         DiskIoMgr io_mgr(num_disks, threads_per_disk, threads_per_disk, MIN_BUFFER_SIZE,
             MAX_BUFFER_SIZE);
-        ASSERT_OK(io_mgr.Init());
+        EXPECT_OK(io_mgr.Init(&mem_tracker));
 
         for (int i = 0; i < NUM_READERS; ++i) {
-          RegisterBufferPoolClient(
-              LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &clients[i]);
-          readers[i] = io_mgr.RegisterContext();
+          readers[i] = io_mgr.RegisterContext(&mem_tracker);
 
           vector<ScanRange*> ranges;
           for (int j = 0; j < DATA_LEN; ++j) {
             int disk_id = j % num_disks;
-            ranges.push_back(InitRange(&tmp_pool, file_names[i].c_str(), j, 1, disk_id, mtimes[i]));
+            ranges.push_back(InitRange(file_names[i].c_str(), j, 1, disk_id, mtimes[i]));
           }
           ASSERT_OK(io_mgr.AddScanRanges(readers[i].get(), ranges));
         }
@@ -993,20 +892,18 @@ TEST_F(DiskIoMgrTest, MultipleReader) {
         for (int i = 0; i < NUM_READERS; ++i) {
           for (int j = 0; j < NUM_THREADS_PER_READER; ++j) {
             threads.add_thread(new thread(ScanRangeThread, &io_mgr, readers[i].get(),
-                &clients[i], data[i].c_str(), data[i].size(), Status::OK(), 0,
-                &num_ranges_processed));
+                data[i].c_str(), data[i].size(), Status::OK(), 0, &num_ranges_processed));
           }
         }
         threads.join_all();
         EXPECT_EQ(num_ranges_processed.Load(), DATA_LEN * NUM_READERS);
         for (int i = 0; i < NUM_READERS; ++i) {
           io_mgr.UnregisterContext(readers[i].get());
-          buffer_pool()->DeregisterClient(&clients[i]);
         }
       }
     }
   }
-  EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+  EXPECT_EQ(mem_tracker.consumption(), 0);
 }
 
 // Stress test for multiple clients with cancellation
@@ -1018,148 +915,115 @@ TEST_F(DiskIoMgrTest, StressTest) {
   test.Run(2); // In seconds
 }
 
-// IMPALA-2366: handle partial read where range goes past end of file.
-TEST_F(DiskIoMgrTest, PartialRead) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
-  const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
-  const char* data = "the quick brown fox jumped over the lazy dog";
-  int len = strlen(data);
-  int read_len = len + 1000; // Read past end of file.
-  // Test with various buffer sizes to exercise different code paths, e.g.
-  // * the truncated data ends exactly on a buffer boundary
-  // * the data is split between many buffers
-  // * the data fits in one buffer
-  const int64_t MIN_BUFFER_SIZE = 2;
-  vector<int64_t> max_buffer_sizes{4, 16, 32, 128, 1024, 4096};
-  CreateTempFile(tmp_file, data);
-
-  // Get mtime for file
-  struct stat stat_val;
-  stat(tmp_file, &stat_val);
-
-  for (int64_t max_buffer_size : max_buffer_sizes) {
-    DiskIoMgr io_mgr(1, 1, 1, MIN_BUFFER_SIZE, max_buffer_size);
-    ASSERT_OK(io_mgr.Init());
-    unique_ptr<RequestContext> reader;
-    reader = io_mgr.RegisterContext();
-
-    BufferPool::ClientHandle read_client;
-    RegisterBufferPoolClient(
-        LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
-
-    // We should not read past the end of file.
-    ScanRange* range = InitRange(&pool_, tmp_file, 0, read_len, 0, stat_val.st_mtime);
-    unique_ptr<BufferDescriptor> buffer;
-    bool needs_buffers;
-    ASSERT_OK(io_mgr.StartScanRange(reader.get(), range, &needs_buffers));
-    if (needs_buffers) {
-      ASSERT_OK(io_mgr.AllocateBuffersForRange(
-          reader.get(), &read_client, range, 3 * max_buffer_size));
-    }
-
-    int64_t bytes_read = 0;
-    bool eosr = false;
-    do {
-      ASSERT_OK(range->GetNext(&buffer));
-      ASSERT_GE(buffer->buffer_len(), MIN_BUFFER_SIZE);
-      ASSERT_LE(buffer->buffer_len(), max_buffer_size);
-      ASSERT_LE(buffer->len(), len - bytes_read);
-      ASSERT_TRUE(memcmp(buffer->buffer(), data + bytes_read, buffer->len()) == 0);
-      bytes_read += buffer->len();
-      eosr = buffer->eosr();
-      // Should see eosr if we've read past the end of the file. If the data is an exact
-      // multiple of the max buffer size then we may read to the end of the file without
-      // noticing that it is eosr. Eosr will be returned on the next read in that case.
-      ASSERT_TRUE(bytes_read < len || buffer->eosr()
-          || (buffer->len() == max_buffer_size && len % max_buffer_size == 0))
-          << "max_buffer_size " << max_buffer_size << " bytes_read " << bytes_read
-          << "len " << len << " buffer->len() " << buffer->len()
-          << " buffer->buffer_len() " << buffer->buffer_len();
-      ASSERT_TRUE(buffer->len() > 0 || buffer->eosr());
-      range->ReturnBuffer(move(buffer));
-    } while (!eosr);
-
-    io_mgr.UnregisterContext(reader.get());
-    EXPECT_EQ(read_client.GetUsedReservation(), 0);
-    buffer_pool()->DeregisterClient(&read_client);
-  }
-}
-
-// Test zero-length scan range.
-TEST_F(DiskIoMgrTest, ZeroLengthScanRange) {
-  const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
-  const char* data = "the quick brown fox jumped over the lazy dog";
-  const int64_t MIN_BUFFER_SIZE = 2;
-  const int64_t MAX_BUFFER_SIZE = 1024;
-  CreateTempFile(tmp_file, data);
-
-  // Get mtime for file
-  struct stat stat_val;
-  stat(tmp_file, &stat_val);
-
-  DiskIoMgr io_mgr(1, 1, 1, MIN_BUFFER_SIZE, MAX_BUFFER_SIZE);
-
-  ASSERT_OK(io_mgr.Init());
-  unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
-
-  ScanRange* range = InitRange(&pool_, tmp_file, 0, 0, 0, stat_val.st_mtime);
-  bool needs_buffers;
-  Status status = io_mgr.StartScanRange(reader.get(), range, &needs_buffers);
-  ASSERT_EQ(TErrorCode::DISK_IO_ERROR, status.code());
-
-  status = io_mgr.AddScanRanges(reader.get(), vector<ScanRange*>({range}));
-  ASSERT_EQ(TErrorCode::DISK_IO_ERROR, status.code());
-
+TEST_F(DiskIoMgrTest, Buffers) {
+  // Test default min/max buffer size
+  int min_buffer_size = 1024;
+  int max_buffer_size = 8 * 1024 * 1024; // 8 MB
+  MemTracker root_mem_tracker(max_buffer_size * 2);
+
+  DiskIoMgr io_mgr(1, 1, 1, min_buffer_size, max_buffer_size);
+  ASSERT_OK(io_mgr.Init(&root_mem_tracker));
+  ASSERT_EQ(root_mem_tracker.consumption(), 0);
+
+  MemTracker reader_mem_tracker(-1, "Reader", &root_mem_tracker);
+  unique_ptr<RequestContext> reader;
+  reader = io_mgr.RegisterContext(&reader_mem_tracker);
+
+  ScanRange* dummy_range = InitRange("dummy", 0, 0, 0, 0);
+
+  // buffer length should be rounded up to min buffer size
+  int64_t buffer_len = 1;
+  unique_ptr<BufferDescriptor> buffer_desc;
+  buffer_desc = io_mgr.GetFreeBuffer(reader.get(), dummy_range, buffer_len);
+  EXPECT_TRUE(buffer_desc->buffer() != nullptr);
+  EXPECT_EQ(min_buffer_size, buffer_desc->buffer_len());
+  EXPECT_EQ(1, io_mgr.num_allocated_buffers_.Load());
+  io_mgr.FreeBufferMemory(buffer_desc.get());
+  io_mgr.ReturnBuffer(move(buffer_desc));
+  EXPECT_EQ(min_buffer_size, root_mem_tracker.consumption());
+
+  // reuse buffer
+  buffer_len = min_buffer_size;
+  buffer_desc = io_mgr.GetFreeBuffer(reader.get(), dummy_range, buffer_len);
+  EXPECT_TRUE(buffer_desc->buffer() != nullptr);
+  EXPECT_EQ(min_buffer_size, buffer_desc->buffer_len());
+  EXPECT_EQ(1, io_mgr.num_allocated_buffers_.Load());
+  io_mgr.FreeBufferMemory(buffer_desc.get());
+  io_mgr.ReturnBuffer(move(buffer_desc));
+  EXPECT_EQ(min_buffer_size, root_mem_tracker.consumption());
+
+  // bump up to next buffer size
+  buffer_len = min_buffer_size + 1;
+  buffer_desc = io_mgr.GetFreeBuffer(reader.get(), dummy_range, buffer_len);
+  EXPECT_TRUE(buffer_desc->buffer() != nullptr);
+  EXPECT_EQ(min_buffer_size * 2, buffer_desc->buffer_len());
+  EXPECT_EQ(2, io_mgr.num_allocated_buffers_.Load());
+  EXPECT_EQ(min_buffer_size * 3, root_mem_tracker.consumption());
+
+  // gc unused buffer
+  io_mgr.GcIoBuffers();
+  EXPECT_EQ(1, io_mgr.num_allocated_buffers_.Load());
+  EXPECT_EQ(min_buffer_size * 2, root_mem_tracker.consumption());
+
+  io_mgr.FreeBufferMemory(buffer_desc.get());
+  io_mgr.ReturnBuffer(move(buffer_desc));
+
+  // max buffer size
+  buffer_len = max_buffer_size;
+  buffer_desc = io_mgr.GetFreeBuffer(reader.get(), dummy_range, buffer_len);
+  EXPECT_TRUE(buffer_desc->buffer() != nullptr);
+  EXPECT_EQ(max_buffer_size, buffer_desc->buffer_len());
+  EXPECT_EQ(2, io_mgr.num_allocated_buffers_.Load());
+  io_mgr.FreeBufferMemory(buffer_desc.get());
+  io_mgr.ReturnBuffer(move(buffer_desc));
+  EXPECT_EQ(min_buffer_size * 2 + max_buffer_size, root_mem_tracker.consumption());
+
+  // gc buffers
+  io_mgr.GcIoBuffers();
+  EXPECT_EQ(io_mgr.num_allocated_buffers_.Load(), 0);
+  EXPECT_EQ(root_mem_tracker.consumption(), 0);
   io_mgr.UnregisterContext(reader.get());
 }
 
-// Test what happens if don't call AllocateBuffersForRange() after trying to start a
-// range.
-TEST_F(DiskIoMgrTest, SkipAllocateBuffers) {
+// IMPALA-2366: handle partial read where range goes past end of file.
+TEST_F(DiskIoMgrTest, PartialRead) {
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
   const char* data = "the quick brown fox jumped over the lazy dog";
   int len = strlen(data);
-  const int64_t MIN_BUFFER_SIZE = 2;
-  const int64_t MAX_BUFFER_SIZE = 1024;
+  int read_len = len + 1000; // Read past end of file.
   CreateTempFile(tmp_file, data);
 
   // Get mtime for file
   struct stat stat_val;
   stat(tmp_file, &stat_val);
 
-  DiskIoMgr io_mgr(1, 1, 1, MIN_BUFFER_SIZE, MAX_BUFFER_SIZE);
+  scoped_ptr<DiskIoMgr> io_mgr(new DiskIoMgr(1, 1, 1, read_len, read_len));
 
-  ASSERT_OK(io_mgr.Init());
+  ASSERT_OK(io_mgr->Init(&mem_tracker));
   MemTracker reader_mem_tracker;
-  unique_ptr<RequestContext> reader = io_mgr.RegisterContext();
+  unique_ptr<RequestContext> reader;
+  reader = io_mgr->RegisterContext(&reader_mem_tracker);
 
   // We should not read past the end of file.
-  vector<ScanRange*> ranges;
-  for (int i = 0; i < 4; ++i) {
-    ranges.push_back(InitRange(&pool_, tmp_file, 0, len, 0, stat_val.st_mtime));
-  }
-  bool needs_buffers;
-  // Test StartScanRange().
-  ASSERT_OK(io_mgr.StartScanRange(reader.get(), ranges[0], &needs_buffers));
-  EXPECT_TRUE(needs_buffers);
-  ASSERT_OK(io_mgr.StartScanRange(reader.get(), ranges[1], &needs_buffers));
-  EXPECT_TRUE(needs_buffers);
-
-  // Test AddScanRanges()/GetNextUnstartedRange().
-  ASSERT_OK(
-      io_mgr.AddScanRanges(reader.get(), vector<ScanRange*>({ranges[2], ranges[3]})));
-
-  // Cancel two directly, cancel the other two indirectly via the context.
-  ranges[0]->Cancel(Status::CANCELLED);
-  ranges[2]->Cancel(Status::CANCELLED);
-  reader->Cancel();
+  ScanRange* range = InitRange(tmp_file, 0, read_len, 0, stat_val.st_mtime);
+  unique_ptr<BufferDescriptor> buffer;
+  ASSERT_OK(io_mgr->Read(reader.get(), range, &buffer));
+  ASSERT_TRUE(buffer->eosr());
+  ASSERT_EQ(len, buffer->len());
+  ASSERT_TRUE(memcmp(buffer->buffer(), data, len) == 0);
+  io_mgr->ReturnBuffer(move(buffer));
 
-  io_mgr.UnregisterContext(reader.get());
+  io_mgr->UnregisterContext(reader.get());
+  pool_.Clear();
+  io_mgr.reset();
+  EXPECT_EQ(reader_mem_tracker.consumption(), 0);
+  EXPECT_EQ(mem_tracker.consumption(), 0);
 }
 
 // Test reading into a client-allocated buffer.
 TEST_F(DiskIoMgrTest, ReadIntoClientBuffer) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   const char* tmp_file = "/tmp/disk_io_mgr_test.txt";
   const char* data = "the quick brown fox jumped over the lazy dog";
   int len = strlen(data);
@@ -1168,19 +1032,19 @@ TEST_F(DiskIoMgrTest, ReadIntoClientBuffer) {
 
   scoped_ptr<DiskIoMgr> io_mgr(new DiskIoMgr(1, 1, 1, read_len, read_len));
 
-  ASSERT_OK(io_mgr->Init());
-  // Reader doesn't need to provide client if it's providing buffers.
-  unique_ptr<RequestContext> reader = io_mgr->RegisterContext();
+  ASSERT_OK(io_mgr->Init(&mem_tracker));
+  // Reader doesn't need to provide mem tracker if it's providing buffers.
+  MemTracker* reader_mem_tracker = nullptr;
+  unique_ptr<RequestContext> reader;
+  reader = io_mgr->RegisterContext(reader_mem_tracker);
 
   for (int buffer_len : vector<int>({len - 1, len, len + 1})) {
     vector<uint8_t> client_buffer(buffer_len);
     int scan_len = min(len, buffer_len);
-    ScanRange* range = pool_.Add(new ScanRange);
+    ScanRange* range = AllocateRange();
     range->Reset(nullptr, tmp_file, scan_len, 0, 0, true,
         BufferOpts::ReadInto(client_buffer.data(), buffer_len));
-    bool needs_buffers;
-    ASSERT_OK(io_mgr->StartScanRange(reader.get(), range, &needs_buffers));
-    ASSERT_FALSE(needs_buffers);
+    ASSERT_OK(io_mgr->AddScanRange(reader.get(), range, true));
 
     unique_ptr<BufferDescriptor> io_buffer;
     ASSERT_OK(range->GetNext(&io_buffer));
@@ -1190,133 +1054,76 @@ TEST_F(DiskIoMgrTest, ReadIntoClientBuffer) {
     ASSERT_EQ(memcmp(io_buffer->buffer(), data, scan_len), 0);
 
     // DiskIoMgr should not have allocated memory.
-    EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
-    range->ReturnBuffer(move(io_buffer));
+    EXPECT_EQ(mem_tracker.consumption(), 0);
+    io_mgr->ReturnBuffer(move(io_buffer));
   }
 
   io_mgr->UnregisterContext(reader.get());
-  EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+  pool_.Clear();
+  io_mgr.reset();
+  EXPECT_EQ(mem_tracker.consumption(), 0);
 }
 
 // Test reading into a client-allocated buffer where the read fails.
 TEST_F(DiskIoMgrTest, ReadIntoClientBufferError) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   const char* tmp_file = "/file/that/does/not/exist";
   const int SCAN_LEN = 128;
 
   scoped_ptr<DiskIoMgr> io_mgr(new DiskIoMgr(1, 1, 1, SCAN_LEN, SCAN_LEN));
 
-  ASSERT_OK(io_mgr->Init());
+  ASSERT_OK(io_mgr->Init(&mem_tracker));
+  // Reader doesn't need to provide mem tracker if it's providing buffers.
+  MemTracker* reader_mem_tracker = nullptr;
+  unique_ptr<RequestContext> reader;
   vector<uint8_t> client_buffer(SCAN_LEN);
   for (int i = 0; i < 1000; ++i) {
-    // Reader doesn't need to provide mem tracker if it's providing buffers.
-    BufferPool::ClientHandle read_client;
-    RegisterBufferPoolClient(
-        LARGE_RESERVATION_LIMIT, LARGE_INITIAL_RESERVATION, &read_client);
-    unique_ptr<RequestContext> reader = io_mgr->RegisterContext();
-    ScanRange* range = pool_.Add(new ScanRange);
+    reader = io_mgr->RegisterContext(reader_mem_tracker);
+    ScanRange* range = AllocateRange();
     range->Reset(nullptr, tmp_file, SCAN_LEN, 0, 0, true,
         BufferOpts::ReadInto(client_buffer.data(), SCAN_LEN));
-    bool needs_buffers;
-    ASSERT_OK(io_mgr->StartScanRange(reader.get(), range, &needs_buffers));
-    ASSERT_FALSE(needs_buffers);
+    ASSERT_OK(io_mgr->AddScanRange(reader.get(), range, true));
 
     /// Also test the cancellation path. Run multiple iterations since it is racy whether
     /// the read fails before the cancellation.
-    if (i >= 1) reader->Cancel();
+    if (i >= 1) io_mgr->CancelContext(reader.get());
 
     unique_ptr<BufferDescriptor> io_buffer;
     ASSERT_FALSE(range->GetNext(&io_buffer).ok());
 
     // DiskIoMgr should not have allocated memory.
-    EXPECT_EQ(read_client.GetUsedReservation(), 0);
+    EXPECT_EQ(mem_tracker.consumption(), 0);
 
     io_mgr->UnregisterContext(reader.get());
-    EXPECT_EQ(read_client.GetUsedReservation(), 0);
-    buffer_pool()->DeregisterClient(&read_client);
   }
 
-  EXPECT_EQ(root_reservation_.GetChildReservations(), 0);
+  pool_.Clear();
+  io_mgr.reset();
+  EXPECT_EQ(mem_tracker.consumption(), 0);
 }
 
 // Test to verify configuration parameters for number of I/O threads per disk.
 TEST_F(DiskIoMgrTest, VerifyNumThreadsParameter) {
-  InitRootReservation(LARGE_RESERVATION_LIMIT);
   const int num_io_threads_for_remote_disks = FLAGS_num_remote_hdfs_io_threads
       + FLAGS_num_s3_io_threads + FLAGS_num_adls_io_threads;
 
   // Verify num_io_threads_per_rotational_disk and num_io_threads_per_solid_state_disk.
   // Since we do not have control over which disk is used, we check for either type
   // (rotational/solid state)
+  MemTracker mem_tracker(LARGE_MEM_LIMIT);
   const int num_io_threads_per_rotational_or_ssd = 2;
   DiskIoMgr io_mgr(1, num_io_threads_per_rotational_or_ssd,
       num_io_threads_per_rotational_or_ssd, 1, 10);
-  ASSERT_OK(io_mgr.Init());
+  ASSERT_OK(io_mgr.Init(&mem_tracker));
   const int num_io_threads = io_mgr.disk_thread_group_.Size();
   ASSERT_TRUE(num_io_threads ==
       num_io_threads_per_rotational_or_ssd + num_io_threads_for_remote_disks);
 }
-
-// Test to verify that the correct buffer sizes are chosen given different
-// of scan range lengths and max_bytes values.
-TEST_F(DiskIoMgrTest, BufferSizeSelection) {
-  DiskIoMgr io_mgr(1, 1, 1, MIN_BUFFER_SIZE, MAX_BUFFER_SIZE);
-  ASSERT_OK(io_mgr.Init());
-
-  // Scan range doesn't fit in max_bytes - allocate as many max-sized buffers as possible.
-  EXPECT_EQ(vector<int64_t>(3, MAX_BUFFER_SIZE),
-      io_mgr.ChooseBufferSizes(10 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE));
-  EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE}),
-      io_mgr.ChooseBufferSizes(10 * MAX_BUFFER_SIZE, MAX_BUFFER_SIZE));
-  EXPECT_EQ(vector<int64_t>(4, MAX_BUFFER_SIZE),
-      io_mgr.ChooseBufferSizes(10 * MAX_BUFFER_SIZE, 4 * MAX_BUFFER_SIZE));
-
-  // Scan range fits in max_bytes - allocate as many max-sized buffers as possible, then
-  // a smaller buffer to fit the remainder.
-  EXPECT_EQ(vector<int64_t>(2, MAX_BUFFER_SIZE),
-      io_mgr.ChooseBufferSizes(2 * MAX_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE));
-  EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE, MAX_BUFFER_SIZE, MIN_BUFFER_SIZE}),
-      io_mgr.ChooseBufferSizes(2 * MAX_BUFFER_SIZE + 1, 3 * MAX_BUFFER_SIZE));
-  EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE, MAX_BUFFER_SIZE, 2 * MIN_BUFFER_SIZE}),
-      io_mgr.ChooseBufferSizes(
-        2 * MAX_BUFFER_SIZE + MIN_BUFFER_SIZE + 1, 3 * MAX_BUFFER_SIZE));
-  EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE, MAX_BUFFER_SIZE, 2 * MIN_BUFFER_SIZE}),
-      io_mgr.ChooseBufferSizes(
-        2 * MAX_BUFFER_SIZE + 2 * MIN_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE));
-
-  // Scan range is smaller than max buffer size - allocate a single buffer that fits
-  // the range.
-  EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE}),
-      io_mgr.ChooseBufferSizes(MAX_BUFFER_SIZE - 1, 3 * MAX_BUFFER_SIZE));
-  EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE / 2}),
-      io_mgr.ChooseBufferSizes(MAX_BUFFER_SIZE - 1, MAX_BUFFER_SIZE / 2));
-  EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE / 2}),
-      io_mgr.ChooseBufferSizes(MAX_BUFFER_SIZE / 2 - 1, 3 * MAX_BUFFER_SIZE));
-  EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE / 2}),
-      io_mgr.ChooseBufferSizes(MAX_BUFFER_SIZE / 2- 1, MAX_BUFFER_SIZE / 2));
-  EXPECT_EQ(vector<int64_t>({MIN_BUFFER_SIZE}),
-      io_mgr.ChooseBufferSizes(MIN_BUFFER_SIZE, 3 * MAX_BUFFER_SIZE));
-  EXPECT_EQ(vector<int64_t>({MIN_BUFFER_SIZE}),
-      io_mgr.ChooseBufferSizes(MIN_BUFFER_SIZE, MIN_BUFFER_SIZE));
-
-  // Scan range is smaller than max buffer size and max bytes is smaller still -
-  // should allocate a single smaller buffer.
-  EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE / 4}),
-      io_mgr.ChooseBufferSizes(MAX_BUFFER_SIZE / 2, MAX_BUFFER_SIZE / 2 - 1));
-
-  // Non power-of-two size > max buffer size.
-  EXPECT_EQ(vector<int64_t>({MAX_BUFFER_SIZE, MIN_BUFFER_SIZE}),
-      io_mgr.ChooseBufferSizes(MAX_BUFFER_SIZE + 7, 3 * MAX_BUFFER_SIZE));
-  // Non power-of-two size < min buffer size.
-  EXPECT_EQ(vector<int64_t>({MIN_BUFFER_SIZE}),
-      io_mgr.ChooseBufferSizes(MIN_BUFFER_SIZE - 7, 3 * MAX_BUFFER_SIZE));
-}
 }
 }
 
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);
   impala::InitCommonRuntime(argc, argv, true, impala::TestInfo::BE_TEST);
-  impala::InitFeSupport();
   return RUN_ALL_TESTS();
 }


[8/9] impala git commit: Revert IMPALA-4835 and dependent changes

Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/scanner-context.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/scanner-context.cc b/be/src/exec/scanner-context.cc
index c669e65..abdde07 100644
--- a/be/src/exec/scanner-context.cc
+++ b/be/src/exec/scanner-context.cc
@@ -41,15 +41,14 @@ static const int64_t INIT_READ_PAST_SIZE_BYTES = 64 * 1024;
 const int64_t ScannerContext::Stream::OUTPUT_BUFFER_BYTES_LEFT_INIT;
 
 ScannerContext::ScannerContext(RuntimeState* state, HdfsScanNodeBase* scan_node,
-    BufferPool::ClientHandle* bp_client, HdfsPartitionDescriptor* partition_desc,
-    const vector<FilterContext>& filter_ctxs,
-    MemPool* expr_results_pool)
+    HdfsPartitionDescriptor* partition_desc, ScanRange* scan_range,
+    const vector<FilterContext>& filter_ctxs, MemPool* expr_results_pool)
   : state_(state),
     scan_node_(scan_node),
-    bp_client_(bp_client),
     partition_desc_(partition_desc),
     filter_ctxs_(filter_ctxs),
     expr_results_pool_(expr_results_pool) {
+  AddStream(scan_range);
 }
 
 ScannerContext::~ScannerContext() {
@@ -67,20 +66,19 @@ void ScannerContext::ClearStreams() {
 }
 
 ScannerContext::Stream::Stream(ScannerContext* parent, ScanRange* scan_range,
-    int64_t reservation, const HdfsFileDesc* file_desc)
+    const HdfsFileDesc* file_desc)
   : parent_(parent),
     scan_range_(scan_range),
     file_desc_(file_desc),
-    reservation_(reservation),
     file_len_(file_desc->file_length),
     next_read_past_size_bytes_(INIT_READ_PAST_SIZE_BYTES),
     boundary_pool_(new MemPool(parent->scan_node_->mem_tracker())),
     boundary_buffer_(new StringBuffer(boundary_pool_.get())) {
 }
 
-ScannerContext::Stream* ScannerContext::AddStream(ScanRange* range, int64_t reservation) {
-  streams_.emplace_back(new Stream(this, range, reservation,
-      scan_node_->GetFileDesc(partition_desc_->id(), range->file())));
+ScannerContext::Stream* ScannerContext::AddStream(ScanRange* range) {
+  streams_.emplace_back(new Stream(
+      this, range, scan_node_->GetFileDesc(partition_desc_->id(), range->file())));
   return streams_.back().get();
 }
 
@@ -103,7 +101,6 @@ void ScannerContext::Stream::ReleaseCompletedResources(bool done) {
 
 Status ScannerContext::Stream::GetNextBuffer(int64_t read_past_size) {
   DCHECK_EQ(0, io_buffer_bytes_left_);
-  DiskIoMgr* io_mgr = ExecEnv::GetInstance()->disk_io_mgr();
   if (UNLIKELY(parent_->cancelled())) return Status::CANCELLED;
   if (io_buffer_ != nullptr) ReturnIoBuffer();
 
@@ -124,7 +121,7 @@ Status ScannerContext::Stream::GetNextBuffer(int64_t read_past_size) {
     SCOPED_TIMER(parent_->state_->total_storage_wait_timer());
 
     int64_t read_past_buffer_size = 0;
-    int64_t max_buffer_size = io_mgr->max_buffer_size();
+    int64_t max_buffer_size = parent_->state_->io_mgr()->max_read_buffer_size();
     if (!read_past_size_cb_.empty()) read_past_buffer_size = read_past_size_cb_(offset);
     if (read_past_buffer_size <= 0) {
       // Either no callback was set or the callback did not return an estimate. Use
@@ -136,7 +133,6 @@ Status ScannerContext::Stream::GetNextBuffer(int64_t read_past_size) {
     read_past_buffer_size = ::max(read_past_buffer_size, read_past_size);
     read_past_buffer_size = ::min(read_past_buffer_size, file_bytes_remaining);
     read_past_buffer_size = ::min(read_past_buffer_size, max_buffer_size);
-    read_past_buffer_size = ::min(read_past_buffer_size, reservation_);
     // We're reading past the scan range. Be careful not to read past the end of file.
     DCHECK_GE(read_past_buffer_size, 0);
     if (read_past_buffer_size == 0) {
@@ -147,23 +143,8 @@ Status ScannerContext::Stream::GetNextBuffer(int64_t read_past_size) {
     ScanRange* range = parent_->scan_node_->AllocateScanRange(
         scan_range_->fs(), filename(), read_past_buffer_size, offset, partition_id,
         scan_range_->disk_id(), false, BufferOpts::Uncached());
-    bool needs_buffers;
-    RETURN_IF_ERROR(io_mgr->StartScanRange(
-        parent_->scan_node_->reader_context(), range, &needs_buffers));
-    if (needs_buffers) {
-      // Allocate fresh buffers. The buffers for 'scan_range_' should be released now
-      // since we hit EOS.
-      if (reservation_ < io_mgr->min_buffer_size()) {
-        return Status(Substitute("Could not read past end of scan range in file '$0'. "
-            "Reservation provided $1 was < the minimum I/O buffer size",
-            reservation_, io_mgr->min_buffer_size()));
-      }
-      RETURN_IF_ERROR(io_mgr->AllocateBuffersForRange(
-          parent_->scan_node_->reader_context(), parent_->bp_client_, range,
-          reservation_));
-    }
-    RETURN_IF_ERROR(range->GetNext(&io_buffer_));
-    DCHECK(io_buffer_->eosr());
+    RETURN_IF_ERROR(parent_->state_->io_mgr()->Read(
+        parent_->scan_node_->reader_context(), range, &io_buffer_));
   }
 
   DCHECK(io_buffer_ != nullptr);
@@ -343,8 +324,7 @@ Status ScannerContext::Stream::CopyIoToBoundary(int64_t num_bytes) {
 
 void ScannerContext::Stream::ReturnIoBuffer() {
   DCHECK(io_buffer_ != nullptr);
-  ScanRange* range = io_buffer_->scan_range();
-  range->ReturnBuffer(move(io_buffer_));
+  ExecEnv::GetInstance()->disk_io_mgr()->ReturnBuffer(move(io_buffer_));
   io_buffer_pos_ = nullptr;
   io_buffer_bytes_left_ = 0;
 }

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/exec/scanner-context.h
----------------------------------------------------------------------
diff --git a/be/src/exec/scanner-context.h b/be/src/exec/scanner-context.h
index 6292486..a131d3f 100644
--- a/be/src/exec/scanner-context.h
+++ b/be/src/exec/scanner-context.h
@@ -27,7 +27,6 @@
 #include "common/compiler-util.h"
 #include "common/status.h"
 #include "exec/filter-context.h"
-#include "runtime/bufferpool/buffer-pool.h"
 #include "runtime/io/request-ranges.h"
 
 namespace impala {
@@ -85,12 +84,10 @@ class TupleRow;
 class ScannerContext {
  public:
   /// Create a scanner context with the parent scan_node (where materialized row batches
-  /// get pushed to) and the scan range to process. Buffers are allocated using
-  /// 'bp_client'.
-  ScannerContext(RuntimeState* state, HdfsScanNodeBase* scan_node,
-      BufferPool::ClientHandle* bp_client,
-      HdfsPartitionDescriptor* partition_desc,
-      const std::vector<FilterContext>& filter_ctxs,
+  /// get pushed to) and the scan range to process.
+  /// This context starts with 1 stream.
+  ScannerContext(RuntimeState*, HdfsScanNodeBase*, HdfsPartitionDescriptor*,
+      io::ScanRange* scan_range, const std::vector<FilterContext>& filter_ctxs,
       MemPool* expr_results_pool);
   /// Destructor verifies that all stream objects have been released.
   ~ScannerContext();
@@ -153,7 +150,6 @@ class ScannerContext {
     const char* filename() { return scan_range_->file(); }
     const io::ScanRange* scan_range() { return scan_range_; }
     const HdfsFileDesc* file_desc() { return file_desc_; }
-    int64_t reservation() const { return reservation_; }
 
     /// Returns the buffer's current offset in the file.
     int64_t file_offset() const { return scan_range_->offset() + total_bytes_returned_; }
@@ -215,15 +211,9 @@ class ScannerContext {
 
    private:
     friend class ScannerContext;
-    ScannerContext* const parent_;
-    io::ScanRange* const scan_range_;
-    const HdfsFileDesc* const file_desc_;
-
-    /// Reservation given to this stream for allocating I/O buffers. The reservation is
-    /// shared with 'scan_range_', so the context must be careful not to use this until
-    /// all of 'scan_ranges_'s buffers have been freed. Must be >= the minimum IoMgr
-    /// buffer size to allow reading past the end of 'scan_range_'.
-    const int64_t reservation_;
+    ScannerContext* parent_;
+    io::ScanRange* scan_range_;
+    const HdfsFileDesc* file_desc_;
 
     /// Total number of bytes returned from GetBytes()
     int64_t total_bytes_returned_ = 0;
@@ -282,8 +272,7 @@ class ScannerContext {
     /// output_buffer_bytes_left_ will be set to something else.
     static const int64_t OUTPUT_BUFFER_BYTES_LEFT_INIT = 0;
 
-    /// Private constructor. See AddStream() for public API.
-    Stream(ScannerContext* parent, io::ScanRange* scan_range, int64_t reservation,
+    Stream(ScannerContext* parent, io::ScanRange* scan_range,
         const HdfsFileDesc* file_desc);
 
     /// GetBytes helper to handle the slow path.
@@ -366,37 +355,24 @@ class ScannerContext {
   /// size to 0.
   void ClearStreams();
 
-  /// Add a stream to this ScannerContext for 'range'. 'range' must already have any
-  /// buffers that it needs allocated. 'reservation' is the amount of reservation that
-  /// is given to this stream for allocating I/O buffers. The reservation is shared with
-  /// 'range', so the context must be careful not to use this until all of 'range's
-  /// buffers have been freed. Must be >= the minimum IoMgr buffer size o allow reading
-  /// past the end of 'range'.
-  ///
-  /// Returns the added stream. The returned stream is owned by this context.
-  Stream* AddStream(io::ScanRange* range, int64_t reservation);
+  /// Add a stream to this ScannerContext for 'range'. The stream is owned by this
+  /// context.
+  Stream* AddStream(io::ScanRange* range);
 
   /// Returns true if RuntimeState::is_cancelled() is true, or if scan node is not
   /// multi-threaded and is done (finished, cancelled or reached it's limit).
   /// In all other cases returns false.
   bool cancelled() const;
 
-  BufferPool::ClientHandle* bp_client() const { return bp_client_; }
-  HdfsPartitionDescriptor* partition_descriptor() const { return partition_desc_; }
+  HdfsPartitionDescriptor* partition_descriptor() { return partition_desc_; }
   const std::vector<FilterContext>& filter_ctxs() const { return filter_ctxs_; }
   MemPool* expr_results_pool() const { return expr_results_pool_; }
  private:
   friend class Stream;
 
-  RuntimeState* const state_;
-  HdfsScanNodeBase* const scan_node_;
-
-  /// Buffer pool client used to allocate I/O buffers. This is accessed by multiple
-  /// threads in the multi-threaded scan node, so those threads must take care to only
-  /// call thread-safe BufferPool methods with this client.
-  BufferPool::ClientHandle* const bp_client_;
-
-  HdfsPartitionDescriptor* const partition_desc_;
+  RuntimeState* state_;
+  HdfsScanNodeBase* scan_node_;
+  HdfsPartitionDescriptor* partition_desc_;
 
   /// Vector of streams. Non-columnar formats will always have one stream per context.
   std::vector<std::unique_ptr<Stream>> streams_;

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/bufferpool/buffer-pool.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/bufferpool/buffer-pool.h b/be/src/runtime/bufferpool/buffer-pool.h
index d14da63..285aacb 100644
--- a/be/src/runtime/bufferpool/buffer-pool.h
+++ b/be/src/runtime/bufferpool/buffer-pool.h
@@ -37,7 +37,6 @@
 
 namespace impala {
 
-class MemTracker;
 class ReservationTracker;
 class RuntimeProfile;
 class SystemAllocator;

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/bufferpool/reservation-tracker-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/bufferpool/reservation-tracker-test.cc b/be/src/runtime/bufferpool/reservation-tracker-test.cc
index e441402..c46c5ea 100644
--- a/be/src/runtime/bufferpool/reservation-tracker-test.cc
+++ b/be/src/runtime/bufferpool/reservation-tracker-test.cc
@@ -522,15 +522,15 @@ TEST_F(ReservationTrackerTest, TransferReservation) {
 TEST_F(ReservationTrackerTest, ReservationUtil) {
   const int64_t MEG = 1024 * 1024;
   const int64_t GIG = 1024 * 1024 * 1024;
-  EXPECT_EQ(32 * MEG, ReservationUtil::RESERVATION_MEM_MIN_REMAINING);
+  EXPECT_EQ(75 * MEG, ReservationUtil::RESERVATION_MEM_MIN_REMAINING);
 
   EXPECT_EQ(0, ReservationUtil::GetReservationLimitFromMemLimit(0));
   EXPECT_EQ(0, ReservationUtil::GetReservationLimitFromMemLimit(-1));
-  EXPECT_EQ(0, ReservationUtil::GetReservationLimitFromMemLimit(32 * MEG));
+  EXPECT_EQ(0, ReservationUtil::GetReservationLimitFromMemLimit(75 * MEG));
   EXPECT_EQ(8 * GIG, ReservationUtil::GetReservationLimitFromMemLimit(10 * GIG));
 
-  EXPECT_EQ(32 * MEG, ReservationUtil::GetMinMemLimitFromReservation(0));
-  EXPECT_EQ(32 * MEG, ReservationUtil::GetMinMemLimitFromReservation(-1));
+  EXPECT_EQ(75 * MEG, ReservationUtil::GetMinMemLimitFromReservation(0));
+  EXPECT_EQ(75 * MEG, ReservationUtil::GetMinMemLimitFromReservation(-1));
   EXPECT_EQ(500 * MEG, ReservationUtil::GetMinMemLimitFromReservation(400 * MEG));
   EXPECT_EQ(5 * GIG, ReservationUtil::GetMinMemLimitFromReservation(4 * GIG));
 

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/bufferpool/reservation-util.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/bufferpool/reservation-util.cc b/be/src/runtime/bufferpool/reservation-util.cc
index a27ab9d..85718ab 100644
--- a/be/src/runtime/bufferpool/reservation-util.cc
+++ b/be/src/runtime/bufferpool/reservation-util.cc
@@ -24,7 +24,7 @@ namespace impala {
 // Most operators that accumulate memory use reservations, so the majority of memory
 // should be allocated to buffer reservations, as a heuristic.
 const double ReservationUtil::RESERVATION_MEM_FRACTION = 0.8;
-const int64_t ReservationUtil::RESERVATION_MEM_MIN_REMAINING = 32 * 1024 * 1024;
+const int64_t ReservationUtil::RESERVATION_MEM_MIN_REMAINING = 75 * 1024 * 1024;
 
 int64_t ReservationUtil::GetReservationLimitFromMemLimit(int64_t mem_limit) {
   int64_t max_reservation = std::min<int64_t>(

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/exec-env.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/exec-env.cc b/be/src/runtime/exec-env.cc
index 2f19786..51eece5 100644
--- a/be/src/runtime/exec-env.cc
+++ b/be/src/runtime/exec-env.cc
@@ -36,9 +36,9 @@
 #include "runtime/client-cache.h"
 #include "runtime/coordinator.h"
 #include "runtime/data-stream-mgr.h"
+#include "runtime/io/disk-io-mgr.h"
 #include "runtime/hbase-table-factory.h"
 #include "runtime/hdfs-fs-cache.h"
-#include "runtime/io/disk-io-mgr.h"
 #include "runtime/krpc-data-stream-mgr.h"
 #include "runtime/lib-cache.h"
 #include "runtime/mem-tracker.h"
@@ -367,7 +367,10 @@ Status ExecEnv::Init() {
   LOG(INFO) << "Buffer pool limit: "
             << PrettyPrinter::Print(buffer_pool_limit, TUnit::BYTES);
 
-  RETURN_IF_ERROR(disk_io_mgr_->Init());
+  RETURN_IF_ERROR(disk_io_mgr_->Init(mem_tracker_.get()));
+
+  mem_tracker_->AddGcFunction(
+      [this](int64_t bytes_to_free) { disk_io_mgr_->GcIoBuffers(bytes_to_free); });
 
   // Start services in order to ensure that dependencies between them are met
   if (enable_webserver_) {

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr-internal.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr-internal.h b/be/src/runtime/io/disk-io-mgr-internal.h
index 2d32487..3fc3895 100644
--- a/be/src/runtime/io/disk-io-mgr-internal.h
+++ b/be/src/runtime/io/disk-io-mgr-internal.h
@@ -35,25 +35,9 @@
 #include "util/filesystem-util.h"
 #include "util/hdfs-util.h"
 #include "util/impalad-metrics.h"
-#include "util/runtime-profile-counters.h"
 
 /// This file contains internal structures shared between submodules of the IoMgr. Users
 /// of the IoMgr do not need to include this file.
-
-// Macros to work around counters sometimes not being provided.
-// TODO: fix things so that counters are always non-NULL.
-#define COUNTER_ADD_IF_NOT_NULL(c, v) \
-  do { \
-    ::impala::RuntimeProfile::Counter* __ctr__ = (c); \
-    if (__ctr__ != nullptr) __ctr__->Add(v); \
- } while (false);
-
-#define COUNTER_BITOR_IF_NOT_NULL(c, v) \
-  do { \
-    ::impala::RuntimeProfile::Counter* __ctr__ = (c); \
-    if (__ctr__ != nullptr) __ctr__->BitOr(v); \
- } while (false);
-
 namespace impala {
 namespace io {
 

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr-stress-test.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr-stress-test.cc b/be/src/runtime/io/disk-io-mgr-stress-test.cc
index 2ec1d09..45b36ed 100644
--- a/be/src/runtime/io/disk-io-mgr-stress-test.cc
+++ b/be/src/runtime/io/disk-io-mgr-stress-test.cc
@@ -15,13 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gflags/gflags.h>
-
-#include "common/init.h"
 #include "runtime/io/disk-io-mgr-stress.h"
-#include "common/init.h"
-#include "runtime/test-env.h"
-#include "service/fe-support.h"
+#include "util/cpu-info.h"
 #include "util/string-parser.h"
 
 #include "common/names.h"
@@ -33,32 +28,34 @@ using namespace impala::io;
 // can be passed to control how long to run this test (0 for forever).
 
 // TODO: make these configurable once we decide how to run BE tests with args
-constexpr int DEFAULT_DURATION_SEC = 1;
+const int DEFAULT_DURATION_SEC = 1;
 const int NUM_DISKS = 5;
 const int NUM_THREADS_PER_DISK = 5;
 const int NUM_CLIENTS = 10;
 const bool TEST_CANCELLATION = true;
-const int64_t BUFFER_POOL_CAPACITY = 1024L * 1024L * 1024L * 4L;
-
-DEFINE_int64(duration_sec, DEFAULT_DURATION_SEC,
-    "Disk I/O Manager stress test duration in seconds. 0 means run indefinitely.");
 
 int main(int argc, char** argv) {
-  impala::InitCommonRuntime(argc, argv, true, impala::TestInfo::BE_TEST);
-  impala::InitFeSupport();
-
-  if (FLAGS_duration_sec != 0) {
-    printf("Running stress test for %ld seconds.\n", FLAGS_duration_sec);
+  google::InitGoogleLogging(argv[0]);
+  CpuInfo::Init();
+  OsInfo::Init();
+  impala::InitThreading();
+  int duration_sec = DEFAULT_DURATION_SEC;
+
+  if (argc == 2) {
+    StringParser::ParseResult status;
+    duration_sec = StringParser::StringToInt<int>(argv[1], strlen(argv[1]), &status);
+    if (status != StringParser::PARSE_SUCCESS) {
+      printf("Invalid arg: %s\n", argv[1]);
+      return 1;
+    }
+  }
+  if (duration_sec != 0) {
+    printf("Running stress test for %d seconds.\n", duration_sec);
   } else {
     printf("Running stress test indefinitely.\n");
   }
-
-  TestEnv test_env;
-  // Tests try to allocate arbitrarily small buffers. Ensure Buffer Pool allows it.
-  test_env.SetBufferPoolArgs(DiskIoMgrStress::MIN_READ_BUFFER_SIZE, BUFFER_POOL_CAPACITY);
-  Status status = test_env.Init();
-  CHECK(status.ok()) << status.GetDetail();
   DiskIoMgrStress test(NUM_DISKS, NUM_THREADS_PER_DISK, NUM_CLIENTS, TEST_CANCELLATION);
-  test.Run(FLAGS_duration_sec);
+  test.Run(duration_sec);
+
   return 0;
 }

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr-stress.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr-stress.cc b/be/src/runtime/io/disk-io-mgr-stress.cc
index 3fd33de..8815357 100644
--- a/be/src/runtime/io/disk-io-mgr-stress.cc
+++ b/be/src/runtime/io/disk-io-mgr-stress.cc
@@ -19,8 +19,6 @@
 
 #include "runtime/io/disk-io-mgr-stress.h"
 
-#include "runtime/bufferpool/reservation-tracker.h"
-#include "runtime/exec-env.h"
 #include "runtime/io/request-context.h"
 #include "util/time.h"
 
@@ -29,20 +27,18 @@
 using namespace impala;
 using namespace impala::io;
 
-constexpr float DiskIoMgrStress::ABORT_CHANCE;
-const int DiskIoMgrStress::MIN_READ_LEN;
-const int DiskIoMgrStress::MAX_READ_LEN;
+static const float ABORT_CHANCE = .10f;
+static const int MIN_READ_LEN = 1;
+static const int MAX_READ_LEN = 20;
 
-const int DiskIoMgrStress::MIN_FILE_LEN;
-const int DiskIoMgrStress::MAX_FILE_LEN;
+static const int MIN_FILE_LEN = 10;
+static const int MAX_FILE_LEN = 1024;
 
 // Make sure this is between MIN/MAX FILE_LEN to test more cases
-const int DiskIoMgrStress::MIN_READ_BUFFER_SIZE;
-const int DiskIoMgrStress::MAX_READ_BUFFER_SIZE;
+static const int MIN_READ_BUFFER_SIZE = 64;
+static const int MAX_READ_BUFFER_SIZE = 128;
 
-const int DiskIoMgrStress::MAX_BUFFER_BYTES_PER_SCAN_RANGE;
-
-const int DiskIoMgrStress::CANCEL_READER_PERIOD_MS;
+static const int CANCEL_READER_PERIOD_MS = 20;  // in ms
 
 static void CreateTempFile(const char* filename, const char* data) {
   FILE* file = fopen(filename, "w");
@@ -51,7 +47,7 @@ static void CreateTempFile(const char* filename, const char* data) {
   fclose(file);
 }
 
-string DiskIoMgrStress::GenerateRandomData() {
+string GenerateRandomData() {
   int rand_len = rand() % (MAX_FILE_LEN - MIN_FILE_LEN) + MIN_FILE_LEN;
   stringstream ss;
   for (int i = 0; i < rand_len; ++i) {
@@ -63,8 +59,6 @@ string DiskIoMgrStress::GenerateRandomData() {
 
 struct DiskIoMgrStress::Client {
   boost::mutex lock;
-  /// Pool for objects that is cleared when the client is (re-)initialized in NewClient().
-  ObjectPool obj_pool;
   unique_ptr<RequestContext> reader;
   int file_idx;
   vector<ScanRange*> scan_ranges;
@@ -83,7 +77,7 @@ DiskIoMgrStress::DiskIoMgrStress(int num_disks, int num_threads_per_disk,
 
   io_mgr_.reset(new DiskIoMgr(num_disks, num_threads_per_disk, num_threads_per_disk,
       MIN_READ_BUFFER_SIZE, MAX_READ_BUFFER_SIZE));
-  Status status = io_mgr_->Init();
+  Status status = io_mgr_->Init(&mem_tracker_);
   CHECK(status.ok());
 
   // Initialize some data files.  It doesn't really matter how many there are.
@@ -98,7 +92,6 @@ DiskIoMgrStress::DiskIoMgrStress(int num_disks, int num_threads_per_disk,
 
   clients_ = new Client[num_clients_];
   client_mem_trackers_.resize(num_clients_);
-  buffer_pool_clients_.reset(new BufferPool::ClientHandle[num_clients_]);
   for (int i = 0; i < num_clients_; ++i) {
     NewClient(i);
   }
@@ -117,16 +110,9 @@ void DiskIoMgrStress::ClientThread(int client_id) {
 
     while (!eos) {
       ScanRange* range;
-      bool needs_buffers;
-      Status status =
-          io_mgr_->GetNextUnstartedRange(client->reader.get(), &range, &needs_buffers);
+      Status status = io_mgr_->GetNextRange(client->reader.get(), &range);
       CHECK(status.ok() || status.IsCancelled());
       if (range == NULL) break;
-      if (needs_buffers) {
-        status = io_mgr_->AllocateBuffersForRange(client->reader.get(),
-            &buffer_pool_clients_[client_id], range, MAX_BUFFER_BYTES_PER_SCAN_RANGE);
-        CHECK(status.ok()) << status.GetDetail();
-      }
 
       while (true) {
         unique_ptr<BufferDescriptor> buffer;
@@ -151,7 +137,7 @@ void DiskIoMgrStress::ClientThread(int client_id) {
 
         // Copy the bytes from this read into the result buffer.
         memcpy(read_buffer + file_offset, buffer->buffer(), buffer->len());
-        range->ReturnBuffer(move(buffer));
+        io_mgr_->ReturnBuffer(move(buffer));
         bytes_read += len;
 
         CHECK_GE(bytes_read, 0);
@@ -173,7 +159,6 @@ void DiskIoMgrStress::ClientThread(int client_id) {
     // Unregister the old client and get a new one
     unique_lock<mutex> lock(client->lock);
     io_mgr_->UnregisterContext(client->reader.get());
-    client->reader.reset();
     NewClient(client_id);
   }
 
@@ -185,9 +170,11 @@ void DiskIoMgrStress::ClientThread(int client_id) {
 // Cancel a random reader
 void DiskIoMgrStress::CancelRandomReader() {
   if (!includes_cancellation_) return;
-  Client* rand_client = &clients_[rand() % num_clients_];
-  unique_lock<mutex> lock(rand_client->lock);
-  rand_client->reader->Cancel();
+
+  int rand_client = rand() % num_clients_;
+
+  unique_lock<mutex> lock(clients_[rand_client].lock);
+  io_mgr_->CancelContext(clients_[rand_client].reader.get());
 }
 
 void DiskIoMgrStress::Run(int sec) {
@@ -212,18 +199,10 @@ void DiskIoMgrStress::Run(int sec) {
 
   for (int i = 0; i < num_clients_; ++i) {
     unique_lock<mutex> lock(clients_[i].lock);
-    if (clients_[i].reader != NULL) clients_[i].reader->Cancel();
+    if (clients_[i].reader != NULL) io_mgr_->CancelContext(clients_[i].reader.get());
   }
-  readers_.join_all();
 
-  for (int i = 0; i < num_clients_; ++i) {
-    if (clients_[i].reader != nullptr) {
-      io_mgr_->UnregisterContext(clients_[i].reader.get());
-    }
-    ExecEnv::GetInstance()->buffer_pool()->DeregisterClient(&buffer_pool_clients_[i]);
-    client_mem_trackers_[i]->Close();
-  }
-  mem_tracker_.Close();
+  readers_.join_all();
 }
 
 // Initialize a client to read one of the files at random.  The scan ranges are
@@ -244,41 +223,25 @@ void DiskIoMgrStress::NewClient(int i) {
     }
   }
 
-  // Clean up leftover state from the previous client (if any).
+  for (int i = 0; i < client.scan_ranges.size(); ++i) {
+    delete client.scan_ranges[i];
+  }
   client.scan_ranges.clear();
-  ExecEnv* exec_env = ExecEnv::GetInstance();
-  exec_env->buffer_pool()->DeregisterClient(&buffer_pool_clients_[i]);
-  if (client_mem_trackers_[i] != nullptr) client_mem_trackers_[i]->Close();
-  client.obj_pool.Clear();
 
   int assigned_len = 0;
   while (assigned_len < file_len) {
     int range_len = rand() % (MAX_READ_LEN - MIN_READ_LEN) + MIN_READ_LEN;
     range_len = min(range_len, file_len - assigned_len);
 
-    ScanRange* range = client.obj_pool.Add(new ScanRange);
+    ScanRange* range = new ScanRange();
     range->Reset(NULL, files_[client.file_idx].filename.c_str(), range_len, assigned_len,
         0, false, BufferOpts::Uncached());
     client.scan_ranges.push_back(range);
     assigned_len += range_len;
   }
 
-  string client_name = Substitute("Client $0", i);
-  client_mem_trackers_[i].reset(new MemTracker(-1, client_name, &mem_tracker_));
-  Status status = exec_env->buffer_pool()->RegisterClient(client_name, nullptr,
-      exec_env->buffer_reservation(), client_mem_trackers_[i].get(),
-      numeric_limits<int64_t>::max(), RuntimeProfile::Create(&client.obj_pool, client_name),
-      &buffer_pool_clients_[i]);
-  CHECK(status.ok());
-  // Reserve enough memory for 3 buffers per range, which should be enough to guarantee
-  // progress.
-  CHECK(buffer_pool_clients_[i].IncreaseReservationToFit(
-      MAX_BUFFER_BYTES_PER_SCAN_RANGE * client.scan_ranges.size()))
-      << buffer_pool_clients_[i].DebugString() << "\n"
-      << exec_env->buffer_pool()->DebugString() << "\n"
-      << exec_env->buffer_reservation()->DebugString();
-
-  client.reader = io_mgr_->RegisterContext();
-  status = io_mgr_->AddScanRanges(client.reader.get(), client.scan_ranges);
+  client_mem_trackers_[i].reset(new MemTracker(-1, "", &mem_tracker_));
+  client.reader = io_mgr_->RegisterContext(client_mem_trackers_[i].get());
+  Status status = io_mgr_->AddScanRanges(client.reader.get(), client.scan_ranges);
   CHECK(status.ok());
 }

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/be/src/runtime/io/disk-io-mgr-stress.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/io/disk-io-mgr-stress.h b/be/src/runtime/io/disk-io-mgr-stress.h
index 574b58c..b872694 100644
--- a/be/src/runtime/io/disk-io-mgr-stress.h
+++ b/be/src/runtime/io/disk-io-mgr-stress.h
@@ -22,11 +22,8 @@
 #include <memory>
 #include <vector>
 #include <boost/scoped_ptr.hpp>
-#include <boost/thread/condition_variable.hpp>
 #include <boost/thread/thread.hpp>
 
-#include "common/object-pool.h"
-#include "runtime/bufferpool/buffer-pool.h"
 #include "runtime/io/disk-io-mgr.h"
 #include "runtime/mem-tracker.h"
 #include "runtime/thread-resource-mgr.h"
@@ -46,29 +43,15 @@ class DiskIoMgrStress {
   /// Run the test for 'sec'.  If 0, run forever
   void Run(int sec);
 
-  static constexpr float ABORT_CHANCE = .10f;
-  static const int MIN_READ_LEN = 1;
-  static const int MAX_READ_LEN = 20;
-
-  static const int MIN_FILE_LEN = 10;
-  static const int MAX_FILE_LEN = 1024;
-
-  // Make sure this is between MIN/MAX FILE_LEN to test more cases
-  static const int MIN_READ_BUFFER_SIZE = 64;
-  static const int MAX_READ_BUFFER_SIZE = 128;
-
-  // Maximum bytes to allocate per scan range.
-  static const int MAX_BUFFER_BYTES_PER_SCAN_RANGE = MAX_READ_BUFFER_SIZE * 3;
-
-  static const int CANCEL_READER_PERIOD_MS = 20;
  private:
   struct Client;
 
   struct File {
     std::string filename;
-    std::string data; // the data in the file, used to validate
+    std::string data;  // the data in the file, used to validate
   };
 
+
   /// Files used for testing.  These are created at startup and recycled
   /// during the test
   std::vector<File> files_;
@@ -89,9 +72,6 @@ class DiskIoMgrStress {
   /// Client MemTrackers, one per client.
   std::vector<std::unique_ptr<MemTracker>> client_mem_trackers_;
 
-  /// Buffer pool clients, one per client.
-  std::unique_ptr<BufferPool::ClientHandle[]> buffer_pool_clients_;
-
   /// If true, tests cancelling readers
   bool includes_cancellation_;
 
@@ -108,8 +88,6 @@ class DiskIoMgrStress {
 
   /// Possibly cancels a random reader.
   void CancelRandomReader();
-
-  static std::string GenerateRandomData();
 };
 }
 }


[3/9] impala git commit: Revert IMPALA-4835 and dependent changes

Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
index 003e68f..261e8fe 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
@@ -1,795 +1,24 @@
 # Parquet scan
 select * from tpch_parquet.lineitem
 ---- PLAN
-Max Per-Host Resource Reservation: Memory=72.00MB
-Per-Host Resource Estimates: Memory=80.00MB
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=80.00MB mem-reservation=72.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_parquet.lineitem]
-   partitions=1/1 files=3 size=193.71MB
-   stored statistics:
-     table: rows=6001215 size=193.71MB
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
-   tuple-ids=0 row-size=263B cardinality=6001215
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=72.00MB
-Per-Host Resource Estimates: Memory=80.00MB
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=263B cardinality=6001215
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=80.00MB mem-reservation=72.00MB
-00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
-   stored statistics:
-     table: rows=6001215 size=193.71MB
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
-   tuple-ids=0 row-size=263B cardinality=6001215
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=144.00MB
-Per-Host Resource Estimates: Memory=160.00MB
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=263B cardinality=6001215
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=160.00MB mem-reservation=144.00MB
-00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
-   stored statistics:
-     table: rows=6001215 size=193.71MB
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
-   tuple-ids=0 row-size=263B cardinality=6001215
-====
-# Single column parquet scan - memory reservation is reduced compared to multi-column
-# scan.
-select l_comment from tpch_parquet.lineitem
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=80.00MB
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_parquet.lineitem]
-   partitions=1/1 files=3 size=193.71MB
-   stored statistics:
-     table: rows=6001215 size=193.71MB
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=42B cardinality=6001215
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=80.00MB
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=42B cardinality=6001215
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB
-00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
-   stored statistics:
-     table: rows=6001215 size=193.71MB
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=42B cardinality=6001215
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=160.00MB
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=42B cardinality=6001215
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=160.00MB mem-reservation=16.00MB
-00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
-   stored statistics:
-     table: rows=6001215 size=193.71MB
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=42B cardinality=6001215
-====
-# Small parquet files - memory reservation is reduced because of small file size.
-select string_col from functional_parquet.alltypes;
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=8.00KB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [functional_parquet.alltypes]
-   partitions=24/24 files=24 size=174.62KB
-   stored statistics:
-     table: rows=unavailable size=unavailable
-     partitions: 0/24 rows=unavailable
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=8.00KB
-   tuple-ids=0 row-size=16B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=16B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=8.00KB
-00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
-   partitions=24/24 files=24 size=174.62KB
-   stored statistics:
-     table: rows=unavailable size=unavailable
-     partitions: 0/24 rows=unavailable
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=8.00KB
-   tuple-ids=0 row-size=16B cardinality=unavailable
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00KB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=16B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=16.00KB
-00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
-   partitions=24/24 files=24 size=174.62KB
-   stored statistics:
-     table: rows=unavailable size=unavailable
-     partitions: 0/24 rows=unavailable
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=8.00KB
-   tuple-ids=0 row-size=16B cardinality=unavailable
-====
-# Multi-column parquet scan with small files - memory reservation is reduced because of
-# small file size but a minimum amount is reserved per column.
-select int_col, float_col, string_col from functional_parquet.alltypes;
----- PLAN
-Max Per-Host Resource Reservation: Memory=24.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=24.00KB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [functional_parquet.alltypes]
-   partitions=24/24 files=24 size=174.62KB
-   stored statistics:
-     table: rows=unavailable size=unavailable
-     partitions: 0/24 rows=unavailable
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=24.00KB
-   tuple-ids=0 row-size=24B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=24.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=24B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=24.00KB
-00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
-   partitions=24/24 files=24 size=174.62KB
-   stored statistics:
-     table: rows=unavailable size=unavailable
-     partitions: 0/24 rows=unavailable
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=24.00KB
-   tuple-ids=0 row-size=24B cardinality=unavailable
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=48.00KB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=24B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=48.00KB
-00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
-   partitions=24/24 files=24 size=174.62KB
-   stored statistics:
-     table: rows=unavailable size=unavailable
-     partitions: 0/24 rows=unavailable
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=24.00KB
-   tuple-ids=0 row-size=24B cardinality=unavailable
-====
-# Parquet scan with no materialized columns. Need reservation to scan levels to determine
-# row count.
-select 'foo' from functional_parquet.alltypes
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00KB
-Per-Host Resource Estimates: Memory=10.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=1.00MB mem-reservation=8.00KB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [functional_parquet.alltypes]
-   partitions=24/24 files=24 size=174.62KB
-   stored statistics:
-     table: rows=unavailable size=unavailable
-     partitions: 0/24 rows=unavailable
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=1.00MB mem-reservation=8.00KB
-   tuple-ids=0 row-size=0B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00KB
-Per-Host Resource Estimates: Memory=10.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=0B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=1.00MB mem-reservation=8.00KB
-00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
-   partitions=24/24 files=24 size=174.62KB
-   stored statistics:
-     table: rows=unavailable size=unavailable
-     partitions: 0/24 rows=unavailable
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=1.00MB mem-reservation=8.00KB
-   tuple-ids=0 row-size=0B cardinality=unavailable
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00KB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional_parquet.alltypes
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=0B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=16.00KB
-00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
-   partitions=24/24 files=24 size=174.62KB
-   stored statistics:
-     table: rows=unavailable size=unavailable
-     partitions: 0/24 rows=unavailable
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=8.00KB
-   tuple-ids=0 row-size=0B cardinality=unavailable
-====
-# Parquet nested types, unnested in scan - should reserve memory for each column.
-select o_orderkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority, o_clerk
-from tpch_nested_parquet.customer.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=48.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=48.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_nested_parquet.customer.c_orders]
-   partitions=1/1 files=4 size=292.36MB
-   stored statistics:
-     table: rows=150000 size=292.36MB
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=48.00MB
-   tuple-ids=0 row-size=80B cardinality=1500000
-====
-# Parquet nested types, unnested in scan - don't reserve extra memory for "pos" virtual
-# column that piggy-backs on another column.
-select o_orderkey, pos
-from tpch_nested_parquet.customer.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_nested_parquet.customer.c_orders]
-   partitions=1/1 files=4 size=292.36MB
-   stored statistics:
-     table: rows=150000 size=292.36MB
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=16B cardinality=1500000
-====
-# Parquet nested types, unnested in scan - reserve memory for "pos" virtual column if it
-# is the only column materialized.
-select pos
-from tpch_nested_parquet.customer.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_nested_parquet.customer.c_orders]
-   partitions=1/1 files=4 size=292.36MB
-   stored statistics:
-     table: rows=150000 size=292.36MB
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=8B cardinality=1500000
-====
-# Parquet nested types, left nested in scan - should reserve memory for each scalar
-# column in the nested collection plus the top-level column.
-select c_custkey, o_orderkey, o_orderstatus, o_totalprice, o_orderdate, o_orderpriority,
-    o_clerk
-from tpch_nested_parquet.customer c, c.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=56.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_nested_parquet.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=56.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:SUBPLAN
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=1,0 row-size=104B cardinality=1500000
-|
-|--04:NESTED LOOP JOIN [CROSS JOIN]
-|  |  mem-estimate=24B mem-reservation=0B
-|  |  tuple-ids=1,0 row-size=104B cardinality=10
-|  |
-|  |--02:SINGULAR ROW SRC
-|  |     parent-subplan=01
-|  |     mem-estimate=0B mem-reservation=0B
-|  |     tuple-ids=0 row-size=24B cardinality=1
-|  |
-|  03:UNNEST [c.c_orders]
-|     parent-subplan=01
-|     mem-estimate=0B mem-reservation=0B
-|     tuple-ids=1 row-size=0B cardinality=10
-|
-00:SCAN HDFS [tpch_nested_parquet.customer c]
-   partitions=1/1 files=4 size=292.36MB
-   predicates: !empty(c.c_orders)
-   stored statistics:
-     table: rows=150000 size=292.36MB
-     columns missing stats: c_orders
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=56.00MB
-   tuple-ids=0 row-size=24B cardinality=150000
-====
-# Parquet nested types, left nested in scan - should reserve memory for each scalar
-# column (excluding pos) in the nested collection plus the top-level column.
-select c_custkey, o_orderkey, pos
-from tpch_nested_parquet.customer c, c.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_nested_parquet.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=16.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:SUBPLAN
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=1,0 row-size=40B cardinality=1500000
-|
-|--04:NESTED LOOP JOIN [CROSS JOIN]
-|  |  mem-estimate=24B mem-reservation=0B
-|  |  tuple-ids=1,0 row-size=40B cardinality=10
-|  |
-|  |--02:SINGULAR ROW SRC
-|  |     parent-subplan=01
-|  |     mem-estimate=0B mem-reservation=0B
-|  |     tuple-ids=0 row-size=24B cardinality=1
-|  |
-|  03:UNNEST [c.c_orders]
-|     parent-subplan=01
-|     mem-estimate=0B mem-reservation=0B
-|     tuple-ids=1 row-size=0B cardinality=10
-|
-00:SCAN HDFS [tpch_nested_parquet.customer c]
-   partitions=1/1 files=4 size=292.36MB
-   predicates: !empty(c.c_orders)
-   stored statistics:
-     table: rows=150000 size=292.36MB
-     columns missing stats: c_orders
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=16.00MB
-   tuple-ids=0 row-size=24B cardinality=150000
-====
-# Parquet nested types, left nested in scan - should reserve memory for virtual
-# pos column in the nested collection because nothing else is materialized from
-# the collection, plus the top-level column.
-select c_custkey, pos
-from tpch_nested_parquet.customer c, c.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_nested_parquet.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=16.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:SUBPLAN
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=1,0 row-size=32B cardinality=1500000
-|
-|--04:NESTED LOOP JOIN [CROSS JOIN]
-|  |  mem-estimate=24B mem-reservation=0B
-|  |  tuple-ids=1,0 row-size=32B cardinality=10
-|  |
-|  |--02:SINGULAR ROW SRC
-|  |     parent-subplan=01
-|  |     mem-estimate=0B mem-reservation=0B
-|  |     tuple-ids=0 row-size=24B cardinality=1
-|  |
-|  03:UNNEST [c.c_orders]
-|     parent-subplan=01
-|     mem-estimate=0B mem-reservation=0B
-|     tuple-ids=1 row-size=0B cardinality=10
-|
-00:SCAN HDFS [tpch_nested_parquet.customer c]
-   partitions=1/1 files=4 size=292.36MB
-   predicates: !empty(c.c_orders)
-   stored statistics:
-     table: rows=150000 size=292.36MB
-     columns missing stats: c_orders
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=16.00MB
-   tuple-ids=0 row-size=24B cardinality=150000
-====
-# Parquet nested types, left nested in scan - should reserve memory for nested column
-# plus top-level column even though nested fields are not materialized because the
-# column levels still need to be scanned to determine the # of elements in the collection.
-select c_custkey
-from tpch_nested_parquet.customer c, c.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_nested_parquet.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=16.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:SUBPLAN
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=1,0 row-size=24B cardinality=1500000
-|
-|--04:NESTED LOOP JOIN [CROSS JOIN]
-|  |  mem-estimate=24B mem-reservation=0B
-|  |  tuple-ids=1,0 row-size=24B cardinality=10
-|  |
-|  |--02:SINGULAR ROW SRC
-|  |     parent-subplan=01
-|  |     mem-estimate=0B mem-reservation=0B
-|  |     tuple-ids=0 row-size=24B cardinality=1
-|  |
-|  03:UNNEST [c.c_orders]
-|     parent-subplan=01
-|     mem-estimate=0B mem-reservation=0B
-|     tuple-ids=1 row-size=0B cardinality=10
-|
-00:SCAN HDFS [tpch_nested_parquet.customer c]
-   partitions=1/1 files=4 size=292.36MB
-   predicates: !empty(c.c_orders)
-   stored statistics:
-     table: rows=150000 size=292.36MB
-     columns missing stats: c_orders
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=16.00MB
-   tuple-ids=0 row-size=24B cardinality=150000
-====
-# Parquet nested types, left nested in scan - should reserve memory for nested column
-# only when no top-level column is materialized.
-select o_orderkey
-from tpch_nested_parquet.customer c, c.c_orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_nested_parquet.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:SUBPLAN
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=1,0 row-size=24B cardinality=1500000
-|
-|--04:NESTED LOOP JOIN [CROSS JOIN]
-|  |  mem-estimate=16B mem-reservation=0B
-|  |  tuple-ids=1,0 row-size=24B cardinality=10
-|  |
-|  |--02:SINGULAR ROW SRC
-|  |     parent-subplan=01
-|  |     mem-estimate=0B mem-reservation=0B
-|  |     tuple-ids=0 row-size=16B cardinality=1
-|  |
-|  03:UNNEST [c.c_orders]
-|     parent-subplan=01
-|     mem-estimate=0B mem-reservation=0B
-|     tuple-ids=1 row-size=0B cardinality=10
-|
-00:SCAN HDFS [tpch_nested_parquet.customer c]
-   partitions=1/1 files=4 size=292.36MB
-   predicates: !empty(c.c_orders)
-   stored statistics:
-     table: rows=150000 size=292.36MB
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=16B cardinality=150000
-====
-# Parquet nested types with two levels of nesting materialized in scan. Should
-# reserve memory for columns at all three levels
-select c_custkey, o_orderkey, l_comment
-from tpch_nested_parquet.customer c, c.c_orders o, o.o_lineitems
----- PLAN
-Max Per-Host Resource Reservation: Memory=24.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_nested_parquet.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=24.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:SUBPLAN
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=2,1,0 row-size=64B cardinality=15000000
-|
-|--08:NESTED LOOP JOIN [CROSS JOIN]
-|  |  mem-estimate=24B mem-reservation=0B
-|  |  tuple-ids=2,1,0 row-size=64B cardinality=100
-|  |
-|  |--02:SINGULAR ROW SRC
-|  |     parent-subplan=01
-|  |     mem-estimate=0B mem-reservation=0B
-|  |     tuple-ids=0 row-size=24B cardinality=1
-|  |
-|  04:SUBPLAN
-|  |  mem-estimate=0B mem-reservation=0B
-|  |  tuple-ids=2,1 row-size=40B cardinality=100
-|  |
-|  |--07:NESTED LOOP JOIN [CROSS JOIN]
-|  |  |  mem-estimate=24B mem-reservation=0B
-|  |  |  tuple-ids=2,1 row-size=40B cardinality=10
-|  |  |
-|  |  |--05:SINGULAR ROW SRC
-|  |  |     parent-subplan=04
-|  |  |     mem-estimate=0B mem-reservation=0B
-|  |  |     tuple-ids=1 row-size=24B cardinality=1
-|  |  |
-|  |  06:UNNEST [o.o_lineitems]
-|  |     parent-subplan=04
-|  |     mem-estimate=0B mem-reservation=0B
-|  |     tuple-ids=2 row-size=0B cardinality=10
-|  |
-|  03:UNNEST [c.c_orders o]
-|     parent-subplan=01
-|     mem-estimate=0B mem-reservation=0B
-|     tuple-ids=1 row-size=0B cardinality=10
-|
-00:SCAN HDFS [tpch_nested_parquet.customer c]
-   partitions=1/1 files=4 size=292.36MB
-   predicates: !empty(c.c_orders)
-   predicates on o: !empty(o.o_lineitems)
-   stored statistics:
-     table: rows=150000 size=292.36MB
-     columns missing stats: c_orders
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=24.00MB
-   tuple-ids=0 row-size=24B cardinality=150000
-====
-# Text scan
-select * from tpch.lineitem
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch.lineitem]
-   partitions=1/1 files=1 size=718.94MB
-   stored statistics:
-     table: rows=6001215 size=718.94MB
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=263B cardinality=6001215
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=263B cardinality=6001215
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-00:SCAN HDFS [tpch.lineitem, RANDOM]
-   partitions=1/1 files=1 size=718.94MB
-   stored statistics:
-     table: rows=6001215 size=718.94MB
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=263B cardinality=6001215
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=176.00MB
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=263B cardinality=6001215
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
-00:SCAN HDFS [tpch.lineitem, RANDOM]
-   partitions=1/1 files=1 size=718.94MB
-   stored statistics:
-     table: rows=6001215 size=718.94MB
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=263B cardinality=6001215
-====
-# Single column text scan - memory reservation is same as multi-column scan.
-select l_comment from tpch_parquet.lineitem
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=80.00MB
 
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB
+|  Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
 00:SCAN HDFS [tpch_parquet.lineitem]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=42B cardinality=6001215
+   mem-estimate=80.00MB mem-reservation=0B
+   tuple-ids=0 row-size=263B cardinality=6001215
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=80.00MB
 
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -799,20 +28,20 @@ PLAN-ROOT SINK
 |
 01:EXCHANGE [UNPARTITIONED]
 |  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=42B cardinality=6001215
+|  tuple-ids=0 row-size=263B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=80.00MB mem-reservation=8.00MB
+Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=42B cardinality=6001215
+   mem-estimate=80.00MB mem-reservation=0B
+   tuple-ids=0 row-size=263B cardinality=6001215
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=160.00MB
 
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -822,44 +51,41 @@ PLAN-ROOT SINK
 |
 01:EXCHANGE [UNPARTITIONED]
 |  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=42B cardinality=6001215
+|  tuple-ids=0 row-size=263B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=160.00MB mem-reservation=16.00MB
+Per-Host Resources: mem-estimate=160.00MB mem-reservation=0B
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=42B cardinality=6001215
+   mem-estimate=80.00MB mem-reservation=0B
+   tuple-ids=0 row-size=263B cardinality=6001215
 ====
-# Text scan on small files - memory reservation is reduced.
-select * from functional.alltypes
+# Text scan
+select * from tpch.lineitem;
 ---- PLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-Codegen disabled by planner
+Max Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=88.00MB
 
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
+|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
-00:SCAN HDFS [functional.alltypes]
-   partitions=24/24 files=24 size=478.45KB
+00:SCAN HDFS [tpch.lineitem]
+   partitions=1/1 files=1 size=718.94MB
    stored statistics:
-     table: rows=7300 size=478.45KB
-     partitions: 24/24 rows=7300
+     table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=32.00KB
-   tuple-ids=0 row-size=97B cardinality=7300
+   mem-estimate=88.00MB mem-reservation=0B
+   tuple-ids=0 row-size=263B cardinality=6001215
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-Codegen disabled by planner
+Max Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=88.00MB
 
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
 |  Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -868,314 +94,21 @@ PLAN-ROOT SINK
 |
 01:EXCHANGE [UNPARTITIONED]
 |  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=97B cardinality=7300
+|  tuple-ids=0 row-size=263B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
-00:SCAN HDFS [functional.alltypes, RANDOM]
-   partitions=24/24 files=24 size=478.45KB
-   stored statistics:
-     table: rows=7300 size=478.45KB
-     partitions: 24/24 rows=7300
-     columns: all
-   extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=32.00KB
-   tuple-ids=0 row-size=97B cardinality=7300
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=64.00KB
-Per-Host Resource Estimates: Memory=32.00MB
-Codegen disabled by planner
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=97B cardinality=7300
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=64.00KB
-00:SCAN HDFS [functional.alltypes, RANDOM]
-   partitions=24/24 files=24 size=478.45KB
+Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
+00:SCAN HDFS [tpch.lineitem, RANDOM]
+   partitions=1/1 files=1 size=718.94MB
    stored statistics:
-     table: rows=7300 size=478.45KB
-     partitions: 24/24 rows=7300
+     table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=32.00KB
-   tuple-ids=0 row-size=97B cardinality=7300
-====
-# Avro scan.
-select * from tpch_avro.orders
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_avro.orders
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_avro.orders]
-   partitions=1/1 files=2 size=156.92MB
-   stored statistics:
-     table: rows=unavailable size=156.92MB
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=108B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=88.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_avro.orders
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=108B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
-00:SCAN HDFS [tpch_avro.orders, RANDOM]
-   partitions=1/1 files=2 size=156.92MB
-   stored statistics:
-     table: rows=unavailable size=156.92MB
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=108B cardinality=unavailable
+   mem-estimate=88.00MB mem-reservation=0B
+   tuple-ids=0 row-size=263B cardinality=6001215
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=176.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_avro.orders
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=108B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
-00:SCAN HDFS [tpch_avro.orders, RANDOM]
-   partitions=1/1 files=2 size=156.92MB
-   stored statistics:
-     table: rows=unavailable size=156.92MB
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=108B cardinality=unavailable
-====
-# RC scan.
-select * from tpch_rc.customer
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_rc.customer
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpch_rc.customer]
-   partitions=1/1 files=1 size=22.48MB
-   stored statistics:
-     table: rows=unavailable size=22.48MB
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=32.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=98B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_rc.customer
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=98B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=8.00MB
-00:SCAN HDFS [tpch_rc.customer, RANDOM]
-   partitions=1/1 files=1 size=22.48MB
-   stored statistics:
-     table: rows=unavailable size=22.48MB
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=32.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=98B cardinality=unavailable
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=64.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpch_rc.customer
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=98B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-Per-Host Resources: mem-estimate=64.00MB mem-reservation=16.00MB
-00:SCAN HDFS [tpch_rc.customer, RANDOM]
-   partitions=1/1 files=1 size=22.48MB
-   stored statistics:
-     table: rows=unavailable size=22.48MB
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=32.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=98B cardinality=unavailable
-====
-# Seq scan.
-select * from tpcds_seq_snap.web_returns
----- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpcds_seq_snap.web_returns
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=8.00MB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [tpcds_seq_snap.web_returns]
-   partitions=1/1 files=1 size=6.55MB
-   stored statistics:
-     table: rows=unavailable size=6.55MB
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=104B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpcds_seq_snap.web_returns
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=104B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=8.00MB
-00:SCAN HDFS [tpcds_seq_snap.web_returns, RANDOM]
-   partitions=1/1 files=1 size=6.55MB
-   stored statistics:
-     table: rows=unavailable size=6.55MB
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=104B cardinality=unavailable
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-tpcds_seq_snap.web_returns
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=104B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=16.00MB
-00:SCAN HDFS [tpcds_seq_snap.web_returns, RANDOM]
-   partitions=1/1 files=1 size=6.55MB
-   stored statistics:
-     table: rows=unavailable size=6.55MB
-     columns: unavailable
-   extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=8.00MB
-   tuple-ids=0 row-size=104B cardinality=unavailable
-====
-# Mixed table format scan
-select * from functional.alltypesmixedformat
----- PLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional.alltypesmixedformat
-
-F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-00:SCAN HDFS [functional.alltypesmixedformat]
-   partitions=3/3 files=3 size=58.36KB
-   stored statistics:
-     table: rows=unavailable size=unavailable
-     partitions: 0/3 rows=unavailable
-     columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col
-   extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=32.00KB
-   tuple-ids=0 row-size=88B cardinality=unavailable
----- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=32.00KB
-Per-Host Resource Estimates: Memory=16.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional.alltypesmixedformat
-
-F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=0B mem-reservation=0B
-PLAN-ROOT SINK
-|  mem-estimate=0B mem-reservation=0B
-|
-01:EXCHANGE [UNPARTITIONED]
-|  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=88B cardinality=unavailable
-|
-F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
-00:SCAN HDFS [functional.alltypesmixedformat, RANDOM]
-   partitions=3/3 files=3 size=58.36KB
-   stored statistics:
-     table: rows=unavailable size=unavailable
-     partitions: 0/3 rows=unavailable
-     columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col
-   extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=32.00KB
-   tuple-ids=0 row-size=88B cardinality=unavailable
----- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=64.00KB
-Per-Host Resource Estimates: Memory=32.00MB
-WARNING: The following tables are missing relevant table and/or column statistics.
-functional.alltypesmixedformat
 
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
 |  Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -1184,19 +117,18 @@ PLAN-ROOT SINK
 |
 01:EXCHANGE [UNPARTITIONED]
 |  mem-estimate=0B mem-reservation=0B
-|  tuple-ids=0 row-size=88B cardinality=unavailable
+|  tuple-ids=0 row-size=263B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=64.00KB
-00:SCAN HDFS [functional.alltypesmixedformat, RANDOM]
-   partitions=3/3 files=3 size=58.36KB
+Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
+00:SCAN HDFS [tpch.lineitem, RANDOM]
+   partitions=1/1 files=1 size=718.94MB
    stored statistics:
-     table: rows=unavailable size=unavailable
-     partitions: 0/3 rows=unavailable
-     columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col
+     table: rows=6001215 size=718.94MB
+     columns: all
    extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=32.00KB
-   tuple-ids=0 row-size=88B cardinality=unavailable
+   mem-estimate=88.00MB mem-reservation=0B
+   tuple-ids=0 row-size=263B cardinality=6001215
 ====
 # HBase scan
 select * from functional_hbase.alltypes
@@ -1326,11 +258,11 @@ select * from tpch.lineitem
 union all
 select * from tpch.lineitem
 ---- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=88.00MB
 
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
+|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -1345,7 +277,7 @@ PLAN-ROOT SINK
 |       table: rows=6001215 size=718.94MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=88.00MB mem-reservation=8.00MB
+|     mem-estimate=88.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=263B cardinality=6001215
 |
 01:SCAN HDFS [tpch.lineitem]
@@ -1354,10 +286,10 @@ PLAN-ROOT SINK
      table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=88.00MB
 
 F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1370,7 +302,7 @@ PLAN-ROOT SINK
 |  tuple-ids=2 row-size=263B cardinality=12002430
 |
 F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
+Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
 00:UNION
 |  pass-through-operands: all
 |  mem-estimate=0B mem-reservation=0B
@@ -1382,7 +314,7 @@ Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
 |       table: rows=6001215 size=718.94MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=88.00MB mem-reservation=8.00MB
+|     mem-estimate=88.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=263B cardinality=6001215
 |
 01:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -1391,10 +323,10 @@ Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
      table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=176.00MB
 
 F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1407,7 +339,7 @@ PLAN-ROOT SINK
 |  tuple-ids=2 row-size=263B cardinality=12002430
 |
 F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
+Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
 00:UNION
 |  pass-through-operands: all
 |  mem-estimate=0B mem-reservation=0B
@@ -1419,7 +351,7 @@ Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
 |       table: rows=6001215 size=718.94MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=88.00MB mem-reservation=8.00MB
+|     mem-estimate=88.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=263B cardinality=6001215
 |
 01:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -1428,7 +360,7 @@ Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
      table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ====
 # Grouping aggregation
@@ -1436,11 +368,11 @@ select l_orderkey, count(*)
 from tpch_parquet.lineitem
 group by l_orderkey
 ---- PLAN
-Max Per-Host Resource Reservation: Memory=42.00MB
+Max Per-Host Resource Reservation: Memory=34.00MB
 Per-Host Resource Estimates: Memory=114.00MB
 
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=114.00MB mem-reservation=42.00MB
+|  Per-Host Resources: mem-estimate=114.00MB mem-reservation=34.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -1451,15 +383,15 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=16B cardinality=1563438
 |
 00:SCAN HDFS [tpch_parquet.lineitem]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=6001215
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=50.50MB
+Max Per-Host Resource Reservation: Memory=42.50MB
 Per-Host Resource Estimates: Memory=124.00MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1484,7 +416,7 @@ Per-Host Resources: mem-estimate=10.00MB mem-reservation=8.50MB
 |  tuple-ids=1 row-size=16B cardinality=1563438
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=114.00MB mem-reservation=42.00MB
+Per-Host Resources: mem-estimate=114.00MB mem-reservation=34.00MB
 01:AGGREGATE [STREAMING]
 |  output: count(*)
 |  group by: l_orderkey
@@ -1492,15 +424,15 @@ Per-Host Resources: mem-estimate=114.00MB mem-reservation=42.00MB
 |  tuple-ids=1 row-size=16B cardinality=1563438
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=6001215
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=93.50MB
+Max Per-Host Resource Reservation: Memory=77.50MB
 Per-Host Resource Estimates: Memory=248.00MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1525,7 +457,7 @@ Per-Host Resources: mem-estimate=20.00MB mem-reservation=9.50MB
 |  tuple-ids=1 row-size=16B cardinality=1563438
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=228.00MB mem-reservation=84.00MB
+Per-Host Resources: mem-estimate=228.00MB mem-reservation=68.00MB
 01:AGGREGATE [STREAMING]
 |  output: count(*)
 |  group by: l_orderkey
@@ -1533,22 +465,22 @@ Per-Host Resources: mem-estimate=228.00MB mem-reservation=84.00MB
 |  tuple-ids=1 row-size=16B cardinality=1563438
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=6001215
 ====
 # Non-grouping aggregation with zero-slot parquet scan
 select count(*) from tpch_parquet.lineitem
 ---- PLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=18.00MB
+Max Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=11.00MB
 
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=18.00MB mem-reservation=8.00MB
+|  Per-Host Resources: mem-estimate=11.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -1558,16 +490,16 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=8B cardinality=1
 |
 00:SCAN HDFS [tpch_parquet.lineitem]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=8.00MB mem-reservation=8.00MB
+   mem-estimate=1.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=6001215
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=8.00MB
-Per-Host Resource Estimates: Memory=28.00MB
+Max Per-Host Resource Reservation: Memory=0B
+Per-Host Resource Estimates: Memory=21.00MB
 
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
 |  Per-Host Resources: mem-estimate=10.00MB mem-reservation=0B
@@ -1584,22 +516,22 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=8B cardinality=1
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=18.00MB mem-reservation=8.00MB
+Per-Host Resources: mem-estimate=11.00MB mem-reservation=0B
 01:AGGREGATE
 |  output: sum_init_zero(tpch_parquet.lineitem.parquet-stats: num_rows)
 |  mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
 |  tuple-ids=1 row-size=8B cardinality=1
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=8.00MB mem-reservation=8.00MB
+   mem-estimate=1.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=6001215
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=16.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=190.00MB
 
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1617,19 +549,19 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=8B cardinality=1
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=180.00MB mem-reservation=16.00MB
+Per-Host Resources: mem-estimate=180.00MB mem-reservation=0B
 01:AGGREGATE
 |  output: sum_init_zero(tpch_parquet.lineitem.parquet-stats: num_rows)
 |  mem-estimate=10.00MB mem-reservation=0B spill-buffer=2.00MB
 |  tuple-ids=1 row-size=8B cardinality=1
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=6001215
 ====
 # Sort
@@ -1637,11 +569,11 @@ select *
 from tpch_parquet.lineitem
 order by l_comment
 ---- PLAN
-Max Per-Host Resource Reservation: Memory=84.00MB
+Max Per-Host Resource Reservation: Memory=12.00MB
 Per-Host Resource Estimates: Memory=120.00MB
 
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=120.00MB mem-reservation=84.00MB
+|  Per-Host Resources: mem-estimate=120.00MB mem-reservation=12.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -1651,15 +583,15 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=263B cardinality=6001215
 |
 00:SCAN HDFS [tpch_parquet.lineitem]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=84.00MB
+Max Per-Host Resource Reservation: Memory=12.00MB
 Per-Host Resource Estimates: Memory=120.00MB
 
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1673,22 +605,22 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=263B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=120.00MB mem-reservation=84.00MB
+Per-Host Resources: mem-estimate=120.00MB mem-reservation=12.00MB
 01:SORT
 |  order by: l_comment ASC
 |  mem-estimate=40.00MB mem-reservation=12.00MB spill-buffer=2.00MB
 |  tuple-ids=1 row-size=263B cardinality=6001215
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=168.00MB
+Max Per-Host Resource Reservation: Memory=24.00MB
 Per-Host Resource Estimates: Memory=240.00MB
 
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1702,19 +634,19 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=263B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=240.00MB mem-reservation=168.00MB
+Per-Host Resources: mem-estimate=240.00MB mem-reservation=24.00MB
 01:SORT
 |  order by: l_comment ASC
 |  mem-estimate=40.00MB mem-reservation=12.00MB spill-buffer=2.00MB
 |  tuple-ids=1 row-size=263B cardinality=6001215
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ====
 # TOP-N
@@ -1723,11 +655,11 @@ from tpch_parquet.lineitem
 order by l_comment
 limit 100
 ---- PLAN
-Max Per-Host Resource Reservation: Memory=72.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=80.03MB
 
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=80.03MB mem-reservation=72.00MB
+|  Per-Host Resources: mem-estimate=80.03MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -1737,15 +669,15 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=263B cardinality=100
 |
 00:SCAN HDFS [tpch_parquet.lineitem]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=72.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=80.03MB
 
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1760,22 +692,22 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=263B cardinality=100
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=80.03MB mem-reservation=72.00MB
+Per-Host Resources: mem-estimate=80.03MB mem-reservation=0B
 01:TOP-N [LIMIT=100]
 |  order by: l_comment ASC
 |  mem-estimate=25.66KB mem-reservation=0B
 |  tuple-ids=1 row-size=263B cardinality=100
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=144.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=160.05MB
 
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1790,30 +722,30 @@ PLAN-ROOT SINK
 |  tuple-ids=1 row-size=263B cardinality=100
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=160.05MB mem-reservation=144.00MB
+Per-Host Resources: mem-estimate=160.05MB mem-reservation=0B
 01:TOP-N [LIMIT=100]
 |  order by: l_comment ASC
 |  mem-estimate=25.66KB mem-reservation=0B
 |  tuple-ids=1 row-size=263B cardinality=100
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ====
 # Broadcast Hash Join
 select *
 from tpch.lineitem inner join tpch.orders on l_orderkey = o_orderkey
 ---- PLAN
-Max Per-Host Resource Reservation: Memory=51.00MB
+Max Per-Host Resource Reservation: Memory=35.00MB
 Per-Host Resource Estimates: Memory=477.41MB
 
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=477.41MB mem-reservation=51.00MB runtime-filters-memory=1.00MB
+|  Per-Host Resources: mem-estimate=477.41MB mem-reservation=35.00MB runtime-filters-memory=1.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -1830,7 +762,7 @@ PLAN-ROOT SINK
 |       table: rows=1500000 size=162.56MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=88.00MB mem-reservation=8.00MB
+|     mem-estimate=88.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=191B cardinality=1500000
 |
 00:SCAN HDFS [tpch.lineitem]
@@ -1840,10 +772,10 @@ PLAN-ROOT SINK
      table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=51.00MB
+Max Per-Host Resource Reservation: Memory=35.00MB
 Per-Host Resource Estimates: Memory=477.41MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1856,7 +788,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1 row-size=454B cardinality=5757710
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=389.41MB mem-reservation=43.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=389.41MB mem-reservation=35.00MB runtime-filters-memory=1.00MB
 02:HASH JOIN [INNER JOIN, BROADCAST]
 |  hash predicates: l_orderkey = o_orderkey
 |  fk/pk conjuncts: l_orderkey = o_orderkey
@@ -1869,14 +801,14 @@ Per-Host Resources: mem-estimate=389.41MB mem-reservation=43.00MB runtime-filter
 |  |  tuple-ids=1 row-size=191B cardinality=1500000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
+|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch.orders, RANDOM]
 |     partitions=1/1 files=1 size=162.56MB
 |     stored statistics:
 |       table: rows=1500000 size=162.56MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=88.00MB mem-reservation=8.00MB
+|     mem-estimate=88.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=191B cardinality=1500000
 |
 00:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -1886,10 +818,10 @@ Per-Host Resources: mem-estimate=389.41MB mem-reservation=43.00MB runtime-filter
      table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=102.00MB
+Max Per-Host Resource Reservation: Memory=70.00MB
 Per-Host Resource Estimates: Memory=954.83MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -1902,7 +834,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1 row-size=454B cardinality=5757710
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=778.83MB mem-reservation=86.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=778.83MB mem-reservation=70.00MB runtime-filters-memory=1.00MB
 02:HASH JOIN [INNER JOIN, BROADCAST]
 |  hash-table-id=00
 |  hash predicates: l_orderkey = o_orderkey
@@ -1923,14 +855,14 @@ Per-Host Resources: mem-estimate=778.83MB mem-reservation=86.00MB runtime-filter
 |  |  tuple-ids=1 row-size=191B cardinality=1500000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-|  Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
+|  Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch.orders, RANDOM]
 |     partitions=1/1 files=1 size=162.56MB
 |     stored statistics:
 |       table: rows=1500000 size=162.56MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=88.00MB mem-reservation=8.00MB
+|     mem-estimate=88.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=191B cardinality=1500000
 |
 00:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -1940,18 +872,18 @@ Per-Host Resources: mem-estimate=778.83MB mem-reservation=86.00MB runtime-filter
      table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ====
 # Shuffle Hash Join
 select *
 from tpch.lineitem inner join /* +shuffle */ tpch.orders on l_orderkey = o_orderkey
 ---- PLAN
-Max Per-Host Resource Reservation: Memory=51.00MB
+Max Per-Host Resource Reservation: Memory=35.00MB
 Per-Host Resource Estimates: Memory=477.41MB
 
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=477.41MB mem-reservation=51.00MB runtime-filters-memory=1.00MB
+|  Per-Host Resources: mem-estimate=477.41MB mem-reservation=35.00MB runtime-filters-memory=1.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -1968,7 +900,7 @@ PLAN-ROOT SINK
 |       table: rows=1500000 size=162.56MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=88.00MB mem-reservation=8.00MB
+|     mem-estimate=88.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=191B cardinality=1500000
 |
 00:SCAN HDFS [tpch.lineitem]
@@ -1978,10 +910,10 @@ PLAN-ROOT SINK
      table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=52.00MB
+Max Per-Host Resource Reservation: Memory=36.00MB
 Per-Host Resource Estimates: Memory=278.14MB
 
 F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2007,14 +939,14 @@ Per-Host Resources: mem-estimate=101.14MB mem-reservation=35.00MB runtime-filter
 |  |  tuple-ids=1 row-size=191B cardinality=1500000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
+|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch.orders, RANDOM]
 |     partitions=1/1 files=1 size=162.56MB
 |     stored statistics:
 |       table: rows=1500000 size=162.56MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=88.00MB mem-reservation=8.00MB
+|     mem-estimate=88.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=191B cardinality=1500000
 |
 03:EXCHANGE [HASH(l_orderkey)]
@@ -2022,7 +954,7 @@ Per-Host Resources: mem-estimate=101.14MB mem-reservation=35.00MB runtime-filter
 |  tuple-ids=0 row-size=263B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=89.00MB mem-reservation=9.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=89.00MB mem-reservation=1.00MB runtime-filters-memory=1.00MB
 00:SCAN HDFS [tpch.lineitem, RANDOM]
    partitions=1/1 files=1 size=718.94MB
    runtime filters: RF000[bloom] -> l_orderkey
@@ -2030,10 +962,10 @@ Per-Host Resources: mem-estimate=89.00MB mem-reservation=9.00MB runtime-filters-
      table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=104.00MB
+Max Per-Host Resource Reservation: Memory=72.00MB
 Per-Host Resource Estimates: Memory=456.14MB
 
 F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2067,14 +999,14 @@ Per-Host Resources: mem-estimate=102.14MB mem-reservation=70.00MB runtime-filter
 |  |  tuple-ids=1 row-size=191B cardinality=1500000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-|  Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
+|  Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch.orders, RANDOM]
 |     partitions=1/1 files=1 size=162.56MB
 |     stored statistics:
 |       table: rows=1500000 size=162.56MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=88.00MB mem-reservation=8.00MB
+|     mem-estimate=88.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=191B cardinality=1500000
 |
 03:EXCHANGE [HASH(l_orderkey)]
@@ -2082,7 +1014,7 @@ Per-Host Resources: mem-estimate=102.14MB mem-reservation=70.00MB runtime-filter
 |  tuple-ids=0 row-size=263B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=178.00MB mem-reservation=18.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=178.00MB mem-reservation=2.00MB runtime-filters-memory=1.00MB
 00:SCAN HDFS [tpch.lineitem, RANDOM]
    partitions=1/1 files=1 size=718.94MB
    runtime filters: RF000[bloom] -> l_orderkey
@@ -2090,18 +1022,18 @@ Per-Host Resources: mem-estimate=178.00MB mem-reservation=18.00MB runtime-filter
      table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ====
 # Nested loop join
 select *
 from tpch.lineitem, tpch.orders
 ---- PLAN
-Max Per-Host Resource Reservation: Memory=16.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=449.10MB
 
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=449.10MB mem-reservation=16.00MB
+|  Per-Host Resources: mem-estimate=449.10MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -2115,7 +1047,7 @@ PLAN-ROOT SINK
 |       table: rows=1500000 size=162.56MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=88.00MB mem-reservation=8.00MB
+|     mem-estimate=88.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=191B cardinality=1500000
 |
 00:SCAN HDFS [tpch.lineitem]
@@ -2124,10 +1056,10 @@ PLAN-ROOT SINK
      table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=16.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=449.10MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2140,7 +1072,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1 row-size=454B cardinality=9001822500000
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=361.10MB mem-reservation=8.00MB
+Per-Host Resources: mem-estimate=361.10MB mem-reservation=0B
 02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
 |  mem-estimate=273.10MB mem-reservation=0B
 |  tuple-ids=0,1 row-size=454B cardinality=9001822500000
@@ -2150,14 +1082,14 @@ Per-Host Resources: mem-estimate=361.10MB mem-reservation=8.00MB
 |  |  tuple-ids=1 row-size=191B cardinality=1500000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=8.00MB
+|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch.orders, RANDOM]
 |     partitions=1/1 files=1 size=162.56MB
 |     stored statistics:
 |       table: rows=1500000 size=162.56MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=88.00MB mem-reservation=8.00MB
+|     mem-estimate=88.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=191B cardinality=1500000
 |
 00:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -2166,10 +1098,10 @@ Per-Host Resources: mem-estimate=361.10MB mem-reservation=8.00MB
      table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=32.00MB
+Max Per-Host Resource Reservation: Memory=0B
 Per-Host Resource Estimates: Memory=898.21MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2182,7 +1114,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1 row-size=454B cardinality=9001822500000
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=722.21MB mem-reservation=16.00MB
+Per-Host Resources: mem-estimate=722.21MB mem-reservation=0B
 02:NESTED LOOP JOIN [CROSS JOIN, BROADCAST]
 |  join table id: 00
 |  mem-estimate=273.10MB mem-reservation=0B
@@ -2199,14 +1131,14 @@ Per-Host Resources: mem-estimate=722.21MB mem-reservation=16.00MB
 |  |  tuple-ids=1 row-size=191B cardinality=1500000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-|  Per-Host Resources: mem-estimate=176.00MB mem-reservation=16.00MB
+|  Per-Host Resources: mem-estimate=176.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch.orders, RANDOM]
 |     partitions=1/1 files=1 size=162.56MB
 |     stored statistics:
 |       table: rows=1500000 size=162.56MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=88.00MB mem-reservation=8.00MB
+|     mem-estimate=88.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=191B cardinality=1500000
 |
 00:SCAN HDFS [tpch.lineitem, RANDOM]
@@ -2215,7 +1147,7 @@ Per-Host Resources: mem-estimate=722.21MB mem-reservation=16.00MB
      table: rows=6001215 size=718.94MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=88.00MB mem-reservation=8.00MB
+   mem-estimate=88.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ====
 # Empty set node
@@ -2264,12 +1196,12 @@ PLAN-ROOT SINK
 select max(tinyint_col) over(partition by int_col)
 from functional.alltypes
 ---- PLAN
-Max Per-Host Resource Reservation: Memory=10.03MB
+Max Per-Host Resource Reservation: Memory=10.00MB
 Per-Host Resource Estimates: Memory=26.00MB
 Codegen disabled by planner
 
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=26.00MB mem-reservation=10.03MB
+|  Per-Host Resources: mem-estimate=26.00MB mem-reservation=10.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -2291,10 +1223,10 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=32.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=5B cardinality=7300
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=10.03MB
+Max Per-Host Resource Reservation: Memory=10.00MB
 Per-Host Resource Estimates: Memory=26.00MB
 Codegen disabled by planner
 
@@ -2325,7 +1257,7 @@ Per-Host Resources: mem-estimate=10.00MB mem-reservation=10.00MB
 |  tuple-ids=0 row-size=5B cardinality=7300
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
+Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
 00:SCAN HDFS [functional.alltypes, RANDOM]
    partitions=24/24 files=24 size=478.45KB
    stored statistics:
@@ -2333,10 +1265,10 @@ Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=32.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=5B cardinality=7300
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=20.06MB
+Max Per-Host Resource Reservation: Memory=20.00MB
 Per-Host Resource Estimates: Memory=52.00MB
 Codegen disabled by planner
 
@@ -2367,7 +1299,7 @@ Per-Host Resources: mem-estimate=20.00MB mem-reservation=20.00MB
 |  tuple-ids=0 row-size=5B cardinality=7300
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=32.00MB mem-reservation=64.00KB
+Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
 00:SCAN HDFS [functional.alltypes, RANDOM]
    partitions=24/24 files=24 size=478.45KB
    stored statistics:
@@ -2375,7 +1307,7 @@ Per-Host Resources: mem-estimate=32.00MB mem-reservation=64.00KB
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=32.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=5B cardinality=7300
 ====
 # Pipeline of blocking operators from analytic fns. Blocking operators break
@@ -2385,11 +1317,11 @@ select *, row_number() over (order by o_totalprice) rnum_price,
   row_number() over (order by o_orderpriority) rnum_priority
 from tpch_parquet.orders
 ---- PLAN
-Max Per-Host Resource Reservation: Memory=72.00MB
-Per-Host Resource Estimates: Memory=78.00MB
+Max Per-Host Resource Reservation: Memory=36.00MB
+Per-Host Resource Estimates: Memory=70.00MB
 
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=78.00MB mem-reservation=72.00MB
+|  Per-Host Resources: mem-estimate=70.00MB mem-reservation=36.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -2435,11 +1367,11 @@ PLAN-ROOT SINK
      table: rows=1500000 size=54.07MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=48.00MB mem-reservation=48.00MB
+   mem-estimate=40.00MB mem-reservation=0B
    tuple-ids=0 row-size=191B cardinality=1500000
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=92.00MB
-Per-Host Resource Estimates: Memory=110.00MB
+Max Per-Host Resource Reservation: Memory=44.00MB
+Per-Host Resource Estimates: Memory=102.00MB
 
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
 |  Per-Host Resources: mem-estimate=44.00MB mem-reservation=32.00MB
@@ -2483,7 +1415,7 @@ PLAN-ROOT SINK
 |  tuple-ids=6 row-size=191B cardinality=1500000
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-Per-Host Resources: mem-estimate=66.00MB mem-reservation=60.00MB
+Per-Host Resources: mem-estimate=58.00MB mem-reservation=12.00MB
 01:SORT
 |  order by: o_totalprice ASC
 |  mem-estimate=18.00MB mem-reservation=12.00MB spill-buffer=2.00MB
@@ -2495,11 +1427,11 @@ Per-Host Resources: mem-estimate=66.00MB mem-reservation=60.00MB
      table: rows=1500000 size=54.07MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=48.00MB mem-reservation=48.00MB
+   mem-estimate=40.00MB mem-reservation=0B
    tuple-ids=0 row-size=191B cardinality=1500000
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=152.00MB
-Per-Host Resource Estimates: Memory=176.00MB
+Max Per-Host Resource Reservation: Memory=56.00MB
+Per-Host Resource Estimates: Memory=160.00MB
 
 F01:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
 |  Per-Host Resources: mem-estimate=44.00MB mem-reservation=32.00MB
@@ -2543,7 +1475,7 @@ PLAN-ROOT SINK
 |  tuple-ids=6 row-size=191B cardinality=1500000
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-Per-Host Resources: mem-estimate=132.00MB mem-reservation=120.00MB
+Per-Host Resources: mem-estimate=116.00MB mem-reservation=24.00MB
 01:SORT
 |  order by: o_totalprice ASC
 |  mem-estimate=18.00MB mem-reservation=12.00MB spill-buffer=2.00MB
@@ -2555,7 +1487,7 @@ Per-Host Resources: mem-estimate=132.00MB mem-reservation=120.00MB
      table: rows=1500000 size=54.07MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=48.00MB mem-reservation=48.00MB
+   mem-estimate=40.00MB mem-reservation=0B
    tuple-ids=0 row-size=191B cardinality=1500000
 ====
 # Union with non-trivial branches: each branch executes sequentially within fragment.
@@ -2571,11 +1503,11 @@ select l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment
 from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
 where l_shipmode = 'F'
 ---- PLAN
-Max Per-Host Resource Reservation: Memory=102.00MB
+Max Per-Host Resource Reservation: Memory=54.00MB
 Per-Host Resource Estimates: Memory=142.58MB
 
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=142.58MB mem-reservation=102.00MB runtime-filters-memory=3.00MB
+|  Per-Host Resources: mem-estimate=142.58MB mem-reservation=54.00MB runtime-filters-memory=3.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -2597,20 +1529,20 @@ PLAN-ROOT SINK
 |  |       table: rows=1500000 size=54.07MB
 |  |       columns: all
 |  |     extrapolated-rows=disabled
-|  |     mem-estimate=40.00MB mem-reservation=8.00MB
+|  |     mem-estimate=40.00MB mem-reservation=0B
 |  |     tuple-ids=6 row-size=8B cardinality=1500000
 |  |
 |  08:SCAN HDFS [tpch_parquet.lineitem]
-|     partitions=1/1 files=3 size=193.71MB
+|     partitions=1/1 files=3 size=193.73MB
 |     predicates: l_shipmode = 'F'
 |     runtime filters: RF004[bloom] -> l_orderkey
 |     stored statistics:
-|       table: rows=6001215 size=193.71MB
+|       table: rows=6001215 size=193.73MB
 |       columns: all
 |     extrapolated-rows=disabled
 |     parquet statistics predicates: l_shipmode = 'F'
 |     parquet dictionary predicates: l_shipmode = 'F'
-|     mem-estimate=80.00MB mem-reservation=48.00MB
+|     mem-estimate=80.00MB mem-reservation=0B
 |     tuple-ids=5 row-size=91B cardinality=857316
 |
 |--07:HASH JOIN [INNER JOIN]
@@ -2629,17 +1561,17 @@ PLAN-ROOT SINK
 |  |     extrapolated-rows=disabled
 |  |     parquet statistics predicates: o_orderpriority = '2-HIGH'
 |  |     parquet dictionary predicates: o_orderpriority = '2-HIGH'
-|  |     mem-estimate=40.00MB mem-reservation=16.00MB
+|  |     mem-estimate=40.00MB mem-reservation=0B
 |  |     tuple-ids=4 row-size=32B cardinality=300000
 |  |
 |  05:SCAN HDFS [tpch_parquet.lineitem]
-|     partitions=1/1 files=3 size=193.71MB
+|     partitions=1/1 files=3 size=193.73MB
 |     runtime filters: RF002[bloom] -> l_orderkey
 |     stored statistics:
-|       table: rows=6001215 size=193.71MB
+|       table: rows=6001215 size=193.73MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=80.00MB mem-reservation=40.00MB
+|     mem-estimate=80.00MB mem-reservation=0B
 |     tuple-ids=3 row-size=70B cardinality=6001215
 |
 04:AGGREGATE [FINALIZE]
@@ -2660,23 +1592,23 @@ PLAN-ROOT SINK
 |       table: rows=1500000 size=54.07MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=40.00MB mem-reservation=8.00MB
+|     mem-estimate=40.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=8B cardinality=1500000
 |
 01:SCAN HDFS [tpch_parquet.lineitem]
-   partitions=1/1 files=3 size=193.71MB
+   partitions=1/1 files=3 size=193.73MB
    predicates: l_tax > 10
    runtime filters: RF000[bloom] -> l_orderkey
    stored statistics:
-     table: rows=6001215 size=193.71MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
    parquet statistics predicates: l_tax > 10
    parquet dictionary predicates: l_tax > 10
-   mem-estimate=80.00MB mem-reservation=48.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=78B cardinality=600122
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=187.75MB
+Max Per-Host Resource Reservation: Memory=76.75MB
 Per-Host Resource Estimates: Memory=348.33MB
 
 F09:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -2689,7 +1621,7 @@ PLAN-ROOT SINK
 |  tuple-ids=7 row-size=70B cardinality=2549844
 |
 F08:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=99.00MB mem-reservation=67.00MB runtime-filters-memory=2.00MB
+Per-Host Resources: mem-estimate=99.00MB mem-reservation=36.00MB runtime-filters-memory=2.00MB
 00:UNION
 |  pass-through-operands: 14
 |  mem-estimate=0B mem-reservation=0B
@@ -2707,27 +1639,27 @@ Per-Host Resources: mem-estimate=99.00MB mem-reservation=67.00MB runtime-filters
 |  |  |  tuple-ids=6 row-size=8B cardinality=1500000
 |  |  |
 |  |  F07:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-|  |  Per-Host Resources: mem-estimate=40.00MB mem-reservation=8.00MB
+|  |  Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
 |  |  09:SCAN HDFS [tpch_parquet.orders, RANDOM]
 |  |     partitions=1/1 files=2 size=54.07MB
 |  |     stored statistics:
 |  |       table: rows=1500000 size=54.07MB
 |  |       columns: all
 |  |     extrapolated-rows=disabled
-|  |     mem-estimate=40.00MB mem-reservation=8.00MB
+|  |     mem-estimate=40.00MB mem-reservation=0B
 |  |     tuple-ids=6 row-size=8B cardinality=1500000
 |  |
 |  08:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-|     partitions=1/1 files=3 size=193.71MB
+|     partitions=1/1 files=3 size=193.73MB
 |     predicates: l_shipmode = 'F'
 |     runtime filters: RF004[bloom] -> l_orderkey
 |     stored statistics:
-|       table: rows=6001215 size=193.71MB
+|       table: rows=6001215 size=193.73MB
 |       columns: all
 |     extrapolated-rows=disabled
 |     parquet statistics predicates: l_shipmode = 'F'
 |     parquet dictionary predicates: l_shipmode = 'F'
-|     mem-estimate=80.00MB mem-reservation=48.00MB
+|     mem-estimate=80.00MB mem-reservation=0B
 |     tuple-ids=5 row-size=91B cardinality=857316
 |
 |--07:HASH JOIN [INNER JOIN, BROADCAST]
@@ -2742,7 +1674,7 @@ Per-Host Resources: mem-estimate=99.00MB mem-reservation=67.00MB runtime-filters
 |  |  |  tuple-ids=4 row-size=32B cardinality=300000
 |  |  |
 |  |  F05:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-|  |  Per-Host Resources: mem-estimate=40.00MB mem-reservation=16.00MB
+|  |  Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
 |  |  06:SCAN HDFS [tpch_parquet.orders, RANDOM]
 |  |     partitions=1/1 files=2 size=54.07MB
 |  |     predicates: o_orderpriority = '2-HIGH'
@@ -2752,17 +1684,17 @@ Per-Host Resources: mem-estimate=99.00MB mem-reservation=67.00MB runtime-filters
 |  |     extrapolated-rows=disabled
 |  |     parquet statistics predicates: o_

<TRUNCATED>

[2/9] impala git commit: Revert IMPALA-4835 and dependent changes

Posted by lv...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test
index b10c5cd..146524a 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/sort-expr-materialization.test
@@ -2,7 +2,7 @@
 select * from functional.alltypes order by random()
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
+|  Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -19,14 +19,14 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=97B cardinality=7300
 ====
 # sort on a deterministic expr that exceeds the cost threshold
 select * from functional.alltypes order by abs(id) + abs(id)
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
+|  Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -43,14 +43,14 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=97B cardinality=7300
 ====
 # sort on a deterministic expr that doesn't exceed the cost threshold
 select * from functional.alltypes order by tinyint_col + 1
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
+|  Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -66,7 +66,7 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=97B cardinality=7300
 ====
 # sort on multiple exprs, subset is materialized
@@ -74,7 +74,7 @@ select * from functional.alltypes
 order by dayofweek(timestamp_col), true, id + 1, string_col = date_string_col, id = tinyint_col
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
+|  Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -91,7 +91,7 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=97B cardinality=7300
 ====
 # expensive analytic order by expr gets materialized
@@ -99,7 +99,7 @@ select last_value(id) over (order by to_date(timestamp_col), bool_col is null)
 from functional.alltypes
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.03MB
+|  Per-Host Resources: mem-estimate=144.00MB mem-reservation=16.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -123,7 +123,7 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=21B cardinality=7300
 ====
 # expensive order by expr in top-n gets materialized
@@ -131,7 +131,7 @@ select id from functional.alltypes order by string_col like 'a.*b', id * bigint_
 regexp_replace(string_col, 'a.*b', 'c') limit 10
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
+|  Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -148,14 +148,14 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=29B cardinality=7300
 ====
 # sort on udf, gets materialized
 select * from functional.alltypes order by TestFn(double_col)
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
+|  Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -172,14 +172,14 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=97B cardinality=7300
 ====
 # sort expr contains SlotRefs that don't need to be materialized separately
 select concat(date_string_col, string_col) c from functional.alltypes order by c
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.03MB
+|  Per-Host Resources: mem-estimate=140.00MB mem-reservation=12.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -196,6 +196,6 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=41B cardinality=7300
 ====

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
index f57e578..03273bc 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/spillable-buffer-sizing.test
@@ -3,7 +3,7 @@ select straight_join *
 from tpch_parquet.customer
     inner join tpch_parquet.nation on c_nationkey = n_nationkey
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=18.97MB
+Max Per-Host Resource Reservation: Memory=2.94MB
 Per-Host Resource Estimates: Memory=42.94MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -16,7 +16,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1 row-size=355B cardinality=150000
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-Per-Host Resources: mem-estimate=26.94MB mem-reservation=18.94MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=26.94MB mem-reservation=2.94MB runtime-filters-memory=1.00MB
 02:HASH JOIN [INNER JOIN, BROADCAST]
 |  hash predicates: c_nationkey = n_nationkey
 |  fk/pk conjuncts: c_nationkey = n_nationkey
@@ -29,14 +29,14 @@ Per-Host Resources: mem-estimate=26.94MB mem-reservation=18.94MB runtime-filters
 |  |  tuple-ids=1 row-size=117B cardinality=25
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=32.00KB
+|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.nation, RANDOM]
 |     partitions=1/1 files=1 size=2.74KB
 |     stored statistics:
 |       table: rows=25 size=2.74KB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=16.00MB mem-reservation=32.00KB
+|     mem-estimate=16.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=117B cardinality=25
 |
 00:SCAN HDFS [tpch_parquet.customer, RANDOM]
@@ -46,10 +46,10 @@ Per-Host Resources: mem-estimate=26.94MB mem-reservation=18.94MB runtime-filters
      table: rows=150000 size=12.31MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=24.00MB mem-reservation=16.00MB
+   mem-estimate=24.00MB mem-reservation=0B
    tuple-ids=0 row-size=238B cardinality=150000
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=37.94MB
+Max Per-Host Resource Reservation: Memory=5.88MB
 Per-Host Resource Estimates: Memory=85.88MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -62,7 +62,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1 row-size=355B cardinality=150000
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-Per-Host Resources: mem-estimate=53.88MB mem-reservation=37.88MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=53.88MB mem-reservation=5.88MB runtime-filters-memory=1.00MB
 02:HASH JOIN [INNER JOIN, BROADCAST]
 |  hash-table-id=00
 |  hash predicates: c_nationkey = n_nationkey
@@ -83,14 +83,14 @@ Per-Host Resources: mem-estimate=53.88MB mem-reservation=37.88MB runtime-filters
 |  |  tuple-ids=1 row-size=117B cardinality=25
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=64.00KB
+|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.nation, RANDOM]
 |     partitions=1/1 files=1 size=2.74KB
 |     stored statistics:
 |       table: rows=25 size=2.74KB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=16.00MB mem-reservation=32.00KB
+|     mem-estimate=16.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=117B cardinality=25
 |
 00:SCAN HDFS [tpch_parquet.customer, RANDOM]
@@ -100,7 +100,7 @@ Per-Host Resources: mem-estimate=53.88MB mem-reservation=37.88MB runtime-filters
      table: rows=150000 size=12.31MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=24.00MB mem-reservation=16.00MB
+   mem-estimate=24.00MB mem-reservation=0B
    tuple-ids=0 row-size=238B cardinality=150000
 ====
 # Join with large build side - should use default-sized buffers.
@@ -108,8 +108,8 @@ select straight_join *
 from tpch_parquet.lineitem
     left join tpch_parquet.orders on l_orderkey = o_orderkey
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=154.00MB
-Per-Host Resource Estimates: Memory=428.41MB
+Max Per-Host Resource Reservation: Memory=34.00MB
+Per-Host Resource Estimates: Memory=420.41MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
 |  Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -121,7 +121,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1N row-size=454B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=380.41MB mem-reservation=106.00MB
+Per-Host Resources: mem-estimate=380.41MB mem-reservation=34.00MB
 02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
 |  hash predicates: l_orderkey = o_orderkey
 |  fk/pk conjuncts: l_orderkey = o_orderkey
@@ -133,27 +133,27 @@ Per-Host Resources: mem-estimate=380.41MB mem-reservation=106.00MB
 |  |  tuple-ids=1 row-size=191B cardinality=1500000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-|  Per-Host Resources: mem-estimate=48.00MB mem-reservation=48.00MB
+|  Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.orders, RANDOM]
 |     partitions=1/1 files=2 size=54.07MB
 |     stored statistics:
 |       table: rows=1500000 size=54.07MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=48.00MB mem-reservation=48.00MB
+|     mem-estimate=40.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=191B cardinality=1500000
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.72MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.72MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=308.00MB
-Per-Host Resource Estimates: Memory=856.83MB
+Max Per-Host Resource Reservation: Memory=68.00MB
+Per-Host Resource Estimates: Memory=840.83MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
 |  Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -165,7 +165,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1N row-size=454B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=760.83MB mem-reservation=212.00MB
+Per-Host Resources: mem-estimate=760.83MB mem-reservation=68.00MB
 02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
 |  hash-table-id=00
 |  hash predicates: l_orderkey = o_orderkey
@@ -185,23 +185,23 @@ Per-Host Resources: mem-estimate=760.83MB mem-reservation=212.00MB
 |  |  tuple-ids=1 row-size=191B cardinality=1500000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-|  Per-Host Resources: mem-estimate=96.00MB mem-reservation=96.00MB
+|  Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.orders, RANDOM]
 |     partitions=1/1 files=2 size=54.07MB
 |     stored statistics:
 |       table: rows=1500000 size=54.07MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=48.00MB mem-reservation=48.00MB
+|     mem-estimate=40.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=191B cardinality=1500000
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.72MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.72MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ====
 # Shuffle join with mid-sized input.
@@ -209,8 +209,8 @@ select straight_join *
 from tpch_parquet.orders
     join /*+shuffle*/ tpch_parquet.customer on o_custkey = c_custkey
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=100.00MB
-Per-Host Resource Estimates: Memory=108.00MB
+Max Per-Host Resource Reservation: Memory=36.00MB
+Per-Host Resource Estimates: Memory=100.00MB
 
 F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
 |  Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -235,14 +235,14 @@ Per-Host Resources: mem-estimate=35.00MB mem-reservation=35.00MB runtime-filters
 |  |  tuple-ids=1 row-size=238B cardinality=150000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=24.00MB mem-reservation=16.00MB
+|  Per-Host Resources: mem-estimate=24.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.customer, RANDOM]
 |     partitions=1/1 files=1 size=12.31MB
 |     stored statistics:
 |       table: rows=150000 size=12.31MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=24.00MB mem-reservation=16.00MB
+|     mem-estimate=24.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=238B cardinality=150000
 |
 03:EXCHANGE [HASH(o_custkey)]
@@ -250,7 +250,7 @@ Per-Host Resources: mem-estimate=35.00MB mem-reservation=35.00MB runtime-filters
 |  tuple-ids=0 row-size=191B cardinality=1500000
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-Per-Host Resources: mem-estimate=49.00MB mem-reservation=49.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=41.00MB mem-reservation=1.00MB runtime-filters-memory=1.00MB
 00:SCAN HDFS [tpch_parquet.orders, RANDOM]
    partitions=1/1 files=2 size=54.07MB
    runtime filters: RF000[bloom] -> o_custkey
@@ -258,11 +258,11 @@ Per-Host Resources: mem-estimate=49.00MB mem-reservation=49.00MB runtime-filters
      table: rows=1500000 size=54.07MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=48.00MB mem-reservation=48.00MB
+   mem-estimate=40.00MB mem-reservation=0B
    tuple-ids=0 row-size=191B cardinality=1500000
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=166.00MB
-Per-Host Resource Estimates: Memory=182.00MB
+Max Per-Host Resource Reservation: Memory=38.00MB
+Per-Host Resource Estimates: Memory=166.00MB
 
 F03:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
 |  Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -295,14 +295,14 @@ Per-Host Resources: mem-estimate=36.00MB mem-reservation=36.00MB runtime-filters
 |  |  tuple-ids=1 row-size=238B cardinality=150000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-|  Per-Host Resources: mem-estimate=48.00MB mem-reservation=32.00MB
+|  Per-Host Resources: mem-estimate=48.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.customer, RANDOM]
 |     partitions=1/1 files=1 size=12.31MB
 |     stored statistics:
 |       table: rows=150000 size=12.31MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=24.00MB mem-reservation=16.00MB
+|     mem-estimate=24.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=238B cardinality=150000
 |
 03:EXCHANGE [HASH(o_custkey)]
@@ -310,7 +310,7 @@ Per-Host Resources: mem-estimate=36.00MB mem-reservation=36.00MB runtime-filters
 |  tuple-ids=0 row-size=191B cardinality=1500000
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-Per-Host Resources: mem-estimate=98.00MB mem-reservation=98.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=82.00MB mem-reservation=2.00MB runtime-filters-memory=1.00MB
 00:SCAN HDFS [tpch_parquet.orders, RANDOM]
    partitions=1/1 files=2 size=54.07MB
    runtime filters: RF000[bloom] -> o_custkey
@@ -318,7 +318,7 @@ Per-Host Resources: mem-estimate=98.00MB mem-reservation=98.00MB runtime-filters
      table: rows=1500000 size=54.07MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=48.00MB mem-reservation=48.00MB
+   mem-estimate=40.00MB mem-reservation=0B
    tuple-ids=0 row-size=191B cardinality=1500000
 ====
 # Broadcast join with mid-sized input - should use larger buffers than shuffle join.
@@ -326,8 +326,8 @@ select straight_join *
 from tpch_parquet.orders
     join /*+broadcast*/ tpch_parquet.customer on o_custkey = c_custkey
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=99.00MB
-Per-Host Resource Estimates: Memory=110.38MB
+Max Per-Host Resource Reservation: Memory=35.00MB
+Per-Host Resource Estimates: Memory=102.38MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
 |  Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -339,7 +339,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1 row-size=428B cardinality=1500000
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-Per-Host Resources: mem-estimate=86.38MB mem-reservation=83.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=78.38MB mem-reservation=35.00MB runtime-filters-memory=1.00MB
 02:HASH JOIN [INNER JOIN, BROADCAST]
 |  hash predicates: o_custkey = c_custkey
 |  fk/pk conjuncts: o_custkey = c_custkey
@@ -352,14 +352,14 @@ Per-Host Resources: mem-estimate=86.38MB mem-reservation=83.00MB runtime-filters
 |  |  tuple-ids=1 row-size=238B cardinality=150000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=24.00MB mem-reservation=16.00MB
+|  Per-Host Resources: mem-estimate=24.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.customer, RANDOM]
 |     partitions=1/1 files=1 size=12.31MB
 |     stored statistics:
 |       table: rows=150000 size=12.31MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=24.00MB mem-reservation=16.00MB
+|     mem-estimate=24.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=238B cardinality=150000
 |
 00:SCAN HDFS [tpch_parquet.orders, RANDOM]
@@ -369,11 +369,11 @@ Per-Host Resources: mem-estimate=86.38MB mem-reservation=83.00MB runtime-filters
      table: rows=1500000 size=54.07MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=48.00MB mem-reservation=48.00MB
+   mem-estimate=40.00MB mem-reservation=0B
    tuple-ids=0 row-size=191B cardinality=1500000
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=198.00MB
-Per-Host Resource Estimates: Memory=220.76MB
+Max Per-Host Resource Reservation: Memory=70.00MB
+Per-Host Resource Estimates: Memory=204.76MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
 |  Per-Host Resources: mem-estimate=0B mem-reservation=0B
@@ -385,7 +385,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1 row-size=428B cardinality=1500000
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-Per-Host Resources: mem-estimate=172.76MB mem-reservation=166.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=156.76MB mem-reservation=70.00MB runtime-filters-memory=1.00MB
 02:HASH JOIN [INNER JOIN, BROADCAST]
 |  hash-table-id=00
 |  hash predicates: o_custkey = c_custkey
@@ -406,14 +406,14 @@ Per-Host Resources: mem-estimate=172.76MB mem-reservation=166.00MB runtime-filte
 |  |  tuple-ids=1 row-size=238B cardinality=150000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-|  Per-Host Resources: mem-estimate=48.00MB mem-reservation=32.00MB
+|  Per-Host Resources: mem-estimate=48.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.customer, RANDOM]
 |     partitions=1/1 files=1 size=12.31MB
 |     stored statistics:
 |       table: rows=150000 size=12.31MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=24.00MB mem-reservation=16.00MB
+|     mem-estimate=24.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=238B cardinality=150000
 |
 00:SCAN HDFS [tpch_parquet.orders, RANDOM]
@@ -423,7 +423,7 @@ Per-Host Resources: mem-estimate=172.76MB mem-reservation=166.00MB runtime-filte
      table: rows=1500000 size=54.07MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=48.00MB mem-reservation=48.00MB
+   mem-estimate=40.00MB mem-reservation=0B
    tuple-ids=0 row-size=191B cardinality=1500000
 ====
 # Join with no stats for right input - should use default buffers.
@@ -431,7 +431,7 @@ select straight_join *
 from functional_parquet.alltypes
     left join functional_parquet.alltypestiny on alltypes.id = alltypestiny.id
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=34.17MB
+Max Per-Host Resource Reservation: Memory=34.00MB
 Per-Host Resource Estimates: Memory=2.03GB
 WARNING: The following tables are missing relevant table and/or column statistics.
 functional_parquet.alltypes, functional_parquet.alltypestiny
@@ -446,7 +446,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1N row-size=176B cardinality=unavailable
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.09MB
+Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.00MB
 02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
 |  hash predicates: alltypes.id = alltypestiny.id
 |  fk/pk conjuncts: assumed fk/pk
@@ -458,7 +458,7 @@ Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.09MB
 |  |  tuple-ids=1 row-size=88B cardinality=unavailable
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=88.00KB
+|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
 |  01:SCAN HDFS [functional_parquet.alltypestiny, RANDOM]
 |     partitions=4/4 files=4 size=9.75KB
 |     stored statistics:
@@ -466,7 +466,7 @@ Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.09MB
 |       partitions: 0/4 rows=unavailable
 |       columns: unavailable
 |     extrapolated-rows=disabled
-|     mem-estimate=16.00MB mem-reservation=88.00KB
+|     mem-estimate=16.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=88B cardinality=unavailable
 |
 00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
@@ -476,10 +476,10 @@ Per-Host Resources: mem-estimate=2.02GB mem-reservation=34.09MB
      partitions: 0/24 rows=unavailable
      columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col
    extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=88.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=88B cardinality=unavailable
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=68.34MB
+Max Per-Host Resource Reservation: Memory=68.00MB
 Per-Host Resource Estimates: Memory=4.06GB
 WARNING: The following tables are missing relevant table and/or column statistics.
 functional_parquet.alltypestiny
@@ -494,7 +494,7 @@ PLAN-ROOT SINK
 |  tuple-ids=0,1N row-size=176B cardinality=unavailable
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.17MB
+Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.00MB
 02:HASH JOIN [LEFT OUTER JOIN, BROADCAST]
 |  hash-table-id=00
 |  hash predicates: alltypes.id = alltypestiny.id
@@ -514,7 +514,7 @@ Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.17MB
 |  |  tuple-ids=1 row-size=88B cardinality=unavailable
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=176.00KB
+|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
 |  01:SCAN HDFS [functional_parquet.alltypestiny, RANDOM]
 |     partitions=4/4 files=4 size=9.75KB
 |     stored statistics:
@@ -522,7 +522,7 @@ Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.17MB
 |       partitions: 0/4 rows=unavailable
 |       columns: unavailable
 |     extrapolated-rows=disabled
-|     mem-estimate=16.00MB mem-reservation=88.00KB
+|     mem-estimate=16.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=88B cardinality=unavailable
 |
 00:SCAN HDFS [functional_parquet.alltypes, RANDOM]
@@ -532,7 +532,7 @@ Per-Host Resources: mem-estimate=4.03GB mem-reservation=68.17MB
      partitions: 0/24 rows=unavailable
      columns missing stats: id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col
    extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=88.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=88B cardinality=unavailable
 ====
 # Low NDV aggregation - should scale down buffers to minimum.
@@ -540,7 +540,7 @@ select c_nationkey, avg(c_acctbal)
 from tpch_parquet.customer
 group by c_nationkey
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=19.94MB
+Max Per-Host Resource Reservation: Memory=3.94MB
 Per-Host Resource Estimates: Memory=44.00MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -565,7 +565,7 @@ Per-Host Resources: mem-estimate=10.00MB mem-reservation=1.94MB
 |  tuple-ids=1 row-size=10B cardinality=25
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=1
-Per-Host Resources: mem-estimate=34.00MB mem-reservation=18.00MB
+Per-Host Resources: mem-estimate=34.00MB mem-reservation=2.00MB
 01:AGGREGATE [STREAMING]
 |  output: avg(c_acctbal)
 |  group by: c_nationkey
@@ -578,10 +578,10 @@ Per-Host Resources: mem-estimate=34.00MB mem-reservation=18.00MB
      table: rows=150000 size=12.31MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=24.00MB mem-reservation=16.00MB
+   mem-estimate=24.00MB mem-reservation=0B
    tuple-ids=0 row-size=10B cardinality=150000
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=39.88MB
+Max Per-Host Resource Reservation: Memory=7.88MB
 Per-Host Resource Estimates: Memory=88.00MB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -606,7 +606,7 @@ Per-Host Resources: mem-estimate=20.00MB mem-reservation=3.88MB
 |  tuple-ids=1 row-size=10B cardinality=25
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=1 instances=2
-Per-Host Resources: mem-estimate=68.00MB mem-reservation=36.00MB
+Per-Host Resources: mem-estimate=68.00MB mem-reservation=4.00MB
 01:AGGREGATE [STREAMING]
 |  output: avg(c_acctbal)
 |  group by: c_nationkey
@@ -619,7 +619,7 @@ Per-Host Resources: mem-estimate=68.00MB mem-reservation=36.00MB
      table: rows=150000 size=12.31MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=24.00MB mem-reservation=16.00MB
+   mem-estimate=24.00MB mem-reservation=0B
    tuple-ids=0 row-size=10B cardinality=150000
 ====
 # Mid NDV aggregation - should scale down buffers to intermediate size.
@@ -629,7 +629,7 @@ from tpch_parquet.lineitem
 group by 1, 2
 having count(*) = 1
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=111.00MB
+Max Per-Host Resource Reservation: Memory=87.00MB
 Per-Host Resource Estimates: Memory=227.12MB
 
 F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -674,14 +674,14 @@ Per-Host Resources: mem-estimate=72.12MB mem-reservation=52.00MB runtime-filters
 |  |  tuple-ids=1 row-size=25B cardinality=1500000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-|  Per-Host Resources: mem-estimate=40.00MB mem-reservation=16.00MB
+|  Per-Host Resources: mem-estimate=40.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.orders, RANDOM]
 |     partitions=1/1 files=2 size=54.07MB
 |     stored statistics:
 |       table: rows=1500000 size=54.07MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=40.00MB mem-reservation=16.00MB
+|     mem-estimate=40.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=25B cardinality=1500000
 |
 04:EXCHANGE [HASH(l_orderkey)]
@@ -689,18 +689,18 @@ Per-Host Resources: mem-estimate=72.12MB mem-reservation=52.00MB runtime-filters
 |  tuple-ids=0 row-size=8B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=81.00MB mem-reservation=9.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=81.00MB mem-reservation=1.00MB runtime-filters-memory=1.00MB
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.72MB
+   partitions=1/1 files=3 size=193.73MB
    runtime filters: RF000[bloom] -> l_orderkey
    stored statistics:
-     table: rows=6001215 size=193.72MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=6001215
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=171.00MB
+Max Per-Host Resource Reservation: Memory=123.00MB
 Per-Host Resource Estimates: Memory=363.00MB
 
 F04:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -753,14 +753,14 @@ Per-Host Resources: mem-estimate=87.00MB mem-reservation=87.00MB runtime-filters
 |  |  tuple-ids=1 row-size=25B cardinality=1500000
 |  |
 |  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=4
-|  Per-Host Resources: mem-estimate=80.00MB mem-reservation=32.00MB
+|  Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
 |  01:SCAN HDFS [tpch_parquet.orders, RANDOM]
 |     partitions=1/1 files=2 size=54.07MB
 |     stored statistics:
 |       table: rows=1500000 size=54.07MB
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=40.00MB mem-reservation=16.00MB
+|     mem-estimate=40.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=25B cardinality=1500000
 |
 04:EXCHANGE [HASH(l_orderkey)]
@@ -768,22 +768,22 @@ Per-Host Resources: mem-estimate=87.00MB mem-reservation=87.00MB runtime-filters
 |  tuple-ids=0 row-size=8B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=162.00MB mem-reservation=18.00MB runtime-filters-memory=1.00MB
+Per-Host Resources: mem-estimate=162.00MB mem-reservation=2.00MB runtime-filters-memory=1.00MB
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.72MB
+   partitions=1/1 files=3 size=193.73MB
    runtime filters: RF000[bloom] -> l_orderkey
    stored statistics:
-     table: rows=6001215 size=193.72MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=8.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=8B cardinality=6001215
 ====
 # High NDV aggregation - should use default buffer size.
 select distinct *
 from tpch_parquet.lineitem
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=140.00MB
+Max Per-Host Resource Reservation: Memory=68.00MB
 Per-Host Resource Estimates: Memory=3.31GB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -807,22 +807,22 @@ Per-Host Resources: mem-estimate=1.62GB mem-reservation=34.00MB
 |  tuple-ids=1 row-size=263B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=1.69GB mem-reservation=106.00MB
+Per-Host Resources: mem-estimate=1.69GB mem-reservation=34.00MB
 01:AGGREGATE [STREAMING]
 |  group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
 |  mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
 |  tuple-ids=1 row-size=263B cardinality=6001215
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.72MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.72MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=280.00MB
+Max Per-Host Resource Reservation: Memory=136.00MB
 Per-Host Resource Estimates: Memory=6.62GB
 
 F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
@@ -846,19 +846,19 @@ Per-Host Resources: mem-estimate=3.23GB mem-reservation=68.00MB
 |  tuple-ids=1 row-size=263B cardinality=6001215
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=3.39GB mem-reservation=212.00MB
+Per-Host Resources: mem-estimate=3.39GB mem-reservation=68.00MB
 01:AGGREGATE [STREAMING]
 |  group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
 |  mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB
 |  tuple-ids=1 row-size=263B cardinality=6001215
 |
 00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
-   partitions=1/1 files=3 size=193.72MB
+   partitions=1/1 files=3 size=193.73MB
    stored statistics:
-     table: rows=6001215 size=193.72MB
+     table: rows=6001215 size=193.73MB
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=72.00MB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=263B cardinality=6001215
 ====
 # Aggregation with unknown input - should use default buffer size.
@@ -866,7 +866,7 @@ select string_col, count(*)
 from functional_parquet.alltypestiny
 group by string_col
 ---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=68.01MB
+Max Per-Host Resource Reservation: Memory=68.00MB
 Per-Host Resource Estimates: Memory=272.00MB
 WARNING: The following tables are missing relevant table and/or column statistics.
 functional_parquet.alltypestiny
@@ -893,7 +893,7 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=34.00MB
 |  tuple-ids=1 row-size=24B cardinality=unavailable
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.01MB
+Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.00MB
 01:AGGREGATE [STREAMING]
 |  output: count(*)
 |  group by: string_col
@@ -907,10 +907,10 @@ Per-Host Resources: mem-estimate=144.00MB mem-reservation=34.01MB
      partitions: 0/4 rows=unavailable
      columns: unavailable
    extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=8.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=16B cardinality=unavailable
 ---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=136.02MB
+Max Per-Host Resource Reservation: Memory=136.00MB
 Per-Host Resource Estimates: Memory=544.00MB
 WARNING: The following tables are missing relevant table and/or column statistics.
 functional_parquet.alltypestiny
@@ -937,7 +937,7 @@ Per-Host Resources: mem-estimate=256.00MB mem-reservation=68.00MB
 |  tuple-ids=1 row-size=24B cardinality=unavailable
 |
 F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=6
-Per-Host Resources: mem-estimate=288.00MB mem-reservation=68.02MB
+Per-Host Resources: mem-estimate=288.00MB mem-reservation=68.00MB
 01:AGGREGATE [STREAMING]
 |  output: count(*)
 |  group by: string_col
@@ -951,6 +951,6 @@ Per-Host Resources: mem-estimate=288.00MB mem-reservation=68.02MB
      partitions: 0/4 rows=unavailable
      columns: unavailable
    extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=8.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=16B cardinality=unavailable
 ====

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
index 1844f11..48163ec 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tablesample.test
@@ -2,7 +2,7 @@
 select * from functional.alltypes tablesample system(10) repeatable(1234)
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=32.00KB
+|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -13,14 +13,14 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=32.00MB mem-reservation=32.00KB
+   mem-estimate=32.00MB mem-reservation=0B
    tuple-ids=0 row-size=97B cardinality=730
 ====
 # Sample 50%
 select * from functional.alltypes tablesample system(50) repeatable(1234)
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=80.00MB mem-reservation=32.00KB
+|  Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -31,7 +31,7 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=80.00MB mem-reservation=32.00KB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=97B cardinality=3650
 ====
 # Sampling and scan predicates. Scan predicates are evaluated after sampling and
@@ -40,7 +40,7 @@ select * from functional.alltypes tablesample system(50) repeatable(1234)
 where id < 10
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=80.00MB mem-reservation=32.00KB
+|  Per-Host Resources: mem-estimate=80.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -53,7 +53,7 @@ PLAN-ROOT SINK
      columns: all
    extrapolated-rows=disabled
    parquet dictionary predicates: id < 10
-   mem-estimate=80.00MB mem-reservation=32.00KB
+   mem-estimate=80.00MB mem-reservation=0B
    tuple-ids=0 row-size=97B cardinality=365
 ====
 # Partition pruning + sampling. Partition pruning happens before sampling.
@@ -61,7 +61,7 @@ select * from functional.alltypes tablesample system(50) repeatable(1234)
 where year = 2009
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=48.00MB mem-reservation=32.00KB
+|  Per-Host Resources: mem-estimate=48.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -72,7 +72,7 @@ PLAN-ROOT SINK
      partitions: 12/12 rows=3650
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=48.00MB mem-reservation=32.00KB
+   mem-estimate=48.00MB mem-reservation=0B
    tuple-ids=0 row-size=97B cardinality=1825
 ====
 # Edge case: sample 0%, no files should be selected
@@ -97,7 +97,7 @@ PLAN-ROOT SINK
 select * from functional.alltypes tablesample system(1) repeatable(1234)
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=32.00KB
+|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -108,7 +108,7 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=32.00MB mem-reservation=32.00KB
+   mem-estimate=32.00MB mem-reservation=0B
    tuple-ids=0 row-size=97B cardinality=73
 ====
 # Edge case: sample 1% and prune partitions, at least one file should be selected
@@ -116,7 +116,7 @@ select * from functional.alltypes tablesample system(1) repeatable(1234)
 where year = 2010
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=32.00KB
+|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -127,14 +127,14 @@ PLAN-ROOT SINK
      partitions: 12/12 rows=3650
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=32.00MB mem-reservation=32.00KB
+   mem-estimate=32.00MB mem-reservation=0B
    tuple-ids=0 row-size=97B cardinality=37
 ====
 # Edge case: sample 100%, all files should be selected
 select * from functional.alltypes tablesample system (100)
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
+|  Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -145,25 +145,25 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=97B cardinality=7300
 ====
 # Table that has no stats.
 select id from functional_parquet.alltypes tablesample system(10) repeatable(1234)
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=8.00KB
+|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
 00:SCAN HDFS [functional_parquet.alltypes]
-   partitions=3/24 files=3 size=22.10KB
+   partitions=3/24 files=3 size=22.07KB
    stored statistics:
      table: rows=unavailable size=unavailable
      partitions: 0/24 rows=unavailable
      columns: unavailable
    extrapolated-rows=disabled
-   mem-estimate=16.00MB mem-reservation=8.00KB
+   mem-estimate=16.00MB mem-reservation=0B
    tuple-ids=0 row-size=4B cardinality=unavailable
 ====
 # Sampling in a subquery.
@@ -172,7 +172,7 @@ select id from functional.alltypes t1 where exists (
   where t1.id = t2.id)
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=162.94MB mem-reservation=2.98MB runtime-filters-memory=1.00MB
+|  Per-Host Resources: mem-estimate=162.94MB mem-reservation=2.94MB runtime-filters-memory=1.00MB
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -189,7 +189,7 @@ PLAN-ROOT SINK
 |       partitions: 4/4 rows=100
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=32.00MB mem-reservation=8.00KB
+|     mem-estimate=32.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=4B cardinality=10
 |
 00:SCAN HDFS [functional.alltypes t1]
@@ -200,7 +200,7 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=4B cardinality=7300
 ====
 # Sampling in WITH-clause view.
@@ -208,7 +208,7 @@ with t as (select * from functional.alltypes tablesample system(10) repeatable(1
 select id from t
 ---- PLAN
 F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=32.00KB
+|  Per-Host Resources: mem-estimate=32.00MB mem-reservation=0B
 PLAN-ROOT SINK
 |  mem-estimate=0B mem-reservation=0B
 |
@@ -219,6 +219,6 @@ PLAN-ROOT SINK
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=32.00MB mem-reservation=32.00KB
+   mem-estimate=32.00MB mem-reservation=0B
    tuple-ids=0 row-size=4B cardinality=730
 ====

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-planner/queries/PlannerTest/union.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/union.test b/testdata/workloads/functional-planner/queries/PlannerTest/union.test
index 09ac11b..a0d0c26 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/union.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/union.test
@@ -3233,7 +3233,7 @@ PLAN-ROOT SINK
 |  tuple-ids=2 row-size=4B cardinality=7300
 |
 F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
+Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
 00:UNION
 |  pass-through-operands: all
 |  mem-estimate=0B mem-reservation=0B
@@ -3246,7 +3246,7 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
 |       partitions: 24/24 rows=7300
 |       columns: all
 |     extrapolated-rows=disabled
-|     mem-estimate=128.00MB mem-reservation=32.00KB
+|     mem-estimate=128.00MB mem-reservation=0B
 |     tuple-ids=1 row-size=4B cardinality=7300
 |
 01:SCAN HDFS [functional.emptytable, RANDOM]
@@ -3277,7 +3277,7 @@ PLAN-ROOT SINK
 |  tuple-ids=2 row-size=4B cardinality=7300
 |
 F02:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
+Per-Host Resources: mem-estimate=128.00MB mem-reservation=0B
 00:UNION
 |  pass-through-operands: all
 |  mem-estimate=0B mem-reservation=0B
@@ -3300,6 +3300,6 @@ Per-Host Resources: mem-estimate=128.00MB mem-reservation=32.00KB
      partitions: 24/24 rows=7300
      columns: all
    extrapolated-rows=disabled
-   mem-estimate=128.00MB mem-reservation=32.00KB
+   mem-estimate=128.00MB mem-reservation=0B
    tuple-ids=0 row-size=4B cardinality=7300
 ====

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
index de12db1..119c2fd 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/admission-reject-min-reservation.test
@@ -5,7 +5,7 @@ select distinct * from functional_parquet.alltypesagg
 ---- CATCH
 minimum memory reservation is greater than memory available to the
  query for buffer reservations. Memory reservation needed given the
- current plan: 68.09 MB. Set mem_limit to at least 100.09 MB.
+ current plan: 68.00 MB. Set mem_limit to at least 143.00 MB.
 ====
 ---- QUERY
 set mem_limit=150mb;
@@ -13,14 +13,14 @@ select distinct * from functional_parquet.alltypesagg
 ---- CATCH
 minimum memory reservation needed is greater than pool max mem resources.
  Pool max mem resources: 10.00 MB.
- Cluster-wide memory reservation needed: 204.26 MB
+ Cluster-wide memory reservation needed: 204.00 MB
 ====
 ---- QUERY
 set buffer_pool_limit=10mb;
 select distinct * from functional_parquet.alltypesagg
 ---- CATCH
 minimum memory reservation is greater than memory available to the query
- for buffer reservations. Increase the buffer_pool_limit to 68.09 MB.
+ for buffer reservations. Increase the buffer_pool_limit to 68.00 MB.
 ====
 ---- QUERY
 set mem_limit=1024;
@@ -29,14 +29,14 @@ from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
 ---- CATCH
 minimum memory reservation is greater than memory available to the
  query for buffer reservations. Memory reservation needed given the
- current plan: 22.75 MB. Set mem_limit to at least 54.75 MB.
+ current plan: 6.75 MB. Set mem_limit to at least 81.75 MB.
 ====
 ---- QUERY
-set mem_limit=50mb;
+set mem_limit=80mb;
 select count(*)
 from tpch_parquet.lineitem join tpch_parquet.orders on l_orderkey = o_orderkey
 ---- CATCH
 minimum memory reservation is greater than memory available to the
  query for buffer reservations. Memory reservation needed given the
- current plan: 34.00 MB. Set mem_limit to at least 66.00 MB.
+ current plan: 18.00 MB. Set mem_limit to at least 93.00 MB.
 ====

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test b/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
index d725391..f8a024a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
@@ -1584,10 +1584,9 @@ int, bigint, bigint, double
 ====
 ---- QUERY
 # Check that the a large analytic query can succeed with the minimum buffers (3 buffers
-# for sort, 2 buffer for analytic, plus 32KB extra for the I/O buffers required to scan
-# the < 16KB functional_parquet.alltypes files).
+# for sort, 2 buffer for analytic).
 SET default_spillable_buffer_size=8m;
-SET buffer_pool_limit=47m;
+SET buffer_pool_limit=40m;
 SELECT lag(-180, 13) over (ORDER BY t1.int_col ASC, t2.int_col ASC) AS int_col
 FROM functional_parquet.alltypes t1 CROSS JOIN functional_parquet.alltypes t2 LIMIT 10;
 ---- TYPES

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test b/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test
index f7013ae..dea6066 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/codegen-mem-limit.test
@@ -1,8 +1,9 @@
 =====
 ---- QUERY
-set exec_single_node_rows_threshold=0;
 set mem_limit=100k;
-with t as (values(1), (2), (3), (4)) select min(t1.`1` + t2.`1`) from t t1 join t t2;
+select *
+from alltypes
+where substr(string_col, 1) = "";
 ---- CATCH
 Codegen failed to reserve
 =====

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test b/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test
index 48649e5..2c4fede 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/disk-spill-encryption.test
@@ -1,6 +1,6 @@
 ====
 ---- QUERY
-set buffer_pool_limit=90m;
+set buffer_pool_limit=10m;
 set default_spillable_buffer_size=64k;
 select count(*)
 from (select distinct o_orderdate, o_custkey, o_comment

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
index 53afc32..92abd5f 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level0.test
@@ -5,8 +5,8 @@ explain
 select *
 from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
 ---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.*
-row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
+'Max Per-Host Resource Reservation: Memory=35.00MB'
+'Per-Host Resource Estimates: Memory=477.41MB'
 ''
 'PLAN-ROOT SINK'
 '04:EXCHANGE [UNPARTITIONED]'

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
index 6d58d29..e65745a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level1.test
@@ -5,8 +5,8 @@ explain
 select *
 from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
 ---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.*
-row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
+'Max Per-Host Resource Reservation: Memory=35.00MB'
+'Per-Host Resource Estimates: Memory=477.41MB'
 ''
 'PLAN-ROOT SINK'
 '|'

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
index 17fdd10..b1c58f4 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level2.test
@@ -5,8 +5,8 @@ explain
 select *
 from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
 ---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.*
-row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
+'Max Per-Host Resource Reservation: Memory=35.00MB'
+'Per-Host Resource Estimates: Memory=477.41MB'
 ''
 'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
 '|  Per-Host Resources: mem-estimate=0B mem-reservation=0B'
@@ -18,12 +18,12 @@ row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
 '|  tuple-ids=0,1 row-size=454B cardinality=5757710'
 '|'
 'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3'
-row_regex:.*Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+'Per-Host Resources: mem-estimate=389.41MB mem-reservation=35.00MB runtime-filters-memory=1.00MB'
 '02:HASH JOIN [INNER JOIN, BROADCAST]'
 '|  hash predicates: l_orderkey = o_orderkey'
 '|  fk/pk conjuncts: l_orderkey = o_orderkey'
 '|  runtime filters: RF000[bloom] <- o_orderkey'
-row_regex:.*|  mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB spill-buffer=[0-9.]*MB.*
+'|  mem-estimate=300.41MB mem-reservation=34.00MB spill-buffer=2.00MB'
 '|  tuple-ids=0,1 row-size=454B cardinality=5757710'
 '|'
 '|--03:EXCHANGE [BROADCAST]'
@@ -31,14 +31,14 @@ row_regex:.*|  mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB spill-buffer=[0-
 '|  |  tuple-ids=1 row-size=191B cardinality=1500000'
 '|  |'
 '|  F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2'
-row_regex:.*|  Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+'|  Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B'
 '|  01:SCAN HDFS [tpch.orders, RANDOM]'
 row_regex:.*partitions=1/1 files=1 size=.*
 '|     stored statistics:'
 row_regex:.*table: rows=1500000 size=.*
 '|       columns: all'
 '|     extrapolated-rows=disabled'
-row_regex:.*|     mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+'|     mem-estimate=88.00MB mem-reservation=0B'
 '|     tuple-ids=1 row-size=191B cardinality=1500000'
 '|'
 '00:SCAN HDFS [tpch.lineitem, RANDOM]'
@@ -48,7 +48,7 @@ row_regex:.*partitions=1/1 files=1 size=.*
 row_regex:.*table: rows=6001215 size=.*
 '     columns: all'
 '   extrapolated-rows=disabled'
-row_regex:.*   mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+'   mem-estimate=88.00MB mem-reservation=0B'
 '   tuple-ids=0 row-size=263B cardinality=6001215'
 ====
 ---- QUERY

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
index 4ef3524..625e2b0 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/explain-level3.test
@@ -5,8 +5,8 @@ explain
 select *
 from tpch.lineitem join tpch.orders on l_orderkey = o_orderkey;
 ---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=[0-9.]*MB.*
-row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
+'Max Per-Host Resource Reservation: Memory=35.00MB'
+'Per-Host Resource Estimates: Memory=477.41MB'
 ''
 'F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
 'Per-Host Resources: mem-estimate=0B mem-reservation=0B'
@@ -18,14 +18,14 @@ row_regex:.*Per-Host Resource Estimates: Memory=[0-9.]*MB.*
 '     tuple-ids=0,1 row-size=454B cardinality=5757710'
 ''
 'F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3'
-row_regex:.*Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+'Per-Host Resources: mem-estimate=389.41MB mem-reservation=35.00MB runtime-filters-memory=1.00MB'
 '  DATASTREAM SINK [FRAGMENT=F02, EXCHANGE=04, UNPARTITIONED]'
 '  |  mem-estimate=0B mem-reservation=0B'
 '  02:HASH JOIN [INNER JOIN, BROADCAST]'
 '  |  hash predicates: l_orderkey = o_orderkey'
 '  |  fk/pk conjuncts: l_orderkey = o_orderkey'
 '  |  runtime filters: RF000[bloom] <- o_orderkey'
-row_regex:.*  |  mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB spill-buffer=[0-9.]*MB.*
+'  |  mem-estimate=300.41MB mem-reservation=34.00MB spill-buffer=2.00MB'
 '  |  tuple-ids=0,1 row-size=454B cardinality=5757710'
 '  |'
 '  |--03:EXCHANGE [BROADCAST]'
@@ -39,11 +39,11 @@ row_regex:.*partitions=1/1 files=1 size=.*
 row_regex:.*table: rows=6001215 size=.*
 '       columns: all'
 '     extrapolated-rows=disabled'
-row_regex:.*     mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+'     mem-estimate=88.00MB mem-reservation=0B'
 '     tuple-ids=0 row-size=263B cardinality=6001215'
 ''
 'F01:PLAN FRAGMENT [RANDOM] hosts=2 instances=2'
-row_regex:.*Per-Host Resources: mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+'Per-Host Resources: mem-estimate=88.00MB mem-reservation=0B'
 '  DATASTREAM SINK [FRAGMENT=F00, EXCHANGE=03, BROADCAST]'
 '  |  mem-estimate=0B mem-reservation=0B'
 '  01:SCAN HDFS [tpch.orders, RANDOM]'
@@ -52,7 +52,7 @@ row_regex:.*partitions=1/1 files=1 size=.*
 row_regex:.*table: rows=1500000 size=.*
 '       columns: all'
 '     extrapolated-rows=disabled'
-row_regex:.*     mem-estimate=[0-9.]*MB mem-reservation=[0-9.]*MB.*
+'     mem-estimate=88.00MB mem-reservation=0B'
 '     tuple-ids=1 row-size=191B cardinality=1500000'
 ====
 ---- QUERY

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
index 6b213eb..c8a80b2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/nested-types-tpch.test
@@ -162,7 +162,7 @@ BIGINT,BIGINT
 ---- QUERY
 # IMPALA-2376
 # Set memory limit low enough to get the below query to consistently fail.
-# This was originally a regression test that hit an error like:
+# TODO: change query and/or table so we can consistently check for the following error:
 # Failed to allocate buffer for collection '...'.
 set mem_limit=4m;
 select max(cnt) from customer c,
@@ -170,7 +170,7 @@ select max(cnt) from customer c,
 ---- TYPES
 BIGINT
 ---- CATCH
-Rejected query from pool default-pool: minimum memory reservation is greater than memory available to the query for buffer reservations.
+Memory limit exceeded
 ====
 ---- QUERY
 # IMPALA-2473: Scan query with large row size leading to oversized batches.
@@ -237,7 +237,7 @@ bigint, bigint
 # This was originally a regression test for IMPALA-5446: dropped status from
 # Sorter::Reset() when sort cannot get reserved buffer. However with the
 # IMPALA-3200 changes it now succeeds.
-set buffer_pool_limit=180m;
+set buffer_pool_limit=100m;
 select c_custkey, c_name, c_address, c_phone, c_acctbal, c_mktsegment, c_comment,
        o_orderdate, sum(o_totalprice), min(rnum)
 from customer c,

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
index fdca977..3d92acb 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
@@ -345,7 +345,7 @@ where l_orderkey = 965 and l_extendedprice * l_tax = p_retailprice;
 # consumption / spilling behaviour.
 ####################################################
 
-SET BUFFER_POOL_LIMIT=80m;
+SET BUFFER_POOL_LIMIT=40m;
 SET RUNTIME_FILTER_MODE=GLOBAL;
 SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS;
 SET RUNTIME_BLOOM_FILTER_SIZE=16M;
@@ -396,14 +396,15 @@ SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS;
 SET RUNTIME_FILTER_MIN_SIZE=128MB;
 SET RUNTIME_FILTER_MAX_SIZE=500MB;
 # Query would have been admitted if memory for runtime filters was not accounted for.
-SET BUFFER_POOL_LIMIT=290MB;
+SET BUFFER_POOL_LIMIT=140MB;
 select STRAIGHT_JOIN * from alltypes a join [SHUFFLE] alltypes b
     on a.month = b.id and b.int_col = -3
 ---- RESULTS
 ---- CATCH
-Rejected query from pool default-pool: minimum memory reservation is
+ImpalaBeeswaxException: INNER EXCEPTION: <class 'beeswaxd.ttypes.BeeswaxException'>
+ MESSAGE: Rejected query from pool default-pool: minimum memory reservation is
  greater than memory available to the query for buffer reservations. Increase
- the buffer_pool_limit to 290.17 MB. See the query profile for more information
+ the buffer_pool_limit to 290.00 MB. See the query profile for more information
  about the per-node memory requirements.
 ====
 ---- QUERY
@@ -413,7 +414,7 @@ SET RUNTIME_FILTER_WAIT_TIME_MS=$RUNTIME_FILTER_WAIT_TIME_MS;
 SET RUNTIME_FILTER_MIN_SIZE=128MB;
 SET RUNTIME_FILTER_MAX_SIZE=500MB;
 # This would run perfectly with just enough memory provided by the buffer pool.
-SET BUFFER_POOL_LIMIT=295MB;
+SET BUFFER_POOL_LIMIT=290MB;
 select STRAIGHT_JOIN * from alltypes a join [SHUFFLE] alltypes b
     on a.month = b.id and b.int_col = -3
 ---- RESULTS

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/scanners.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/scanners.test b/testdata/workloads/functional-query/queries/QueryTest/scanners.test
index a000a15..bd5f496 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/scanners.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/scanners.test
@@ -110,10 +110,3 @@ on (v.x = t3.bool_col) where t3.bool_col = true
 ---- TYPES
 BIGINT
 ====
----- QUERY
-# Scan an empty unpartitioned table.
-select * from emptytable;
----- RESULTS
----- TYPES
-STRING,INT
-====

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/set.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/set.test b/testdata/workloads/functional-query/queries/QueryTest/set.test
index 7983224..32ad938 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/set.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/set.test
@@ -129,7 +129,7 @@ select 1
 set mem_limit=1;
 select count(string_col) from functional.alltypestiny
 ---- CATCH
-Rejected query from pool default-pool: minimum memory reservation
+Memory limit exceeded
 ====
 ---- QUERY
 # Set mem_limit back to unlimited and query should succeed again.

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
index 20813c7..05552d2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-aggs.test
@@ -1,6 +1,6 @@
 ====
 ---- QUERY
-set buffer_pool_limit=34m;
+set buffer_pool_limit=10m;
 select l_orderkey, count(*)
 from lineitem
 group by 1
@@ -25,7 +25,7 @@ row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
 ====
 ---- QUERY
 # Test query with string grouping column and string agg columns
-set buffer_pool_limit=82m;
+set buffer_pool_limit=10m;
 set num_nodes=1;
 select l_returnflag, l_orderkey, avg(l_tax), min(l_shipmode)
 from lineitem
@@ -39,9 +39,10 @@ order by 1,2 limit 3
 STRING, BIGINT, DECIMAL, STRING
 ---- RUNTIME_PROFILE
 row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
+row_regex: .*NumRepartitions: .* \([1-9][0-9]*\)
 ====
 ---- QUERY
-set buffer_pool_limit=34m;
+set buffer_pool_limit=10m;
 select l_orderkey, count(*)
 from lineitem
 group by 1
@@ -65,7 +66,7 @@ row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
 ====
 ---- QUERY
 # Test query with string grouping column
-set buffer_pool_limit=34m;
+set buffer_pool_limit=10m;
 set num_nodes=1;
 select l_comment, count(*)
 from lineitem
@@ -81,10 +82,11 @@ order by count(*) desc limit 5
 STRING, BIGINT
 ---- RUNTIME_PROFILE
 row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
+row_regex: .*NumRepartitions: .* \([1-9][0-9]*\)
 ====
 ---- QUERY
 # Test query with string grouping column and string agg columns
-set buffer_pool_limit=82m;
+set buffer_pool_limit=10m;
 set num_nodes=1;
 select l_returnflag, l_orderkey, round(avg(l_tax),2), min(l_shipmode)
 from lineitem
@@ -99,10 +101,11 @@ STRING, BIGINT, DECIMAL, STRING
 ---- RUNTIME_PROFILE
 # Verify that spilling happened in the agg.
 row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
+row_regex: .*NumRepartitions: .* \([1-9][0-9]*\)
 ====
 ---- QUERY
 # Test with non-scalar intermediate state (avg() uses fixed intermediate value).
-set buffer_pool_limit=34m;
+set buffer_pool_limit=10m;
 select l_orderkey, avg(l_orderkey)
 from lineitem
 group by 1
@@ -122,7 +125,7 @@ row_regex: .*RowsPassedThrough: .* \([1-9][0-9]*\)
 ====
 ---- QUERY
 # Test aggregation spill with group_concat distinct
-set buffer_pool_limit=78m;
+set buffer_pool_limit=30m;
 select l_orderkey, count(*), group_concat(distinct l_linestatus, '|')
 from lineitem
 group by 1
@@ -168,7 +171,7 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
 # Merge aggregation requires 17 buffers and preaggregation requires 16 buffers
 # plus 1mb of hash tables. The buffer size is 256k for this test.
 set max_row_size=256k;
-set buffer_pool_limit="57.25M";
+set buffer_pool_limit=9472k;
 select count(*)
 from (select distinct * from orders) t
 ---- TYPES

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test
index 7678675..420e721 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj-no-deny-reservation.test
@@ -13,7 +13,7 @@
 # except:
 # * even-numbered l_orderkey values, because there is a NULL on the build side.
 # * l_orderkey values > 5500000, because there is a NULL on the probe side
-set buffer_pool_limit=110m;
+set buffer_pool_limit=12m;
 select l_orderkey, l_partkey, l_suppkey, l_linenumber
 from lineitem
 where l_suppkey = 4162 and l_shipmode = 'AIR' and l_returnflag = 'A' and
@@ -54,9 +54,8 @@ order by 1,2,3,4
 BIGINT,BIGINT,BIGINT,INT
 ====
 ---- QUERY
-set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
-# Execute NAAJ query 1 with a debug action so that it can not get enough memory to fit
-# the null build rows in memory.
+set buffer_pool_limit=7m;
+# Execute NAAJ query 1 without enough memory to fit the null build rows in memory.
 select l_orderkey, l_partkey, l_suppkey, l_linenumber
 from lineitem
 where l_suppkey = 4162 and l_shipmode = 'AIR' and l_returnflag = 'A' and

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test
index 39ae2bb..4c0d790 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-naaj.test
@@ -21,7 +21,7 @@
 #   where l_orderkey % 2 = 1 and p_retailprice = l_extendedprice * l_tax
 #   order by 1,2,3,4 limit 5
 #
-set buffer_pool_limit=90m;
+set buffer_pool_limit=10m;
 select l_orderkey, l_partkey, l_suppkey, l_linenumber
 from lineitem
 where if(l_orderkey % 2 = 0, NULL, l_partkey) not in (
@@ -41,7 +41,7 @@ BIGINT,BIGINT,BIGINT,INT
 ---- QUERY
 # Execute NAAJ query 2 in-memory only without enough memory to complete.
 set scratch_limit=0;
-set buffer_pool_limit=90m;
+set buffer_pool_limit=10m;
 select l_orderkey, l_partkey, l_suppkey, l_linenumber
 from lineitem
 where if(l_orderkey % 2 = 0, NULL, l_partkey) not in (
@@ -65,7 +65,7 @@ Could not free memory by spilling to disk: scratch_limit is 0
 #   where p_partkey = 2 and l_partkey % 2 = 1
 #   order by 1,2,3,4
 #
-set buffer_pool_limit=90m;
+set buffer_pool_limit=10m;
 select l_orderkey, l_partkey, l_suppkey, l_linenumber
 from lineitem
 where l_partkey not in (
@@ -86,7 +86,7 @@ BIGINT,BIGINT,BIGINT,INT
 # =======================================================================================
 # This returns one less row than the previous query because l_partkey=116009 is
 # replaced with a NULL.
-set buffer_pool_limit=90m;
+set buffer_pool_limit=10m;
 select l_orderkey, l_partkey, l_suppkey, l_linenumber
 from lineitem
 where if(l_partkey % 5 != 1, NULL, l_partkey) not in (

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test
deleted file mode 100644
index f89e537..0000000
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-no-debug-action.test
+++ /dev/null
@@ -1,66 +0,0 @@
-# This file contains tests where we don't want the python test framework to supply the
-# debug_action value because the test won't succeed with all possible debug_action values.
-====
----- QUERY
-# Tests for the case where a spilled partition has 0 probe rows and so we don't build the
-# hash table in a partitioned hash join. Always runs with the minimum reservation to force
-# spilling.
-# INNER JOIN
-set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
-select straight_join count(*)
-from
-lineitem a, lineitem b
-where
-a.l_partkey = 1 and
-a.l_orderkey = b.l_orderkey;
----- TYPES
-BIGINT
----- RESULTS
-173
----- RUNTIME_PROFILE
-row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
-====
----- QUERY
-# spilled partition with 0 probe rows, NULL AWARE LEFT ANTI JOIN
-set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
-select straight_join count(*)
-from
-lineitem a
-where
-a.l_partkey not in (select l_partkey from lineitem where l_partkey > 10)
-and a.l_partkey < 1000;
----- TYPES
-BIGINT
----- RESULTS
-287
----- RUNTIME_PROFILE
-row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
-====
----- QUERY
-# spilled partition with 0 probe rows, RIGHT OUTER JOIN
-set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
-select straight_join count(*)
-from
-supplier right outer join lineitem on s_suppkey = l_suppkey
-where s_acctbal > 0 and s_acctbal < 10;
----- TYPES
-BIGINT
----- RESULTS
-12138
----- RUNTIME_PROFILE
-row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
-====
----- QUERY
-# spilled partition with 0 probe rows, RIGHT ANTI JOIN
-set debug_action="-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0";
-with x as (select * from supplier limit 10)
-select straight_join count(*)
-from
-x right anti join lineitem on s_suppkey + 100 = l_suppkey;
----- TYPES
-BIGINT
----- RESULTS
-5995258
----- RUNTIME_PROFILE
-row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
-====

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test b/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
index 7d65523..415fe5d 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling-sorts-exhaustive.test
@@ -3,7 +3,7 @@
 # Test sort with small char column materialized by exprs.
 # Set low memory limit to force spilling.
 # IMPALA-3332: comparator makes local allocations that cause runaway memory consumption.
-set buffer_pool_limit=28m;
+set buffer_pool_limit=4m;
 set mem_limit=200m;
 set disable_outermost_topn=1;
 select cast(l_comment as char(50))
@@ -41,7 +41,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
 ---- QUERY
 # Test sort with small input char column materialized before sort.
 set mem_limit=200m;
-set buffer_pool_limit=28m;
+set buffer_pool_limit=4m;
 set disable_outermost_topn=1;
 select char_col
 from (select cast(l_comment as char(50)) char_col
@@ -80,7 +80,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
 # Test sort with large input char column materialized before sort.
 # Set low memory limit to force spilling.
 set mem_limit=200m;
-set buffer_pool_limit=28m;
+set buffer_pool_limit=4m;
 set disable_outermost_topn=1;
 select char_col
 from (select cast(l_comment as char(200)) char_col
@@ -118,7 +118,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
 ---- QUERY
 # Test sort with varchar column materialized by exprs.
 # Set low memory limit to force spilling.
-set buffer_pool_limit=28m;
+set buffer_pool_limit=4m;
 # IMPALA-3332: comparator makes local allocations that cause runaway memory consumption.
 set mem_limit=200m;
 set disable_outermost_topn=1;
@@ -158,7 +158,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
 # Test sort with input varchar column materialized before sort.
 # Set low memory limit to force spilling.
 set mem_limit=200m;
-set buffer_pool_limit=28m;
+set buffer_pool_limit=4m;
 set disable_outermost_topn=1;
 select char_col
 from (select cast(l_comment as varchar(50)) char_col

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/spilling.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/spilling.test b/testdata/workloads/functional-query/queries/QueryTest/spilling.test
index 23d9271..6639d82 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/spilling.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/spilling.test
@@ -1,6 +1,6 @@
 ====
 ---- QUERY
-set buffer_pool_limit=215m;
+set buffer_pool_limit=15m;
 select count(l1.l_tax)
 from
 lineitem l1,
@@ -22,7 +22,7 @@ BIGINT
 row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
 ====
 ---- QUERY
-set buffer_pool_limit=50m;
+set buffer_pool_limit=2m;
 select max(t1.total_count), max(t1.l_shipinstruct), max(t1.l_comment) from
 (select l_shipinstruct, l_comment, count(*) over () total_count from lineitem) t1
 ---- RESULTS
@@ -60,7 +60,7 @@ row_regex: .*SpilledPartitions: 0 .*
 # Adding TPCH-Q21 in the spilling test to check for IMPALA-1471 (spilling left anti
 # and left outer joins were returning wrong results).
 # Q21 - Suppliers Who Kept Orders Waiting Query
-set buffer_pool_limit=200m;
+set buffer_pool_limit=26m;
 select
   s_name,
   count(*) as numwait
@@ -211,7 +211,7 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
 ---- QUERY
 # IMPALA-1346/IMPALA-1546: fix sorter memory management so that it can complete
 # successfully when in same pipeline as a spilling join.
-set buffer_pool_limit=170m;
+set buffer_pool_limit=50m;
 set disable_outermost_topn=1;
 select * from lineitem
   inner join orders on l_orderkey = o_orderkey
@@ -246,6 +246,68 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
 row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
 ====
 ---- QUERY
+# Tests for the case where a spilled partition has 0 probe rows and so we don't build the
+# hash table in a partitioned hash join.
+# INNER JOIN
+set buffer_pool_limit=10m;
+select straight_join count(*)
+from
+lineitem a, lineitem b
+where
+a.l_partkey = 1 and
+a.l_orderkey = b.l_orderkey;
+---- TYPES
+BIGINT
+---- RESULTS
+173
+---- RUNTIME_PROFILE
+row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# spilled partition with 0 probe rows, NULL AWARE LEFT ANTI JOIN
+set buffer_pool_limit=10m;
+select straight_join count(*)
+from
+lineitem a
+where
+a.l_partkey not in (select l_partkey from lineitem where l_partkey > 10)
+and a.l_partkey < 1000;
+---- TYPES
+BIGINT
+---- RESULTS
+287
+---- RUNTIME_PROFILE
+row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# spilled partition with 0 probe rows, RIGHT OUTER JOIN
+set buffer_pool_limit=10m;
+select straight_join count(*)
+from
+supplier right outer join lineitem on s_suppkey = l_suppkey
+where s_acctbal > 0 and s_acctbal < 10;
+---- TYPES
+BIGINT
+---- RESULTS
+12138
+---- RUNTIME_PROFILE
+row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
+====
+---- QUERY
+# spilled partition with 0 probe rows, RIGHT ANTI JOIN
+set buffer_pool_limit=30m;
+with x as (select * from supplier limit 10)
+select straight_join count(*)
+from
+x right anti join lineitem on s_suppkey + 100 = l_suppkey;
+---- TYPES
+BIGINT
+---- RESULTS
+5995258
+---- RUNTIME_PROFILE
+row_regex: .*NumHashTableBuildsSkipped: .* \([1-9][0-9]*\)
+====
+---- QUERY
 # IMPALA-5173: spilling hash join feeding into right side of nested loop join.
 # Equivalent to:
 #   select *
@@ -253,7 +315,7 @@ row_regex: .*TotalMergesPerformed: .* \([1-9][0-9]*\)
 #   where l1.l_quantity = 31.0 and l1.l_tax = 0.03 and l1.l_orderkey <= 100000
 # order by l_orderkey, l_partkey, l_suppkey, l_linenumber
 # limit 5
-set buffer_pool_limit=177m;
+set buffer_pool_limit=9m;
 set num_nodes=1;
 select straight_join l.*
 from
@@ -283,7 +345,7 @@ bigint,bigint,bigint,int,decimal,decimal,decimal,decimal,string,string,string,st
 ====
 ---- QUERY
 # Test spilling aggregation when grouping by nondeterministic expression
-set buffer_pool_limit=79m;
+set buffer_pool_limit=5m;
 set num_nodes=1;
 select l_orderkey, l_partkey, l_suppkey, l_linenumber, l_comment
 from tpch_parquet.lineitem
@@ -297,7 +359,7 @@ row_regex: .*SpilledPartitions: .* \([1-9][0-9]*\)
 ---- QUERY
 # Test spilling join with many duplicates in join key. We don't expect this to succeed
 # with a memory constraint: see IMPALA-4857.
-set buffer_pool_limit=167m;
+set buffer_pool_limit=20m;
 select *
 from lineitem l1 join lineitem l2 on l1.l_linenumber = l2.l_linenumber
 ---- CATCH

http://git-wip-us.apache.org/repos/asf/impala/blob/e5689fb5/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test b/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
index 836ed0a..7da7baf 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/stats-extrapolation.test
@@ -15,6 +15,7 @@ explain select id from alltypes;
 '     partitions: 0/12 rows=unavailable'
 '     columns: unavailable'
 '   extrapolated-rows=unavailable'
+'   mem-estimate=16.00MB mem-reservation=0B'
 '   tuple-ids=0 row-size=4B cardinality=unavailable'
 ---- TYPES
 STRING
@@ -52,14 +53,14 @@ STRING,STRING,BIGINT,BIGINT,BIGINT,STRING,STRING,STRING,STRING,STRING,STRING
 # Stats are available now.
 explain select id from alltypes;
 ---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=.*
-row_regex:.*Per-Host Resource Estimates: Memory=.*
+'Max Per-Host Resource Reservation: Memory=0B'
+'Per-Host Resource Estimates: Memory=16.00MB'
 'Codegen disabled by planner'
 ''
 'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
+'|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
 'PLAN-ROOT SINK'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'|  mem-estimate=0B mem-reservation=0B'
 '|'
 '00:SCAN HDFS [$DATABASE.alltypes]'
 row_regex:.*partitions=12/12 files=12 size=.*
@@ -68,7 +69,7 @@ row_regex:.*table: rows=3650 size=.*
 '     partitions: 0/12 rows=unavailable'
 '     columns: all'
 '   extrapolated-rows=3650'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'   mem-estimate=16.00MB mem-reservation=0B'
 '   tuple-ids=0 row-size=4B cardinality=3650'
 ---- TYPES
 STRING
@@ -77,14 +78,14 @@ STRING
 # Select a subset of partitions.
 explain select id from alltypes where month in (1, 2, 3);
 ---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=.*
-row_regex:.*Per-Host Resource Estimates: Memory=.*
+'Max Per-Host Resource Reservation: Memory=0B'
+'Per-Host Resource Estimates: Memory=16.00MB'
 'Codegen disabled by planner'
 ''
 'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
 'PLAN-ROOT SINK'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'|  mem-estimate=0B mem-reservation=0B'
 '|'
 '00:SCAN HDFS [$DATABASE.alltypes]'
 row_regex:.*partitions=3/12 files=3 size=.*
@@ -93,7 +94,7 @@ row_regex:.*table: rows=3650 size=.*
 '     partitions: 0/3 rows=unavailable'
 '     columns: all'
 '   extrapolated-rows=904'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'   mem-estimate=16.00MB mem-reservation=0B'
 '   tuple-ids=0 row-size=4B cardinality=904'
 ---- TYPES
 STRING
@@ -104,13 +105,13 @@ insert into alltypes partition(year, month)
 select * from functional_parquet.alltypes where year = 2009;
 explain select id from alltypes;
 ---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=.*
-row_regex:.*Per-Host Resource Estimates: Memory=.*
+'Max Per-Host Resource Reservation: Memory=0B'
+'Per-Host Resource Estimates: Memory=16.00MB'
 ''
 'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
+'|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
 'PLAN-ROOT SINK'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'|  mem-estimate=0B mem-reservation=0B'
 '|'
 '00:SCAN HDFS [$DATABASE.alltypes]'
 row_regex:.*partitions=12/12 files=24 size=.*
@@ -119,7 +120,7 @@ row_regex:.*table: rows=3650 size=.*
 '     partitions: 0/12 rows=unavailable'
 '     columns: all'
 '   extrapolated-rows=7300'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'   mem-estimate=16.00MB mem-reservation=0B'
 '   tuple-ids=0 row-size=4B cardinality=7300'
 ---- TYPES
 STRING
@@ -130,14 +131,14 @@ insert into alltypes partition(year, month)
 select * from functional_parquet.alltypes where year = 2010;
 explain select id from alltypes where year = 2010;
 ---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=.*
-row_regex:.*Per-Host Resource Estimates: Memory=16.00MB'
+'Max Per-Host Resource Reservation: Memory=0B'
+'Per-Host Resource Estimates: Memory=16.00MB'
 'Codegen disabled by planner'
 ''
 'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
+'|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
 'PLAN-ROOT SINK'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'|  mem-estimate=0B mem-reservation=0B'
 '|'
 '00:SCAN HDFS [$DATABASE.alltypes]'
 row_regex:.*partitions=12/24 files=12 size=.*
@@ -146,7 +147,7 @@ row_regex:.*table: rows=3650 size=.*
 '     partitions: 0/12 rows=unavailable'
 '     columns: all'
 '   extrapolated-rows=3651'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'   mem-estimate=16.00MB mem-reservation=0B'
 '   tuple-ids=0 row-size=4B cardinality=3651'
 ---- TYPES
 STRING
@@ -156,14 +157,14 @@ STRING
 compute stats alltypes;
 explain select id from alltypes where year = 2010;
 ---- RESULTS: VERIFY_IS_EQUAL
-row_regex:.*Max Per-Host Resource Reservation: Memory=.*
-row_regex:.*Per-Host Resource Estimates: Memory=16.00MB'
+'Max Per-Host Resource Reservation: Memory=0B'
+'Per-Host Resource Estimates: Memory=16.00MB'
 'Codegen disabled by planner'
 ''
 'F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1'
-row_regex:.*Per-Host Resources: mem-estimate=.* mem-reservation=.*
+'|  Per-Host Resources: mem-estimate=16.00MB mem-reservation=0B'
 'PLAN-ROOT SINK'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'|  mem-estimate=0B mem-reservation=0B'
 '|'
 '00:SCAN HDFS [$DATABASE.alltypes]'
 row_regex:.*partitions=12/24 files=12 size=.*
@@ -172,7 +173,7 @@ row_regex:.*table: rows=10950 size=.*
 '     partitions: 0/12 rows=unavailable'
 '     columns: all'
 '   extrapolated-rows=3651'
-row_regex:.*mem-estimate=.* mem-reservation=.*
+'   mem-estimate=16.00MB mem-reservation=0B'
 '   tuple-ids=0 row-size=4B cardinality=3651'
 ---- TYPES
 STRING
@@ -187,6 +188,7 @@ explain select id from alltypes;
 '     partitions: 0/24 rows=unavailable'
 '     columns: unavailable'
 '   extrapolated-rows=unavailable'
+'   mem-estimate=16.00MB mem-reservation=0B'
 '   tuple-ids=0 row-size=4B cardinality=unavailable'
 ---- TYPES
 STRING