You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/10/29 20:52:16 UTC
[1/2] impala git commit: IMPALA-7742: Store the Sentry user names in
a case sensitive way
Repository: impala
Updated Branches:
refs/heads/master 449fe73d2 -> 44e69e818
IMPALA-7742: Store the Sentry user names in a case sensitive way
SENTRY-2432 changes the way it stores user names by making them case
sensitive. This patch updates Impala to match the behavior in Sentry
to make the catalog store the user names in a case sensitive way.
Testing:
- Ran all FE tests
- Ran all authorization E2E tests
- Added a new E2E test
Change-Id: I04bec045e3f70fc4f41b16b9b5c55eeb60bd63b8
Reviewed-on: http://gerrit.cloudera.org:8080/11762
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Fredy Wijaya <fw...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/bf7bb58d
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/bf7bb58d
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/bf7bb58d
Branch: refs/heads/master
Commit: bf7bb58d0bbce971cda02dc4e6cfe9d359e43922
Parents: 449fe73
Author: Fredy Wijaya <fw...@cloudera.com>
Authored: Tue Oct 23 11:23:26 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Sat Oct 27 18:07:00 2018 +0000
----------------------------------------------------------------------
bin/impala-config.sh | 2 +-
.../impala/catalog/AuthorizationPolicy.java | 4 +-
.../org/apache/impala/catalog/Principal.java | 7 +--
.../java/org/apache/impala/catalog/Role.java | 7 ---
.../java/org/apache/impala/catalog/User.java | 7 ---
.../service/CustomClusterGroupMapper.java | 5 +-
tests/authorization/test_owner_privileges.py | 59 ++++++++++++++++++++
7 files changed, 67 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/bf7bb58d/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 57538e5..c0030ca 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -158,7 +158,7 @@ fi
: ${CDH_DOWNLOAD_HOST:=native-toolchain.s3.amazonaws.com}
export CDH_DOWNLOAD_HOST
export CDH_MAJOR_VERSION=6
-export CDH_BUILD_NUMBER=632827
+export CDH_BUILD_NUMBER=663310
export IMPALA_HADOOP_VERSION=3.0.0-cdh6.x-SNAPSHOT
export IMPALA_HBASE_VERSION=2.1.0-cdh6.x-SNAPSHOT
export IMPALA_HIVE_VERSION=2.1.1-cdh6.x-SNAPSHOT
http://git-wip-us.apache.org/repos/asf/impala/blob/bf7bb58d/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java b/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java
index decca8e..4934505 100644
--- a/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java
+++ b/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java
@@ -76,8 +76,8 @@ public class AuthorizationPolicy implements PrivilegeCache {
// Cache of role names (case-insensitive) to role objects.
private final CatalogObjectCache<Role> roleCache_ = new CatalogObjectCache<>();
- // Cache of user names (case-insensitive) to user objects.
- private final CatalogObjectCache<User> userCache_ = new CatalogObjectCache<>();
+ // Cache of user names (case-sensitive) to user objects.
+ private final CatalogObjectCache<User> userCache_ = new CatalogObjectCache<>(false);
// Map of principal ID -> user/role name. Used to match privileges to users/roles.
private final Map<Integer, String> principalIds_ = Maps.newHashMap();
http://git-wip-us.apache.org/repos/asf/impala/blob/bf7bb58d/fe/src/main/java/org/apache/impala/catalog/Principal.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Principal.java b/fe/src/main/java/org/apache/impala/catalog/Principal.java
index 9f3f6d4..d048d10 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Principal.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Principal.java
@@ -39,7 +39,7 @@ public abstract class Principal extends CatalogObjectImpl {
private static AtomicInteger principalId_ = new AtomicInteger(0);
private final CatalogObjectCache<PrincipalPrivilege> principalPrivileges_ =
- new CatalogObjectCache<>(isCaseInsensitiveKeys());
+ new CatalogObjectCache<>();
protected Principal(String principalName, TPrincipalType type,
Set<String> grantGroups) {
@@ -55,11 +55,6 @@ public abstract class Principal extends CatalogObjectImpl {
}
/**
- * Returns true if the keys in the catalog to be treated as case insensitive.
- */
- protected abstract boolean isCaseInsensitiveKeys();
-
- /**
* Adds a privilege to the principal. Returns true if the privilege was added
* successfully or false if there was a newer version of the privilege already added
* to the principal.
http://git-wip-us.apache.org/repos/asf/impala/blob/bf7bb58d/fe/src/main/java/org/apache/impala/catalog/Role.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Role.java b/fe/src/main/java/org/apache/impala/catalog/Role.java
index d2e841a..9cee7d2 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Role.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Role.java
@@ -36,11 +36,4 @@ public class Role extends Principal {
Preconditions.checkArgument(
thriftPrincipal.getPrincipal_type() == TPrincipalType.ROLE);
}
-
- @Override
- protected boolean isCaseInsensitiveKeys() {
- // If Sentry changes the role name to be case sensitive, make sure to update
- // this code to return false.
- return true;
- }
}
http://git-wip-us.apache.org/repos/asf/impala/blob/bf7bb58d/fe/src/main/java/org/apache/impala/catalog/User.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/User.java b/fe/src/main/java/org/apache/impala/catalog/User.java
index 9bcd7fb..2845670 100644
--- a/fe/src/main/java/org/apache/impala/catalog/User.java
+++ b/fe/src/main/java/org/apache/impala/catalog/User.java
@@ -36,11 +36,4 @@ public class User extends Principal {
Preconditions.checkArgument(
thriftPrincipal.getPrincipal_type() == TPrincipalType.USER);
}
-
- @Override
- protected boolean isCaseInsensitiveKeys() {
- // If Sentry changes the user name to be case sensitive, make sure to update
- // this code to return false.
- return true;
- }
}
http://git-wip-us.apache.org/repos/asf/impala/blob/bf7bb58d/fe/src/test/java/org/apache/impala/service/CustomClusterGroupMapper.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/service/CustomClusterGroupMapper.java b/fe/src/test/java/org/apache/impala/service/CustomClusterGroupMapper.java
index 977b500..097c0fc 100644
--- a/fe/src/test/java/org/apache/impala/service/CustomClusterGroupMapper.java
+++ b/fe/src/test/java/org/apache/impala/service/CustomClusterGroupMapper.java
@@ -51,9 +51,12 @@ public class CustomClusterGroupMapper implements GroupMappingService {
groupsMap_.put("user1_shared2", Sets.newHashSet("group_4a","group_4b"));
groupsMap_.put("user2_shared2", Sets.newHashSet("group_4a"));
- // User to grpups for test_owner_privilege tests.
+ // User to groups for test_owner_privilege tests.
groupsMap_.put("oo_user1", Sets.newHashSet("oo_group1"));
groupsMap_.put("oo_user2", Sets.newHashSet("oo_group2"));
+
+ groupsMap_.put("foobar", Sets.newHashSet("foobar"));
+ groupsMap_.put("FOOBAR", Sets.newHashSet("FOOBAR"));
}
@Override
http://git-wip-us.apache.org/repos/asf/impala/blob/bf7bb58d/tests/authorization/test_owner_privileges.py
----------------------------------------------------------------------
diff --git a/tests/authorization/test_owner_privileges.py b/tests/authorization/test_owner_privileges.py
index 6520cd0..1cbce98 100644
--- a/tests/authorization/test_owner_privileges.py
+++ b/tests/authorization/test_owner_privileges.py
@@ -401,3 +401,62 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
test_obj.obj_name), user="oo_user1", delay_s=sentry_refresh_timeout_s)
assert self._validate_user_privilege_count(self.oo_user1_impalad_client,
"show grant user oo_user1", "oo_user1", sentry_refresh_timeout_s, 0)
+
+ @pytest.mark.execute_serially
+ @SentryCacheTestSuite.with_args(
+ impalad_args="--server_name=server1 --sentry_config={0} "
+ "--authorization_policy_provider_class="
+ "org.apache.impala.service.CustomClusterResourceAuthorizationProvider"
+ .format(SENTRY_CONFIG_FILE_OO),
+ catalogd_args="--sentry_config={0} --sentry_catalog_polling_frequency_s={1} "
+ "--authorization_policy_provider_class="
+ "org.apache.impala.service.CustomClusterResourceAuthorizationProvider"
+ .format(SENTRY_CONFIG_FILE_OO, str(SENTRY_POLLING_FREQUENCY_S)),
+ sentry_config=SENTRY_CONFIG_FILE_OO,
+ sentry_log_dir="{0}/test_owner_privileges_different_cases"
+ .format(SENTRY_BASE_LOG_DIR))
+ def test_owner_privileges_different_cases(self, vector, unique_database):
+ """IMPALA-7742: Tests that only user names that differ only in case are not
+ authorized to access the database/table/view unless the user is the owner."""
+ # Use two different clients so that the sessions will use two different user names.
+ foobar_impalad_client = self.create_impala_client()
+ FOOBAR_impalad_client = self.create_impala_client()
+ role_name = "owner_priv_diff_cases_role"
+ try:
+ self.execute_query("create role %s" % role_name)
+ self.execute_query("grant role %s to group foobar" % role_name)
+ self.execute_query("grant all on server to role %s" % role_name)
+
+ self.user_query(foobar_impalad_client, "create database %s_db" %
+ unique_database, user="foobar")
+ # FOOBAR user should not be allowed to create a table in the foobar's database.
+ self.user_query(FOOBAR_impalad_client, "create table %s_db.test_tbl(i int)" %
+ unique_database, user="FOOBAR",
+ error_msg="User 'FOOBAR' does not have privileges to execute "
+ "'CREATE' on: %s_db" % unique_database)
+
+ self.user_query(foobar_impalad_client, "create table %s.owner_case_tbl(i int)" %
+ unique_database, user="foobar")
+ # FOOBAR user should not be allowed to select foobar's table.
+ self.user_query(FOOBAR_impalad_client, "select * from %s.owner_case_tbl" %
+ unique_database, user="FOOBAR",
+ error_msg="User 'FOOBAR' does not have privileges to execute "
+ "'SELECT' on: %s.owner_case_tbl" % unique_database)
+
+ self.user_query(foobar_impalad_client,
+ "create view %s.owner_case_view as select 1" % unique_database,
+ user="foobar")
+ # FOOBAR user should not be allowed to select foobar's view.
+ self.user_query(FOOBAR_impalad_client, "select * from %s.owner_case_view" %
+ unique_database, user="FOOBAR",
+ error_msg="User 'FOOBAR' does not have privileges to execute "
+ "'SELECT' on: %s.owner_case_view" % unique_database)
+
+ # FOOBAR user should not be allowed to see foobar's privileges.
+ self.user_query(FOOBAR_impalad_client, "show grant user foobar", user="FOOBAR",
+ error_msg="User 'FOOBAR' does not have privileges to access the "
+ "requested policy metadata")
+ finally:
+ self.user_query(foobar_impalad_client, "drop database %s_db cascade" %
+ unique_database, user="foobar")
+ self.execute_query("drop role %s" % role_name)
[2/2] impala git commit: IMPALA-7749: Compute AggregationNode's
memory estimate using input cardinality
Posted by ta...@apache.org.
IMPALA-7749: Compute AggregationNode's memory estimate using input cardinality
Prior to this change, the AggregationNode's perInstanceCardinality
was influenced by the node's selectivity and limit. This was
incorrect because the hash table is constructed over the entire
input stream before any row batches are produced. This change
ensures that the input cardinality is used to determine the
perInstanceCardinality.
Testing:
Added a planner test which ensures that an AggregationNode with a
limit estimates memory based on the input cardinality.
Ran front-end and end-to-end tests affected by this change.
Change-Id: Ifd95d2ad5b677fca459c9c32b98f6176842161fc
Reviewed-on: http://gerrit.cloudera.org:8080/11806
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/44e69e81
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/44e69e81
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/44e69e81
Branch: refs/heads/master
Commit: 44e69e8182954db90b22809b5440bba59ed8d0ae
Parents: bf7bb58
Author: poojanilangekar <po...@cloudera.com>
Authored: Fri Oct 26 16:29:22 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Mon Oct 29 20:40:50 2018 +0000
----------------------------------------------------------------------
.../apache/impala/planner/AggregationNode.java | 7 +-
.../PlannerTest/resource-requirements.test | 101 ++++++++++++++++---
.../queries/PlannerTest/tpch-all.test | 12 +--
3 files changed, 99 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/44e69e81/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/AggregationNode.java b/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
index c18d7de..ab234e4 100644
--- a/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
@@ -471,9 +471,10 @@ public class AggregationNode extends PlanNode {
if (perInstanceCardinality == -1) {
perInstanceMemEstimate = DEFAULT_PER_INSTANCE_MEM;
} else {
- // Per-instance cardinality cannot be greater than the total output cardinality.
- if (cardinality_ != -1) {
- perInstanceCardinality = Math.min(perInstanceCardinality, cardinality_);
+ // Per-instance cardinality cannot be greater than the total input cardinality.
+ long inputCardinality = getChild(0).getCardinality();
+ if (inputCardinality != -1) {
+ perInstanceCardinality = Math.min(perInstanceCardinality, inputCardinality);
}
perInstanceDataBytes = (long)Math.ceil(perInstanceCardinality * avgRowSize_);
perInstanceMemEstimate = (long)Math.max(perInstanceDataBytes *
http://git-wip-us.apache.org/repos/asf/impala/blob/44e69e81/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
index 71040ad..1f41712 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/resource-requirements.test
@@ -3514,11 +3514,11 @@ order by
o_orderdate
limit 100
---- PLAN
-Max Per-Host Resource Reservation: Memory=92.25MB Threads=5
-Per-Host Resource Estimates: Memory=396MB
+Max Per-Host Resource Reservation: Memory=111.50MB Threads=5
+Per-Host Resource Estimates: Memory=426MB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=396.18MB mem-reservation=92.25MB thread-reservation=5 runtime-filters-memory=3.00MB
+| Per-Host Resources: mem-estimate=425.54MB mem-reservation=111.50MB thread-reservation=5 runtime-filters-memory=3.00MB
PLAN-ROOT SINK
| mem-estimate=0B mem-reservation=0B thread-reservation=0
|
@@ -3546,7 +3546,7 @@ PLAN-ROOT SINK
| | output: sum(l_quantity)
| | group by: l_orderkey
| | having: sum(l_quantity) > 300
-| | mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB thread-reservation=0
+| | mem-estimate=39.36MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
| | tuple-ids=4 row-size=24B cardinality=156344
| | in pipelines: 04(GETNEXT), 03(OPEN)
| |
@@ -3608,8 +3608,8 @@ PLAN-ROOT SINK
tuple-ids=2 row-size=16B cardinality=6001215
in pipelines: 02(GETNEXT)
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=191.12MB Threads=11
-Per-Host Resource Estimates: Memory=567MB
+Max Per-Host Resource Reservation: Memory=220.38MB Threads=11
+Per-Host Resource Estimates: Memory=597MB
F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=33.72KB mem-reservation=0B thread-reservation=1
@@ -3644,7 +3644,7 @@ Per-Host Resources: mem-estimate=73.26MB mem-reservation=34.00MB thread-reservat
| in pipelines: 02(GETNEXT)
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=3
-Per-Host Resources: mem-estimate=154.75MB mem-reservation=87.12MB thread-reservation=1 runtime-filters-memory=3.00MB
+Per-Host Resources: mem-estimate=184.12MB mem-reservation=116.38MB thread-reservation=1 runtime-filters-memory=3.00MB
08:AGGREGATE [STREAMING]
| output: sum(l_quantity)
| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice
@@ -3663,7 +3663,7 @@ Per-Host Resources: mem-estimate=154.75MB mem-reservation=87.12MB thread-reserva
| | output: sum:merge(l_quantity)
| | group by: l_orderkey
| | having: sum(l_quantity) > 300
-| | mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB thread-reservation=0
+| | mem-estimate=39.36MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
| | tuple-ids=4 row-size=24B cardinality=156344
| | in pipelines: 14(GETNEXT), 03(OPEN)
| |
@@ -3760,8 +3760,8 @@ Per-Host Resources: mem-estimate=90.00MB mem-reservation=10.00MB thread-reservat
tuple-ids=2 row-size=16B cardinality=6001215
in pipelines: 02(GETNEXT)
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=336.88MB Threads=13
-Per-Host Resource Estimates: Memory=1.01GB
+Max Per-Host Resource Reservation: Memory=353.88MB Threads=13
+Per-Host Resource Estimates: Memory=1.03GB
F07:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=64.19KB mem-reservation=0B thread-reservation=1
@@ -3796,7 +3796,7 @@ Per-Host Resources: mem-estimate=147.13MB mem-reservation=68.00MB thread-reserva
| in pipelines: 02(GETNEXT)
|
F02:PLAN FRAGMENT [HASH(l_orderkey)] hosts=3 instances=6
-Per-Host Resources: mem-estimate=207.21MB mem-reservation=128.88MB thread-reservation=2 runtime-filters-memory=3.00MB
+Per-Host Resources: mem-estimate=230.96MB mem-reservation=145.88MB thread-reservation=2 runtime-filters-memory=3.00MB
08:AGGREGATE [STREAMING]
| output: sum(l_quantity)
| group by: c_name, c_custkey, o_orderkey, o_orderdate, o_totalprice
@@ -3823,7 +3823,7 @@ Per-Host Resources: mem-estimate=207.21MB mem-reservation=128.88MB thread-reserv
| | output: sum:merge(l_quantity)
| | group by: l_orderkey
| | having: sum(l_quantity) > 300
-| | mem-estimate=10.00MB mem-reservation=4.75MB spill-buffer=256.00KB thread-reservation=0
+| | mem-estimate=39.36MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
| | tuple-ids=4 row-size=24B cardinality=156344
| | in pipelines: 14(GETNEXT), 03(OPEN)
| |
@@ -5470,3 +5470,80 @@ PLAN-ROOT SINK
tuple-ids=0 row-size=117B cardinality=25
in pipelines: 00(GETNEXT)
====
+# IMPALA-7749: Aggregation with a limit should compute memory estimates based on input
+# cardinality.
+select distinct *
+from tpch_parquet.lineitem
+limit 5
+---- PLAN
+Max Per-Host Resource Reservation: Memory=74.00MB Threads=2
+Per-Host Resource Estimates: Memory=1.69GB
+
+F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Host Resources: mem-estimate=1.69GB mem-reservation=74.00MB thread-reservation=2
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+|
+01:AGGREGATE [FINALIZE]
+| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
+| limit: 5
+| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=263B cardinality=5
+| in pipelines: 01(GETNEXT), 00(OPEN)
+|
+00:SCAN HDFS [tpch_parquet.lineitem]
+ partitions=1/1 files=3 size=193.71MB
+ stored statistics:
+ table: rows=6001215 size=193.71MB
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=2141674
+ mem-estimate=80.00MB mem-reservation=40.00MB thread-reservation=1
+ tuple-ids=0 row-size=263B cardinality=6001215
+ in pipelines: 00(GETNEXT)
+---- DISTRIBUTEDPLAN
+Max Per-Host Resource Reservation: Memory=108.00MB Threads=4
+Per-Host Resource Estimates: Memory=3.32GB
+
+F02:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
+| Per-Host Resources: mem-estimate=16.00KB mem-reservation=0B thread-reservation=1
+PLAN-ROOT SINK
+| mem-estimate=0B mem-reservation=0B thread-reservation=0
+|
+04:EXCHANGE [UNPARTITIONED]
+| limit: 5
+| mem-estimate=16.00KB mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=263B cardinality=5
+| in pipelines: 03(GETNEXT)
+|
+F01:PLAN FRAGMENT [HASH(tpch_parquet.lineitem.l_orderkey,tpch_parquet.lineitem.l_partkey,tpch_parquet.lineitem.l_suppkey,tpch_parquet.lineitem.l_linenumber,tpch_parquet.lineitem.l_quantity,tpch_parquet.lineitem.l_extendedprice,tpch_parquet.lineitem.l_discount,tpch_parquet.lineitem.l_tax,tpch_parquet.lineitem.l_returnflag,tpch_parquet.lineitem.l_linestatus,tpch_parquet.lineitem.l_shipdate,tpch_parquet.lineitem.l_commitdate,tpch_parquet.lineitem.l_receiptdate,tpch_parquet.lineitem.l_shipinstruct,tpch_parquet.lineitem.l_shipmode,tpch_parquet.lineitem.l_comment)] hosts=3 instances=3
+Per-Host Resources: mem-estimate=1.63GB mem-reservation=34.00MB thread-reservation=1
+03:AGGREGATE [FINALIZE]
+| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
+| limit: 5
+| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=263B cardinality=5
+| in pipelines: 03(GETNEXT), 00(OPEN)
+|
+02:EXCHANGE [HASH(tpch_parquet.lineitem.l_orderkey,tpch_parquet.lineitem.l_partkey,tpch_parquet.lineitem.l_suppkey,tpch_parquet.lineitem.l_linenumber,tpch_parquet.lineitem.l_quantity,tpch_parquet.lineitem.l_extendedprice,tpch_parquet.lineitem.l_discount,tpch_parquet.lineitem.l_tax,tpch_parquet.lineitem.l_returnflag,tpch_parquet.lineitem.l_linestatus,tpch_parquet.lineitem.l_shipdate,tpch_parquet.lineitem.l_commitdate,tpch_parquet.lineitem.l_receiptdate,tpch_parquet.lineitem.l_shipinstruct,tpch_parquet.lineitem.l_shipmode,tpch_parquet.lineitem.l_comment)]
+| mem-estimate=10.78MB mem-reservation=0B thread-reservation=0
+| tuple-ids=1 row-size=263B cardinality=6001215
+| in pipelines: 00(GETNEXT)
+|
+F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
+Per-Host Resources: mem-estimate=1.69GB mem-reservation=74.00MB thread-reservation=2
+01:AGGREGATE [STREAMING]
+| group by: tpch_parquet.lineitem.l_orderkey, tpch_parquet.lineitem.l_partkey, tpch_parquet.lineitem.l_suppkey, tpch_parquet.lineitem.l_linenumber, tpch_parquet.lineitem.l_quantity, tpch_parquet.lineitem.l_extendedprice, tpch_parquet.lineitem.l_discount, tpch_parquet.lineitem.l_tax, tpch_parquet.lineitem.l_returnflag, tpch_parquet.lineitem.l_linestatus, tpch_parquet.lineitem.l_shipdate, tpch_parquet.lineitem.l_commitdate, tpch_parquet.lineitem.l_receiptdate, tpch_parquet.lineitem.l_shipinstruct, tpch_parquet.lineitem.l_shipmode, tpch_parquet.lineitem.l_comment
+| mem-estimate=1.62GB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| tuple-ids=1 row-size=263B cardinality=6001215
+| in pipelines: 00(GETNEXT)
+|
+00:SCAN HDFS [tpch_parquet.lineitem, RANDOM]
+ partitions=1/1 files=3 size=193.71MB
+ stored statistics:
+ table: rows=6001215 size=193.71MB
+ columns: all
+ extrapolated-rows=disabled max-scan-range-rows=2141674
+ mem-estimate=80.00MB mem-reservation=40.00MB thread-reservation=1
+ tuple-ids=0 row-size=263B cardinality=6001215
+ in pipelines: 00(GETNEXT)
+====
http://git-wip-us.apache.org/repos/asf/impala/blob/44e69e81/testdata/workloads/functional-planner/queries/PlannerTest/tpch-all.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpch-all.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpch-all.test
index 080e84a..5db8c0c 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/tpch-all.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpch-all.test
@@ -3318,8 +3318,8 @@ order by
o_orderdate
limit 100
---- PLAN
-Max Per-Host Resource Reservation: Memory=92.25MB Threads=5
-Per-Host Resource Estimates: Memory=868MB
+Max Per-Host Resource Reservation: Memory=111.50MB Threads=5
+Per-Host Resource Estimates: Memory=898MB
PLAN-ROOT SINK
|
09:TOP-N [LIMIT=100]
@@ -3360,8 +3360,8 @@ PLAN-ROOT SINK
partitions=1/1 files=1 size=718.94MB
runtime filters: RF000 -> tpch.lineitem.l_orderkey, RF004 -> l_orderkey
---- DISTRIBUTEDPLAN
-Max Per-Host Resource Reservation: Memory=191.12MB Threads=11
-Per-Host Resource Estimates: Memory=1.02GB
+Max Per-Host Resource Reservation: Memory=220.38MB Threads=11
+Per-Host Resource Estimates: Memory=1.04GB
PLAN-ROOT SINK
|
17:MERGING-EXCHANGE [UNPARTITIONED]
@@ -3424,8 +3424,8 @@ PLAN-ROOT SINK
partitions=1/1 files=1 size=718.94MB
runtime filters: RF000 -> tpch.lineitem.l_orderkey, RF004 -> l_orderkey
---- PARALLELPLANS
-Max Per-Host Resource Reservation: Memory=336.88MB Threads=13
-Per-Host Resource Estimates: Memory=1.01GB
+Max Per-Host Resource Reservation: Memory=353.88MB Threads=13
+Per-Host Resource Estimates: Memory=1.03GB
PLAN-ROOT SINK
|
17:MERGING-EXCHANGE [UNPARTITIONED]