You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jr...@apache.org on 2017/09/29 19:14:33 UTC
[3/6] incubator-impala git commit: Move tests related to the old join
node.
Move tests related to the old join node.
No tests were added/dropped or modified. They are consolidated into
fewer .test files.
Change-Id: Idda4b34b5e6e9b5012b177a4c00077aa7fec394c
Reviewed-on: http://gerrit.cloudera.org:8080/8153
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/c1781b73
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/c1781b73
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/c1781b73
Branch: refs/heads/master
Commit: c1781b73b35899ed9374befa2e04e14747224be7
Parents: 4d49099
Author: Alex Behm <al...@cloudera.com>
Authored: Tue Sep 26 22:47:55 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Thu Sep 28 18:36:17 2017 +0000
----------------------------------------------------------------------
.../queries/QueryTest/exprs.test | 12 +++
.../queries/QueryTest/inline-view.test | 19 ++++
.../queries/QueryTest/joins-partitioned.test | 36 --------
.../queries/QueryTest/runtime_row_filters.test | 95 ++++++++++++++++++++
.../QueryTest/runtime_row_filters_phj.test | 94 -------------------
tests/query_test/test_join_queries.py | 3 -
tests/query_test/test_runtime_filters.py | 3 -
7 files changed, 126 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/testdata/workloads/functional-query/queries/QueryTest/exprs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/exprs.test b/testdata/workloads/functional-query/queries/QueryTest/exprs.test
index 811a169..552dea2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/exprs.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/exprs.test
@@ -876,6 +876,18 @@ order by int_col limit 20
int,string
====
---- QUERY
+# IMPALA-2495: In predicate containing a TupleIsNullPredicate
+select count(id) from functional.alltypestiny t1
+left join (select coalesce(1, 10) as int_col
+ from functional.alltypessmall) t2
+on t1.id = t2.int_col
+where t2.int_col in (t2.int_col, 10)
+---- RESULTS
+100
+---- TYPES
+BIGINT
+====
+---- QUERY
select count(*) from alltypesagg
where true in (bool_col, tinyint_col)
---- RESULTS
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/testdata/workloads/functional-query/queries/QueryTest/inline-view.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/inline-view.test b/testdata/workloads/functional-query/queries/QueryTest/inline-view.test
index 7c7616d..dd0094b 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/inline-view.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/inline-view.test
@@ -499,3 +499,22 @@ true
---- TYPES
boolean
====
+---- QUERY
+# IMPALA-1987: Equi-join predicates of outer joins contain TupleIsNullPredicate exprs.
+select t1.int_col, t2.int_col, t3.id
+from alltypestiny t1 left outer join
+ (select coalesce(int_col, 384) as int_col from alltypestiny) t2
+ on t1.int_col = t2.int_col
+left outer join
+ (select 0 as id from alltypestiny) t3
+ on t1.int_col = t3.id
+order by 1 limit 5
+---- RESULTS
+0,0,0
+0,0,0
+0,0,0
+0,0,0
+0,0,0
+---- TYPES
+INT,INT,TINYINT
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/testdata/workloads/functional-query/queries/QueryTest/joins-partitioned.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/joins-partitioned.test b/testdata/workloads/functional-query/queries/QueryTest/joins-partitioned.test
deleted file mode 100644
index d8acb9a..0000000
--- a/testdata/workloads/functional-query/queries/QueryTest/joins-partitioned.test
+++ /dev/null
@@ -1,36 +0,0 @@
-====
----- QUERY
-# Regression test for IMPALA-1987. Equi-join predicates of outer joins contain
-# TupleIsNullPredicate exprs.
-# TODO: Move back to inline-views.test when this works with the non-partitioned hash
-# join (IMPALA-2375).
-select t1.int_col, t2.int_col, t3.id
-from alltypestiny t1 left outer join
- (select coalesce(int_col, 384) as int_col from alltypestiny) t2
- on t1.int_col = t2.int_col
-left outer join
- (select 0 as id from alltypestiny) t3
- on t1.int_col = t3.id
-order by 1 limit 5
----- RESULTS
-0,0,0
-0,0,0
-0,0,0
-0,0,0
-0,0,0
----- TYPES
-INT,INT,TINYINT
-====
----- QUERY
-# Regression test for IMPALA-2495: Crash: impala::InPredicate::SetLookupPrepare
-# TODO: Move back to exprs.test when this works with the non-partitioned hash join
-# (IMPALA-2375).
-select count(id) from functional.alltypestiny t1
-left join (select coalesce(1, 10) as int_col
-from functional.alltypessmall) t2 on t1.id = t2.int_col
-where t2.int_col in (t2.int_col, 10);
----- RESULTS
-100
----- TYPES
-BIGINT
-====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
index 7cb5884..3af07af 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters.test
@@ -322,10 +322,12 @@ from tpch_parquet.lineitem l1 join tpch_parquet.lineitem l2
---- QUERY
+####################################################
# Test case 15: filter with a predicate that has different decimal precision between
# lhs expr and rhs expr.
# IMPALA-5597: Runtime filter should be generated and assigned successfully when the
# source expr and target expr have different decimal types.
+####################################################
select count(*)
from tpch_parquet.lineitem
@@ -334,3 +336,96 @@ where l_orderkey = 965 and l_extendedprice * l_tax = p_retailprice;
---- RESULTS
1
====
+
+
+---- QUERY
+####################################################
+# Test case 16: PHJ nodes that spill should still produce filters.
+# Run this for Parquet only to avoid variable memory
+# consumption / spilling behaviour.
+####################################################
+
+SET BUFFER_POOL_LIMIT=40m;
+SET RUNTIME_FILTER_MODE=GLOBAL;
+SET RUNTIME_FILTER_WAIT_TIME_MS=30000;
+SET RUNTIME_BLOOM_FILTER_SIZE=16M;
+select STRAIGHT_JOIN count(a.l_comment)
+ from tpch_parquet.lineitem a join tpch_parquet.lineitem b
+ on a.l_comment = b.l_comment;
+---- RESULTS
+51495713
+---- TYPES
+BIGINT
+---- RUNTIME_PROFILE
+row_regex: .*SpilledPartitions: [1-9]\d* .*
+row_regex: .*Rows processed: 16.38K.*
+row_regex: .*Rows rejected: 0 .*
+row_regex: .*1 of 1 Runtime Filter Published.*
+====
+
+
+---- QUERY
+####################################################
+# Test case 17: Filters are still effective inside subplans
+# (in certain cases). The query has a HJ-scan pair inside a
+# subplan (on the LHS).
+####################################################
+SET RUNTIME_FILTER_MODE=GLOBAL;
+SET RUNTIME_FILTER_WAIT_TIME_MS=30000;
+select straight_join count(1)
+ from alltypes a join complextypestbl b
+ # b.id + 10 confuses planner, so it doesn't think it
+ # can transitively push a.id < 20 to scan of a.
+ on a.id = b.id + 10 join b.int_array where b.id < 10
+---- RESULTS
+10
+---- RUNTIME_PROFILE
+row_regex: .*1 of 1 Runtime Filter Published.*
+row_regex: .*Rows rejected: 2.43K \(2432\).*
+====
+
+
+---- QUERY
+####################################################
+# Test case 18: Filters will not be used if they exceed
+# the configured memory limit on the coordinator.
+# To test this, we need to construct a query where memory
+# consumption on the coordinator exceeds MEM_LIMIT, but
+# not on the backends (because otherwise they will disable
+# the filters through another path). We set MEM_LIMIT to
+# the minimum possible then set filter size to be roughly
+# half that: since the coordinator must aggregate two of
+# these filters (and indeed must create one as well), it
+# will exceed the memory limit. This is checked for
+# indirectly by confirming that the filter had no effect
+# (when usually it would be selective).
+####################################################
+SET RUNTIME_FILTER_MODE=GLOBAL;
+SET RUNTIME_FILTER_WAIT_TIME_MS=30000;
+SET RUNTIME_FILTER_MIN_SIZE=128MB;
+SET RUNTIME_FILTER_MAX_SIZE=500MB;
+SET MEM_LIMIT=140MB;
+select STRAIGHT_JOIN * from alltypes a join [SHUFFLE] alltypes b
+ on a.month = b.id and b.int_col = -3
+---- RESULTS
+---- RUNTIME_PROFILE
+row_regex: .*Filter 0 \(128.00 MB\).*
+row_regex: .*Files processed: 8.*
+row_regex: .*Files rejected: 0.*
+====
+---- QUERY
+# Confirm that with broadcast join, memory limit is not hit.
+SET RUNTIME_FILTER_MODE=GLOBAL;
+SET RUNTIME_FILTER_WAIT_TIME_MS=30000;
+SET RUNTIME_FILTER_MIN_SIZE=128MB;
+SET RUNTIME_FILTER_MAX_SIZE=500MB;
+# Allocate enough memory for the join + filter + scan
+SET MEM_LIMIT=170MB;
+select STRAIGHT_JOIN * from alltypes a join [BROADCAST] alltypes b
+ on a.month = b.id and b.int_col = -3
+---- RESULTS
+---- RUNTIME_PROFILE
+row_regex: .*Filter 0 \(128.00 MB\).*
+row_regex: .*Files processed: 8.*
+row_regex: .*Files rejected: 8.*
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters_phj.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters_phj.test b/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters_phj.test
deleted file mode 100644
index 66391a5..0000000
--- a/testdata/workloads/functional-query/queries/QueryTest/runtime_row_filters_phj.test
+++ /dev/null
@@ -1,94 +0,0 @@
-====
----- QUERY
-# Test cases that are only valid for PHJ joins are in this file.
-
-####################################################
-# Test case 1: PHJ nodes that spill should still produce filters.
-# Run this for Parquet only to avoid variable memory
-# consumption / spilling behaviour.
-####################################################
-
-SET BUFFER_POOL_LIMIT=40m;
-SET RUNTIME_FILTER_MODE=GLOBAL;
-SET RUNTIME_FILTER_WAIT_TIME_MS=30000;
-SET RUNTIME_BLOOM_FILTER_SIZE=16M;
-select STRAIGHT_JOIN count(a.l_comment)
- from tpch_parquet.lineitem a join tpch_parquet.lineitem b
- on a.l_comment = b.l_comment;
----- RESULTS
-51495713
----- TYPES
-BIGINT
----- RUNTIME_PROFILE
-row_regex: .*SpilledPartitions: [1-9]\d* .*
-row_regex: .*Rows processed: 16.38K.*
-row_regex: .*Rows rejected: 0 .*
-row_regex: .*1 of 1 Runtime Filter Published.*
-====
-
-
----- QUERY
-####################################################
-# Test case 2: Filters are still effective inside subplans
-# (in certain cases). The query has a HJ-scan pair inside a
-# subplan (on the LHS).
-####################################################
-SET RUNTIME_FILTER_MODE=GLOBAL;
-SET RUNTIME_FILTER_WAIT_TIME_MS=30000;
-select straight_join count(1)
- from alltypes a join complextypestbl b
- # b.id + 10 confuses planner, so it doesn't think it
- # can transitively push a.id < 20 to scan of a.
- on a.id = b.id + 10 join b.int_array where b.id < 10
----- RESULTS
-10
----- RUNTIME_PROFILE
-row_regex: .*1 of 1 Runtime Filter Published.*
-row_regex: .*Rows rejected: 2.43K \(2432\).*
-====
-
-
----- QUERY
-####################################################
-# Test case 3: Filters will not be used if they exceed
-# the configured memory limit on the coordinator.
-# To test this, we need to construct a query where memory
-# consumption on the coordinator exceeds MEM_LIMIT, but
-# not on the backends (because otherwise they will disable
-# the filters through another path). We set MEM_LIMIT to
-# the minimum possible then set filter size to be roughly
-# half that: since the coordinator must aggregate two of
-# these filters (and indeed must create one as well), it
-# will exceed the memory limit. This is checked for
-# indirectly by confirming that the filter had no effect
-# (when usually it would be selective).
-####################################################
-SET RUNTIME_FILTER_MODE=GLOBAL;
-SET RUNTIME_FILTER_WAIT_TIME_MS=30000;
-SET RUNTIME_FILTER_MIN_SIZE=128MB;
-SET RUNTIME_FILTER_MAX_SIZE=500MB;
-SET MEM_LIMIT=140MB;
-select STRAIGHT_JOIN * from alltypes a join [SHUFFLE] alltypes b
- on a.month = b.id and b.int_col = -3
----- RESULTS
----- RUNTIME_PROFILE
-row_regex: .*Filter 0 \(128.00 MB\).*
-row_regex: .*Files processed: 8.*
-row_regex: .*Files rejected: 0.*
-====
----- QUERY
-# Confirm that with broadcast join, memory limit is not hit.
-SET RUNTIME_FILTER_MODE=GLOBAL;
-SET RUNTIME_FILTER_WAIT_TIME_MS=30000;
-SET RUNTIME_FILTER_MIN_SIZE=128MB;
-SET RUNTIME_FILTER_MAX_SIZE=500MB;
-# Allocate enough memory for the join + filter + scan
-SET MEM_LIMIT=170MB;
-select STRAIGHT_JOIN * from alltypes a join [BROADCAST] alltypes b
- on a.month = b.id and b.int_col = -3
----- RESULTS
----- RUNTIME_PROFILE
-row_regex: .*Filter 0 \(128.00 MB\).*
-row_regex: .*Files processed: 8.*
-row_regex: .*Files rejected: 8.*
-====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/tests/query_test/test_join_queries.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_join_queries.py b/tests/query_test/test_join_queries.py
index a8f2be0..30c659c 100644
--- a/tests/query_test/test_join_queries.py
+++ b/tests/query_test/test_join_queries.py
@@ -61,9 +61,6 @@ class TestJoinQueries(ImpalaTestSuite):
new_vector.get_value('exec_option')['num_nodes'] = 1
self.run_test_case('QueryTest/single-node-joins-with-limits-exhaustive', new_vector)
- def test_partitioned_joins(self, vector):
- self.run_test_case('QueryTest/joins-partitioned', vector)
-
@SkipIfS3.hbase
@SkipIfADLS.hbase
@SkipIfIsilon.hbase
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c1781b73/tests/query_test/test_runtime_filters.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_runtime_filters.py b/tests/query_test/test_runtime_filters.py
index 7710f43..82b5792 100644
--- a/tests/query_test/test_runtime_filters.py
+++ b/tests/query_test/test_runtime_filters.py
@@ -62,6 +62,3 @@ class TestRuntimeRowFilters(ImpalaTestSuite):
def test_row_filters(self, vector):
self.run_test_case('QueryTest/runtime_row_filters', vector)
-
- def test_row_filters_phj_only(self, vector):
- self.run_test_case('QueryTest/runtime_row_filters_phj', vector)