You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bi...@apache.org on 2018/10/09 23:18:20 UTC
[4/8] impala git commit: IMPALA-7647: Add HS2/Impyla dimension to
TestQueries
IMPALA-7647: Add HS2/Impyla dimension to TestQueries
I used some ideas from Alex Leblang's abandoned patch:
https://gerrit.cloudera.org/#/c/137/ in order to run .test files through
HS2. The advantage of using Impyla is that much of the code will be
reusable for any Python client implementing the standard Python dbapi
and does not require us implementing yet another thrift client.
This gives us better coverage of non-trivial result sets from HS2,
including handling of NULLs, error logs and more interesting result
sets than the basic HS2 tests.
I added HS2 coverage to TestQueries, which has a reasonable variety of
queries and covers the data types in alltypes. I also added
TestDecimalQueries, TestStringQuery and TestCharFormats to get coverage
of DECIMAL, CHAR and VARCHAR that aren't in alltypes. Coverage of
results sets with NULLs was limited so I added a couple of queries.
Places where results differ from Beeswax:
* Impyla is a Python dbapi client so must convert timestamps into python datetime
objects, which only have microsecond precision. Therefore result
timestamps within nanosecond precision are truncated.
* The HS2 interface reports the NULL type as BOOLEAN as a workaround for
IMPALA-914.
* The Beeswax interface reported VARCHAR as STRING, but HS2 reports
VARCHAR.
I dealt with different results by adding additional result sections so
that the expected differences between the clients/protocols were
explicit.
Limitations:
* Not all of the same methods are implemented as for beeswax, so some
tests that have more complicated interactions with the client will not
work with HS2 yet.
* We don't have a way to get the affected row count for inserts.
I also simplified the ImpalaConnection API by removing some unnecessary
methods and moved some generic methods to the base class.
Testing:
* Confirmed that it detected IMPALA-7588 by re-applying the buggy patch.
* Ran exhaustive and CentOS6 tests.
Change-Id: I9908ccc4d3df50365be8043b883cacafca52661e
Reviewed-on: http://gerrit.cloudera.org:8080/11546
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/d05f73f4
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/d05f73f4
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/d05f73f4
Branch: refs/heads/master
Commit: d05f73f41588dc1ba08684df63c2da85055be84a
Parents: fec2d64
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Sat Sep 29 12:13:35 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Oct 9 00:45:10 2018 +0000
----------------------------------------------------------------------
bin/start-impala-cluster.py | 16 +-
.../queries/QueryTest/analytic-fns.test | 37 +++
.../queries/QueryTest/chars-formats.test | 4 +
.../queries/QueryTest/chars.test | 46 ++++
.../queries/QueryTest/data-source-tables.test | 10 +
.../queries/QueryTest/decimal.test | 33 +++
.../queries/QueryTest/inline-view-limit.test | 18 ++
.../queries/QueryTest/inline-view.test | 6 +
.../queries/QueryTest/limit.test | 55 +++++
.../queries/QueryTest/top-n.test | 125 ++++++++++
.../queries/QueryTest/union.test | 8 +
tests/beeswax/impala_beeswax.py | 14 +-
tests/common/impala_connection.py | 247 ++++++++++++++++---
tests/common/impala_service.py | 36 ++-
tests/common/impala_test_suite.py | 102 +++++---
tests/common/test_dimensions.py | 20 ++
tests/common/test_result_verifier.py | 46 ++--
tests/conftest.py | 6 +-
tests/custom_cluster/test_coordinators.py | 10 +-
tests/custom_cluster/test_session_expiration.py | 18 +-
tests/metadata/test_views_compatibility.py | 4 +-
tests/query_test/test_aggregation.py | 6 +-
tests/query_test/test_chars.py | 13 +-
tests/query_test/test_decimal_queries.py | 8 +-
tests/query_test/test_insert.py | 6 +-
tests/query_test/test_queries.py | 9 +-
tests/query_test/test_scanners.py | 6 +-
tests/util/test_file_parser.py | 2 +-
28 files changed, 762 insertions(+), 149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/bin/start-impala-cluster.py
----------------------------------------------------------------------
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index 208cc35..ebf8fa5 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -395,27 +395,25 @@ def wait_for_catalog(impalad, timeout_in_seconds=CLUSTER_WAIT_TIMEOUT_IN_SECONDS
"""Waits for a catalog copy to be received by the impalad. When its received,
additionally waits for client ports to be opened."""
start_time = time()
- client_beeswax = None
- client_hs2 = None
+ beeswax_port_is_open = False
+ hs2_port_is_open = False
num_dbs = 0
num_tbls = 0
- while (time() - start_time < timeout_in_seconds):
+ while ((time() - start_time < timeout_in_seconds) and
+ not (beeswax_port_is_open and hs2_port_is_open)):
try:
num_dbs, num_tbls = impalad.service.get_metric_values(
["catalog.num-databases", "catalog.num-tables"])
- client_beeswax = impalad.service.create_beeswax_client()
- client_hs2 = impalad.service.create_hs2_client()
- break
+ beeswax_port_is_open = impalad.service.beeswax_port_is_open()
+ hs2_port_is_open = impalad.service.hs2_port_is_open()
except Exception as e:
LOG.exception(("Client services not ready. Waiting for catalog cache: "
"({num_dbs} DBs / {num_tbls} tables). Trying again ...").format(
num_dbs=num_dbs,
num_tbls=num_tbls))
- finally:
- if client_beeswax is not None: client_beeswax.close()
sleep(0.5)
- if client_beeswax is None or client_hs2 is None:
+ if not hs2_port_is_open or not beeswax_port_is_open:
raise RuntimeError("Unable to open client ports within {num_seconds} seconds.".format(
num_seconds=timeout_in_seconds))
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test b/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
index adf1320..0c6207c 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
@@ -27,6 +27,25 @@ where id % 4 = 0 and month != 1;
2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666746,'1','9'
2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666746,'1','9'
2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666746,'1','9'
+---- DBAPI_RESULTS: VERIFY_IS_EQUAL_SORTED
+2009,2,6,4.666666666666667,2009-02-01 20:13:00.541666,'1','9'
+2009,2,6,4.666666666666667,2009-02-01 20:13:00.541666,'1','9'
+2009,2,6,4.666666666666667,2009-02-01 20:13:00.541666,'1','9'
+2009,2,6,4.666666666666667,2009-02-01 20:13:00.541666,'1','9'
+2009,2,6,4.666666666666667,2009-02-01 20:13:00.541666,'1','9'
+2009,2,6,4.666666666666667,2009-02-01 20:13:00.541666,'1','9'
+2009,3,6,3.666666666666667,2009-03-01 20:12:00.475000,'0','8'
+2009,3,6,3.666666666666667,2009-03-01 20:12:00.475000,'0','8'
+2009,3,6,3.666666666666667,2009-03-01 20:12:00.475000,'0','8'
+2009,3,6,3.666666666666667,2009-03-01 20:12:00.475000,'0','8'
+2009,3,6,3.666666666666667,2009-03-01 20:12:00.475000,'0','8'
+2009,3,6,3.666666666666667,2009-03-01 20:12:00.475000,'0','8'
+2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666,'1','9'
+2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666,'1','9'
+2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666,'1','9'
+2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666,'1','9'
+2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666,'1','9'
+2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666,'1','9'
---- TYPES
INT, INT, BIGINT, DOUBLE, TIMESTAMP, STRING, STRING
====
@@ -589,6 +608,15 @@ from alltypes where id < 8
5,15,50.49999999999999,5,25.25,2009-01-01 00:07:00.210000038
6,18,70.69999999999999,6,35.34999999999999,NULL
7,13,90.89999999999999,6.5,45.45,NULL
+---- DBAPI_RESULTS: VERIFY_IS_EQUAL_SORTED
+0,1,NULL,0.5,NULL,2009-01-01 00:02:30.199999
+1,3,NULL,1,NULL,2009-01-01 00:03:30.449999
+2,6,0,2,0,2009-01-01 00:04:30.349999
+3,9,10.1,3,5.05,2009-01-01 00:05:30.124999
+4,12,30.3,4,15.15,2009-01-01 00:06:30.180000
+5,15,50.49999999999999,5,25.25,2009-01-01 00:07:00.210000
+6,18,70.69999999999999,6,35.34999999999999,NULL
+7,13,90.89999999999999,6.5,45.45,NULL
---- TYPES
INT, BIGINT, DOUBLE, DOUBLE, DOUBLE, TIMESTAMP
====
@@ -1562,6 +1590,8 @@ from (
) t3
---- RESULTS
2010-01-10 18:02:05.234931468,2010-01-10 18:02:05.215156078
+---- DBAPI_RESULTS
+2010-01-10 18:02:05.234931,2010-01-10 18:02:05.215156
---- TYPES
TIMESTAMP, TIMESTAMP
====
@@ -1848,6 +1878,13 @@ INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, TIMESTAMP
2,true,2,2,2,20,2.200000047683716,20.2,'2',2010-01-01 00:02:00.100000000
3,false,3,3,3,30,3.299999952316284,30.3,'3',2010-01-01 00:03:00.300000000
4,true,4,4,4,40,4.400000095367432,40.4,'4',2010-01-01 00:04:00.600000000
+---- DBAPI_RESULTS: VERIFY_IS_EQUAL_SORTED
+0,true,NULL,NULL,NULL,NULL,NULL,NULL,'0',2010-01-01 00:00:00
+0,true,NULL,NULL,NULL,NULL,NULL,NULL,'0',2010-01-01 00:00:00
+1,false,1,1,1,10,1.100000023841858,10.1,'1',2010-01-01 00:01:00
+2,true,2,2,2,20,2.200000047683716,20.2,'2',2010-01-01 00:02:00.100000
+3,false,3,3,3,30,3.299999952316284,30.3,'3',2010-01-01 00:03:00.300000
+4,true,4,4,4,40,4.400000095367432,40.4,'4',2010-01-01 00:04:00.600000
====
---- QUERY
# Test all possible types for first_value (except decimal).
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/chars-formats.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/chars-formats.test b/testdata/workloads/functional-query/queries/QueryTest/chars-formats.test
index 2ed822d..33938b2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/chars-formats.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/chars-formats.test
@@ -3,6 +3,8 @@
select * from chars_formats order by vc;
---- TYPES
char, char, string
+---- HS2_TYPES
+char, char, varchar
---- RESULTS
'abc ','8d3fffddf79e9a232ffd19f9ccaa4d6b37a6a243dbe0f23137b108a043d9da13121a9b505c804956b22e93c7f93969f4a7ba8ddea45bf4aab0bebc8f814e09918d3fffddf79e','abc'
'abcde','68f8c4575da360c32abb46689e58193a0eeaa905ae6f4a5e6c702a6ae1db35a6f86f8222b7a5489d96eb0466c755b677a64160d074617096a8c6279038bc720468f8c4575da3','b2fe9d4638503a57f93396098f24103a'
@@ -30,6 +32,8 @@ char
select vc from chars_formats order by vc;
---- TYPES
string
+---- HS2_TYPES
+varchar
---- RESULTS
'abc'
'b2fe9d4638503a57f93396098f24103a'
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/chars.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/chars.test b/testdata/workloads/functional-query/queries/QueryTest/chars.test
index cd915ce..6802647 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/chars.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/chars.test
@@ -15,6 +15,8 @@ bigint,bigint,bigint,bigint
select * from chars_tiny where cs = cast('6a' as CHAR(2))
---- TYPES
char,char,string
+---- HS2_TYPES
+char,char,varchar
---- RESULTS
'6a ','6b ','6c'
'6a ','6b ','6c'
@@ -75,6 +77,8 @@ char
select A.vc from chars_tiny as A join chars_tiny using (vc) order by A.vc
---- TYPES
string
+---- HS2_TYPES
+varchar
---- RESULTS
'1cccc'
'2cccccc'
@@ -109,6 +113,8 @@ string, string, bigint, bigint, string, string
select t1.vc, COUNT(1) FROM chars_tiny t1 GROUP BY 1 ORDER BY t1.vc
---- TYPES
string, bigint
+---- HS2_TYPES
+varchar, bigint
---- RESULTS
'1cccc',1
'2cccccc',1
@@ -151,3 +157,43 @@ char, string
'a ','a '
'NULL','NULL'
====
+---- QUERY
+# Test returning mix of nulls and non-nulls.
+WITH numbered AS (
+ SELECT *, row_number() over (order by cs) as rn
+ FROM chars_tiny)
+SELECT *
+FROM (
+ SELECT CASE WHEN rn % 2 = 0 THEN cs END cs,
+ CASE WHEN rn % 2 = 1 THEN cl END cl,
+ CASE WHEN rn % 3 = 0 THEN vc END vc
+ FROM numbered
+ UNION ALL
+ SELECT CASE WHEN rn % 2 = 1 THEN cs END cs,
+ CASE WHEN rn % 2 = 0 THEN cl END cl,
+ CASE WHEN rn % 3 = 1 THEN vc END vc
+ FROM numbered) v
+---- TYPES
+char, char, string
+---- HS2_TYPES
+char, char, varchar
+---- RESULTS
+'NULL','1bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb','NULL'
+'2aaaa','NULL','NULL'
+'NULL','3bbbbb ','3ccc'
+'4aa ','NULL','NULL'
+'NULL','5bbb ','NULL'
+'6a ','NULL','6c'
+'NULL','6b ','NULL'
+'a ','NULL','NULL'
+'NULL','NULL','NULL'
+'1aaaa','NULL','1cccc'
+'NULL','2bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb','NULL'
+'3aaa ','NULL','NULL'
+'NULL','4bbbb ','4cc'
+'5a ','NULL','NULL'
+'NULL','6b ','NULL'
+'6a ','NULL','6c'
+'NULL','b ','NULL'
+'NULL','NULL','NULL'
+====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test b/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test
index e88d928..62ba37b 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test
@@ -13,6 +13,12 @@ where float_col != 0 and
1992,true,2,92,1992,19920,2191.199951171875,1992,1970-01-01 00:00:01.992000000,'1992',-999998007,-9999998007,-9999999999.9999998007,-9.9999999999999999999999999999999998007,-99999.98007
1993,false,3,93,1993,19930,2192.300048828125,1993,1970-01-01 00:00:01.993000000,'1993',999998006,9999998006,9999999999.9999998006,9.9999999999999999999999999999999998006,99999.98006
1994,true,4,94,1994,19940,2193.39990234375,1994,1970-01-01 00:00:01.994000000,'1994',-999998005,-9999998005,-9999999999.9999998005,-9.9999999999999999999999999999999998005,-99999.98005
+---- DBAPI_RESULTS
+1990,true,0,90,1990,19900,2189,1990,1970-01-01 00:00:01.990000,'NULL',-999998009,-9999998009,-9999999999.9999998009,-9.9999999999999999999999999999999998009,-99999.98009
+1991,false,1,91,1991,19910,2190.10009765625,1991,1970-01-01 00:00:01.991000,'1991',999998008,9999998008,9999999999.9999998008,9.9999999999999999999999999999999998008,99999.98008
+1992,true,2,92,1992,19920,2191.199951171875,1992,1970-01-01 00:00:01.992000,'1992',-999998007,-9999998007,-9999999999.9999998007,-9.9999999999999999999999999999999998007,-99999.98007
+1993,false,3,93,1993,19930,2192.300048828125,1993,1970-01-01 00:00:01.993000,'1993',999998006,9999998006,9999999999.9999998006,9.9999999999999999999999999999999998006,99999.98006
+1994,true,4,94,1994,19940,2193.39990234375,1994,1970-01-01 00:00:01.994000,'1994',-999998005,-9999998005,-9999999999.9999998005,-9.9999999999999999999999999999999998005,-99999.98005
---- TYPES
INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, TIMESTAMP, STRING, DECIMAL, DECIMAL, DECIMAL, DECIMAL, DECIMAL
====
@@ -25,6 +31,10 @@ where double_col != 0 and int_col >= 1990 limit 3
19900,1970-01-01 00:00:01.990000000,1990
19910,1970-01-01 00:00:01.991000000,1991
19920,1970-01-01 00:00:01.992000000,1992
+---- DBAPI_RESULTS
+19900,1970-01-01 00:00:01.990000,1990
+19910,1970-01-01 00:00:01.991000,1991
+19920,1970-01-01 00:00:01.992000,1992
---- TYPES
BIGINT, TIMESTAMP, DOUBLE
====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/decimal.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/decimal.test b/testdata/workloads/functional-query/queries/QueryTest/decimal.test
index ac7f1c1..1e1affc 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/decimal.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/decimal.test
@@ -385,3 +385,36 @@ NULL,0.0000
---- TYPES
DECIMAL, DECIMAL
====
+---- QUERY
+# Query return mixed NULL and non-NULL decimal values.
+select case when rn % 2 = 0 then c1 end,
+ case when rn % 3 = 0 then c2 end,
+ case when rn % 4 = 0 then c3 end
+from (
+ select *, row_number() over (order by c1) rn
+ from decimal_tiny) v
+where rn <= 20
+---- RESULTS
+NULL,NULL,NULL
+0.1111,NULL,NULL
+NULL,102.44444,NULL
+0.3333,NULL,0.3
+NULL,NULL,NULL
+0.5555,106.11110,NULL
+NULL,NULL,NULL
+0.7777,NULL,0.7
+NULL,109.77776,NULL
+0.9999,NULL,NULL
+NULL,NULL,NULL
+1.2221,113.44442,0.1
+NULL,NULL,NULL
+1.4443,NULL,NULL
+NULL,117.11108,NULL
+1.6665,NULL,0.5
+NULL,NULL,NULL
+1.8887,120.77774,NULL
+NULL,NULL,NULL
+2.1109,NULL,0.9
+---- TYPES
+DECIMAL, DECIMAL, DECIMAL
+====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/inline-view-limit.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/inline-view-limit.test b/testdata/workloads/functional-query/queries/QueryTest/inline-view-limit.test
index 2c3a72b..96e2fcc 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/inline-view-limit.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/inline-view-limit.test
@@ -21,6 +21,12 @@ INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, STRING, TIM
4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000000,2009,1
6,true,6,6,6,60,6.599999904632568,60.6,'01/01/09','6',2009-01-01 00:06:00.150000000,2009,1
8,true,8,8,8,80,8.800000190734863,80.8,'01/01/09','8',2009-01-01 00:08:00.280000000,2009,1
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+2,true,2,2,2,20,2.200000047683716,20.2,'01/01/09','2',2009-01-01 00:02:00.100000,2009,1
+4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000,2009,1
+6,true,6,6,6,60,6.599999904632568,60.6,'01/01/09','6',2009-01-01 00:06:00.150000,2009,1
+8,true,8,8,8,80,8.800000190734863,80.8,'01/01/09','8',2009-01-01 00:08:00.280000,2009,1
====
---- QUERY
# order by/limit in subquery, followed by addititional selection predicate;
@@ -39,6 +45,12 @@ INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, STRING, TIM
4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000000,2009,1
6,true,6,6,6,60,6.599999904632568,60.6,'01/01/09','6',2009-01-01 00:06:00.150000000,2009,1
8,true,8,8,8,80,8.800000190734863,80.8,'01/01/09','8',2009-01-01 00:08:00.280000000,2009,1
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+2,true,2,2,2,20,2.200000047683716,20.2,'01/01/09','2',2009-01-01 00:02:00.100000,2009,1
+4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000,2009,1
+6,true,6,6,6,60,6.599999904632568,60.6,'01/01/09','6',2009-01-01 00:06:00.150000,2009,1
+8,true,8,8,8,80,8.800000190734863,80.8,'01/01/09','8',2009-01-01 00:08:00.280000,2009,1
====
---- QUERY
# join against subquery with limit creates a merge fragment that applies the limit
@@ -102,6 +114,12 @@ double_col, date_string_col, string_col, timestamp_col, year, month from
2,true,2,2,2,20,2.200000047683716,20.2,'01/01/09','2',2009-01-01 00:02:00.100000000,2009,1
3,false,3,3,3,30,3.299999952316284,30.3,'01/01/09','3',2009-01-01 00:03:00.300000000,2009,1
4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000000,2009,1
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,2,2,2,20,2.200000047683716,20.2,'01/01/09','2',2009-01-01 00:02:00.100000,2009,1
+3,false,3,3,3,30,3.299999952316284,30.3,'01/01/09','3',2009-01-01 00:03:00.300000,2009,1
+4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000,2009,1
---- TYPES
INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, STRING, TIMESTAMP, INT, INT
====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/inline-view.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/inline-view.test b/testdata/workloads/functional-query/queries/QueryTest/inline-view.test
index dd0094b..49b2e91 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/inline-view.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/inline-view.test
@@ -151,6 +151,12 @@ where month = 1
2,3,2,true,2,2,2,20,2.200000047683716,20.2,'01/01/09','2',2009-01-01 00:02:00.100000000,2009,1
3,3,3,false,3,3,3,30,3.299999952316284,30.3,'01/01/09','3',2009-01-01 00:03:00.300000000,2009,1
4,3,4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000000,2009,1
+---- DBAPI_RESULTS
+0,3,0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,3,1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,3,2,true,2,2,2,20,2.200000047683716,20.2,'01/01/09','2',2009-01-01 00:02:00.100000,2009,1
+3,3,3,false,3,3,3,30,3.299999952316284,30.3,'01/01/09','3',2009-01-01 00:03:00.300000,2009,1
+4,3,4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000,2009,1
---- TYPES
int, bigint, int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/limit.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/limit.test b/testdata/workloads/functional-query/queries/QueryTest/limit.test
index 334ed45..49c6129 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/limit.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/limit.test
@@ -46,6 +46,17 @@ select * from (select * from alltypesagg where day = 1 order by id limit 10) a l
7,false,7,7,7,70,7.699999809265137,70.7,'01/01/10','7',2010-01-01 00:07:00.210000000,2010,1,1
8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000000,2010,1,1
9,false,9,9,9,90,9.899999618530273,90.90000000000001,'01/01/10','9',2010-01-01 00:09:00.360000000,2010,1,1
+---- DBAPI_RESULTS : VERIFY_IS_EQUAL_SORTED
+0,true,NULL,NULL,NULL,NULL,NULL,NULL,'01/01/10','0',2010-01-01 00:00:00,2010,1,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/10','1',2010-01-01 00:01:00,2010,1,1
+2,true,2,2,2,20,2.200000047683,20.2,'01/01/10','2',2010-01-01 00:02:00.100000,2010,1,1
+3,false,3,3,3,30,3.299999952316,30.3,'01/01/10','3',2010-01-01 00:03:00.300000,2010,1,1
+4,true,4,4,4,40,4.400000095367,40.4,'01/01/10','4',2010-01-01 00:04:00.600000,2010,1,1
+5,false,5,5,5,50,5.5,50.5,'01/01/10','5',2010-01-01 00:05:00.100000,2010,1,1
+6,true,6,6,6,60,6.599999904632,60.6,'01/01/10','6',2010-01-01 00:06:00.150000,2010,1,1
+7,false,7,7,7,70,7.699999809265,70.7,'01/01/10','7',2010-01-01 00:07:00.210000,2010,1,1
+8,true,8,8,8,80,8.800000190734,80.8,'01/01/10','8',2010-01-01 00:08:00.280000,2010,1,1
+9,false,9,9,9,90,9.899999618530,90.90000000000001,'01/01/10','9',2010-01-01 00:09:00.360000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
@@ -63,6 +74,17 @@ select * from alltypesagg where day = 1 order by id limit 10
7,false,7,7,7,70,7.699999809265137,70.7,'01/01/10','7',2010-01-01 00:07:00.210000000,2010,1,1
8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000000,2010,1,1
9,false,9,9,9,90,9.899999618530273,90.90000000000001,'01/01/10','9',2010-01-01 00:09:00.360000000,2010,1,1
+---- DBAPI_RESULTS
+0,true,NULL,NULL,NULL,NULL,NULL,NULL,'01/01/10','0',2010-01-01 00:00:00,2010,1,1
+1,false,1,1,1,10,1.100000023841858,10.1,'01/01/10','1',2010-01-01 00:01:00,2010,1,1
+2,true,2,2,2,20,2.200000047683716,20.2,'01/01/10','2',2010-01-01 00:02:00.100000,2010,1,1
+3,false,3,3,3,30,3.299999952316284,30.3,'01/01/10','3',2010-01-01 00:03:00.300000,2010,1,1
+4,true,4,4,4,40,4.400000095367432,40.4,'01/01/10','4',2010-01-01 00:04:00.600000,2010,1,1
+5,false,5,5,5,50,5.5,50.5,'01/01/10','5',2010-01-01 00:05:00.100000,2010,1,1
+6,true,6,6,6,60,6.599999904632568,60.6,'01/01/10','6',2010-01-01 00:06:00.150000,2010,1,1
+7,false,7,7,7,70,7.699999809265137,70.7,'01/01/10','7',2010-01-01 00:07:00.210000,2010,1,1
+8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000,2010,1,1
+9,false,9,9,9,90,9.899999618530273,90.90000000000001,'01/01/10','9',2010-01-01 00:09:00.360000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
@@ -80,6 +102,17 @@ select * from alltypesagg where tinyint_col is null and day = 1 order by id limi
70,true,NULL,70,70,700,77,707,'01/01/10','70',2010-01-01 01:10:24.150000000,2010,1,1
80,true,NULL,80,80,800,88,808,'01/01/10','80',2010-01-01 01:20:31.600000000,2010,1,1
90,true,NULL,90,90,900,99,909,'01/01/10','90',2010-01-01 01:30:40.500000000,2010,1,1
+---- DBAPI_RESULTS
+0,true,NULL,NULL,NULL,NULL,NULL,NULL,'01/01/10','0',2010-01-01 00:00:00,2010,1,1
+10,true,NULL,10,10,100,11,101,'01/01/10','10',2010-01-01 00:10:00.450000,2010,1,1
+20,true,NULL,20,20,200,22,202,'01/01/10','20',2010-01-01 00:20:01.900000,2010,1,1
+30,true,NULL,30,30,300,33,303,'01/01/10','30',2010-01-01 00:30:04.350000,2010,1,1
+40,true,NULL,40,40,400,44,404,'01/01/10','40',2010-01-01 00:40:07.800000,2010,1,1
+50,true,NULL,50,50,500,55,505,'01/01/10','50',2010-01-01 00:50:12.250000,2010,1,1
+60,true,NULL,60,60,600,66,606,'01/01/10','60',2010-01-01 01:00:17.700000,2010,1,1
+70,true,NULL,70,70,700,77,707,'01/01/10','70',2010-01-01 01:10:24.150000,2010,1,1
+80,true,NULL,80,80,800,88,808,'01/01/10','80',2010-01-01 01:20:31.600000,2010,1,1
+90,true,NULL,90,90,900,99,909,'01/01/10','90',2010-01-01 01:30:40.500000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
@@ -247,6 +280,17 @@ select * from alltypesagg where id != 0 and id != 10 order by id limit 10 offset
8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000000,2010,1,1
9,false,9,9,9,90,9.899999618530273,90.89999999999999,'01/01/10','9',2010-01-01 00:09:00.360000000,2010,1,1
11,false,1,11,11,110,12.10000038146973,111.1,'01/01/10','11',2010-01-01 00:11:00.550000000,2010,1,1
+---- DBAPI_RESULTS
+1,false,1,1,1,10,1.100000023841858,10.1,'01/01/10','1',2010-01-01 00:01:00,2010,1,1
+2,true,2,2,2,20,2.200000047683716,20.2,'01/01/10','2',2010-01-01 00:02:00.100000,2010,1,1
+3,false,3,3,3,30,3.299999952316284,30.3,'01/01/10','3',2010-01-01 00:03:00.300000,2010,1,1
+4,true,4,4,4,40,4.400000095367432,40.4,'01/01/10','4',2010-01-01 00:04:00.600000,2010,1,1
+5,false,5,5,5,50,5.5,50.5,'01/01/10','5',2010-01-01 00:05:00.100000,2010,1,1
+6,true,6,6,6,60,6.599999904632568,60.59999999999999,'01/01/10','6',2010-01-01 00:06:00.150000,2010,1,1
+7,false,7,7,7,70,7.699999809265137,70.7,'01/01/10','7',2010-01-01 00:07:00.210000,2010,1,1
+8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000,2010,1,1
+9,false,9,9,9,90,9.899999618530273,90.89999999999999,'01/01/10','9',2010-01-01 00:09:00.360000,2010,1,1
+11,false,1,11,11,110,12.10000038146973,111.1,'01/01/10','11',2010-01-01 00:11:00.550000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
@@ -264,6 +308,17 @@ select * from alltypesagg where id != 0 and id != 10 order by id limit 10 offset
14,true,4,14,14,140,15.39999961853027,141.4,'01/01/10','14',2010-01-01 00:14:00.910000000,2010,1,1
15,false,5,15,15,150,16.5,151.5,'01/01/10','15',2010-01-01 00:15:01.500000000,2010,1,1
16,true,6,16,16,160,17.60000038146973,161.6,'01/01/10','16',2010-01-01 00:16:01.200000000,2010,1,1
+---- DBAPI_RESULTS
+6,true,6,6,6,60,6.599999904632568,60.59999999999999,'01/01/10','6',2010-01-01 00:06:00.150000,2010,1,1
+7,false,7,7,7,70,7.699999809265137,70.7,'01/01/10','7',2010-01-01 00:07:00.210000,2010,1,1
+8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000,2010,1,1
+9,false,9,9,9,90,9.899999618530273,90.89999999999999,'01/01/10','9',2010-01-01 00:09:00.360000,2010,1,1
+11,false,1,11,11,110,12.10000038146973,111.1,'01/01/10','11',2010-01-01 00:11:00.550000,2010,1,1
+12,true,2,12,12,120,13.19999980926514,121.2,'01/01/10','12',2010-01-01 00:12:00.660000,2010,1,1
+13,false,3,13,13,130,14.30000019073486,131.3,'01/01/10','13',2010-01-01 00:13:00.780000,2010,1,1
+14,true,4,14,14,140,15.39999961853027,141.4,'01/01/10','14',2010-01-01 00:14:00.910000,2010,1,1
+15,false,5,15,15,150,16.5,151.5,'01/01/10','15',2010-01-01 00:15:01.500000,2010,1,1
+16,true,6,16,16,160,17.60000038146973,161.6,'01/01/10','16',2010-01-01 00:16:01.200000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/top-n.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/top-n.test b/testdata/workloads/functional-query/queries/QueryTest/top-n.test
index 7d52731..32b1504 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/top-n.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/top-n.test
@@ -736,6 +736,27 @@ from alltypessmall order by 1 limit 20
2008-12-20 00:19:00.810000000,2009-02-02 00:19:00.810000000,44
2008-12-20 00:20:00.900000000,2009-02-03 00:20:00.900000000,45
2008-12-20 00:23:00.930000000,2009-03-03 00:23:00.930000000,73
+---- DBAPI_RESULTS
+2008-12-10 00:24:00.960000,2009-01-03 00:24:00.960000,24
+2008-12-11 00:23:00.930000,2009-01-03 00:23:00.930000,23
+2008-12-12 00:22:00.910000,2009-01-03 00:22:00.910000,22
+2008-12-13 00:21:00.900000,2009-01-03 00:21:00.900000,21
+2008-12-14 00:19:00.810000,2009-01-02 00:19:00.810000,19
+2008-12-14 00:20:00.900000,2009-01-03 00:20:00.900000,20
+2008-12-15 00:18:00.730000,2009-01-02 00:18:00.730000,18
+2008-12-16 00:17:00.660000,2009-01-02 00:17:00.660000,17
+2008-12-16 00:24:00.960000,2009-02-03 00:24:00.960000,49
+2008-12-17 00:16:00.600000,2009-01-02 00:16:00.600000,16
+2008-12-17 00:23:00.930000,2009-02-03 00:23:00.930000,48
+2008-12-18 00:15:00.550000,2009-01-02 00:15:00.550000,15
+2008-12-18 00:22:00.910000,2009-02-03 00:22:00.910000,47
+2008-12-19 00:14:00.510000,2009-01-02 00:14:00.510000,14
+2008-12-19 00:21:00.900000,2009-02-03 00:21:00.900000,46
+2008-12-19 00:24:00.960000,2009-03-03 00:24:00.960000,74
+2008-12-20 00:13:00.480000,2009-01-02 00:13:00.480000,13
+2008-12-20 00:19:00.810000,2009-02-02 00:19:00.810000,44
+2008-12-20 00:20:00.900000,2009-02-03 00:20:00.900000,45
+2008-12-20 00:23:00.930000,2009-03-03 00:23:00.930000,73
---- TYPES
TIMESTAMP, TIMESTAMP, INT
====
@@ -816,6 +837,17 @@ select * from alltypesagg where day is not null order by id limit 10 offset 0
7,false,7,7,7,70,7.699999809265137,70.7,'01/01/10','7',2010-01-01 00:07:00.210000000,2010,1,1
8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000000,2010,1,1
9,false,9,9,9,90,9.899999618530273,90.89999999999999,'01/01/10','9',2010-01-01 00:09:00.360000000,2010,1,1
+---- DBAPI_RESULTS
+0,true,NULL,NULL,NULL,NULL,NULL,NULL,'01/01/10','0',2010-01-01 00:00:00,2010,1,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/10','1',2010-01-01 00:01:00,2010,1,1
+2,true,2,2,2,20,2.200000047683,20.2,'01/01/10','2',2010-01-01 00:02:00.100000,2010,1,1
+3,false,3,3,3,30,3.299999952316,30.3,'01/01/10','3',2010-01-01 00:03:00.300000,2010,1,1
+4,true,4,4,4,40,4.400000095367,40.4,'01/01/10','4',2010-01-01 00:04:00.600000,2010,1,1
+5,false,5,5,5,50,5.5,50.5,'01/01/10','5',2010-01-01 00:05:00.100000,2010,1,1
+6,true,6,6,6,60,6.599999904632,60.59999999999999,'01/01/10','6',2010-01-01 00:06:00.150000,2010,1,1
+7,false,7,7,7,70,7.699999809265,70.7,'01/01/10','7',2010-01-01 00:07:00.210000,2010,1,1
+8,true,8,8,8,80,8.800000190734,80.8,'01/01/10','8',2010-01-01 00:08:00.280000,2010,1,1
+9,false,9,9,9,90,9.899999618530,90.89999999999999,'01/01/10','9',2010-01-01 00:09:00.360000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
@@ -834,6 +866,17 @@ select * from alltypesagg where day is not null order by id limit 10 offset 5
12,true,2,12,12,120,13.19999980926514,121.2,'01/01/10','12',2010-01-01 00:12:00.660000000,2010,1,1
13,false,3,13,13,130,14.30000019073486,131.3,'01/01/10','13',2010-01-01 00:13:00.780000000,2010,1,1
14,true,4,14,14,140,15.39999961853027,141.4,'01/01/10','14',2010-01-01 00:14:00.910000000,2010,1,1
+---- DBAPI_RESULTS
+5,false,5,5,5,50,5.5,50.5,'01/01/10','5',2010-01-01 00:05:00.100000,2010,1,1
+6,true,6,6,6,60,6.599999904632,60.59999999999999,'01/01/10','6',2010-01-01 00:06:00.150000,2010,1,1
+7,false,7,7,7,70,7.699999809265,70.7,'01/01/10','7',2010-01-01 00:07:00.210000,2010,1,1
+8,true,8,8,8,80,8.800000190734,80.8,'01/01/10','8',2010-01-01 00:08:00.280000,2010,1,1
+9,false,9,9,9,90,9.899999618530,90.89999999999999,'01/01/10','9',2010-01-01 00:09:00.360000,2010,1,1
+10,true,NULL,10,10,100,11,101,'01/01/10','10',2010-01-01 00:10:00.450000,2010,1,1
+11,false,1,11,11,110,12.10000038146,111.1,'01/01/10','11',2010-01-01 00:11:00.550000,2010,1,1
+12,true,2,12,12,120,13.19999980926,121.2,'01/01/10','12',2010-01-01 00:12:00.660000,2010,1,1
+13,false,3,13,13,130,14.30000019073,131.3,'01/01/10','13',2010-01-01 00:13:00.780000,2010,1,1
+14,true,4,14,14,140,15.39999961853,141.4,'01/01/10','14',2010-01-01 00:14:00.910000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
@@ -901,6 +944,9 @@ limit 6
20,NULL,NULL
---- TYPES
int, tinyint, null
+---- HS2_TYPES
+# HS2 maps NULL to BOOLEAN
+int, tinyint, boolean
====
---- QUERY
# check cross joins within a subquery
@@ -963,6 +1009,27 @@ limit 100
83,83,3,3,983,8.800000190734863,'3',1
91,91,1,1,991,6.599999904632568,'1',0
91,91,1,1,991,6.599999904632568,'1',1
+---- DBAPI_RESULTS
+31,31,1,1,931,6.599999904632,'1',0
+31,31,1,1,931,6.599999904632,'1',1
+32,32,2,2,932,7.699999809265,'2',0
+32,32,2,2,932,7.699999809265,'2',1
+33,33,3,3,933,8.800000190734,'3',0
+33,33,3,3,933,8.800000190734,'3',1
+41,41,1,1,941,6.599999904632,'1',0
+41,41,1,1,941,6.599999904632,'1',1
+42,42,2,2,942,7.699999809265,'2',0
+42,42,2,2,942,7.699999809265,'2',1
+43,43,3,3,943,8.800000190734,'3',0
+43,43,3,3,943,8.800000190734,'3',1
+81,81,1,1,981,6.599999904632,'1',0
+81,81,1,1,981,6.599999904632,'1',1
+82,82,2,2,982,7.699999809265,'2',0
+82,82,2,2,982,7.699999809265,'2',1
+83,83,3,3,983,8.800000190734,'3',0
+83,83,3,3,983,8.800000190734,'3',1
+91,91,1,1,991,6.599999904632,'1',0
+91,91,1,1,991,6.599999904632,'1',1
---- TYPES
SMALLINT, INT, TINYINT, INT, INT, FLOAT, STRING, INT
====
@@ -1006,6 +1073,10 @@ order by 1 limit 3
0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1024,6 +1095,10 @@ order by 1,2 limit 3
0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1043,6 +1118,11 @@ order by 1,2 limit 4
1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1069,6 +1149,14 @@ limit 20
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
3,false,1,1,1,10,1.100000023841858,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2
4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+3,false,1,1,1,10,1.10000002384158,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2
+4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1093,6 +1181,12 @@ limit 20
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
3,false,1,1,1,10,1.100000023841858,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2
4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+3,false,1,1,1,10,1.100000023841,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2
+4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1132,6 +1226,18 @@ limit 20
6,true,0,0,0,0,0,0,'04/01/09','0',2009-04-01 00:00:00,2009,4
7,false,1,1,1,10,1.100000023841858,10.1,'04/01/09','1',2009-04-01 00:01:00,2009,4
7,false,1,1,1,10,1.100000023841858,10.1,'04/01/09','1',2009-04-01 00:01:00,2009,4
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+3,false,1,1,1,10,1.100000023841,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2
+4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
+4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
+5,false,1,1,1,10,1.100000023841,10.1,'03/01/09','1',2009-03-01 00:01:00,2009,3
+6,true,0,0,0,0,0,0,'04/01/09','0',2009-04-01 00:00:00,2009,4
+6,true,0,0,0,0,0,0,'04/01/09','0',2009-04-01 00:00:00,2009,4
+7,false,1,1,1,10,1.100000023841,10.1,'04/01/09','1',2009-04-01 00:01:00,2009,4
+7,false,1,1,1,10,1.100000023841,10.1,'04/01/09','1',2009-04-01 00:01:00,2009,4
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1151,6 +1257,12 @@ order by 1 limit 5
1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1168,6 +1280,10 @@ order by 1 limit 3
0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1243,6 +1359,15 @@ select * from functional.alltypestiny order by 2,1 limit 10
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
6,true,0,0,0,0,0,0,'04/01/09','0',2009-04-01 00:00:00,2009,4
+---- DBAPI_RESULTS
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+3,false,1,1,1,10,1.100000023841,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2
+5,false,1,1,1,10,1.100000023841,10.1,'03/01/09','1',2009-03-01 00:01:00,2009,3
+7,false,1,1,1,10,1.100000023841,10.1,'04/01/09','1',2009-04-01 00:01:00,2009,4
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
+6,true,0,0,0,0,0,0,'04/01/09','0',2009-04-01 00:00:00,2009,4
---- TYPES
INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, STRING, TIMESTAMP, INT, INT
====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/union.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/union.test b/testdata/workloads/functional-query/queries/QueryTest/union.test
index 7fe4020..6448fc7 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/union.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/union.test
@@ -740,6 +740,8 @@ union all
select 3, 'c', NULL, 30.0
---- TYPES
tinyint, string, null, decimal
+---- HS2_TYPES
+tinyint, string, boolean, decimal
---- RESULTS: VERIFY_IS_EQUAL_SORTED
1,'a',NULL,10.0
2,'b',NULL,20.0
@@ -754,6 +756,8 @@ union distinct
select 1, 'a', NULL, 10.0
---- TYPES
tinyint, string, null, decimal
+---- HS2_TYPES
+tinyint, string, boolean, decimal
---- RESULTS: VERIFY_IS_EQUAL_SORTED
1,'a',NULL,10.0
2,'b',NULL,20.0
@@ -767,6 +771,8 @@ union all
values(3, 'c', NULL, 30.0)
---- TYPES
tinyint, string, null, decimal
+---- HS2_TYPES
+tinyint, string, boolean, decimal
---- RESULTS: VERIFY_IS_EQUAL_SORTED
1,'a',NULL,10.0
2,'b',NULL,20.0
@@ -781,6 +787,8 @@ union distinct
values(1, 'a', NULL, 10.0)
---- TYPES
tinyint, string, null, decimal
+---- HS2_TYPES
+tinyint, string, boolean, decimal
---- RESULTS: VERIFY_IS_EQUAL_SORTED
1,'a',NULL,10.0
2,'b',NULL,20.0
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/beeswax/impala_beeswax.py
----------------------------------------------------------------------
diff --git a/tests/beeswax/impala_beeswax.py b/tests/beeswax/impala_beeswax.py
index 1a3bc65..d8272ed 100644
--- a/tests/beeswax/impala_beeswax.py
+++ b/tests/beeswax/impala_beeswax.py
@@ -74,6 +74,12 @@ class ImpalaBeeswaxResult(object):
self.time_taken = kwargs.get('time_taken', 0)
self.summary = kwargs.get('summary', str())
self.schema = kwargs.get('schema', None)
+ self.column_types = None
+ self.column_labels = None
+ if self.schema is not None:
+ # Extract labels and types so there is a shared interface with HS2ResultSet.
+ self.column_types = [fs.type.upper() for fs in self.schema.fieldSchemas]
+ self.column_labels = [fs.name.upper() for fs in self.schema.fieldSchemas]
self.runtime_profile = kwargs.get('runtime_profile', str())
self.exec_summary = kwargs.get('exec_summary', None)
@@ -420,14 +426,6 @@ class ImpalaBeeswaxClient(object):
def get_log(self, query_handle):
return self.__do_rpc(lambda: self.imp_service.get_log(query_handle))
- def refresh(self):
- """Invalidate the Impalad catalog"""
- return self.execute("invalidate metadata")
-
- def refresh_table(self, db_name, table_name):
- """Refresh a specific table from the catalog"""
- return self.execute("refresh %s.%s" % (db_name, table_name))
-
def fetch_results(self, query_string, query_handle, max_rows = -1):
"""Fetches query results given a handle and query type (insert, use, other)"""
query_type = self.__get_query_type(query_string)
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/common/impala_connection.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_connection.py b/tests/common/impala_connection.py
index b075506..e7dace9 100644
--- a/tests/common/impala_connection.py
+++ b/tests/common/impala_connection.py
@@ -21,7 +21,9 @@
import abc
import logging
+import re
+import impala.dbapi as impyla
from tests.beeswax.impala_beeswax import ImpalaBeeswaxClient
@@ -33,32 +35,37 @@ console_handler.setFormatter(logging.Formatter('%(message)s'))
LOG.addHandler(console_handler)
LOG.propagate = False
+# Regular expression that matches the "progress" entry in the HS2 log.
+PROGRESS_LOG_RE = re.compile(
+ r'^Query [a-z0-9:]+ [0-9]+% Complete \([0-9]+ out of [0-9]+\)$')
+
# Common wrapper around the internal types of HS2/Beeswax operation/query handles.
class OperationHandle(object):
- def __init__(self, handle):
+ def __init__(self, handle, sql_stmt):
self.__handle = handle
+ self.__sql_stmt = sql_stmt
+
+ def get_handle(self):
+ return self.__handle
- def get_handle(self): return self.__handle
+ def sql_stmt(self):
+ return self.__sql_stmt
# Represents an Impala connection.
class ImpalaConnection(object):
__metaclass__ = abc.ABCMeta
-
@abc.abstractmethod
def set_configuration_option(self, name, value):
"""Sets a configuraiton option name to the given value"""
pass
- @abc.abstractmethod
- def get_configuration(self):
- """Returns the configuration (a dictionary of key-value pairs) for this connection"""
- pass
-
- @abc.abstractmethod
- def set_configuration(self, configuration_option_dict):
+ def set_configuration(self, config_option_dict):
"""Replaces existing configuration with the given dictionary"""
- pass
+ assert config_option_dict is not None, "config_option_dict cannot be None"
+ self.clear_configuration()
+ for name, value in config_option_dict.iteritems():
+ self.set_configuration_option(name, value)
@abc.abstractmethod
def clear_configuration(self):
@@ -66,6 +73,13 @@ class ImpalaConnection(object):
pass
@abc.abstractmethod
+ def get_default_configuration(self):
+ """Return the default configuration for the connection, before any modifications are
+ made to the session state. Returns a map with the config variable as the key and a
+ string representation of the default value as the value."""
+ pass
+
+ @abc.abstractmethod
def connect(self):
"""Opens the connection"""
pass
@@ -87,7 +101,7 @@ class ImpalaConnection(object):
@abc.abstractmethod
def get_log(self, operation_handle):
- """Returns the log of an operation"""
+ """Returns the log of an operation as a string, with entries separated by newlines."""
pass
@abc.abstractmethod
@@ -101,7 +115,9 @@ class ImpalaConnection(object):
@abc.abstractmethod
def execute_async(self, sql_stmt):
- """Issues a query and returns the handle to the caller for processing"""
+ """Issues a query and returns the handle to the caller for processing. Only one
+ async operation per connection at a time is supported, due to limitations of the
+ Beeswax protocol and the Impyla client."""
pass
@abc.abstractmethod
@@ -127,17 +143,11 @@ class BeeswaxConnection(ImpalaConnection):
LOG.info('SET %s=%s;' % (name, value))
self.__beeswax_client.set_query_option(name, value)
- def get_configuration(self):
- return self.__beeswax_client.get_query_options()
-
def get_default_configuration(self):
- return self.__beeswax_client.get_default_configuration()
-
- def set_configuration(self, config_option_dict):
- assert config_option_dict is not None, "config_option_dict cannot be None"
- self.clear_configuration()
- for name, value in config_option_dict.iteritems():
- self.set_configuration_option(name, value)
+ result = {}
+ for item in self.__beeswax_client.get_default_configuration():
+ result[item.key] = item.value
+ return result
def clear_configuration(self):
self.__beeswax_client.clear_query_options()
@@ -161,7 +171,8 @@ class BeeswaxConnection(ImpalaConnection):
def execute_async(self, sql_stmt, user=None):
LOG.info("-- executing async: %s\n%s;\n" % (self.__host_port, sql_stmt))
- return OperationHandle(self.__beeswax_client.execute_query_async(sql_stmt, user=user))
+ beeswax_handle = self.__beeswax_client.execute_query_async(sql_stmt, user=user)
+ return OperationHandle(beeswax_handle, sql_stmt)
def cancel(self, operation_handle):
LOG.info("-- canceling operation: %s" % operation_handle)
@@ -197,26 +208,184 @@ class BeeswaxConnection(ImpalaConnection):
LOG.info("-- getting log for operation: %s" % operation_handle)
return self.__beeswax_client.get_log(operation_handle.get_handle())
- def refresh(self):
- """Invalidate the Impalad catalog"""
- return self.execute("invalidate metadata")
-
- def invalidate_table(self, table_name):
- """Invalidate a specific table from the catalog"""
- return self.execute("invalidate metadata %s" % (table_name))
-
- def refresh_table(self, db_name, table_name):
- """Refresh a specific table from the catalog"""
- return self.execute("refresh %s.%s" % (db_name, table_name))
-
def fetch(self, sql_stmt, operation_handle, max_rows = -1):
LOG.info("-- fetching results from: %s" % operation_handle)
return self.__beeswax_client.fetch_results(
sql_stmt, operation_handle.get_handle(), max_rows)
-def create_connection(host_port, use_kerberos=False):
- # TODO: Support HS2 connections.
- return BeeswaxConnection(host_port=host_port, use_kerberos=use_kerberos)
+
+class ImpylaHS2Connection(ImpalaConnection):
+ """Connection to Impala using the impyla client connecting to HS2 endpoint.
+ impyla implements the standard Python dbabi: https://www.python.org/dev/peps/pep-0249/
+ plus Impala-specific extensions, e.g. for fetching runtime profiles.
+ TODO: implement support for kerberos, SSL, etc.
+ """
+ def __init__(self, host_port, use_kerberos=False):
+ self.__host_port = host_port
+ if use_kerberos:
+ raise NotImplementedError("Kerberos support not yet implemented")
+ # Impyla connection and cursor is initialised in connect(). We need to reuse the same
+ # cursor for different operations (as opposed to creating a new cursor per operation)
+ # so that the session is preserved. This means that we can only execute one operation
+ # at a time per connection, which is a limitation also imposed by the Beeswax API.
+ self.__impyla_conn = None
+ self.__cursor = None
+ # Query options to send along with each query.
+ self.__query_options = {}
+
+ def set_configuration_option(self, name, value):
+ self.__query_options[name] = str(value)
+
+ def get_default_configuration(self):
+ return self.__default_query_options.copy()
+
+ def clear_configuration(self):
+ self.__query_options.clear()
+
+ def connect(self):
+ LOG.info("-- connecting to {0} with impyla".format(self.__host_port))
+ host, port = self.__host_port.split(":")
+ self.__impyla_conn = impyla.connect(host=host, port=int(port))
+ LOG.info("Conn {0}".format(self.__impyla_conn))
+ # Get the default query options for the session before any modifications are made.
+ self.__cursor = self.__impyla_conn.cursor()
+ self.__cursor.execute("set all")
+ self.__default_query_options = {}
+ for name, val, _ in self.__cursor:
+ self.__default_query_options[name] = val
+ self.__cursor.close_operation()
+ LOG.debug("Default query options: {0}".format(self.__default_query_options))
+
+ def close(self):
+ LOG.info("-- closing connection to: {0}".format(self.__host_port))
+ self.__impyla_conn.close()
+
+ def close_query(self, operation_handle):
+ LOG.info("-- closing query for operation handle: {0}".format(operation_handle))
+ operation_handle.get_handle().close_operation()
+
+ def execute(self, sql_stmt, user=None):
+ handle = self.execute_async(sql_stmt, user)
+ try:
+ return self.__fetch_results(handle)
+ finally:
+ self.close_query(handle)
+
+ def execute_async(self, sql_stmt, user=None):
+ LOG.info("-- executing: {0}\n{1};\n".format(self.__host_port, sql_stmt))
+ if user is not None:
+ raise NotImplementedError("Not yet implemented for HS2 - authentication")
+ try:
+ self.__cursor.execute(sql_stmt, configuration=self.__query_options)
+ return OperationHandle(self.__cursor, sql_stmt)
+ except Exception:
+ self.__cursor.close_operation()
+ raise
+
+ def cancel(self, operation_handle):
+ LOG.info("-- canceling operation: {0}".format(operation_handle))
+ return self.__beeswax_client.cancel_query(operation_handle.get_handle())
+
+ def get_state(self, operation_handle):
+ LOG.info("-- getting state for operation: {0}".format(operation_handle))
+ raise NotImplementedError("Not yet implemented for HS2 - states differ from beeswax")
+
+ def get_exec_summary(self, operation_handle):
+ LOG.info("-- getting exec summary operation: {0}".format(operation_handle))
+ raise NotImplementedError(
+ "Not yet implemented for HS2 - summary returned is thrift, not string.")
+
+ def get_runtime_profile(self, operation_handle):
+ LOG.info("-- getting runtime profile operation: {0}".format(operation_handle))
+ return self.__cursor.get_profile()
+
+ def wait_for_finished_timeout(self, operation_handle, timeout):
+ LOG.info("-- waiting for query to reach FINISHED state: {0}".format(operation_handle))
+ raise NotImplementedError("Not yet implemented for HS2 - states differ from beeswax")
+
+ def wait_for_admission_control(self, operation_handle):
+ LOG.info("-- waiting for completion of the admission control processing of the "
+ "query: {0}".format(operation_handle))
+ raise NotImplementedError("Not yet implemented for HS2 - states differ from beeswax")
+
+ def get_admission_result(self, operation_handle):
+ LOG.info("-- getting the admission result: {0}".format(operation_handle))
+ raise NotImplementedError("Not yet implemented for HS2 - states differ from beeswax")
+
+ def get_log(self, operation_handle):
+ LOG.info("-- getting log for operation: {0}".format(operation_handle))
+ # HS2 includes non-error log messages that we need to filter out.
+ cursor = operation_handle.get_handle()
+ lines = [line for line in cursor.get_log().split('\n')
+ if not PROGRESS_LOG_RE.match(line)]
+ return '\n'.join(lines)
+
+ def fetch(self, sql_stmt, handle, max_rows=-1):
+ LOG.info("-- fetching results from: {0}".format(handle))
+ return self.__fetch_results(handle, max_rows)
+
+ def __fetch_results(self, handle, max_rows=-1):
+ """Implementation of result fetching from handle."""
+ cursor = handle.get_handle()
+ assert cursor is not None
+ # Don't fetch data for queries with no results.
+ result_tuples = None
+ column_labels = None
+ column_types = None
+ if cursor.has_result_set:
+ desc = cursor.description
+ column_labels = [col_desc[0].upper() for col_desc in desc]
+ column_types = [col_desc[1].upper() for col_desc in desc]
+ if max_rows < 0:
+ result_tuples = cursor.fetchall()
+ else:
+ result_tuples = cursor.fetchmany(max_rows)
+ log = self.get_log(handle)
+ profile = self.get_runtime_profile(handle)
+ return ImpylaHS2ResultSet(success=True, result_tuples=result_tuples,
+ column_labels=column_labels, column_types=column_types,
+ query=handle.sql_stmt(), log=log, profile=profile)
+
+
+class ImpylaHS2ResultSet(object):
+ """This emulates the interface of ImpalaBeeswaxResult so that it can be used in
+ place of it. TODO: when we deprecate/remove Beeswax, clean this up."""
+ def __init__(self, success, result_tuples, column_labels, column_types, query, log,
+ profile):
+ self.success = success
+ self.column_labels = column_labels
+ self.column_types = column_types
+ self.query = query
+ self.log = log
+ self.profile = profile
+ self.__result_tuples = result_tuples
+ # self.data is the data in the ImpalaBeeswaxResult format: a list of rows with each
+ # row represented as a tab-separated string.
+ self.data = None
+ if result_tuples is not None:
+ self.data = [self.__convert_result_row(tuple) for tuple in result_tuples]
+
+ def __convert_result_row(self, result_tuple):
+ """Take primitive values from a result tuple and construct the tab-separated string
+ that would have been returned via beeswax."""
+ return '\t'.join([self.__convert_result_value(val, type)
+ for val, type in zip(result_tuple, self.column_types)])
+
+ def __convert_result_value(self, val, type):
+ """Take a primitive value from a result tuple and its type and construct the string
+ that would have been returned via beeswax."""
+ if val is None:
+ return 'NULL'
+ else:
+ return str(val)
+
+
+def create_connection(host_port, use_kerberos=False, protocol='beeswax'):
+ if protocol == 'beeswax':
+ return BeeswaxConnection(host_port=host_port, use_kerberos=use_kerberos)
+ else:
+ assert protocol == 'hs2'
+ return ImpylaHS2Connection(host_port=host_port, use_kerberos=use_kerberos)
def create_ldap_connection(host_port, user, password, use_ssl=False):
return BeeswaxConnection(host_port=host_port, user=user, password=password,
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/common/impala_service.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_service.py b/tests/common/impala_service.py
index 0ad4496..16419ae 100644
--- a/tests/common/impala_service.py
+++ b/tests/common/impala_service.py
@@ -283,10 +283,21 @@ class ImpaladService(BaseImpalaService):
def create_beeswax_client(self, use_kerberos=False):
"""Creates a new beeswax client connection to the impalad"""
- client = create_connection('%s:%d' % (self.hostname, self.beeswax_port), use_kerberos)
+ client = create_connection('%s:%d' % (self.hostname, self.beeswax_port),
+ use_kerberos, 'beeswax')
client.connect()
return client
+ def beeswax_port_is_open(self):
+ """Test if the beeswax port is open. Does not need to authenticate."""
+ try:
+ # The beeswax client will connect successfully even if not authenticated.
+ client = self.create_beeswax_client()
+ client.close()
+ return True
+ except Exception:
+ return False
+
def create_ldap_beeswax_client(self, user, password, use_ssl=False):
client = create_ldap_connection('%s:%d' % (self.hostname, self.beeswax_port),
user=user, password=password, use_ssl=use_ssl)
@@ -295,14 +306,23 @@ class ImpaladService(BaseImpalaService):
def create_hs2_client(self):
"""Creates a new HS2 client connection to the impalad"""
- host, port = (self.hostname, self.hs2_port)
- socket = TSocket(host, port)
- transport = TBufferedTransport(socket)
- transport.open()
- protocol = TBinaryProtocol.TBinaryProtocol(transport)
- hs2_client = TCLIService.Client(protocol)
- return hs2_client
+ client = create_connection('%s:%d' % (self.hostname, self.hs2_port), protocol='hs2')
+ client.connect()
+ return client
+ def hs2_port_is_open(self):
+ """Test if the HS2 port is open. Does not need to authenticate."""
+ # Impyla will try to authenticate as part of connecting, so preserve previous logic
+ # that uses the HS2 thrift code directly.
+ try:
+ socket = TSocket(self.hostname, self.hs2_port)
+ transport = TBufferedTransport(socket)
+ transport.open()
+ transport.close()
+ return True
+ except Exception, e:
+ LOG.info(e)
+ return False
# Allows for interacting with the StateStore service to perform operations such as
# accessing the debug webpage.
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/common/impala_test_suite.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index 0f18dea..954ca4c 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -122,11 +122,14 @@ class ImpalaTestSuite(BaseTestSuite):
cls.ImpalaTestMatrix.add_dimension(
cls.create_table_info_dimension(cls.exploration_strategy()))
cls.ImpalaTestMatrix.add_dimension(cls.__create_exec_option_dimension())
+ # Execute tests through Beeswax by default. Individual tests that have been converted
+ # to work with the HS2 client can add HS2 in addition to or instead of beeswax.
+ cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension('protocol', 'beeswax'))
@classmethod
def setup_class(cls):
"""Setup section that runs before each test suite"""
- cls.hive_client, cls.client = [None, None]
+ cls.hive_client, cls.client, cls.hs2_client = [None, None, None]
# Create a Hive Metastore Client (used for executing some test SETUP steps
metastore_host, metastore_port = pytest.config.option.metastore_server.split(':')
trans_type = 'buffered'
@@ -141,8 +144,15 @@ class ImpalaTestSuite(BaseTestSuite):
cls.hive_client = ThriftHiveMetastore.Client(protocol)
cls.hive_transport.open()
- # Create a connection to Impala.
- cls.client = cls.create_impala_client(IMPALAD)
+ # Create a connection to Impala, self.client is Beeswax so that existing tests that
+ # assume beeswax do not need modification (yet).
+ cls.client = cls.create_impala_client(protocol='beeswax')
+ try:
+ cls.hs2_client = cls.create_impala_client(protocol='hs2')
+ except Exception, e:
+ # HS2 connection can fail for benign reasons, e.g. running with unsupported auth.
+ LOG.info("HS2 connection setup failed, continuing...: {0}", e)
+
# Default query options are populated on demand.
cls.default_query_options = {}
@@ -165,18 +175,42 @@ class ImpalaTestSuite(BaseTestSuite):
# Cleanup the Impala and Hive Metastore client connections
if cls.hive_transport:
cls.hive_transport.close()
-
if cls.client:
cls.client.close()
+ if cls.hs2_client:
+ cls.hs2_client.close()
@classmethod
- def create_impala_client(cls, host_port=IMPALAD):
+ def create_impala_client(cls, host_port=None, protocol='beeswax'):
+ if host_port is None:
+ host_port = cls.__get_default_host_port(protocol)
client = create_connection(host_port=host_port,
- use_kerberos=pytest.config.option.use_kerberos)
+ use_kerberos=pytest.config.option.use_kerberos, protocol=protocol)
client.connect()
return client
@classmethod
+ def __get_default_host_port(cls, protocol):
+ if protocol == 'beeswax':
+ return IMPALAD
+ else:
+ assert protocol == 'hs2'
+ return IMPALAD_HS2_HOST_PORT
+
+ @classmethod
+ def __get_cluster_host_ports(cls, protocol):
+ """Return a list of host/port combinations for all impalads in the cluster."""
+ if protocol == 'beeswax':
+ return IMPALAD_HOST_PORT_LIST
+ else:
+ assert protocol == 'hs2'
+ # TODO: support running tests against multiple coordinators for HS2. It should work,
+ # we just need to update all test runners to pass in all host/port combinations for
+ # the cluster and then handle it here.
+ raise NotImplementedError(
+ "Not yet implemented: only one HS2 host/port can be configured")
+
+ @classmethod
def create_impala_service(cls, host_port=IMPALAD, webserver_port=25000):
host, port = host_port.split(':')
return ImpaladService(host, beeswax_port=port, webserver_port=webserver_port)
@@ -208,13 +242,9 @@ class ImpalaTestSuite(BaseTestSuite):
"""
# Populate the default query option if it's empty.
if not self.default_query_options:
- try:
- query_options = impalad_client.get_default_configuration()
- for query_option in query_options:
- self.default_query_options[query_option.key.upper()] = query_option.value
- except Exception as e:
- LOG.info('Unexpected exception when getting default query options: ' + str(e))
- return
+ query_options = impalad_client.get_default_configuration()
+ for key, value in query_options.iteritems():
+ self.default_query_options[key.upper()] = value
# Restore all the changed query options.
for query_option in query_options_changed:
query_option = query_option.upper()
@@ -295,7 +325,7 @@ class ImpalaTestSuite(BaseTestSuite):
actual values are easily compared.
"""
replace_filenames_with_placeholder = True
- for section_name in ('RESULTS', 'ERRORS'):
+ for section_name in ('RESULTS', 'DBAPI_RESULTS', 'ERRORS'):
if section_name in test_section:
if "$NAMENODE" in test_section[section_name]:
replace_filenames_with_placeholder = False
@@ -305,8 +335,22 @@ class ImpalaTestSuite(BaseTestSuite):
.replace('$USER', getuser())
if use_db:
test_section[section_name] = test_section[section_name].replace('$DATABASE', use_db)
+ result_section, type_section = 'RESULTS', 'TYPES'
+ if vector.get_value('protocol') == 'hs2':
+ if 'DBAPI_RESULTS' in test_section:
+ assert 'RESULTS' in test_section,\
+ "Base RESULTS section must always be included alongside DBAPI_RESULTS"
+ # In some cases Impyla (the HS2 dbapi client) is expected to return different
+ # results, so use the dbapi-specific section if present.
+ result_section = 'DBAPI_RESULTS'
+ if 'HS2_TYPES' in test_section:
+ assert 'TYPES' in test_section,\
+ "Base TYPES section must always be included alongside HS2_TYPES"
+ # In some cases HS2 types are expected differ from Beeswax types (e.g. see
+ # IMPALA-914), so use the HS2-specific section if present.
+ type_section = 'HS2_TYPES'
verify_raw_results(test_section, result, vector.get_value('table_format').file_format,
- pytest.config.option.update_results,
+ result_section, type_section, pytest.config.option.update_results,
replace_filenames_with_placeholder)
@@ -318,7 +362,10 @@ class ImpalaTestSuite(BaseTestSuite):
Runs the query using targeting the file format/compression specified in the test
vector and the exec options specified in the test vector. If multiple_impalad=True
a connection to a random impalad will be chosen to execute each test section.
- Otherwise, the default impalad client will be used.
+ Otherwise, the default impalad client will be used. If 'protocol' (either 'hs2' or
+ 'beeswax') is set in the vector, a client for that protocol is used. Otherwise we
+ use the default: beeswax.
+
Additionally, the encoding for all test data can be specified using the 'encoding'
parameter. This is useful when data is ingested in a different encoding (ex.
latin). If not set, the default system encoding will be used.
@@ -328,6 +375,7 @@ class ImpalaTestSuite(BaseTestSuite):
"""
table_format_info = vector.get_value('table_format')
exec_options = vector.get_value('exec_option')
+ protocol = vector.get_value('protocol')
# Resolve the current user's primary group name.
group_id = pwd.getpwnam(getuser()).pw_gid
@@ -336,9 +384,14 @@ class ImpalaTestSuite(BaseTestSuite):
target_impalad_clients = list()
if multiple_impalad:
target_impalad_clients =\
- map(ImpalaTestSuite.create_impala_client, IMPALAD_HOST_PORT_LIST)
+ [ImpalaTestSuite.create_impala_client(host_port, protocol=protocol)
+ for host_port in self.__get_cluster_host_ports(protocol)]
else:
- target_impalad_clients = [self.client]
+ if protocol == 'beeswax':
+ target_impalad_clients = [self.client]
+ else:
+ assert protocol == 'hs2'
+ target_impalad_clients = [self.hs2_client]
# Change the database to reflect the file_format, compression codec etc, or the
# user specified database for all targeted impalad.
@@ -402,7 +455,7 @@ class ImpalaTestSuite(BaseTestSuite):
if 'USER' in test_section:
# Create a new client so the session will use the new username.
user = test_section['USER'].strip()
- target_impalad_client = self.create_impala_client()
+ target_impalad_client = self.create_impala_client(protocol=protocol)
for query in query.split(';'):
set_pattern_match = SET_PATTERN.match(query)
if set_pattern_match != None:
@@ -463,8 +516,8 @@ class ImpalaTestSuite(BaseTestSuite):
test_section['DML_RESULTS_TABLE']
dml_result = self.__execute_query(target_impalad_client, dml_results_query)
verify_raw_results(test_section, dml_result,
- vector.get_value('table_format').file_format,
- pytest.config.option.update_results, result_section='DML_RESULTS')
+ vector.get_value('table_format').file_format, result_section='DML_RESULTS',
+ update_section=pytest.config.option.update_results)
if pytest.config.option.update_results:
output_file = os.path.join(EE_TEST_LOGS_DIR,
test_file_name.replace('/','_') + ".test")
@@ -630,13 +683,6 @@ class ImpalaTestSuite(BaseTestSuite):
if query_options is not None: impalad_client.set_configuration(query_options)
return impalad_client.execute(query, user=user)
- def __execute_query_new_client(self, query, query_options=None,
- use_kerberos=False):
- """Executes the given query against the specified Impalad"""
- new_client = self.create_impala_client()
- new_client.set_configuration(query_options)
- return new_client.execute(query)
-
def __reset_table(self, db_name, table_name):
"""Resets a table (drops and recreates the table)"""
table = self.hive_client.get_table(db_name, table_name)
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/common/test_dimensions.py
----------------------------------------------------------------------
diff --git a/tests/common/test_dimensions.py b/tests/common/test_dimensions.py
index af1f8e1..3413266 100644
--- a/tests/common/test_dimensions.py
+++ b/tests/common/test_dimensions.py
@@ -113,6 +113,26 @@ def create_avro_snappy_dimension(workload):
return ImpalaTestDimension('table_format',
TableFormatInfo.create_from_string(dataset, 'avro/snap/block'))
+
+def create_beeswax_hs2_dimension():
+ return ImpalaTestDimension('protocol', 'beeswax', 'hs2')
+
+
+def hs2_parquet_constraint(v):
+ """Constraint function, used to only run HS2 against Parquet format, because file format
+ and the client protocol are orthogonal."""
+ return (v.get_value('protocol') == 'beeswax' or
+ v.get_value('table_format').file_format == 'parquet' and
+ v.get_value('table_format').compression_codec == 'none')
+
+
+def hs2_text_constraint(v):
+ """Constraint function, used to only run HS2 against uncompressed text, because file
+ format and the client protocol are orthogonal."""
+ return (v.get_value('protocol') == 'beeswax' or
+ v.get_value('table_format').file_format == 'text' and
+ v.get_value('table_format').compression_codec == 'none')
+
# Common sets of values for the exec option vectors
ALL_BATCH_SIZES = [0]
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/common/test_result_verifier.py
----------------------------------------------------------------------
diff --git a/tests/common/test_result_verifier.py b/tests/common/test_result_verifier.py
index f071e6b..285b875 100644
--- a/tests/common/test_result_verifier.py
+++ b/tests/common/test_result_verifier.py
@@ -85,7 +85,7 @@ class ResultRow(object):
self.regex = try_compile_regex(row_string)
def __parse_row(self, row_string, column_types, column_labels):
- """Parses a row string and build a list of ResultColumn objects"""
+ """Parses a row string (from Beeswax) and build a list of ResultColumn objects"""
column_values = list()
if not row_string:
return column_values
@@ -314,8 +314,10 @@ def apply_error_match_filter(error_list, replace_filenames=True):
return re.sub(r'Backend \d+:', '', row)
return [replace_fn(row) for row in error_list]
-def verify_raw_results(test_section, exec_result, file_format, update_section=False,
- replace_filenames=True, result_section='RESULTS'):
+
+def verify_raw_results(test_section, exec_result, file_format, result_section,
+ type_section='TYPES', update_section=False,
+ replace_filenames=True):
"""
Accepts a raw exec_result object and verifies it matches the expected results,
including checking the ERRORS, TYPES, and LABELS test sections.
@@ -328,6 +330,9 @@ def verify_raw_results(test_section, exec_result, file_format, update_section=Fa
The result_section parameter can be used to make this function check the results in
a DML_RESULTS section instead of the regular RESULTS section.
+
+ The 'type_section' parameter can be used to make this function check the types against
+ an alternative section from the default TYPES.
TODO: separate out the handling of sections like ERRORS from checking of query results
to allow regular RESULTS/ERRORS sections in tests with DML_RESULTS (IMPALA-4471).
"""
@@ -350,11 +355,11 @@ def verify_raw_results(test_section, exec_result, file_format, update_section=Fa
else:
raise
- if 'TYPES' in test_section:
+ if type_section in test_section:
# Distinguish between an empty list and a list with an empty string.
- expected_types = list()
- if test_section.get('TYPES'):
- expected_types = [c.strip().upper() for c in test_section['TYPES'].rstrip('\n').split(',')]
+ section = test_section[type_section]
+ expected_types = [c.strip().upper()
+ for c in remove_comments(section).rstrip('\n').split(',')]
# Avro and Kudu represent TIMESTAMP columns as strings, so tests using TIMESTAMP are
# skipped because results will be wrong.
@@ -371,7 +376,7 @@ def verify_raw_results(test_section, exec_result, file_format, update_section=Fa
LOG.info("Skipping type verification of Avro-format table.")
actual_types = expected_types
else:
- actual_types = parse_column_types(exec_result.schema)
+ actual_types = exec_result.column_types
try:
verify_results(expected_types, actual_types, order_matters=True)
@@ -386,8 +391,8 @@ def verify_raw_results(test_section, exec_result, file_format, update_section=Fa
actual_types = ['BIGINT']
actual_labels = ['DUMMY_LABEL']
- if exec_result and exec_result.schema:
- actual_labels = parse_column_labels(exec_result.schema)
+ if exec_result and exec_result.column_labels:
+ actual_labels = exec_result.column_labels
if 'LABELS' in test_section:
assert actual_labels is not None
@@ -441,22 +446,11 @@ def contains_order_by(query):
"""Returns true of the query contains an 'order by' clause"""
return re.search( r'order\s+by\b', query, re.M|re.I) is not None
-def parse_column_types(schema):
- """Enumerates all field schemas and returns a list of column type strings"""
- return [fs.type.upper() for fs in schema.fieldSchemas]
-
-def parse_column_labels(schema):
- """Enumerates all field schemas and returns a list of column label strings"""
- # This is an statement doesn't return a field schema (insert statement).
- if schema is None or schema.fieldSchemas is None: return list()
- return [fs.name.upper() for fs in schema.fieldSchemas]
-
def create_query_result(exec_result, order_matters=False):
"""Creates query result in the test format from the result returned from a query"""
- col_labels = parse_column_labels(exec_result.schema)
- col_types = parse_column_types(exec_result.schema)
data = parse_result_rows(exec_result)
- return QueryTestResult(data, col_types, col_labels, order_matters)
+ return QueryTestResult(data, exec_result.column_types, exec_result.column_labels,
+ order_matters)
def parse_result_rows(exec_result):
"""
@@ -467,17 +461,17 @@ def parse_result_rows(exec_result):
return []
# If the schema is 'None' assume this is an insert statement
- if exec_result.schema is None:
+ if exec_result.column_labels is None:
return raw_result
result = list()
- col_types = parse_column_types(exec_result.schema)
+ col_types = exec_result.column_types or []
for row in exec_result.data:
cols = row.split('\t')
assert len(cols) == len(col_types)
new_cols = list()
for i in xrange(len(cols)):
- if col_types[i] == 'STRING' or col_types[i] == 'CHAR':
+ if col_types[i] in ['STRING', 'CHAR', 'VARCHAR']:
col = cols[i].encode('unicode_escape')
new_cols.append("'%s'" % col)
else:
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/conftest.py
----------------------------------------------------------------------
diff --git a/tests/conftest.py b/tests/conftest.py
index 58f64a4..1eb5c76 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -78,9 +78,9 @@ def pytest_addoption(parser):
"format: workload:exploration_strategy. Ex: tpch:core,tpcds:pairwise.")
parser.addoption("--impalad", default=DEFAULT_IMPALADS,
- help="A comma-separated list of impalad host:ports to target. Note: "
- "Not all tests make use of all impalad, some tests target just the "
- "first item in the list (it is considered the 'default'")
+ help="A comma-separated list of impalad Beeswax host:ports to target. "
+ "Note: Not all tests make use of all impalad, some tests target just "
+ "the first item in the list (it is considered the 'default'")
parser.addoption("--impalad_hs2_port", default=DEFAULT_IMPALAD_HS2_PORT,
help="The impalad HiveServer2 port.")
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/custom_cluster/test_coordinators.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_coordinators.py b/tests/custom_cluster/test_coordinators.py
index b49e328..703f779 100644
--- a/tests/custom_cluster/test_coordinators.py
+++ b/tests/custom_cluster/test_coordinators.py
@@ -17,6 +17,7 @@
#
# The base class that should be used for almost all Impala tests
+import logging
import pytest
import os
import time
@@ -24,6 +25,9 @@ from subprocess import check_call
from tests.util.filesystem_utils import get_fs_path
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+LOG = logging.getLogger('test_coordinators')
+LOG.setLevel(level=logging.DEBUG)
+
class TestCoordinators(CustomClusterTestSuite):
@pytest.mark.execute_serially
def test_multiple_coordinators(self):
@@ -43,14 +47,16 @@ class TestCoordinators(CustomClusterTestSuite):
beeswax_client = None
try:
beeswax_client = worker.service.create_beeswax_client()
- except: pass
+ except Exception, e:
+ LOG.info("Caught exception {0}".format(e))
finally:
assert beeswax_client is None
hs2_client = None
try:
hs2_client = worker.service.create_hs2_client()
- except: pass
+ except Exception, e:
+ LOG.info("Caught exception {0}".format(e))
finally:
assert hs2_client is None
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/custom_cluster/test_session_expiration.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_session_expiration.py b/tests/custom_cluster/test_session_expiration.py
index 78862ec..027cc41 100644
--- a/tests/custom_cluster/test_session_expiration.py
+++ b/tests/custom_cluster/test_session_expiration.py
@@ -29,12 +29,7 @@ class TestSessionExpiration(CustomClusterTestSuite):
@CustomClusterTestSuite.with_args("--idle_session_timeout=6")
def test_session_expiration(self, vector):
impalad = self.cluster.get_any_impalad()
- # setup_class creates an Impala client to <hostname>:21000 after the cluster starts.
- # The client expires at the same time as the client created below. Since we choose the
- # impalad to connect to randomly, the test becomes flaky, as the metric we expect to
- # be incremented by 1 gets incremented by 2 if both clients are connected to the same
- # Impalad.
- self.client.close()
+ self.__close_default_clients()
num_expired = impalad.service.get_metric_value("impala-server.num-sessions-expired")
client = impalad.service.create_beeswax_client()
# Sleep for half the expiration time to confirm that the session is not expired early
@@ -50,7 +45,7 @@ class TestSessionExpiration(CustomClusterTestSuite):
@CustomClusterTestSuite.with_args("--idle_session_timeout=3")
def test_session_expiration_with_set(self, vector):
impalad = self.cluster.get_any_impalad()
- self.client.close()
+ self.__close_default_clients()
num_expired = impalad.service.get_metric_value("impala-server.num-sessions-expired")
# Test if we can set a shorter timeout than the process-wide option
@@ -72,7 +67,7 @@ class TestSessionExpiration(CustomClusterTestSuite):
@CustomClusterTestSuite.with_args("--idle_session_timeout=5")
def test_unsetting_session_expiration(self, vector):
impalad = self.cluster.get_any_impalad()
- self.client.close()
+ self.__close_default_clients()
num_expired = impalad.service.get_metric_value("impala-server.num-sessions-expired")
# Test unsetting IDLE_SESSION_TIMEOUT
@@ -109,3 +104,10 @@ class TestSessionExpiration(CustomClusterTestSuite):
queued_query_profile = impalad.service.create_beeswax_client().get_runtime_profile(
queued_handle)
assert "Admission result: Cancelled (queued)" in queued_query_profile
+
+ def __close_default_clients(self):
+ """Close the clients that were automatically created by setup_class(). These clients
+ can expire during test, which results in metrics that tests depend on changing. Each
+ test should create its own clients as needed."""
+ self.client.close()
+ self.hs2_client.close()
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/metadata/test_views_compatibility.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_views_compatibility.py b/tests/metadata/test_views_compatibility.py
index f4987c3..ac208d7 100644
--- a/tests/metadata/test_views_compatibility.py
+++ b/tests/metadata/test_views_compatibility.py
@@ -103,7 +103,7 @@ class TestViewCompatibility(ImpalaTestSuite):
# The table may or may not have been created in Hive. And so, "invalidate metadata"
# may throw an exception.
try:
- self.client.invalidate_table(test_case.hive_view_name)
+ self.client.execute("invalidate metadata {0}".format(test_case.hive_view_name))
except ImpalaBeeswaxException as e:
assert "TableNotFoundException" in str(e)
@@ -135,7 +135,7 @@ class TestViewCompatibility(ImpalaTestSuite):
self._exec_in_hive(test_case.get_drop_view_sql('HIVE'),\
test_case.get_create_view_sql('HIVE'), None)
try:
- self.client.invalidate_table(test_case.hive_view_name)
+ self.client.execute("invalidate metadata {0}".format(test_case.hive_view_name))
except ImpalaBeeswaxException as e:
assert "TableNotFoundException" in str(e)
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/query_test/test_aggregation.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_aggregation.py b/tests/query_test/test_aggregation.py
index 55233fa..c62f729 100644
--- a/tests/query_test/test_aggregation.py
+++ b/tests/query_test/test_aggregation.py
@@ -27,8 +27,6 @@ from tests.common.test_dimensions import (
create_uncompressed_text_dimension)
from tests.common.test_result_verifier import (
assert_codegen_enabled,
- parse_column_types,
- parse_column_labels,
QueryTestResult,
parse_result_rows)
from tests.common.test_vector import ImpalaTestDimension
@@ -397,8 +395,8 @@ class TestWideAggregationQueries(ImpalaTestSuite):
# All rows should be distinct.
expected_result = widetable.get_data(1000, 10, quote_strings=True)
- types = parse_column_types(result.schema)
- labels = parse_column_labels(result.schema)
+ types = result.column_types
+ labels = result.column_labels
expected = QueryTestResult(expected_result, types, labels, order_matters=False)
actual = QueryTestResult(parse_result_rows(result), types, labels,
order_matters=False)