You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by as...@apache.org on 2025/02/07 23:38:11 UTC
(impala) 03/04: IMPALA-13672: Migrate query_test/test_kudu.py to use hs2 protocol
This is an automated email from the ASF dual-hosted git repository.
asherman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit c08aff420d020fff35e83df82b8d8a212b6ab58f
Author: Riza Suminto <ri...@cloudera.com>
AuthorDate: Sun Jan 12 08:46:10 2025 -0800
IMPALA-13672: Migrate query_test/test_kudu.py to use hs2 protocol
This patch migrate query_test/test_kudu.py to use hs2 client protocol.
Here are the steps taken:
- Override default_test_protocol() to return 'hs2'.
See documentation in ImpalaTestSuite about what this method does.
- Remove usage of deprecated cursor and unique_cursor fixture.
- Replace all direct ImpalaTestSuite.client usage with helper
function call such as execute_query() or execute_query_using_vector().
- Remove all "SET" query invocation and replace it with passing
exec_option dictionary to helper method.
- Replace veryfing kudu modified / inserted rows from reading query
output to reading runtime profile counters.
- Add HS2_TYPES section at test cases where only TYPES exist.
- Remove all drop_impala_table_after_context() calls and replace it with
proper use of unique_database fixture.
KuduTestSuite is fixed with hs2 protocol dimension. Meanwhile,
CustomKuduTest is fixed to use beeswax protocol dimension until proper
migration can be done.
Added following convenience methods:
- ImpalaTestSuite.default_test_protocol() to allow individual test
class to override its default test procol.
- ImpylaHS2ResultSet.tuples() to access the raw HS2 result set that is
a list of tuples.
This patch also added several literal constants around test vector
dimension to help with traceability.
Fixed a bug where "SHOW PARTITIONS" via hs2 over kudu table will shows
NULL number of #Replicas because TResultRowBuilder does not have
overload for int type value. Adjust numFiles variable inside
HdfsTable.getTableStats() from int to long to match Type.BIGINT of
column '#Files'.
Fixed py.test classes that does not inherit BaseTestSuite. Fixed flake8
issues in test_statestore.py.
Testing:
- Run and pass all tests extended from KuduTestSuite in exhaustive mode.
Change-Id: I5f38baf5a0bbde1a1ad0bb4666c300f4f3cabd33
Reviewed-on: http://gerrit.cloudera.org:8080/22358
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../java/org/apache/impala/catalog/FeFsTable.java | 1 -
.../org/apache/impala/catalog/FeKuduTable.java | 2 +-
.../java/org/apache/impala/catalog/HdfsTable.java | 2 +-
.../org/apache/impala/util/TResultRowBuilder.java | 7 +
.../queries/QueryTest/kudu-scan-node.test | 2 +
.../QueryTest/kudu_create_table_like_table.test | 5 +-
.../queries/QueryTest/kudu_delete.test | 54 +-
.../queries/QueryTest/kudu_insert.test | 54 +-
.../queries/QueryTest/kudu_update.test | 40 +-
.../queries/QueryTest/kudu_upsert.test | 42 +-
tests/common/base_test_suite.py | 7 +-
tests/common/impala_connection.py | 7 +-
tests/common/impala_test_suite.py | 134 +-
tests/common/kudu_test_suite.py | 22 +-
tests/common/test_dimensions.py | 36 +-
tests/common/test_result_verifier.py | 13 +-
tests/common/test_vector.py | 45 +-
tests/conftest.py | 15 +-
tests/custom_cluster/test_kudu.py | 9 +-
tests/hs2/test_fetch.py | 21 +-
tests/infra/test_thrift_parser.py | 36 +-
tests/infra/test_utils.py | 80 +-
tests/query_test/test_kudu.py | 1477 +++++++++++---------
tests/query_test/test_observability.py | 4 +-
tests/statestore/test_statestore.py | 56 +-
tests/unittests/test_command.py | 7 +-
tests/verifiers/test_banned_log_messages.py | 4 +-
27 files changed, 1269 insertions(+), 913 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeFsTable.java b/fe/src/main/java/org/apache/impala/catalog/FeFsTable.java
index 8deda66fb..4edb8858f 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeFsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeFsTable.java
@@ -44,7 +44,6 @@ import org.apache.impala.analysis.PartitionKeyValue;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.common.PrintUtils;
-import org.apache.impala.planner.HdfsScanNode;
import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.TColumn;
import org.apache.impala.thrift.TNetworkAddress;
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeKuduTable.java b/fe/src/main/java/org/apache/impala/catalog/FeKuduTable.java
index ec0336bbe..c7bee21a3 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeKuduTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeKuduTable.java
@@ -158,7 +158,7 @@ public interface FeKuduTable extends FeTable {
BackendConfig.INSTANCE.getKuduClientTimeoutMs());
TResultRowBuilder tResultRowBuilder = new TResultRowBuilder();
tResultRowBuilder.add(table.getNumRows());
- tResultRowBuilder.add(tablets.size());
+ tResultRowBuilder.add((long) tablets.size());
tResultRowBuilder.addBytes(kuduTable.getTableStatistics().getOnDiskSize());
tResultRowBuilder.add("KUDU");
tResultRowBuilder.add(table.getKuduMasterHosts());
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index 0f57d4f15..0498cef0e 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -2787,7 +2787,7 @@ public class HdfsTable extends Table implements FeFsTable {
long totalBytes = 0L;
long totalNumFiles = 0L;
for (FeFsPartition p: orderedPartitions) {
- int numFiles = p.getNumFileDescriptors();
+ long numFiles = p.getNumFileDescriptors();
long size = p.getSize();
totalNumFiles += numFiles;
totalBytes += size;
diff --git a/fe/src/main/java/org/apache/impala/util/TResultRowBuilder.java b/fe/src/main/java/org/apache/impala/util/TResultRowBuilder.java
index 836e238de..e5a5d092d 100644
--- a/fe/src/main/java/org/apache/impala/util/TResultRowBuilder.java
+++ b/fe/src/main/java/org/apache/impala/util/TResultRowBuilder.java
@@ -27,6 +27,13 @@ import org.apache.impala.thrift.TResultRow;
public class TResultRowBuilder {
private final TResultRow row_ = new TResultRow();
+ public TResultRowBuilder add(int val) {
+ TColumnValue colVal = new TColumnValue();
+ colVal.setInt_val(val);
+ row_.addToColVals(colVal);
+ return this;
+ }
+
public TResultRowBuilder add(long val) {
TColumnValue colVal = new TColumnValue();
colVal.setLong_val(val);
diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu-scan-node.test b/testdata/workloads/functional-query/queries/QueryTest/kudu-scan-node.test
index 5f3d7c4a3..f806efea4 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu-scan-node.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu-scan-node.test
@@ -161,6 +161,8 @@ order by key;
4,'y'
---- TYPES
INT, STRING
+---- HS2_TYPES
+INT, VARCHAR
====
---- QUERY
# Regression test for IMPALA-6187. Make sure count(*) queries with partition columns only
diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_create_table_like_table.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_create_table_like_table.test
index 96dd13a75..fec21c42d 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu_create_table_like_table.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_create_table_like_table.test
@@ -57,8 +57,9 @@ select id, bool_col, tinyint_col, smallint_col, int_col, bigint_col,
float_col, double_col, date_string_col, string_col, timestamp_col,
year, month
from functional.alltypes where year=2009 and month=4
----- RESULTS
-: 300
+---- RUNTIME_PROFILE
+NumModifiedRows: 300
+NumRowErrors: 0
====
---- QUERY
# Make sure we can read the new data.
diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_delete.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_delete.test
index ecbd926d5..57c2396d1 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu_delete.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_delete.test
@@ -22,8 +22,9 @@ insert into table tdata values
(8, 0, 80, NULL, true, 10, 11, 12, 0.000000013, 0.14, 15, DATE '2019-01-08', NULL),
(9, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL),
(127, 1, 2, '127', false, 3, 4, 5, 0.000000000, 0.00, 0, DATE '2019-01-27', CAST('127' AS VARCHAR(10)))
----- RESULTS
-: 10
+---- RUNTIME_PROFILE
+NumModifiedRows: 10
+NumRowErrors: 0
====
---- QUERY
# single row, predicate on key
@@ -45,6 +46,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
127,1,2,'127',false,3,4,5,0.000000000,0.00,0,2019-01-27,'127'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# predicate on key, NULL
@@ -66,6 +69,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
127,1,2,'127',false,3,4,5,0.000000000,0.00,0,2019-01-27,'127'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# predicate on key, nothing is deleted
@@ -87,6 +92,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
127,1,2,'127',false,3,4,5,0.000000000,0.00,0,2019-01-27,'127'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# predicate on key, boundary value
@@ -107,6 +114,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
9,NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'NULL'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# compound predicate on key
@@ -126,6 +135,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
9,NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'NULL'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# predicate on key, multiple rows
@@ -143,6 +154,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
9,NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'NULL'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# insert new values, including some that were previously deleted
@@ -171,6 +184,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
8,0,80,'NULL',true,10,11,12,0.000000013,0.14,15,2019-01-08,'NULL'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# predicate on non-key, NULL
@@ -190,6 +205,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
8,0,80,'NULL',true,10,11,12,0.000000013,0.14,15,2019-01-08,'NULL'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# predicate on non-key, nothing is deleted
@@ -209,6 +226,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
8,0,80,'NULL',true,10,11,12,0.000000013,0.14,15,2019-01-08,'NULL'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# predicate on non-key, compound predicate
@@ -227,6 +246,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
11,-11,11,'eleven',false,1,11,111,0.000000011,11.00,11,2019-01-11,'eleven'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# predicate on non-key, multiple rows
@@ -243,6 +264,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
11,-11,11,'eleven',false,1,11,111,0.000000011,11.00,11,2019-01-11,'eleven'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# 'from' syntax - the join results in four deletes, 3 of which fail
@@ -258,6 +281,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
1,1.0,1,'one',true,1,1,1,0.000000001,1.11,1,2019-01-01,'one'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# Try to delete a row with a primary key value that is not covered by the existing range
@@ -274,6 +299,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
1,1.0,1,'one',true,1,1,1,0.000000001,1.11,1,2019-01-01,'one'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# Add a couple more rows back
@@ -281,8 +308,9 @@ insert into table tdata values
(3, 0, NULL, 'three', false, 10, 20, 30, 0.000000040, 50.00, 60, DATE '2019-01-03', CAST('three' AS VARCHAR(10))),
(4, 5, 6, 'four', true, 7, 8, 9, 0.000000010, 11.11, 12, DATE '2019-01-04', CAST('four' AS VARCHAR(10))),
(5, 0, 10, 'five', NULL, 15, 20, 25, 0.000000030, 35.35, 40, DATE '2019-01-05', CAST('five' AS VARCHAR(10)))
----- RESULTS
-: 3
+---- RUNTIME_PROFILE
+NumModifiedRows: 3
+NumRowErrors: 0
====
---- QUERY
# predicate on decimal, multiple rows
@@ -298,6 +326,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
5,0,10,'five',NULL,15,20,25,0.000000030,35.35,40,2019-01-05,'five'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# predicate on decimal, single row
@@ -312,14 +342,14 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
4,5,6,'four',true,7,8,9,0.000000010,11.11,12,2019-01-04,'four'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
insert into tdata
select cast(id + 100 as int), float_col, bigint_col, string_col, bool_col, tinyint_col,
smallint_col, double_col, NULL, NULL, NULL, NULL, NULL
from functional_kudu.alltypes
----- RESULTS
-: 7300
---- RUNTIME_PROFILE
NumModifiedRows: 7300
NumRowErrors: 0
@@ -335,6 +365,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16, VALDAT
---- DML_RESULTS: tdata
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
create table multiple_key_cols
@@ -350,8 +382,9 @@ insert into multiple_key_cols values
('b', 1, 2, 3, false, 7, NULL, 9),
('c', 4, 5, 6, true, 0, -1, 0),
('d', 10, 20, 30, false, NULL, 40, 50)
----- RESULTS
-: 4
+---- RUNTIME_PROFILE
+NumModifiedRows: 4
+NumRowErrors: 0
====
---- QUERY
# multiple key cols, predicate on one of them
@@ -393,8 +426,9 @@ insert into impala_3454 values
(1, 1),
(2, 2),
(3, 3)
----- RESULTS
-: 3
+---- RUNTIME_PROFILE
+NumModifiedRows: 3
+NumRowErrors: 0
====
---- QUERY
delete from impala_3454 where key_1 < (select max(key_2) from impala_3454)
diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test
index d869ba217..3fc054e9d 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_insert.test
@@ -23,6 +23,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS, DECIMAL4, DECIMAL8, DECIMAL16,
1,1,1,'one',true,1,1,1,1987-05-19 00:00:00,0.000000001,1.00,1,1970-01-01,'one'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, single row, all target cols, NULL
@@ -37,6 +39,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS, DECIMAL4, DECIMAL8, DECIMAL16,
2,NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'NULL'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, single row, all target cols, boundary values. The timestamp value is the max
@@ -58,6 +62,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS, DECIMAL4, DECIMAL8, DECIMAL16,
3,NaN,9223372036854775807,'',true,-128,32767,-Infinity,9999-12-31 23:59:59.999999000,0.999999999,9999999999999999.99,99999999999999999999999999999999999999,9999-12-31,''
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, single row, subset of target cols
@@ -74,6 +80,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS, DECIMAL4, DECIMAL8, DECIMAL16,
4,NULL,NULL,'NULL',true,NULL,NULL,0,NULL,NULL,NULL,NULL,NULL,'NULL'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, multiple rows, all target cols
@@ -96,6 +104,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS, DECIMAL4, DECIMAL8, DECIMAL16,
7,NULL,10,'NULL',false,127,-7,2,1400-01-01 00:00:00,0.000000000,0.00,0,1970-01-01,'NULL'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, multiple rows, subset of cols
@@ -119,6 +129,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS, DECIMAL4, DECIMAL8, DECIMAL16,
9,-9,-99,'NULL',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'NULL'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# SELECT, single row, all target cols
@@ -144,6 +156,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS, DECIMAL4, DECIMAL8, DECIMAL16,
10,0,0,'0',true,0,0,0,2009-01-02 00:10:00.450000000,NULL,NULL,NULL,NULL,'NULL'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# SELECT, single row, subset of cols
@@ -169,6 +183,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS, DECIMAL4, DECIMAL8, DECIMAL16,
11,NULL,10,'NULL',false,NULL,NULL,10.1,2009-01-02 00:11:00.450000000,NULL,NULL,NULL,NULL,'NULL'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
delete tdata
@@ -190,6 +206,8 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS, DECIMAL4, DECIMAL8, DECIMAL16,
1,1.100000023841858,10,'1',false,1,1,10.1,2009-01-01 00:01:00,NULL,NULL,NULL,NULL,'1'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# SELECT, multiple rows, subset of cols
@@ -209,13 +227,13 @@ ID, VALF, VALI, VALV, VALB, VALT, VALS, VALD, TS, DECIMAL4, DECIMAL8, DECIMAL16,
5,NULL,NULL,'NULL',NULL,5,5,50.5,2009-01-01 00:05:00.100000000,NULL,NULL,NULL,NULL,'NULL'
---- TYPES
INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,TIMESTAMP,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# Make sure we can insert empty strings into string columns and that we can scan them
# back.
insert into tdata values (320, 2.0, 932, cast('' as string), false, 0, 0, 0, NULL, NULL, NULL, NULL, NULL, cast('' as varchar(10)))
----- RESULTS
-: 1
---- RUNTIME_PROFILE
NumModifiedRows: 1
NumRowErrors: 0
@@ -226,12 +244,12 @@ select id, valv, valb, valvc from tdata where id = 320;
320,'',false,''
---- TYPES
INT,STRING,BOOLEAN,STRING
+---- HS2_TYPES
+INT,STRING,BOOLEAN,VARCHAR
====
---- QUERY
insert into tdata values
(666, cast(1.2 as float), 43, cast('z' as string), true, 0, 0, 0, NULL, NULL, NULL, NULL, NULL, NULL)
----- RESULTS
-: 1
---- RUNTIME_PROFILE
NumModifiedRows: 1
NumRowErrors: 0
@@ -240,8 +258,6 @@ NumRowErrors: 0
# insert row with primary key that already exists
insert into tdata values
(666, cast(1.2 as float), 43, cast('z' as VARCHAR(20)), true, 0, 0, 0, NULL, NULL, NULL, NULL, NULL, NULL)
----- RESULTS
-: 0
---- RUNTIME_PROFILE
NumModifiedRows: 0
NumRowErrors: 1
@@ -260,8 +276,6 @@ NumRowErrors: 0
---- QUERY
insert into kudu_test_tbl
select * from functional_kudu.alltypes where id < 100;
----- RESULTS
-: 0
---- RUNTIME_PROFILE
NumModifiedRows: 0
NumRowErrors: 100
@@ -270,8 +284,6 @@ NumRowErrors: 100
# large insert - 100 rows were already inserted above and result in errors
insert into kudu_test_tbl
select * from functional_kudu.alltypes;
----- RESULTS
-: 7200
---- RUNTIME_PROFILE
NumModifiedRows: 7200
NumRowErrors: 100
@@ -303,8 +315,6 @@ create table allkeytypes (i1 tinyint, i2 smallint, i3 int, i4 bigint, name strin
insert into allkeytypes select cast(id as tinyint), smallint_col, int_col,
cast (bigint_col/10 as bigint), string_col, timestamp_col, cast(string_col as varchar(10)), float_col, double_col
from functional.alltypes where id > 1 and id < 10
----- RESULTS
-: 2
---- RUNTIME_PROFILE
NumModifiedRows: 2
NumRowErrors: 6
@@ -313,8 +323,6 @@ NumRowErrors: 6
# IMPALA-5871 - test that a cast is correctly added when inserting a string into a Kudu
# timestamp partition column with distributed exec.
insert into allkeytypes values (1,1,1,1,'1','2009-01-01 00:01:00',cast('1' as varchar(10)),null,null)
----- RESULTS
-: 1
---- RUNTIME_PROFILE
NumModifiedRows: 1
NumRowErrors: 0
@@ -327,8 +335,6 @@ create table timestampkey (t timestamp primary key)
---- QUERY
# Regression test for IMPALA-8909 (used to hit a DCHECK in debug builds).
insert into timestampkey select concat(string_col, " ") from functional.alltypessmall;
----- RESULTS
-: 0
---- RUNTIME_PROFILE
NumModifiedRows: 0
NumRowErrors: 100
@@ -357,6 +363,8 @@ A, B, C, D, E, F, G, H, I, J
4,10,100,1000,NULL,4,'test',true,1111.11,'foo'
---- TYPES
INT,INT,INT,INT,INT,INT,STRING,BOOLEAN,DECIMAL,STRING
+---- HS2_TYPES
+INT,INT,INT,INT,INT,INT,STRING,BOOLEAN,DECIMAL,VARCHAR
====
---- QUERY
insert into tbl_with_defaults values (5, 5, 5, 5, 5, 5, 'row', false, 55555.55, cast('row' as varchar(10)))
@@ -373,6 +381,8 @@ A, B, C, D, E, F, G, H, I, J
5,5,5,5,5,5,'row',false,55555.55,'row'
---- TYPES
INT,INT,INT,INT,INT,INT,STRING,BOOLEAN,DECIMAL,STRING
+---- HS2_TYPES
+INT,INT,INT,INT,INT,INT,STRING,BOOLEAN,DECIMAL,VARCHAR
====
---- QUERY
alter table tbl_with_defaults add columns (k int null, l int not null default 10000)
@@ -389,6 +399,8 @@ select * from tbl_with_defaults
5,5,5,5,5,5,'row',false,55555.55,'row',NULL,10000
---- TYPES
INT,INT,INT,INT,INT,INT,STRING,BOOLEAN,DECIMAL,STRING,INT,INT
+---- HS2_TYPES
+INT,INT,INT,INT,INT,INT,STRING,BOOLEAN,DECIMAL,VARCHAR,INT,INT
====
---- QUERY
insert into tbl_with_defaults values (6,6,6,6,6,6,'another row',false,66666.66,cast('another' as varchar(10)),6,6)
@@ -406,6 +418,8 @@ A, B, C, D, E, F, G, H, I, J, K, L
6,6,6,6,6,6,'another row',false,66666.66,'another',6,6
---- TYPES
INT,INT,INT,INT,INT,INT,STRING,BOOLEAN,DECIMAL,STRING,INT,INT
+---- HS2_TYPES
+INT,INT,INT,INT,INT,INT,STRING,BOOLEAN,DECIMAL,VARCHAR,INT,INT
====
---- QUERY
# IMPALA-5217: Try to insert NULL to a 'NOT NULL' col with a target col list that leaves
@@ -434,6 +448,8 @@ A, B, C, D, E, F, G, H, I, J, K, L
6,6,6,6,6,6,'another row',false,66666.66,'another',6,6
---- TYPES
INT,INT,INT,INT,INT,INT,STRING,BOOLEAN,DECIMAL,STRING,INT,INT
+---- HS2_TYPES
+INT,INT,INT,INT,INT,INT,STRING,BOOLEAN,DECIMAL,VARCHAR,INT,INT
====
---- QUERY
# Test VARCHAR truncation
@@ -454,6 +470,8 @@ A, B, C, D, E, F, G, H, I, J, K, L
7,7,7,7,7,7,'another row',false,77777.77,'sevenseven',7,7
---- TYPES
INT,INT,INT,INT,INT,INT,STRING,BOOLEAN,DECIMAL,STRING,INT,INT
+---- HS2_TYPES
+INT,INT,INT,INT,INT,INT,STRING,BOOLEAN,DECIMAL,VARCHAR,INT,INT
====
---- QUERY
create table multiple_partition_cols (x bigint, y bigint, z string, primary key(x, y))
@@ -479,8 +497,6 @@ BIGINT,BIGINT,STRING
# SELECT with constant NULL
insert into multiple_partition_cols select bigint_col, null, string_col
from functional.alltypes where id = 1
----- RESULTS
-: 0
---- RUNTIME_PROFILE
NumModifiedRows: 0
NumRowErrors: 1
@@ -528,8 +544,6 @@ stored as kudu;
# so that there's enough parallelism to reproduce the bug.
insert into orders_key_only
select o_orderkey from tpch_kudu.orders;
----- RESULTS
-: 1500000
---- RUNTIME_PROFILE
NumModifiedRows: 1500000
NumRowErrors: 0
diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_update.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_update.test
index 2541d1288..0052e1dd7 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu_update.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_update.test
@@ -17,8 +17,6 @@ insert into tdata values
(3, "todd", cast(1.0 as float), 993393939, "c", true, 7, 8, 9, 0.000000007, 8.88, 9,DATE '1970-01-03', cast('todd' as varchar(10))),
(40, "he", cast(0.0 as float), 43, cast('e' as string), false, 50, 60, 70, 0.000000050, 66.60, 70, DATE '1970-01-04', cast('he' as varchar(10))),
(120, "she", cast(0.0 as float), 99, cast('f' as string), true, -1, 0, 1, -0.000000001, 0.00, 1, DATE '1970-01-05', cast('she' as varchar(10)))
----- RESULTS
-: 5
---- RUNTIME_PROFILE
NumModifiedRows: 5
NumRowErrors: 0
@@ -41,6 +39,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,99,'f',true,-1,0,1,-0.000000001,0.00,1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# single row, equality on key, string
@@ -60,6 +60,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,99,'f',true,-1,0,1,-0.000000001,0.00,1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# single row, equality on key, boolean
@@ -77,6 +79,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,99,'f',true,-1,0,1,-0.000000001,0.00,1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# single row, equality on key, set to NULL
@@ -94,6 +98,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,99,'f',true,-1,0,1,-0.000000001,0.00,1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# single row, equlity on key, set from NULL
@@ -111,6 +117,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,99,'f',true,-1,0,1,-0.000000001,0.00,1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# single row, equality on key, boundary value
@@ -128,6 +136,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,9223372036854775807,'f',true,-1,0,1,-0.000000001,0.00,1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# single row, inequality on key, float
@@ -145,6 +155,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,9223372036854775807,'f',true,-1,0,1,-0.000000001,0.00,1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# single row, inequality on key, tinyint
@@ -162,6 +174,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,9223372036854775807,'f',true,10,0,1,-0.000000001,0.00,1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# single row, equality on non-key, string
@@ -179,6 +193,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,9223372036854775807,'f',true,10,0,1,-0.000000001,0.00,1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# single row, inequality on non-key, double
@@ -196,6 +212,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,9223372036854775807,'f',true,10,0,1,-0.000000001,0.00,1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# single row, inequality on non-key, decimal
@@ -213,6 +231,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,9223372036854775807,'f',true,10,0,1,0.000000001,0.00,-1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# multiple rows, predicate on key
@@ -230,6 +250,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,9223372036854775807,'f',true,10,0,1,0.000000001,0.00,-1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# multiple rows, predicate on non-key
@@ -247,6 +269,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,9223372036854775807,'f',false,10,0,1,0.000000001,0.00,-1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# update the same row multiple times: cross join produces 5 identical updates
@@ -264,6 +288,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,9223372036854775807,'f',false,10,0,1,0.000000001,0.00,-1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# value of update is a constant expression
@@ -281,6 +307,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,9223372036854775807,'f',false,10,0,1,0.000000001,0.00,-1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# value of update is a non-constant expression
@@ -298,6 +326,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,9223372036854775807,'f',false,10,0,1,0.000000001,0.00,-1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# error - update key column
@@ -321,6 +351,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,9223372036854775807,'f',false,10,0,1,0.000000001,0.00,-1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
# Try to update a row with a primary key value that is not covered by the existing range
@@ -339,13 +371,13 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC4, VALDEC8, VALDEC16,
120,'she',0.0,9223372036854775807,'f',false,10,0,1,0.000000001,0.00,-1,1970-01-05,'she'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DECIMAL,DECIMAL,DATE,VARCHAR
====
---- QUERY
insert into tdata
select id, string_col, float_col, bigint_col, string_col, bool_col, tinyint_col,
smallint_col, double_col, NULL, NULL, NULL, NULL, NULL from functional_kudu.alltypes
----- RESULTS
-: 7295
---- RUNTIME_PROFILE
NumModifiedRows: 7295
NumRowErrors: 5
diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_upsert.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_upsert.test
index 303d065cc..b94a46798 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu_upsert.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_upsert.test
@@ -15,8 +15,9 @@ insert into table tdata values
(1,'unknown',1,43,'aaaaaaaaaaaaaaaaaaaa',false,-1,-2,0,-3,DATE '1970-01-01',cast('unknown' as varchar(10))),
(2,'david',1,43,'b',false,0,0,0,0,DATE '1970-01-02',cast('david' as varchar(10))),
(3,'todd',1,43,'c',true,3,3,3,3,DATE '1970-01-03',cast('todd' as varchar(10)))
----- RESULTS
-: 4
+---- RUNTIME_PROFILE
+NumModifiedRows: 4
+NumRowErrors: 0
====
---- QUERY
# VALUES, single row, all cols, results in insert
@@ -34,6 +35,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
4,'a',0,1,'b',false,1,2,1.5,4,1970-01-04,'a'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, single row, all cols, results in update
@@ -51,6 +54,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
4,'b',-1,1,'a',true,2,3,2.5,5,1970-01-04,'b'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, single row, all cols, insert NULL all types
@@ -69,6 +74,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
10,'NULL',NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL,NULL,'NULL'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, single row, all cols, update NULL all types
@@ -87,6 +94,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
10,'NULL',NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL,NULL,'NULL'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, single row, all cols, update from NULL all types
@@ -105,6 +114,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
10,'NULL',NULL,NULL,'NULL',NULL,NULL,NULL,NULL,NULL,NULL,'NULL'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, single row, all cols, insert boundary values
@@ -126,6 +137,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
2147483647,'',NaN,-9223372036854775808,'',true,127,-32768,Infinity,999999999,9999-12-31,''
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, single row, all cols, update boundary values
@@ -147,6 +160,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
2147483647,'',-Infinity,9223372036854775807,'',true,-128,32767,NaN,-999999999,0001-01-01,''
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, single row, subset of cols, results in insert
@@ -167,6 +182,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
5,'five',NULL,-5,'NULL',NULL,NULL,NULL,0.5,NULL,NULL,'five'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, single row, subset of cols, results in update
@@ -187,6 +204,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
5,'NULL',0,-5,'six',false,NULL,NULL,0.5,NULL,NULL,'NULL'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, multiple rows, all cols, no errors
@@ -213,6 +232,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
7,'seven',0,-9223372036854775808,'NULL',true,7,1,2,3,1970-01-07,'seven'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, multiple rows, subset of cols, no errors
@@ -241,6 +262,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
9,'nine',NULL,9,'NULL',true,NULL,NULL,NULL,NULL,NULL,'nine'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# SELECT, all cols, single row, no errors
@@ -266,6 +289,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
9,'nine',NULL,9,'NULL',true,NULL,NULL,NULL,NULL,NULL,'nine'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# SELECT, all cols, multiple rows, no errors
@@ -291,6 +316,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
9,'nine',NULL,9,'NULL',true,NULL,NULL,NULL,NULL,NULL,'nine'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# SELECT, subset of cols, single row, no errors
@@ -317,6 +344,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
0,'NULL',NULL,0,'0',NULL,NULL,NULL,NULL,NULL,NULL,'NULL'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# SELECT, subset of cols, multiple rows, no errors
@@ -343,6 +372,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
0,'0',NULL,0,'0',true,0,NULL,NULL,NULL,NULL,'0'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# SELECT, decimal column, multiple rows, no errors
@@ -369,6 +400,8 @@ ID, NAME, VALF, VALI, VALV, VALB, VALT, VALS, VALD, VALDEC, VALDATE, VALVC
0,'0',NULL,0,'0',true,0,NULL,NULL,NULL,NULL,'0'
---- TYPES
INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,STRING
+---- HS2_TYPES
+INT,STRING,FLOAT,BIGINT,STRING,BOOLEAN,TINYINT,SMALLINT,DOUBLE,DECIMAL,DATE,VARCHAR
====
---- QUERY
# VALUES, single row, all cols, null for non-nullable column
@@ -436,8 +469,9 @@ create table multiple_key_cols
insert into multiple_key_cols values
('a', 1, 2, 3, true, 4, 5, NULL),
('b', 1, 2, 3, false, 7, NULL, 9)
----- RESULTS
-: 2
+---- RUNTIME_PROFILE
+NumModifiedRows: 2
+NumRowErrors: 0
====
---- QUERY
# VALUES, multiple key columns, all cols
diff --git a/tests/common/base_test_suite.py b/tests/common/base_test_suite.py
index bd9cb1882..84dc26241 100644
--- a/tests/common/base_test_suite.py
+++ b/tests/common/base_test_suite.py
@@ -19,7 +19,7 @@
from __future__ import absolute_import, division, print_function
import logging
-from tests.common.test_vector import ImpalaTestMatrix
+from tests.common.test_vector import BEESWAX, ImpalaTestMatrix
LOG = logging.getLogger('base_test_suite')
@@ -36,3 +36,8 @@ class BaseTestSuite(object):
add more dimensions or different dimensions they can override this function.
"""
cls.ImpalaTestMatrix = ImpalaTestMatrix()
+
+ @classmethod
+ def default_test_protocol(cls):
+ """See documentation in ImpalaTestSuite.default_test_protocol()."""
+ return BEESWAX
diff --git a/tests/common/impala_connection.py b/tests/common/impala_connection.py
index 20e689bcb..41f4b48f8 100644
--- a/tests/common/impala_connection.py
+++ b/tests/common/impala_connection.py
@@ -594,7 +594,8 @@ class ImpylaHS2ResultSet(object):
self.column_types = column_types
self.query = query
self.log = log
- self.profile = profile
+ # ImpalaBeeswaxResult store profile at runtime_profile field
+ self.runtime_profile = profile
self.query_id = query_id
self.__result_tuples = result_tuples
# self.data is the data in the ImpalaBeeswaxResult format: a list of rows with each
@@ -603,6 +604,10 @@ class ImpylaHS2ResultSet(object):
if result_tuples is not None:
self.data = [self.__convert_result_row(tuple) for tuple in result_tuples]
+ def tuples(self):
+ """Return the raw HS2 result set, which is a list of tuples."""
+ return self.__result_tuples
+
def __convert_result_row(self, result_tuple):
"""Take primitive values from a result tuple and construct the tab-separated string
that would have been returned via beeswax."""
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index fd314d64c..5108a935e 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -54,7 +54,6 @@ from tests.common.test_dimensions import (
ALL_NODES_ONLY,
TableFormatInfo,
create_exec_option_dimension,
- default_client_protocol_dimension,
get_dataset_from_workload,
load_table_info_dimension)
from tests.common.test_result_verifier import (
@@ -63,7 +62,10 @@ from tests.common.test_result_verifier import (
verify_lineage,
verify_raw_results,
verify_runtime_profile)
-from tests.common.test_vector import ImpalaTestDimension, EXEC_OPTION_KEY
+from tests.common.test_vector import (
+ EXEC_OPTION, PROTOCOL, TABLE_FORMAT,
+ BEESWAX, HS2, HS2_HTTP,
+ ImpalaTestDimension)
from tests.performance.query import Query
from tests.performance.query_exec_functions import execute_using_jdbc
from tests.performance.query_executor import JdbcQueryExecConfig
@@ -199,9 +201,32 @@ class ImpalaTestSuite(BaseTestSuite):
cls.ImpalaTestMatrix.add_dimension(
cls.create_table_info_dimension(cls.exploration_strategy()))
cls.ImpalaTestMatrix.add_dimension(cls.__create_exec_option_dimension())
- # Execute tests through Beeswax by default. Individual tests that have been converted
- # to work with the HS2 client can add HS2 in addition to or instead of beeswax.
- cls.ImpalaTestMatrix.add_dimension(default_client_protocol_dimension())
+ cls.ImpalaTestMatrix.add_dimension(
+ ImpalaTestDimension(PROTOCOL, cls.default_test_protocol()))
+
+ @classmethod
+ def default_test_protocol(cls):
+ """This method is used to allow test subclasses to override
+ pytest.config.option.default_test_protocol as the default 'protocol' dimension.
+ If subclass override this method, return value must either be 'beeswax', 'hs2',
+ or 'hs2-http'.
+ The protocol kind of self.client is always match with return value of this method.
+ On the other hand, subclasses can still exercise other 'protocol' dimension
+ of test vector, by overriding add_test_dimensions(). Do note the correctness of
+ 'vector' fixture relies on always querying through helper method that accept
+ 'vector' argument, such as:
+ - run_test_case()
+ - execute_query_using_client()
+ - execute_query_async_using_client()
+ - execute_query_using_vector()
+ - create_impala_client_from_vector()
+ Most method in ImpalaTestSuite that has optional 'protocol' argument will default
+ to the return value of this method.
+ Default to return pytest.config.option.default_test_protocol."""
+ # default_test_protocol is usually 'beeswax', unless user specify otherwise.
+ # Individual tests that have been converted to work with the HS2 client can add HS2
+ # in addition to or instead of beeswax.
+ return pytest.config.option.default_test_protocol
@staticmethod
def create_hive_client(port):
@@ -332,13 +357,13 @@ class ImpalaTestSuite(BaseTestSuite):
pass
@classmethod
- def create_impala_client(cls, host_port=None,
- protocol=pytest.config.option.default_test_protocol,
- is_hive=False):
+ def create_impala_client(cls, host_port=None, protocol=None, is_hive=False):
"""
Create a new ImpalaConnection client.
Make sure to always call this method using a with-as statement or manually close
the returned connection before discarding it."""
+ if protocol is None:
+ protocol = cls.default_test_protocol()
if host_port is None:
host_port = cls.__get_default_host_port(protocol)
client = create_connection(host_port=host_port,
@@ -354,18 +379,18 @@ class ImpalaTestSuite(BaseTestSuite):
Return a client of specified 'protocol' and with cofiguration 'exec_option' set.
Make sure to always call this method using a with-as statement or manually close
the returned connection before discarding it."""
- client = cls.create_impala_client(
- protocol=vector.get_value('protocol'))
+ client = cls.create_impala_client(protocol=vector.get_protocol())
client.set_configuration(vector.get_exec_option_dict())
return client
@classmethod
def get_impalad_cluster_size(cls):
- return len(cls.__get_cluster_host_ports(pytest.config.option.default_test_protocol))
+ return len(cls.__get_cluster_host_ports(cls.default_test_protocol()))
@classmethod
- def create_client_for_nth_impalad(cls, nth=0,
- protocol=pytest.config.option.default_test_protocol):
+ def create_client_for_nth_impalad(cls, nth=0, protocol=None):
+ if protocol is None:
+ protocol = cls.default_test_protocol()
host_ports = cls.__get_cluster_host_ports(protocol)
if nth < len(IMPALAD_HOST_PORT_LIST):
host_port = host_ports[nth]
@@ -383,21 +408,21 @@ class ImpalaTestSuite(BaseTestSuite):
"""Creates Impala clients for all supported protocols."""
# The default connection (self.client) is Beeswax so that existing tests, which assume
# Beeswax do not need modification (yet).
- cls.beeswax_client = cls.create_impala_client(protocol='beeswax')
+ cls.beeswax_client = cls.create_impala_client(protocol=BEESWAX)
cls.hs2_client = None
try:
- cls.hs2_client = cls.create_impala_client(protocol='hs2')
+ cls.hs2_client = cls.create_impala_client(protocol=HS2)
except Exception as e:
# HS2 connection can fail for benign reasons, e.g. running with unsupported auth.
LOG.info("HS2 connection setup failed, continuing...: {0}".format(e))
cls.hs2_http_client = None
try:
- cls.hs2_http_client = cls.create_impala_client(protocol='hs2-http')
+ cls.hs2_http_client = cls.create_impala_client(protocol=HS2_HTTP)
except Exception as e:
# HS2 HTTP connection can fail for benign reasons, e.g. running with unsupported
# auth.
LOG.info("HS2 HTTP connection setup failed, continuing...: {0}".format(e))
- cls.client = cls.default_impala_client(pytest.config.option.default_test_protocol)
+ cls.client = cls.default_impala_client(cls.default_test_protocol())
@classmethod
def __reset_impala_clients(cls):
@@ -428,32 +453,33 @@ class ImpalaTestSuite(BaseTestSuite):
@classmethod
def default_impala_client(cls, protocol):
- if protocol == 'beeswax':
+ if protocol == BEESWAX:
return cls.beeswax_client
- if protocol == 'hs2':
+ if protocol == HS2:
return cls.hs2_client
- if protocol == 'hs2-http':
+ if protocol == HS2_HTTP:
return cls.hs2_http_client
raise Exception("unknown protocol: {0}".format(protocol))
@classmethod
def __get_default_host_port(cls, protocol):
- if protocol == 'beeswax':
+ if protocol == BEESWAX:
return IMPALAD
- elif protocol == 'hs2-http':
+ elif protocol == HS2_HTTP:
return IMPALAD_HS2_HTTP_HOST_PORT
- else:
- assert protocol == 'hs2'
+ elif protocol == HS2:
return IMPALAD_HS2_HOST_PORT
+ else:
+ raise NotImplementedError("Not yet implemented: protocol=" + protocol)
@classmethod
def __get_cluster_host_ports(cls, protocol):
"""Return a list of host/port combinations for all impalads in the cluster."""
- if protocol == 'beeswax':
+ if protocol == BEESWAX:
return IMPALAD_HOST_PORT_LIST
- elif protocol == 'hs2':
+ elif protocol == HS2:
return IMPALAD_HS2_HOST_PORT_LIST
- elif protocol == 'hs2-http':
+ elif protocol == HS2_HTTP:
return IMPALAD_HS2_HTTP_HOST_PORT_LIST
else:
raise NotImplementedError("Not yet implemented: protocol=" + protocol)
@@ -682,15 +708,7 @@ class ImpalaTestSuite(BaseTestSuite):
test_section[section_name] = test_section[section_name].replace(
'$DATABASE', use_db)
result_section, type_section = 'RESULTS', 'TYPES'
- if vector.get_value('protocol').startswith('hs2'):
- # hs2 or hs2-http
- if 'HS2_TYPES' in test_section:
- assert 'TYPES' in test_section,\
- "Base TYPES section must always be included alongside HS2_TYPES"
- # In some cases HS2 types are expected differ from Beeswax types (e.g. see
- # IMPALA-914), so use the HS2-specific section if present.
- type_section = 'HS2_TYPES'
- verify_raw_results(test_section, result, vector.get_value('table_format').file_format,
+ verify_raw_results(test_section, result, vector,
result_section, type_section, pytest.config.option.update_results,
replace_filenames_with_placeholder)
@@ -714,9 +732,9 @@ class ImpalaTestSuite(BaseTestSuite):
names, see 'reserved_keywords' below.
"""
self.validate_exec_option_dimension(vector)
- table_format_info = vector.get_value('table_format')
- exec_options = vector.get_value(EXEC_OPTION_KEY)
- protocol = vector.get_value('protocol')
+ table_format_info = vector.get_table_format()
+ exec_options = vector.get_exec_option_dict()
+ protocol = vector.get_protocol()
target_impalad_clients = list()
if multiple_impalad:
@@ -754,7 +772,7 @@ class ImpalaTestSuite(BaseTestSuite):
if set_pattern_match:
option_name = set_pattern_match.groups()[0].lower()
query_options_changed.append(option_name)
- assert option_name not in vector.get_value(EXEC_OPTION_KEY), (
+ assert option_name not in vector.get_value(EXEC_OPTION), (
"{} cannot be set in the '.test' file since it is in the test vector. "
"Consider deepcopy()-ing the vector and removing this option in the "
"python test.".format(option_name))
@@ -769,7 +787,7 @@ class ImpalaTestSuite(BaseTestSuite):
Helper to execute a query block in Hive. No special handling of query
options is done, since we use a separate session for each block.
"""
- h = ImpalaTestSuite.create_impala_client(HIVE_HS2_HOST_PORT, protocol='hs2',
+ h = ImpalaTestSuite.create_impala_client(HIVE_HS2_HOST_PORT, protocol=HS2,
is_hive=True)
try:
result = None
@@ -913,9 +931,9 @@ class ImpalaTestSuite(BaseTestSuite):
dml_results_query = "select * from %s limit 1000" % \
test_section['DML_RESULTS_TABLE']
dml_result = exec_fn(dml_results_query)
- verify_raw_results(test_section, dml_result,
- vector.get_value('table_format').file_format, result_section='DML_RESULTS',
- update_section=pytest.config.option.update_results)
+ verify_raw_results(test_section, dml_result, vector,
+ result_section='DML_RESULTS',
+ update_section=pytest.config.option.update_results)
except Exception as e:
# When the calcite report mode is off, fail fast when hitting an error.
if not pytest.config.option.calcite_report_mode:
@@ -1012,11 +1030,11 @@ class ImpalaTestSuite(BaseTestSuite):
@wraps(function)
def wrapper(*args, **kwargs):
table_format = None
- if kwargs.get('table_format'):
- table_format = kwargs.get('table_format')
- del kwargs['table_format']
+ if kwargs.get(TABLE_FORMAT):
+ table_format = kwargs.get(TABLE_FORMAT)
+ del kwargs[TABLE_FORMAT]
if kwargs.get('vector'):
- table_format = kwargs.get('vector').get_value('table_format')
+ table_format = kwargs.get('vector').get_table_format()
del kwargs['vector']
# self is the implicit first argument
if table_format is not None:
@@ -1078,15 +1096,15 @@ class ImpalaTestSuite(BaseTestSuite):
def execute_query_using_client(self, client, query, vector):
self.validate_exec_option_dimension(vector)
- self.change_database(client, vector.get_value('table_format'))
- query_options = vector.get_value(EXEC_OPTION_KEY)
+ self.change_database(client, vector.get_table_format())
+ query_options = vector.get_value(EXEC_OPTION)
if query_options is not None: client.set_configuration(query_options)
return client.execute(query)
def execute_query_async_using_client(self, client, query, vector):
self.validate_exec_option_dimension(vector)
- self.change_database(client, vector.get_value('table_format'))
- query_options = vector.get_value(EXEC_OPTION_KEY)
+ self.change_database(client, vector.get_table_format())
+ query_options = vector.get_value(EXEC_OPTION)
if query_options is not None: client.set_configuration(query_options)
return client.execute_async(query)
@@ -1098,7 +1116,7 @@ class ImpalaTestSuite(BaseTestSuite):
'vector' must have 'protocol' and 'exec_option' dimension.
Default ImpalaTestSuite client will be used depending on value of 'protocol'
dimension."""
- client = self.default_impala_client(vector.get_value('protocol'))
+ client = self.default_impala_client(vector.get_protocol())
result = self.execute_query_using_client(client, query, vector)
# Restore client configuration before returning.
modified_configs = vector.get_exec_option_dict().keys()
@@ -1304,7 +1322,7 @@ class ImpalaTestSuite(BaseTestSuite):
for tf in pytest.config.option.table_formats.split(','):
dataset = get_dataset_from_workload(cls.get_workload())
table_formats.append(TableFormatInfo.create_from_string(dataset, tf))
- tf_dimensions = ImpalaTestDimension('table_format', *table_formats)
+ tf_dimensions = ImpalaTestDimension(TABLE_FORMAT, *table_formats)
else:
tf_dimensions = load_table_info_dimension(cls.get_workload(), exploration_strategy)
# If 'skip_hbase' is specified or the filesystem is isilon, s3, GCS(gs), COS(cosn) or
@@ -1578,8 +1596,8 @@ class ImpalaTestSuite(BaseTestSuite):
option_dim_names = []
exec_option = dict()
for vector_value in vector.vector_values:
- if vector_value.name == EXEC_OPTION_KEY:
- exec_option = vector.get_value(EXEC_OPTION_KEY)
+ if vector_value.name == EXEC_OPTION:
+ exec_option = vector.get_value(EXEC_OPTION)
elif vector_value.name.lower() in EXEC_OPTION_NAMES:
option_dim_names.append(vector_value.name.lower())
@@ -1592,10 +1610,10 @@ class ImpalaTestSuite(BaseTestSuite):
pytest.fail("Exec option {} declared as independent dimension but not inserted "
"into {} dimension. Consider using helper function "
"add_exec_option_dimension or add_mandatory_exec_option "
- "to declare it.".format(name, EXEC_OPTION_KEY))
+ "to declare it.".format(name, EXEC_OPTION))
elif vector.get_value(name) != exec_option[name]:
pytest.fail("{}[{}]={} does not match against dimension {}={}.".format(
- EXEC_OPTION_KEY, name, exec_option[name], name, vector.get_value(name)))
+ EXEC_OPTION, name, exec_option[name], name, vector.get_value(name)))
def __build_log_path(self, daemon, level):
"""Builds a path to a log file for a particular daemon. Does not assert that file
diff --git a/tests/common/kudu_test_suite.py b/tests/common/kudu_test_suite.py
index f9cf36f56..d92eb7e1a 100644
--- a/tests/common/kudu_test_suite.py
+++ b/tests/common/kudu_test_suite.py
@@ -40,7 +40,7 @@ from random import choice, sample
from string import ascii_lowercase, digits
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.test_dimensions import create_kudu_dimension
+from tests.common.test_dimensions import HS2, create_kudu_dimension
DEFAULT_KUDU_MASTER_WEBUI_PORT = os.getenv('KUDU_MASTER_WEBUI_PORT', '8051')
@@ -66,6 +66,7 @@ def get_kudu_master_flag(flag):
return split[1]
assert False, "Failed to find Kudu master flag: %s" % flag
+
class KuduTestSuite(ImpalaTestSuite):
# Lazily set.
@@ -74,7 +75,7 @@ class KuduTestSuite(ImpalaTestSuite):
@classmethod
def get_conn_timeout(cls):
# For IMPALA-5079,IMPALA-4454
- return 60 * 5 # 5 minutes
+ return 60 * 5 # 5 minutes
@classmethod
def setup_class(cls):
@@ -90,6 +91,12 @@ class KuduTestSuite(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_dimension(
create_kudu_dimension(cls.get_workload()))
+ @classmethod
+ def default_test_protocol(cls):
+ # run_test_case() can produce different result types between beeswax vs hs2 protocol
+ # in some tests. This fix the test to use hs2 protocol.
+ return HS2
+
@classmethod
def auto_create_db(cls):
return True
@@ -167,19 +174,20 @@ class KuduTestSuite(ImpalaTestSuite):
kudu.delete_table(name)
@contextmanager
- def drop_impala_table_after_context(self, cursor, table_name):
- """For use in a "with" block: The named table will be dropped using the provided
- cursor when the block exits.
+ def drop_impala_table_after_context(self, impala_client, table_name):
+ """DEPRECATED: use unique_database fixture instead.
+ For use in a "with" block: The named table will be dropped using the provided
+ impala_client when the block exits.
cursor.execute("CREATE TABLE foo ...")
- with drop_impala_table_after_context(cursor, "foo"):
+ with drop_impala_table_after_context(impala_client, "foo"):
...
# Now table foo no longer exists.
"""
try:
yield
finally:
- cursor.execute("DROP TABLE %s" % table_name)
+ impala_client.execute("DROP TABLE IF EXISTS %s" % table_name)
def kudu_col_type_to_impala_col_type(self, col_type):
mapping = {BOOL: "BOOLEAN",
diff --git a/tests/common/test_dimensions.py b/tests/common/test_dimensions.py
index 7e8f33a6f..ba3179037 100644
--- a/tests/common/test_dimensions.py
+++ b/tests/common/test_dimensions.py
@@ -25,7 +25,9 @@ import pytest
from itertools import product
from tests.common.test_vector import (
- ImpalaTestDimension, ImpalaTestVector, assert_exec_option_key)
+ EXEC_OPTION, PROTOCOL, TABLE_FORMAT,
+ BEESWAX, HS2, HS2_HTTP,
+ ImpalaTestDimension, ImpalaTestVector, assert_exec_option_key)
from tests.util.filesystem_utils import (
IS_HDFS)
@@ -121,7 +123,7 @@ class TableFormatInfo(object):
def create_table_format_dimension(workload, table_format_string):
dataset = get_dataset_from_workload(workload)
- return ImpalaTestDimension('table_format',
+ return ImpalaTestDimension(TABLE_FORMAT,
TableFormatInfo.create_from_string(dataset, table_format_string))
@@ -150,15 +152,15 @@ def create_kudu_dimension(workload):
def default_client_protocol_dimension():
- return ImpalaTestDimension('protocol', pytest.config.option.default_test_protocol)
+ return ImpalaTestDimension(PROTOCOL, pytest.config.option.default_test_protocol)
def beeswax_client_protocol_dimension():
- return ImpalaTestDimension('protocol', 'beeswax')
+ return ImpalaTestDimension(PROTOCOL, BEESWAX)
def hs2_client_protocol_dimension():
- return ImpalaTestDimension('protocol', 'hs2')
+ return ImpalaTestDimension(PROTOCOL, HS2)
def create_client_protocol_dimension():
@@ -167,12 +169,12 @@ def create_client_protocol_dimension():
# transport.
import ssl
if not hasattr(ssl, "create_default_context"):
- return ImpalaTestDimension('protocol', 'beeswax', 'hs2')
- return ImpalaTestDimension('protocol', 'beeswax', 'hs2', 'hs2-http')
+ return ImpalaTestDimension(PROTOCOL, BEESWAX, HS2)
+ return ImpalaTestDimension(PROTOCOL, BEESWAX, HS2, HS2_HTTP)
def create_client_protocol_http_transport():
- return ImpalaTestDimension('protocol', 'hs2-http')
+ return ImpalaTestDimension(PROTOCOL, HS2_HTTP)
def create_client_protocol_strict_dimension():
@@ -191,22 +193,22 @@ def create_client_protocol_no_strict_dimension():
def hs2_parquet_constraint(v):
"""Constraint function, used to only run HS2 against Parquet format, because file format
and the client protocol are orthogonal."""
- return (v.get_value('protocol') == 'beeswax' or
- v.get_value('table_format').file_format == 'parquet' and
- v.get_value('table_format').compression_codec == 'none')
+ return (v.get_protocol() == BEESWAX
+ or (v.get_table_format().file_format == 'parquet'
+ and v.get_table_format().compression_codec == 'none'))
def hs2_text_constraint(v):
"""Constraint function, used to only run HS2 against uncompressed text, because file
format and the client protocol are orthogonal."""
- return (v.get_value('protocol') == 'beeswax' or
- v.get_value('table_format').file_format == 'text' and
- v.get_value('table_format').compression_codec == 'none')
+ return (v.get_protocol() == BEESWAX
+ or (v.get_table_format().file_format == 'text'
+ and v.get_table_format().compression_codec == 'none'))
def orc_schema_resolution_constraint(v):
""" Constraint to use multiple orc_schema_resolution only in case of orc files"""
- file_format = v.get_value('table_format').file_format
+ file_format = v.get_table_format().file_format
orc_schema_resolution = v.get_value('orc_schema_resolution')
return file_format == 'orc' or orc_schema_resolution == 0
@@ -277,7 +279,7 @@ def create_exec_option_dimension_from_dict(exec_option_dimensions):
exec_option_dimension_values = [dict(zip(keys, prod)) for prod in combinations]
# Build a test vector out of it
- return ImpalaTestDimension('exec_option', *exec_option_dimension_values)
+ return ImpalaTestDimension(EXEC_OPTION, *exec_option_dimension_values)
def add_exec_option_dimension(test_suite, key, values):
@@ -354,7 +356,7 @@ def load_table_info_dimension(workload_name, exploration_strategy, file_formats=
continue
vector_values.append(TableFormatInfo(**vals))
- return ImpalaTestDimension('table_format', *vector_values)
+ return ImpalaTestDimension(TABLE_FORMAT, *vector_values)
def is_supported_insert_format(table_format):
diff --git a/tests/common/test_result_verifier.py b/tests/common/test_result_verifier.py
index 71b196212..0ecc4042d 100644
--- a/tests/common/test_result_verifier.py
+++ b/tests/common/test_result_verifier.py
@@ -376,7 +376,7 @@ def apply_error_match_filter(error_list, replace_filenames=True):
return [replace_fn(row) for row in error_list]
-def verify_raw_results(test_section, exec_result, file_format, result_section,
+def verify_raw_results(test_section, exec_result, vector, result_section,
type_section='TYPES', update_section=False,
replace_filenames=True):
"""
@@ -397,6 +397,8 @@ def verify_raw_results(test_section, exec_result, file_format, result_section,
TODO: separate out the handling of sections like ERRORS from checking of query results
to allow regular RESULTS/ERRORS sections in tests with DML_RESULTS (IMPALA-4471).
"""
+ file_format = vector.get_value('table_format').file_format
+ protocol = vector.get_value('protocol')
expected_results = None
if result_section in test_section:
expected_results = remove_comments(test_section[result_section])
@@ -425,6 +427,15 @@ def verify_raw_results(test_section, exec_result, file_format, result_section,
else:
raise
+ if type_section == 'TYPES' and protocol.startswith('hs2'):
+ # hs2 or hs2-http
+ if 'HS2_TYPES' in test_section:
+ assert 'TYPES' in test_section, \
+ "Base TYPES section must always be included alongside HS2_TYPES"
+ # In some cases HS2 types are expected differ from Beeswax types (e.g. see
+ # IMPALA-914), so use the HS2-specific section if present.
+ type_section = 'HS2_TYPES'
+
if type_section in test_section:
# Distinguish between an empty list and a list with an empty string.
section = test_section[type_section]
diff --git a/tests/common/test_vector.py b/tests/common/test_vector.py
index fc3234b44..900f3a072 100644
--- a/tests/common/test_vector.py
+++ b/tests/common/test_vector.py
@@ -63,7 +63,14 @@ import logging
LOG = logging.getLogger(__name__)
-EXEC_OPTION_KEY = 'exec_option'
+# Literal constants to refer to some standard dimension names.
+EXEC_OPTION = 'exec_option'
+PROTOCOL = 'protocol'
+TABLE_FORMAT = 'table_format'
+# Literal constants to refer protocol names.
+BEESWAX = 'beeswax'
+HS2 = 'hs2'
+HS2_HTTP = 'hs2-http'
def assert_exec_option_key(key):
@@ -95,19 +102,25 @@ class ImpalaTestVector(object):
return vector_value.value
raise ValueError("Test vector does not contain value '%s'" % name)
+ def get_protocol(self):
+ return self.get_value(PROTOCOL)
+
+ def get_table_format(self):
+ return self.get_value(TABLE_FORMAT)
+
def get_exec_option_dict(self):
- return self.get_value(EXEC_OPTION_KEY)
+ return self.get_value(EXEC_OPTION)
def get_exec_option(self, option_name):
- value = self.get_value(EXEC_OPTION_KEY).get(option_name.lower())
+ value = self.get_value(EXEC_OPTION).get(option_name.lower())
assert value is not None
return value
def set_exec_option(self, option_name, option_value):
- self.get_value(EXEC_OPTION_KEY)[option_name.lower()] = str(option_value)
+ self.get_value(EXEC_OPTION)[option_name.lower()] = str(option_value)
def unset_exec_option(self, option_name):
- del self.get_value(EXEC_OPTION_KEY)[option_name.lower()]
+ del self.get_value(EXEC_OPTION)[option_name.lower()]
def __str__(self):
return ' | '.join(['%s' % vector_value for vector_value in self.vector_values])
@@ -133,34 +146,34 @@ class ImpalaTestMatrix(object):
def add_dimension(self, dimension):
self.dimensions[dimension.name] = dimension
- if dimension.name == EXEC_OPTION_KEY:
+ if dimension.name == EXEC_OPTION:
for name in list(self.independent_exec_option_names):
LOG.warn("Reassigning {} dimension will remove exec option {}={} that was "
"independently declared through add_exec_option_dimension.".format(
- EXEC_OPTION_KEY, name, [v.value for v in self.dimensions[name]]))
+ EXEC_OPTION, name, [v.value for v in self.dimensions[name]]))
self.clear_dimension(name)
def assert_unique_exec_option_key(self, key):
"""Assert that 'exec_option' dimension exist and 'key' is not exist yet
in self.dimension['exec_option']."""
assert_exec_option_key(key)
- assert EXEC_OPTION_KEY in self.dimensions, (
- "Must have '" + EXEC_OPTION_KEY + "' dimension previously declared!")
+ assert EXEC_OPTION in self.dimensions, (
+ "Must have '" + EXEC_OPTION + "' dimension previously declared!")
- for vector in self.dimensions[EXEC_OPTION_KEY]:
+ for vector in self.dimensions[EXEC_OPTION]:
assert key not in vector.value, (
- "'{}' is already exist in '{}' dimension!".format(key, EXEC_OPTION_KEY))
+ "'{}' is already exist in '{}' dimension!".format(key, EXEC_OPTION))
# 'key' must not previously declared with add_exec_option_dimension().
assert key not in self.independent_exec_option_names, (
"['{}']['{}'] was previously declared with non-unique value: {}".format(
- EXEC_OPTION_KEY, key, [dim.value for dim in self.dimensions[key]]))
+ EXEC_OPTION, key, [dim.value for dim in self.dimensions[key]]))
def add_mandatory_exec_option(self, key, value):
"""Append key=value pair into 'exec_option' dimension."""
self.assert_unique_exec_option_key(key)
- for vector in self.dimensions[EXEC_OPTION_KEY]:
+ for vector in self.dimensions[EXEC_OPTION]:
vector.value[key] = value
def add_exec_option_dimension(self, dimension):
@@ -208,7 +221,7 @@ class ImpalaTestMatrix(object):
exec_values = []
exec_option = None
for val in vector_values:
- if val.name == EXEC_OPTION_KEY:
+ if val.name == EXEC_OPTION:
# 'exec_option' is a map. We need to deepcopy the value for each vector.
exec_option = deepcopy(val.value)
elif val.name in self.independent_exec_option_names:
@@ -218,11 +231,11 @@ class ImpalaTestMatrix(object):
values.append(val)
if self.independent_exec_option_names:
assert exec_option is not None, (
- "Must have '" + EXEC_OPTION_KEY + "' dimension previously declared!")
+ "Must have '" + EXEC_OPTION + "' dimension previously declared!")
for val in exec_values:
exec_option[val.name] = val.value
if exec_option:
- values.append(ImpalaTestVector.Value(EXEC_OPTION_KEY, exec_option))
+ values.append(ImpalaTestVector.Value(EXEC_OPTION, exec_option))
return values
def __generate_exhaustive_combinations(self):
diff --git a/tests/conftest.py b/tests/conftest.py
index 9366e021c..3f2a5abcb 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -32,14 +32,17 @@ import sys
import tests.common
from impala_py_lib.helpers import find_all_files, is_core_dump
+import tests.common.base_test_suite
from tests.common.environ import build_flavor_timeout
from tests.common.test_result_verifier import QueryTestResult
from tests.common.patterns import is_valid_impala_identifier
+from tests.common.test_vector import BEESWAX, HS2, HS2_HTTP
from tests.comparison.db_connection import ImpalaConnection
from tests.util.filesystem_utils import FILESYSTEM, ISILON_WEBHDFS_PORT, WAREHOUSE
LOG = logging.getLogger('test_configuration')
LOG_FORMAT = "-- %(asctime)s %(levelname)-8s %(threadName)s: %(message)s"
+VALID_TEST_PROTOCOLS = [BEESWAX, HS2, HS2_HTTP]
DEFAULT_CONN_TIMEOUT = 45
DEFAULT_EXPLORATION_STRATEGY = 'core'
@@ -54,7 +57,7 @@ DEFAULT_KUDU_MASTER_HOSTS = os.getenv('KUDU_MASTER_HOSTS', '127.0.0.1')
DEFAULT_KUDU_MASTER_PORT = os.getenv('KUDU_MASTER_PORT', '7051')
DEFAULT_METASTORE_SERVER = 'localhost:9083'
DEFAULT_NAMENODE_ADDR = None
-DEFAULT_TEST_PROTOCOL = os.getenv('DEFAULT_TEST_PROTOCOL', 'beeswax')
+DEFAULT_TEST_PROTOCOL = os.getenv('DEFAULT_TEST_PROTOCOL', BEESWAX)
if FILESYSTEM == 'isilon':
DEFAULT_NAMENODE_ADDR = "{node}:{port}".format(node=os.getenv("ISILON_NAMENODE"),
port=ISILON_WEBHDFS_PORT)
@@ -188,7 +191,7 @@ def pytest_addoption(parser):
"calcite_report_mode. Defaults to ${IMPALA_LOGS_DIR}/calcite_report.")
parser.addoption("--default_test_protocol", default=DEFAULT_TEST_PROTOCOL,
- choices=("beeswax", "hs2", "hs2-http"),
+ choices=VALID_TEST_PROTOCOLS,
help="Impala protocol to run test if test does not specify any.")
@@ -239,6 +242,9 @@ def pytest_generate_tests(metafunc):
If a test has the 'vector' fixture specified, this code is invoked and it will build
a set of test vectors to parameterize the test with.
"""
+ # All Impala tests classes must inherit BaseTestSuite.
+ assert issubclass(metafunc.cls, tests.common.base_test_suite.BaseTestSuite)
+ assert metafunc.cls.default_test_protocol() in VALID_TEST_PROTOCOLS
if 'vector' in metafunc.fixturenames:
metafunc.cls.add_test_dimensions()
vectors = metafunc.cls.ImpalaTestMatrix.generate_test_vectors(
@@ -698,8 +704,9 @@ def pytest_collection_modifyitems(items, config, session):
# We must modify the items list in place for it to take effect.
items[:] = items_selected
- logging.info("pytest shard selection enabled %s. Of %d items, selected %d items by hash.",
- config.option.shard_tests, num_items, len(items))
+ logging.info(
+ "pytest shard selection enabled %s. Of %d items, selected %d items by hash.",
+ config.option.shard_tests, num_items, len(items))
@pytest.hookimpl(trylast=True)
diff --git a/tests/custom_cluster/test_kudu.py b/tests/custom_cluster/test_kudu.py
index 10b4d6af5..e712b7421 100644
--- a/tests/custom_cluster/test_kudu.py
+++ b/tests/custom_cluster/test_kudu.py
@@ -26,7 +26,7 @@ from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
from tests.common.kudu_test_suite import KuduTestSuite
from tests.common.skip import SkipIfKudu, SkipIfBuildType, SkipIf
-from tests.common.test_dimensions import add_mandatory_exec_option
+from tests.common.test_dimensions import BEESWAX, add_mandatory_exec_option
from tests.common.test_result_verifier import error_msg_expected
from tests.util.event_processor_utils import EventProcessorUtils
@@ -40,6 +40,13 @@ class CustomKuduTest(CustomClusterTestSuite, KuduTestSuite):
def get_workload(cls):
return 'functional-query'
+ @classmethod
+ def default_test_protocol(cls):
+ # run_test_case() can produce different result types between beeswax vs hs2 protocol
+ # in some tests. This fix the test to use beeswax protocol until we can migrate
+ # to hs2.
+ return BEESWAX
+
@classmethod
def add_custom_cluster_constraints(cls):
# Override this method to relax the set of constraints added in
diff --git a/tests/hs2/test_fetch.py b/tests/hs2/test_fetch.py
index 1d64d49b3..4392ed5eb 100644
--- a/tests/hs2/test_fetch.py
+++ b/tests/hs2/test_fetch.py
@@ -20,9 +20,6 @@ from __future__ import absolute_import, division, print_function
import pytest
import re
-from time import sleep
-from time import time
-from tests.common.errors import Timeout
from tests.hs2.hs2_test_suite import (HS2TestSuite, needs_session,
create_op_handle_without_secret)
from TCLIService import TCLIService, constants
@@ -36,15 +33,14 @@ class TestFetch(HS2TestSuite):
assert hs2_type.typeDesc.types[0].primitiveEntry.type == expected_type
def __verify_char_max_len(self, t, max_len):
- l = t.typeDesc.types[0].primitiveEntry.typeQualifiers.qualifiers\
- [constants.CHARACTER_MAXIMUM_LENGTH]
- assert l.i32Value == max_len
+ qualifiers = t.typeDesc.types[0].primitiveEntry.typeQualifiers.qualifiers
+ i32val = qualifiers[constants.CHARACTER_MAXIMUM_LENGTH].i32Value
+ assert i32val == max_len
def __verify_decimal_precision_scale(self, hs2_type, precision, scale):
- p = hs2_type.typeDesc.types[0].primitiveEntry.typeQualifiers.qualifiers\
- [constants.PRECISION]
- s = hs2_type.typeDesc.types[0].primitiveEntry.typeQualifiers.qualifiers\
- [constants.SCALE]
+ qualifiers = hs2_type.typeDesc.types[0].primitiveEntry.typeQualifiers.qualifiers
+ p = qualifiers[constants.PRECISION]
+ s = qualifiers[constants.SCALE]
assert p.i32Value == precision
assert s.i32Value == scale
@@ -199,8 +195,9 @@ class TestFetch(HS2TestSuite):
self.column_results_to_string(fetch_results_resp.results.columns)
assert num_rows == 25
# Match whether stats are computed or not
- assert re.match(
- r"2009, 1, -?\d+, -?\d+, \d*\.?\d+KB, NOT CACHED, NOT CACHED, TEXT", result) is not None
+ pattern = r"2009, 1, -?\d+, -?\d+, \d*\.?\d+KB, NOT CACHED, NOT CACHED, TEXT"
+ assert re.match(pattern, result) is not None, \
+ 'Looking for "{}" but none found:\n{}'.format(pattern, result)
@needs_session()
def test_show_column_stats(self):
diff --git a/tests/infra/test_thrift_parser.py b/tests/infra/test_thrift_parser.py
index bc196818c..7fcec9d6c 100644
--- a/tests/infra/test_thrift_parser.py
+++ b/tests/infra/test_thrift_parser.py
@@ -19,20 +19,24 @@ from __future__ import absolute_import
import os
from subprocess import check_output
+from tests.common.base_test_suite import BaseTestSuite
-def test_thrift_parser():
- """Make sure thrift_parser can parse all the thrift files"""
- impala_home = os.getenv("IMPALA_HOME")
- # Only supports python3
- output = check_output([os.path.join(impala_home, "bin/impala-python3"),
- os.path.join(impala_home, "bin/jenkins/thrift_parser.py"),
- os.path.join(impala_home, "common/thrift")],
- universal_newlines=True)
- assert "TStatus" in output
- assert "Field1 required ErrorCodes.TErrorCode status_code" in output
- assert "Field2 optional list<string> error_msgs" in output
- assert "TErrorCode" in output
- assert "EnumItem OK" in output
- assert "EnumItem UNUSED" in output
- assert "EnumItem GENERAL" in output
- assert "EnumItem CANCELLED" in output
+
+class TestThriftParser(BaseTestSuite):
+
+ def test_thrift_parser(self):
+ """Make sure thrift_parser can parse all the thrift files"""
+ impala_home = os.getenv("IMPALA_HOME")
+ # Only supports python3
+ output = check_output([os.path.join(impala_home, "bin/impala-python3"),
+ os.path.join(impala_home, "bin/jenkins/thrift_parser.py"),
+ os.path.join(impala_home, "common/thrift")],
+ universal_newlines=True)
+ assert "TStatus" in output
+ assert "Field1 required ErrorCodes.TErrorCode status_code" in output
+ assert "Field2 optional list<string> error_msgs" in output
+ assert "TErrorCode" in output
+ assert "EnumItem OK" in output
+ assert "EnumItem UNUSED" in output
+ assert "EnumItem GENERAL" in output
+ assert "EnumItem CANCELLED" in output
diff --git a/tests/infra/test_utils.py b/tests/infra/test_utils.py
index 8c93770cf..e1d3b564f 100644
--- a/tests/infra/test_utils.py
+++ b/tests/infra/test_utils.py
@@ -18,51 +18,53 @@
# This module contains tests for some of the tests/util code.
from __future__ import absolute_import, division, print_function
+
+from tests.common.base_test_suite import BaseTestSuite
from tests.util.filesystem_utils import prepend_with_fs
from tests.util.parse_util import get_bytes_summary_stats_counter, \
get_time_summary_stats_counter
-def test_filesystem_utils():
- # Verify that empty FS prefix gives back the same path.
- path = "/fake-warehouse"
- assert prepend_with_fs("", path) == path
-
- # Verify that prepend_with_fs() is idempotent.
- fs = "fakeFs://bucket"
- path = "/fake-warehouse"
- assert prepend_with_fs(fs, path) == fs + path
- assert prepend_with_fs(fs, prepend_with_fs(fs, path)) == fs + path
+class TestUtils(BaseTestSuite):
+ def test_filesystem_utils(self):
+ # Verify that empty FS prefix gives back the same path.
+ path = "/fake-warehouse"
+ assert prepend_with_fs("", path) == path
-def test_get_bytes_summary_stats_counter():
- """Test get_bytes_summary_stats_counter(counter_name, runtime_profile) using a dummy
- runtime profile.
- """
- runtime_profile = "- ExampleCounter: (Avg: 8.00 KB (8192) ; " \
- "Min: 6.00 KB (6144) ; " \
- "Max: 10.00 KB (10240) ; " \
- "Number of samples: 4)"
- summary_stats = get_bytes_summary_stats_counter("ExampleCounter",
- runtime_profile)
- assert len(summary_stats) == 1
- assert summary_stats[0].sum == 32768 and summary_stats[0].min_value == 6144 and \
- summary_stats[0].max_value == 10240 and summary_stats[0].total_num_values == 4
+ # Verify that prepend_with_fs() is idempotent.
+ fs = "fakeFs://bucket"
+ path = "/fake-warehouse"
+ assert prepend_with_fs(fs, path) == fs + path
+ assert prepend_with_fs(fs, prepend_with_fs(fs, path)) == fs + path
+ def test_get_bytes_summary_stats_counter(self):
+ """Test get_bytes_summary_stats_counter(counter_name, runtime_profile) using a dummy
+ runtime profile.
+ """
+ runtime_profile = "- ExampleCounter: (Avg: 8.00 KB (8192) ; " \
+ "Min: 6.00 KB (6144) ; " \
+ "Max: 10.00 KB (10240) ; " \
+ "Number of samples: 4)"
+ summary_stats = get_bytes_summary_stats_counter("ExampleCounter",
+ runtime_profile)
+ assert len(summary_stats) == 1
+ assert summary_stats[0].sum == 32768 and summary_stats[0].min_value == 6144 and \
+ summary_stats[0].max_value == 10240 and summary_stats[0].total_num_values == 4
-def test_get_time_summary_stats_counter():
- """Test get_time_summary_stats_counter(counter_name, runtime_profile) using a dummy
- runtime profile.
- """
- # This is constructed to test the parsing logic for timestamps, so the number don't
- # add up.
- runtime_profile = "- ExampleTimeStats: (Avg: 161.554ms ; " \
- "Min: 101.411us ; " \
- "Max: 1h2m3s4ms5us6ns ; " \
- "Number of samples: 6)"
- summary_stats = get_time_summary_stats_counter("ExampleTimeStats", runtime_profile)
- assert len(summary_stats) == 1
- assert summary_stats[0].sum == 969324000
- assert summary_stats[0].min_value == 101411
- assert summary_stats[0].max_value == 3723004005006
- assert summary_stats[0].total_num_values == 6
+ def test_get_time_summary_stats_counter(self):
+ """Test get_time_summary_stats_counter(counter_name, runtime_profile) using a dummy
+ runtime profile.
+ """
+ # This is constructed to test the parsing logic for timestamps, so the number don't
+ # add up.
+ runtime_profile = "- ExampleTimeStats: (Avg: 161.554ms ; " \
+ "Min: 101.411us ; " \
+ "Max: 1h2m3s4ms5us6ns ; " \
+ "Number of samples: 6)"
+ summary_stats = get_time_summary_stats_counter("ExampleTimeStats", runtime_profile)
+ assert len(summary_stats) == 1
+ assert summary_stats[0].sum == 969324000
+ assert summary_stats[0].min_value == 101411
+ assert summary_stats[0].max_value == 3723004005006
+ assert summary_stats[0].total_num_values == 6
diff --git a/tests/query_test/test_kudu.py b/tests/query_test/test_kudu.py
index bd1be1233..1fb09871b 100644
--- a/tests/query_test/test_kudu.py
+++ b/tests/query_test/test_kudu.py
@@ -41,15 +41,18 @@ import re
import textwrap
import threading
import time
-from datetime import datetime, date
+from datetime import datetime
from pytz import utc
from tests.common.environ import ImpalaTestClusterProperties, HIVE_MAJOR_VERSION
from tests.common.kudu_test_suite import KuduTestSuite
from tests.common.impala_cluster import ImpalaCluster
from tests.common.skip import SkipIfNotHdfsMinicluster, SkipIfKudu, SkipIfHive2
-from tests.common.test_dimensions import (add_exec_option_dimension,
- add_mandatory_exec_option)
+from tests.common.test_dimensions import (
+ HS2,
+ add_exec_option_dimension,
+ add_mandatory_exec_option,
+ create_single_exec_option_dimension)
from tests.verifiers.metric_verifier import MetricVerifier
KUDU_MASTER_HOSTS = pytest.config.option.kudu_master_hosts
@@ -58,6 +61,24 @@ IMPALA_TEST_CLUSTER_PROPERTIES = ImpalaTestClusterProperties.get_instance()
LOG = logging.getLogger(__name__)
+def create_external_kudu_query(db, tbl, kudu_tbl_name, other_props=None):
+ props = {'kudu.table_name': kudu_tbl_name}
+ if other_props is not None:
+ props.update(other_props)
+ props_str = ', '.join(["'{}'='{}'".format(k, v) for k, v in props.items()])
+ return (
+ """
+ CREATE EXTERNAL TABLE {}.{}
+ STORED AS KUDU
+ TBLPROPERTIES ({})""").format(db, tbl, props_str)
+
+
+def set_latest_observed_option(vector, kudu_client):
+ vector.set_exec_option(
+ 'kudu_snapshot_read_timestamp_micros',
+ to_unixtime_micros(kudu_client.latest_observed_timestamp()))
+
+
# TODO(IMPALA-8614): parameterize some tests to run with HMS integration enabled.
class TestKuduBasicDML(KuduTestSuite):
"""
@@ -127,7 +148,7 @@ class TestKuduTimestampConvert(KuduTestSuite):
new_vector = deepcopy(vector)
del new_vector.get_value('exec_option')['timezone'] # .test file sets timezone
self.run_test_case('QueryTest/kudu_runtime_filter_with_timestamp_conversion',
- new_vector)
+ new_vector)
# TODO(IMPALA-8614): parameterize some tests to run with HMS integration enabled.
@@ -139,45 +160,48 @@ class TestKuduOperations(KuduTestSuite):
@classmethod
def add_test_dimensions(cls):
super(TestKuduOperations, cls).add_test_dimensions()
+ cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
# The default read mode of READ_LATEST does not provide high enough consistency for
# these tests.
add_mandatory_exec_option(cls, "kudu_read_mode", "READ_AT_SNAPSHOT")
@SkipIfKudu.no_hybrid_clock()
@SkipIfKudu.hms_integration_enabled()
- def test_out_of_range_timestamps(self, vector, cursor, kudu_client, unique_database):
+ def test_out_of_range_timestamps(self, vector, kudu_client, unique_database):
"""Test timestamp values that are outside of Impala's supported date range."""
- cursor.execute("set kudu_read_mode=READ_AT_SNAPSHOT")
- cursor.execute("""CREATE TABLE %s.times (a INT PRIMARY KEY, ts TIMESTAMP)
- PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % unique_database)
+ tbl = 'times'
+ self.execute_query_using_vector(
+ ("CREATE TABLE {}.{} (a INT PRIMARY KEY, ts TIMESTAMP)"
+ "PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU").format(unique_database, tbl),
+ vector)
assert kudu_client.table_exists(
- KuduTestSuite.to_kudu_table_name(unique_database, "times"))
+ KuduTestSuite.to_kudu_table_name(unique_database, tbl))
- table = kudu_client.table(KuduTestSuite.to_kudu_table_name(unique_database, "times"))
+ table = kudu_client.table(KuduTestSuite.to_kudu_table_name(unique_database, tbl))
session = kudu_client.new_session()
session.apply(table.new_insert((0, datetime(1987, 5, 19, 0, 0, tzinfo=utc))))
# Add a date before 1400
session.apply(table.new_insert((1, datetime(1300, 1, 1, 0, 0, tzinfo=utc))))
# TODO: Add a date after 9999. There isn't a way to represent a date greater than
# 9999 in Python datetime.
- #session.apply(table.new_insert((2, datetime(12000, 1, 1, 0, 0, tzinfo=utc))))
+ # session.apply(table.new_insert((2, datetime(12000, 1, 1, 0, 0, tzinfo=utc))))
session.flush()
- cursor.execute("set kudu_snapshot_read_timestamp_micros=%s" %
- to_unixtime_micros(kudu_client.latest_observed_timestamp()))
+ set_latest_observed_option(vector, kudu_client)
# TODO: The test driver should have a way to specify query options in an 'options'
# section rather than having to split abort_on_error cases into separate files.
- vector.get_value('exec_option')['abort_on_error'] = 0
+ vector.set_exec_option('abort_on_error', 0)
self.run_test_case('QueryTest/kudu-overflow-ts', vector,
- use_db=unique_database)
+ use_db=unique_database)
- vector.get_value('exec_option')['abort_on_error'] = 1
+ vector.set_exec_option('abort_on_error', 1)
self.run_test_case('QueryTest/kudu-overflow-ts-abort-on-error', vector,
- use_db=unique_database)
+ use_db=unique_database)
@SkipIfKudu.no_hybrid_clock()
def test_kudu_scan_node(self, vector, unique_database):
self.run_test_case('QueryTest/kudu-scan-node', vector, use_db=unique_database)
+
@SkipIfNotHdfsMinicluster.tuned_for_minicluster
@SkipIfKudu.no_hybrid_clock()
def test_kudu_insert_mem_limit(self, vector, unique_database):
@@ -207,7 +231,7 @@ class TestKuduOperations(KuduTestSuite):
def test_kudu_limit(self, vector, unique_database):
self.run_test_case('QueryTest/kudu_limit', vector, use_db=unique_database)
- def test_kudu_column_options(self, cursor, kudu_client, unique_database):
+ def test_kudu_column_options(self, vector, kudu_client, unique_database):
"""Test Kudu column options"""
encodings = ["ENCODING PLAIN_ENCODING", ""]
compressions = ["COMPRESSION SNAPPY", ""]
@@ -221,30 +245,39 @@ class TestKuduOperations(KuduTestSuite):
for blocksize in blocksizes:
for nullable in nullability:
impala_tbl_name = "test_column_options_%s" % str(indx)
- cursor.execute("""CREATE TABLE %s.%s (a INT PRIMARY KEY
- %s %s %s %s, b INT %s %s %s %s %s) PARTITION BY HASH (a)
- PARTITIONS 3 STORED AS KUDU""" % (unique_database, impala_tbl_name,
- encoding, compression, default, blocksize, nullable, encoding,
- compression, default, blocksize))
+ self.execute_query_using_vector(
+ """
+ CREATE TABLE {}.{} (
+ a INT PRIMARY KEY {} {} {} {},
+ b INT {} {} {} {} {}
+ ) PARTITION BY HASH (a) PARTITIONS 3 STORED AS KUDU""".format(
+ unique_database, impala_tbl_name,
+ encoding, compression, default, blocksize,
+ nullable, encoding, compression, default, blocksize),
+ vector)
indx = indx + 1
assert kudu_client.table_exists(
KuduTestSuite.to_kudu_table_name(unique_database, impala_tbl_name))
- def test_kudu_col_changed(
- self, cursor, kudu_client, unique_database, cluster_properties):
+ def test_kudu_col_changed(self, vector, kudu_client, unique_database,
+ cluster_properties):
"""Test changing a Kudu column outside of Impala results in a failure on read with
outdated metadata (IMPALA-4828)."""
- cursor.execute("set kudu_read_mode=READ_AT_SNAPSHOT")
- cursor.execute("""CREATE TABLE %s.foo (a INT PRIMARY KEY, s STRING)
- PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % unique_database)
+ tbl = 'foo'
+ db_tbl = unique_database + '.' + tbl
+ self.execute_query_using_vector(
+ """
+ CREATE TABLE {}.{} (a INT PRIMARY KEY, s STRING)
+ PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""".format(unique_database, tbl),
+ vector)
assert kudu_client.table_exists(
- KuduTestSuite.to_kudu_table_name(unique_database, "foo"))
+ KuduTestSuite.to_kudu_table_name(unique_database, tbl))
# Force metadata to be loaded on impalads
- cursor.execute("select * from %s.foo" % (unique_database))
+ self.execute_query_using_vector("select * from %s" % db_tbl, vector)
# Load the table via the Kudu client and change col 's' to be a different type.
- table = kudu_client.table(KuduTestSuite.to_kudu_table_name(unique_database, "foo"))
+ table = kudu_client.table(KuduTestSuite.to_kudu_table_name(unique_database, tbl))
alterer = kudu_client.new_table_alterer(table)
alterer.drop_column("s")
table = alterer.alter()
@@ -259,40 +292,42 @@ class TestKuduOperations(KuduTestSuite):
session.apply(op)
session.flush()
- cursor.execute("set kudu_snapshot_read_timestamp_micros=%s" %
- to_unixtime_micros(kudu_client.latest_observed_timestamp()))
+ set_latest_observed_option(vector, kudu_client)
# Scanning should result in an error with Catalog V1, since the metadata is cached.
try:
- cursor.execute("SELECT * FROM %s.foo" % (unique_database))
- assert cluster_properties.is_catalog_v2_cluster(),\
+ self.execute_query_using_vector("SELECT * FROM %s" % db_tbl, vector)
+ assert cluster_properties.is_catalog_v2_cluster(), \
"Should fail with Catalog V1, which caches metadata"
except Exception as e:
- assert not cluster_properties.is_catalog_v2_cluster(),\
+ assert not cluster_properties.is_catalog_v2_cluster(), \
"Should succeed with Catalog V2, which does not cache metadata"
expected_error = "Column 's' is type INT but Impala expected STRING. The table "\
"metadata in Impala may be outdated and need to be refreshed."
assert expected_error in str(e)
# After a REFRESH the scan should succeed
- cursor.execute("REFRESH %s.foo" % (unique_database))
- cursor.execute("SELECT * FROM %s.foo" % (unique_database))
- assert len(cursor.fetchall()) == 100
+ self.execute_query_using_vector("REFRESH %s" % db_tbl, vector)
+ result = self.execute_query_using_vector("SELECT * FROM %s" % db_tbl, vector)
+ assert len(result.data) == 100
- def test_kudu_col_not_null_changed(
- self, cursor, kudu_client, unique_database, cluster_properties):
+ def test_kudu_col_not_null_changed(self, vector, kudu_client, unique_database,
+ cluster_properties):
"""Test changing a NOT NULL Kudu column outside of Impala results in a failure
on read with outdated metadata (IMPALA-4828)."""
- cursor.execute("set kudu_read_mode=READ_AT_SNAPSHOT")
- cursor.execute("""CREATE TABLE %s.foo (a INT PRIMARY KEY, s STRING NOT NULL)
- PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % unique_database)
+ tbl = 'foo'
+ db_tbl = unique_database + '.' + tbl
+ self.execute_query_using_vector(
+ """CREATE TABLE %s (a INT PRIMARY KEY, s STRING NOT NULL)
+ PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % db_tbl,
+ vector)
assert kudu_client.table_exists(
- KuduTestSuite.to_kudu_table_name(unique_database, "foo"))
+ KuduTestSuite.to_kudu_table_name(unique_database, tbl))
# Force metadata to be loaded on impalads
- cursor.execute("select * from %s.foo" % (unique_database))
+ self.execute_query_using_vector("select * from %s" % db_tbl, vector)
# Load the table via the Kudu client and change col 's' to be a different type.
- table = kudu_client.table(KuduTestSuite.to_kudu_table_name(unique_database, "foo"))
+ table = kudu_client.table(KuduTestSuite.to_kudu_table_name(unique_database, tbl))
alterer = kudu_client.new_table_alterer(table)
alterer.drop_column("s")
table = alterer.alter()
@@ -307,15 +342,14 @@ class TestKuduOperations(KuduTestSuite):
session.apply(op)
session.flush()
- cursor.execute("set kudu_snapshot_read_timestamp_micros=%s" %
- to_unixtime_micros(kudu_client.latest_observed_timestamp()))
+ set_latest_observed_option(vector, kudu_client)
# Scanning should result in an error
try:
- cursor.execute("SELECT * FROM %s.foo" % (unique_database))
- assert cluster_properties.is_catalog_v2_cluster(),\
+ self.execute_query_using_vector("SELECT * FROM %s" % db_tbl, vector)
+ assert cluster_properties.is_catalog_v2_cluster(), \
"Should fail with Catalog V1, which caches metadata"
except Exception as e:
- assert not cluster_properties.is_catalog_v2_cluster(),\
+ assert not cluster_properties.is_catalog_v2_cluster(), \
"Should succeed with Catalog V2, which does not cache metadata"
expected_error = "Column 's' is nullable but Impala expected it to be "\
"not nullable. The table metadata in Impala may be outdated and need to be "\
@@ -323,25 +357,27 @@ class TestKuduOperations(KuduTestSuite):
assert expected_error in str(e)
# After a REFRESH the scan should succeed
- cursor.execute("REFRESH %s.foo" % (unique_database))
- cursor.execute("SELECT * FROM %s.foo" % (unique_database))
- assert len(cursor.fetchall()) == 100
+ self.execute_query_using_vector("REFRESH %s" % db_tbl, vector)
+ result = self.execute_query_using_vector("SELECT * FROM %s" % db_tbl, vector)
+ assert len(result.data) == 100
- def test_kudu_col_null_changed(
- self, cursor, kudu_client, unique_database, cluster_properties):
+ def test_kudu_col_null_changed(self, vector, kudu_client, unique_database,
+ cluster_properties):
"""Test changing a NULL Kudu column outside of Impala results in a failure
on read with outdated metadata (IMPALA-4828)."""
- cursor.execute("set kudu_read_mode=READ_AT_SNAPSHOT")
- cursor.execute("""CREATE TABLE %s.foo (a INT PRIMARY KEY, s STRING NULL)
- PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % unique_database)
+ tbl = 'foo'
+ db_tbl = unique_database + '.' + tbl
+ self.execute_query_using_vector(
+ """CREATE TABLE %s (a INT PRIMARY KEY, s STRING NULL)
+ PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % db_tbl, vector)
assert kudu_client.table_exists(
- KuduTestSuite.to_kudu_table_name(unique_database, "foo"))
+ KuduTestSuite.to_kudu_table_name(unique_database, tbl))
# Force metadata to be loaded on impalads
- cursor.execute("select * from %s.foo" % (unique_database))
+ self.execute_query_using_vector("select * from %s" % db_tbl, vector)
# Load the table via the Kudu client and change col 's' to be a different type.
- table = kudu_client.table(KuduTestSuite.to_kudu_table_name(unique_database, "foo"))
+ table = kudu_client.table(KuduTestSuite.to_kudu_table_name(unique_database, tbl))
alterer = kudu_client.new_table_alterer(table)
alterer.drop_column("s")
table = alterer.alter()
@@ -352,19 +388,18 @@ class TestKuduOperations(KuduTestSuite):
# Add some rows
session = kudu_client.new_session()
for i in range(100):
- op = table.new_insert((i, "foo"))
+ op = table.new_insert((i, "bar"))
session.apply(op)
session.flush()
- cursor.execute("set kudu_snapshot_read_timestamp_micros=%s" %
- to_unixtime_micros(kudu_client.latest_observed_timestamp()))
+ set_latest_observed_option(vector, kudu_client)
# Scanning should result in an error
try:
- cursor.execute("SELECT * FROM %s.foo" % (unique_database))
- assert cluster_properties.is_catalog_v2_cluster(),\
+ self.execute_query_using_vector("SELECT * FROM %s" % db_tbl, vector)
+ assert cluster_properties.is_catalog_v2_cluster(), \
"Should fail with Catalog V1, which caches metadata"
except Exception as e:
- assert not cluster_properties.is_catalog_v2_cluster(),\
+ assert not cluster_properties.is_catalog_v2_cluster(), \
"Should succeed with Catalog V2, which does not cache metadata"
expected_error = "Column 's' is not nullable but Impala expected it to be "\
"nullable. The table metadata in Impala may be outdated and need to be "\
@@ -372,23 +407,25 @@ class TestKuduOperations(KuduTestSuite):
assert expected_error in str(e)
# After a REFRESH the scan should succeed
- cursor.execute("REFRESH %s.foo" % (unique_database))
- cursor.execute("SELECT * FROM %s.foo" % (unique_database))
- assert len(cursor.fetchall()) == 100
+ self.execute_query_using_vector("REFRESH %s" % db_tbl, vector)
+ result = self.execute_query_using_vector("SELECT * FROM %s" % db_tbl, vector)
+ assert len(result.data) == 100
- def test_kudu_col_added(self, cursor, kudu_client, unique_database, cluster_properties):
+ def test_kudu_col_added(self, vector, kudu_client, unique_database,
+ cluster_properties):
"""Test adding a Kudu column outside of Impala."""
- cursor.execute("set kudu_read_mode=READ_AT_SNAPSHOT")
- cursor.execute("""CREATE TABLE %s.foo (a INT PRIMARY KEY)
- PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % unique_database)
+ tbl = 'foo'
+ db_tbl = unique_database + '.' + tbl
+ self.execute_query_using_vector("""CREATE TABLE %s (a INT PRIMARY KEY)
+ PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % db_tbl, vector)
assert kudu_client.table_exists(
- KuduTestSuite.to_kudu_table_name(unique_database, "foo"))
+ KuduTestSuite.to_kudu_table_name(unique_database, tbl))
# Force metadata to be loaded on impalads
- cursor.execute("select * from %s.foo" % (unique_database))
+ self.execute_query_using_vector("select * from %s" % db_tbl, vector)
# Load the table via the Kudu client and add a new col
- table = kudu_client.table(KuduTestSuite.to_kudu_table_name(unique_database, "foo"))
+ table = kudu_client.table(KuduTestSuite.to_kudu_table_name(unique_database, tbl))
alterer = kudu_client.new_table_alterer(table)
alterer.add_column("b", "int32")
table = alterer.alter()
@@ -399,57 +436,57 @@ class TestKuduOperations(KuduTestSuite):
session.apply(op)
session.flush()
- cursor.execute("set kudu_snapshot_read_timestamp_micros=%s" %
- to_unixtime_micros(kudu_client.latest_observed_timestamp()))
- cursor.execute("SELECT * FROM %s.foo" % (unique_database))
+ set_latest_observed_option(vector, kudu_client)
+ result = self.execute_query_using_vector("SELECT * FROM %s" % db_tbl, vector)
if cluster_properties.is_catalog_v2_cluster():
# Changes in Kudu should be immediately visible to Impala with Catalog V2.
- assert cursor.fetchall() == [(0, 0)]
+ assert result.data[0].split('\t') == ['0', '0']
else:
# Only the first col is visible to Impala. Impala will not know about the missing
# column, so '*' is expanded to known columns. This doesn't have a separate check
# because the query can proceed and checking would need to fetch metadata from the
# Kudu master, which is what REFRESH is for.
- assert cursor.fetchall() == [(0, )]
+ assert result.data[0].split('\t') == ['0']
# After a REFRESH both cols should be visible
- cursor.execute("REFRESH %s.foo" % (unique_database))
- cursor.execute("SELECT * FROM %s.foo" % (unique_database))
- assert cursor.fetchall() == [(0, 0)]
+ self.execute_query_using_vector("REFRESH %s" % db_tbl, vector)
+ result = self.execute_query_using_vector("SELECT * FROM %s" % db_tbl, vector)
+ assert result.data[0].split('\t') == ['0', '0']
@SkipIfKudu.no_hybrid_clock()
@SkipIfKudu.hms_integration_enabled()
- def test_kudu_col_removed(self, cursor, kudu_client, unique_database):
+ def test_kudu_col_removed(self, vector, kudu_client, unique_database):
"""Test removing a Kudu column outside of Impala."""
- cursor.execute("set kudu_read_mode=READ_AT_SNAPSHOT")
- cursor.execute("""CREATE TABLE %s.foo (a INT PRIMARY KEY, s STRING)
- PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % unique_database)
+ tbl = 'foo'
+ db_tbl = unique_database + '.' + tbl
+ self.execute_query_using_vector("""CREATE TABLE %s (a INT PRIMARY KEY, s STRING)
+ PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % db_tbl, vector)
assert kudu_client.table_exists(
- KuduTestSuite.to_kudu_table_name(unique_database, "foo"))
+ KuduTestSuite.to_kudu_table_name(unique_database, tbl))
# Force metadata to be loaded on impalads
- cursor.execute("select * from %s.foo" % (unique_database))
- cursor.execute("insert into %s.foo values (0, 'foo')" % (unique_database))
+ self.execute_query_using_vector("select * from %s" % db_tbl, vector)
+ self.execute_query_using_vector("insert into %s values (0, 'foo')" % db_tbl, vector)
# Load the table via the Kudu client and change col 's' to be a different type.
- table = kudu_client.table(KuduTestSuite.to_kudu_table_name(unique_database, "foo"))
+ table = kudu_client.table(KuduTestSuite.to_kudu_table_name(unique_database, tbl))
alterer = kudu_client.new_table_alterer(table)
alterer.drop_column("s")
table = alterer.alter()
# Scanning should result in an error
try:
- cursor.execute("SELECT * FROM %s.foo" % (unique_database))
+ self.execute_query_using_vector("SELECT * FROM %s" % db_tbl, vector)
except Exception as e:
expected_error = "Column 's' not found in kudu table impala::test_kudu_col_removed"
assert expected_error in str(e)
# After a REFRESH the scan should succeed
- cursor.execute("REFRESH %s.foo" % (unique_database))
- cursor.execute("SELECT * FROM %s.foo" % (unique_database))
- assert cursor.fetchall() == [(0, )]
+ self.execute_query_using_vector("REFRESH %s" % db_tbl, vector)
+ result = self.execute_query_using_vector("SELECT * FROM %s" % db_tbl, vector)
+ assert result.data == ['0']
- def test_kudu_show_unbounded_range_partition(self, cursor, kudu_client,
+ def test_kudu_show_unbounded_range_partition(self, vector, kudu_client,
unique_database):
"""Check that a single unbounded range partition gets printed correctly."""
schema_builder = SchemaBuilder()
@@ -457,48 +494,44 @@ class TestKuduOperations(KuduTestSuite):
column_spec.nullable(False)
schema_builder.set_primary_keys(["id"])
schema = schema_builder.build()
-
- name = unique_database + ".unbounded_range_table"
+ tbl = 'unbounded_range_table'
+ name = unique_database + "." + tbl
try:
- kudu_client.create_table(name, schema,
- partitioning=Partitioning().set_range_partition_columns(["id"]))
+ kudu_client.create_table(
+ name, schema, partitioning=Partitioning().set_range_partition_columns(["id"]))
kudu_table = kudu_client.table(name)
- impala_table_name = self.get_kudu_table_base_name(kudu_table.name)
- props = "TBLPROPERTIES('kudu.table_name'='%s')" % kudu_table.name
- cursor.execute("CREATE EXTERNAL TABLE %s STORED AS KUDU %s" % (impala_table_name,
- props))
- with self.drop_impala_table_after_context(cursor, impala_table_name):
- cursor.execute("SHOW RANGE PARTITIONS %s" % impala_table_name)
- assert cursor.description == [
- ('RANGE (id)', 'STRING', None, None, None, None, None)]
- assert cursor.fetchall() == [('UNBOUNDED',)]
+ self.execute_query_using_vector(
+ create_external_kudu_query(unique_database, tbl, kudu_table.name), vector)
+ result = self.execute_query("SHOW RANGE PARTITIONS %s" % name)
+ assert result.column_labels == ['RANGE (ID)']
+ assert result.column_types == ['STRING']
+ assert result.data == ['UNBOUNDED']
finally:
if kudu_client.table_exists(name):
kudu_client.delete_table(name)
@SkipIfKudu.no_hybrid_clock()
- def test_column_storage_attributes(self, cursor, unique_database):
+ def test_column_storage_attributes(self, vector, unique_database):
"""Tests that for every valid combination of column type, encoding, and compression,
we can insert a value and scan it back from Kudu."""
# This test takes about 2min and is unlikely to break, so only run it in exhaustive.
if self.exploration_strategy() != 'exhaustive':
pytest.skip("Only runs in exhaustive to reduce core time.")
table_name = "%s.storage_attrs" % unique_database
- types = ['boolean', 'tinyint', 'smallint', 'int', 'bigint', 'float', 'double', \
- 'string', 'timestamp', 'decimal', 'date', 'varchar(10)']
- cursor.execute("set kudu_read_mode=READ_AT_SNAPSHOT")
+ types = ['boolean', 'tinyint', 'smallint', 'int', 'bigint', 'float', 'double',
+ 'string', 'timestamp', 'decimal', 'date', 'varchar(10)']
create_query = "create table %s (id int primary key" % table_name
for t in types:
# We truncate the type attributes in the column name to keep things simple.
create_query += ", %s_col %s" % (t.split('(')[0], t)
create_query += ") partition by hash(id) partitions 16 stored as kudu"
- cursor.execute(create_query)
+ self.execute_query_using_vector(create_query, vector)
- encodings = ['AUTO_ENCODING', 'PLAIN_ENCODING', 'PREFIX_ENCODING', 'GROUP_VARINT', \
- 'RLE', 'DICT_ENCODING', 'BIT_SHUFFLE']
+ encodings = ['AUTO_ENCODING', 'PLAIN_ENCODING', 'PREFIX_ENCODING', 'GROUP_VARINT',
+ 'RLE', 'DICT_ENCODING', 'BIT_SHUFFLE']
compressions = ['DEFAULT_COMPRESSION', 'NO_COMPRESSION', 'SNAPPY', 'LZ4', 'ZLIB']
i = 0
for e in encodings:
@@ -506,34 +539,45 @@ class TestKuduOperations(KuduTestSuite):
for t in types:
try:
# We truncate the type attributes in the column name to keep things simple.
- cursor.execute("""alter table %s alter column %s_col
- set encoding %s compression %s""" % (table_name, t.split('(')[0], e, c))
+ self.execute_query_using_vector(
+ """
+ alter table {} alter column {}_col
+ set encoding {} compression {}""".format(
+ table_name, t.split('(')[0], e, c),
+ vector)
except Exception as err:
assert "encoding %s not supported for type" % e in str(err)
- cursor.execute("""insert into %s values (%s, true, 0, 0, 0, 0, 0, 0, '0',
- cast('2009-01-01' as timestamp), cast(0 as decimal),
- cast('2010-01-01' as date), cast('' as varchar(10)))""" % (table_name, i))
- cursor.execute("select * from %s where id = %s" % (table_name, i))
- assert cursor.fetchall() == \
- [(i, True, 0, 0, 0, 0, 0.0, 0.0, '0', datetime(2009, 1, 1, 0, 0), 0,
- date(2010, 1, 1), '')]
+ self.execute_query_using_vector(
+ """insert into %s values (%s, true, 0, 0, 0, 0, 0, 0, '0',
+ cast('2009-01-01' as timestamp), cast(0 as decimal),
+ cast('2010-01-01' as date), cast('' as varchar(10)))""" % (table_name, i),
+ vector)
+ result = self.execute_query_using_vector(
+ "select * from %s where id = %s" % (table_name, i), vector)
+ # Beeswax return 'true' while HS2 return 'True'. Lowercase the result.
+ assert result.data[0].lower().split('\t') == [
+ str(i), 'true', '0', '0', '0', '0', '0', '0', '0',
+ '2009-01-01 00:00:00', '0', '2010-01-01', '']
i += 1
- cursor.execute("select count(*) from %s" % table_name)
- print(cursor.fetchall() == [(i, )])
+ result = self.execute_query_using_vector(
+ "select count(*) from %s" % table_name, vector)
+ print(result.data == [str(i)])
- def test_concurrent_schema_change(self, cursor, unique_database):
+ def test_concurrent_schema_change(self, vector, unique_database):
"""Tests that an insert into a Kudu table with a concurrent schema change either
succeeds or fails gracefully."""
table_name = "%s.test_schema_change" % unique_database
- cursor.execute("""create table %s (col0 bigint primary key, col1 bigint)
- partition by hash(col0) partitions 16 stored as kudu""" % table_name)
+ self.execute_query_using_vector(
+ """create table %s (col0 bigint primary key, col1 bigint)
+ partition by hash(col0) partitions 16 stored as kudu""" % table_name, vector)
iters = 5
+
def insert_values():
threading.current_thread().errors = []
- client = self.create_impala_client()
+ client = self.create_impala_client_from_vector(vector)
for i in range(0, iters):
- time.sleep(random.random()) # sleeps for up to one second
+ time.sleep(random.random()) # sleeps for up to one second
try:
client.execute("insert into %s values (0, 0), (1, 1)" % table_name)
except Exception as e:
@@ -543,12 +587,15 @@ class TestKuduOperations(KuduTestSuite):
insert_thread.start()
for i in range(0, iters):
- time.sleep(random.random()) # sleeps for up to one second
- cursor.execute("alter table %s drop column col1" % table_name)
+ time.sleep(random.random()) # sleeps for up to one second
+ self.execute_query_using_vector(
+ "alter table %s drop column col1" % table_name, vector)
if i % 2 == 0:
- cursor.execute("alter table %s add columns (col1 string)" % table_name)
+ self.execute_query_using_vector(
+ "alter table %s add columns (col1 string)" % table_name, vector)
else:
- cursor.execute("alter table %s add columns (col1 bigint)" % table_name)
+ self.execute_query_using_vector(
+ "alter table %s add columns (col1 bigint)" % table_name, vector)
insert_thread.join()
@@ -568,50 +615,57 @@ class TestKuduOperations(KuduTestSuite):
]
assert any(err in msg for err in possible_errors)
- def _retry_query(self, cursor, query, expected):
+ def _retry_query(self, query, vector, expected):
retries = 0
while retries < 3:
- cursor.execute(query)
- result = cursor.fetchall()
- if result == expected:
+ result = self.execute_query_using_vector(query, vector)
+ if result.data == expected:
break
retries += 1
time.sleep(1)
assert retries < 3, \
"Did not get a correct result for %s after 3 retries: %s" % (query, result)
- def test_read_modes(self, cursor, unique_database):
+ def test_read_modes(self, vector, unique_database):
"""Other Kudu tests are run with a scan level of READ_AT_SNAPSHOT to have predicable
scan results. This test verifies that scans work as expected at the scan level of
READ_LATEST by retrying the scan if the results are incorrect."""
+ vector.set_exec_option('kudu_read_mode', 'READ_LATEST')
table_name = "%s.test_read_latest" % unique_database
- cursor.execute("set kudu_read_mode=READ_LATEST")
- cursor.execute("""create table %s (a int primary key, b string) partition by hash(a)
- partitions 8 stored as kudu""" % table_name)
- cursor.execute("insert into %s values (0, 'a'), (1, 'b'), (2, 'c')" % table_name)
- self._retry_query(cursor, "select * from %s order by a" % table_name,
- [(0, 'a'), (1, 'b'), (2, 'c')])
- cursor.execute("""insert into %s select id, string_col from functional.alltypes
- where id > 2 limit 100""" % table_name)
- self._retry_query(cursor, "select count(*) from %s" % table_name, [(103,)])
-
- def test_replica_selection(self, cursor, unique_database):
+ self.execute_query_using_vector(
+ """create table %s (a int primary key, b string)
+ partition by hash(a)
+ partitions 8 stored as kudu""" % table_name, vector)
+ self.execute_query_using_vector(
+ "insert into %s values (0, 'a'), (1, 'b'), (2, 'c')" % table_name, vector)
+ self._retry_query("select * from %s order by a" % table_name, vector,
+ ['0\ta', '1\tb', '2\tc'])
+ self.execute_query_using_vector(
+ """insert into %s select id, string_col from functional.alltypes
+ where id > 2 limit 100""" % table_name, vector)
+ self._retry_query("select count(*) from %s" % table_name, vector, ['103'])
+
+ def test_replica_selection(self, vector, unique_database):
"""This test verifies that scans work as expected with different replica selection.
"""
table_name = "%s.replica_selection" % unique_database
- cursor.execute("set kudu_read_mode=READ_AT_SNAPSHOT")
- cursor.execute("""create table %s (a int primary key, b string) partition by hash(a)
- partitions 8 stored as kudu""" % table_name)
- cursor.execute("""insert into %s select id, string_col from functional.alltypes
- limit 100""" % table_name)
+ self.execute_query_using_vector(
+ """create table %s (a int primary key, b string)
+ partition by hash(a)
+ partitions 8 stored as kudu""" % table_name, vector)
+ self.execute_query_using_vector(
+ """insert into %s select id, string_col from functional.alltypes
+ limit 100""" % table_name, vector)
- cursor.execute("set kudu_replica_selection=LEADER_ONLY")
- cursor.execute("select count(*) from %s" % table_name)
- assert cursor.fetchall() == [(100,)]
+ vector.set_exec_option('kudu_replica_selection', 'LEADER_ONLY')
+ result = self.execute_query_using_vector(
+ "select count(*) from %s" % table_name, vector)
+ assert result.data == ['100']
- cursor.execute("set kudu_replica_selection=CLOSEST_REPLICA")
- cursor.execute("select count(*) from %s" % table_name)
- assert cursor.fetchall() == [(100,)]
+ vector.set_exec_option('kudu_replica_selection', 'CLOSEST_REPLICA')
+ result = self.execute_query_using_vector(
+ "select count(*) from %s" % table_name, vector)
+ assert result.data == ['100']
class TestKuduPartitioning(KuduTestSuite):
@@ -620,8 +674,7 @@ class TestKuduPartitioning(KuduTestSuite):
# Test both the interpreted and the codegen'd path.
super(TestKuduPartitioning, cls).add_test_dimensions()
- def test_partitions_evenly_distributed(self, vector, cursor,
- kudu_client, unique_database):
+ def test_partitions_evenly_distributed(self, vector, kudu_client, unique_database):
"""Sanity check for KuduPartitionExpr. We insert numbers into a table and check that
inserted elements are distributed evenly among the partitions. The assumption here is
that the source distribution is more or less uniform and that hashing retains this
@@ -630,17 +683,16 @@ class TestKuduPartitioning(KuduTestSuite):
table_name = "partitioning"
table_full_name = unique_database + ".partitioning"
- cursor.execute("""CREATE TABLE %s (a INT PRIMARY KEY)
- PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % table_full_name)
+ self.execute_query("""CREATE TABLE %s (a INT PRIMARY KEY)
+ PARTITION BY HASH(a) PARTITIONS 3 STORED AS KUDU""" % table_full_name,
+ vector.get_exec_option_dict())
assert kudu_client.table_exists(KuduTestSuite.to_kudu_table_name(
unique_database, table_name))
query = "INSERT INTO %s SELECT id FROM functional.alltypes" % table_full_name
- exec_options = dict((k, str(v)) for k, v
- in vector.get_value('exec_option').items())
- cursor.execute(query, configuration=exec_options)
+ result = self.execute_query(query, vector.get_exec_option_dict())
- profile = cursor.get_profile()
+ profile = result.runtime_profile
numbers = TestKuduPartitioning.extract_kudu_rows_from_profile(profile)
TestKuduPartitioning.assert_rows_evenly_distributed(numbers)
@@ -666,21 +718,26 @@ class TestKuduPartitioning(KuduTestSuite):
within_kudu_table_sink_section = False
for line in profile.splitlines():
- if within_kudu_table_sink_section:
- match = total_num_rows_re.search(line)
- if match:
- res.append(int(match.group(1)))
- within_kudu_table_sink_section = False
- else:
- if kudu_table_sink in line:
- within_kudu_table_sink_section = True
+ if within_kudu_table_sink_section:
+ match = total_num_rows_re.search(line)
+ if match:
+ res.append(int(match.group(1)))
+ within_kudu_table_sink_section = False
+ else:
+ if kudu_table_sink in line:
+ within_kudu_table_sink_section = True
return res
class TestCreateExternalTable(KuduTestSuite):
+ @classmethod
+ def add_test_dimensions(cls):
+ super(TestCreateExternalTable, cls).add_test_dimensions()
+ cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
+
@SkipIfKudu.hms_integration_enabled()
- def test_external_timestamp_default_value(self, cursor, kudu_client, unique_database):
+ def test_external_timestamp_default_value(self, vector, kudu_client, unique_database):
"""Checks that a Kudu table created outside Impala with a default value on a
UNIXTIME_MICROS column can be loaded by Impala, and validates the DESCRIBE
output is correct."""
@@ -691,194 +748,205 @@ class TestCreateExternalTable(KuduTestSuite):
column_spec.default(datetime(2009, 1, 1, 0, 0, tzinfo=utc))
schema_builder.set_primary_keys(["id"])
schema = schema_builder.build()
- name = unique_database + ".tsdefault"
+ tbl = 'tsdefault'
+ name = unique_database + "." + tbl
try:
- kudu_client.create_table(name, schema,
- partitioning=Partitioning().set_range_partition_columns(["id"]))
+ kudu_client.create_table(
+ name, schema, partitioning=Partitioning().set_range_partition_columns(["id"]))
kudu_table = kudu_client.table(name)
- impala_table_name = self.get_kudu_table_base_name(kudu_table.name)
- props = "TBLPROPERTIES('kudu.table_name'='%s')" % kudu_table.name
- cursor.execute("CREATE EXTERNAL TABLE %s STORED AS KUDU %s" % (impala_table_name,
- props))
- with self.drop_impala_table_after_context(cursor, impala_table_name):
- cursor.execute("DESCRIBE %s" % impala_table_name)
- table_desc = [[col.strip() if col else col for col in row] for row in cursor]
- # Pytest shows truncated output on failure, so print the details just in case.
- LOG.info(table_desc)
- assert ["ts", "timestamp", "", "false", "", "true", "1230768000000000",
- "AUTO_ENCODING", "DEFAULT_COMPRESSION", "0"] in table_desc
+
+ self.execute_query_using_vector(
+ create_external_kudu_query(unique_database, tbl, kudu_table.name), vector)
+ result = self.execute_query_using_vector("DESCRIBE %s" % name, vector)
+ table_desc = [[col.strip() if col else col for col in row.split('\t')]
+ for row in result.data]
+ # Pytest shows truncated output on failure, so print the details just in case.
+ LOG.info(table_desc)
+ assert ["ts", "timestamp", "", "false", "", "true", "1230768000000000",
+ "AUTO_ENCODING", "DEFAULT_COMPRESSION", "0"] in table_desc
finally:
if kudu_client.table_exists(name):
kudu_client.delete_table(name)
@SkipIfKudu.hms_integration_enabled()
- def test_implicit_table_props(self, cursor, kudu_client):
+ def test_implicit_table_props(self, vector, kudu_client, unique_database):
"""Check that table properties added internally during table creation are as
expected.
"""
- with self.temp_kudu_table(kudu_client, [STRING, INT8, BOOL], num_key_cols=2) \
- as kudu_table:
+ with self.temp_kudu_table(
+ kudu_client, [STRING, INT8, BOOL], num_key_cols=2) as kudu_table:
impala_table_name = self.get_kudu_table_base_name(kudu_table.name)
- props = "TBLPROPERTIES('kudu.table_name'='%s')" % kudu_table.name
- cursor.execute("CREATE EXTERNAL TABLE %s STORED AS KUDU %s" % (impala_table_name,
- props))
- with self.drop_impala_table_after_context(cursor, impala_table_name):
- cursor.execute("DESCRIBE FORMATTED %s" % impala_table_name)
- table_desc = [[col.strip() if col else col for col in row] for row in cursor]
- LOG.info(table_desc)
- # Pytest shows truncated output on failure, so print the details just in case.
- assert ["", "EXTERNAL", "TRUE"] in table_desc
- assert ["", "kudu.master_addresses", KUDU_MASTER_HOSTS] in table_desc
- assert ["", "kudu.table_name", kudu_table.name] in table_desc
- assert ["", "storage_handler", "org.apache.hadoop.hive.kudu.KuduStorageHandler"] \
- in table_desc
+
+ self.execute_query_using_vector(
+ create_external_kudu_query(unique_database, impala_table_name, kudu_table.name),
+ vector)
+ result = self.execute_query_using_vector(
+ "DESCRIBE FORMATTED %s.%s" % (unique_database, impala_table_name), vector)
+ table_desc = [[col.strip() if col else col for col in row.split('\t')]
+ for row in result.data]
+ LOG.info(table_desc)
+ # Pytest shows truncated output on failure, so print the details just in case.
+ assert ["", "EXTERNAL", "TRUE"] in table_desc
+ assert ["", "kudu.master_addresses", KUDU_MASTER_HOSTS] in table_desc
+ assert ["", "kudu.table_name", kudu_table.name] in table_desc
+ assert ["", "storage_handler", "org.apache.hadoop.hive.kudu.KuduStorageHandler"] \
+ in table_desc
@SkipIfKudu.hms_integration_enabled()
- def test_col_types(self, cursor, kudu_client):
+ def test_col_types(self, vector, kudu_client, unique_database):
"""Check that a table can be created using all available column types."""
# TODO: Add DECIMAL when the Kudu python client supports decimal
kudu_types = [STRING, BOOL, DOUBLE, FLOAT, INT16, INT32, INT64, INT8, BINARY,
UNIXTIME_MICROS, DATE]
with self.temp_kudu_table(kudu_client, kudu_types) as kudu_table:
impala_table_name = self.get_kudu_table_base_name(kudu_table.name)
- props = "TBLPROPERTIES('kudu.table_name'='%s')" % kudu_table.name
- cursor.execute("CREATE EXTERNAL TABLE %s STORED AS KUDU %s" % (impala_table_name,
- props))
- with self.drop_impala_table_after_context(cursor, impala_table_name):
- cursor.execute("DESCRIBE %s" % impala_table_name)
- kudu_schema = kudu_table.schema
- for i, (col_name, col_type, _, _, _, _, _, _, _, _) in enumerate(cursor):
- kudu_col = kudu_schema[i]
- assert col_name == kudu_col.name
- assert col_type.upper() == \
- self.kudu_col_type_to_impala_col_type(kudu_col.type.type)
+
+ self.execute_query_using_vector(
+ create_external_kudu_query(unique_database, impala_table_name, kudu_table.name),
+ vector)
+ result = self.execute_query_using_vector(
+ "DESCRIBE %s.%s" % (unique_database, impala_table_name), vector)
+ kudu_schema = kudu_table.schema
+ for i, row in enumerate(result.data):
+ cols = row.split('\t')
+ kudu_col = kudu_schema[i]
+ assert cols[0] == kudu_col.name
+ assert cols[1].upper() == \
+ self.kudu_col_type_to_impala_col_type(kudu_col.type.type)
@SkipIfKudu.hms_integration_enabled()
- def test_drop_external_table(self, cursor, kudu_client):
+ def test_drop_external_table(self, vector, kudu_client, unique_database):
"""Check that dropping an external table only affects the catalog and does not delete
the table in Kudu.
"""
with self.temp_kudu_table(kudu_client, [INT32]) as kudu_table:
impala_table_name = self.get_kudu_table_base_name(kudu_table.name)
- props = "TBLPROPERTIES('kudu.table_name'='%s')" % kudu_table.name
- cursor.execute("CREATE EXTERNAL TABLE %s STORED AS KUDU %s" % (impala_table_name,
- props))
- with self.drop_impala_table_after_context(cursor, impala_table_name):
- cursor.execute("SELECT COUNT(*) FROM %s" % impala_table_name)
- assert cursor.fetchall() == [(0, )]
+
+ self.execute_query_using_vector(
+ create_external_kudu_query(unique_database, impala_table_name, kudu_table.name),
+ vector)
+ result = self.execute_query_using_vector(
+ "SELECT COUNT(*) FROM %s.%s" % (unique_database, impala_table_name), vector)
+ assert result.data == ['0']
+ result = self.execute_query_using_vector(
+ "DROP TABLE %s.%s" % (unique_database, impala_table_name), vector)
+ assert result.success
try:
- cursor.execute("SELECT COUNT(*) FROM %s" % impala_table_name)
+ self.execute_query_using_vector(
+ "SELECT COUNT(*) FROM %s.%s" % (unique_database, impala_table_name), vector)
assert False
except Exception as e:
assert "Could not resolve table reference" in str(e)
assert kudu_client.table_exists(kudu_table.name)
@SkipIfKudu.hms_integration_enabled()
- def test_explicit_name(self, cursor, kudu_client):
+ def test_explicit_name(self, vector, kudu_client, unique_database):
"""Check that a Kudu table can be specified using a table property."""
with self.temp_kudu_table(kudu_client, [INT32]) as kudu_table:
table_name = self.random_table_name()
- cursor.execute("""
- CREATE EXTERNAL TABLE %s
- STORED AS KUDU
- TBLPROPERTIES('kudu.table_name' = '%s')""" % (table_name, kudu_table.name))
- with self.drop_impala_table_after_context(cursor, table_name):
- cursor.execute("SELECT * FROM %s" % table_name)
- assert len(cursor.fetchall()) == 0
+
+ self.execute_query_using_vector(
+ create_external_kudu_query(unique_database, table_name, kudu_table.name),
+ vector)
+ result = self.execute_query_using_vector(
+ "SELECT * FROM %s.%s" % (unique_database, table_name), vector)
+ assert len(result.data) == 0
@SkipIfKudu.hms_integration_enabled()
- def test_explicit_name_preference(self, cursor, kudu_client):
+ def test_explicit_name_preference(self, vector, kudu_client, unique_database):
"""Check that the table name from a table property is used when a table of the
implied name also exists.
"""
with self.temp_kudu_table(kudu_client, [INT64]) as preferred_kudu_table:
with self.temp_kudu_table(kudu_client, [INT8]) as other_kudu_table:
impala_table_name = self.get_kudu_table_base_name(other_kudu_table.name)
- cursor.execute("""
- CREATE EXTERNAL TABLE %s
- STORED AS KUDU
- TBLPROPERTIES('kudu.table_name' = '%s')""" % (
- impala_table_name, preferred_kudu_table.name))
- with self.drop_impala_table_after_context(cursor, impala_table_name):
- cursor.execute("DESCRIBE %s" % impala_table_name)
- assert cursor.fetchall() == \
- [("a", "bigint", "", "true", "true", "false", "", "AUTO_ENCODING",
- "DEFAULT_COMPRESSION", "0")]
+
+ self.execute_query_using_vector(
+ create_external_kudu_query(
+ unique_database, impala_table_name, preferred_kudu_table.name),
+ vector)
+ result = self.execute_query_using_vector(
+ "DESCRIBE %s.%s" % (unique_database, impala_table_name), vector)
+ assert result.data[0].split('\t') == \
+ ["a", "bigint", "", "true", "true", "false", "", "AUTO_ENCODING",
+ "DEFAULT_COMPRESSION", "0"]
@SkipIfKudu.hms_integration_enabled()
- def test_explicit_name_doesnt_exist(self, cursor, kudu_client):
+ def test_explicit_name_doesnt_exist(self, vector, unique_database):
kudu_table_name = self.random_table_name()
try:
- cursor.execute("""
- CREATE EXTERNAL TABLE %s
- STORED AS KUDU
- TBLPROPERTIES('kudu.table_name' = '%s')""" % (
- self.random_table_name(), kudu_table_name))
+ self.execute_query_using_vector(
+ create_external_kudu_query(
+ unique_database, self.random_table_name(), kudu_table_name),
+ vector)
assert False
except Exception as e:
assert "Table does not exist in Kudu: '%s'" % kudu_table_name in str(e)
@SkipIfKudu.hms_integration_enabled()
- def test_explicit_name_doesnt_exist_but_implicit_does(self, cursor, kudu_client):
+ def test_explicit_name_doesnt_exist_but_implicit_does(self, vector, kudu_client,
+ unique_database):
"""Check that when an explicit table name is given but that table doesn't exist,
there is no fall-through to an existing implicit table.
"""
with self.temp_kudu_table(kudu_client, [INT32]) as kudu_table:
table_name = self.random_table_name()
try:
- cursor.execute("""
- CREATE EXTERNAL TABLE %s
- STORED AS KUDU
- TBLPROPERTIES('kudu.table_name' = '%s')""" % (
- self.get_kudu_table_base_name(kudu_table.name), table_name))
+ self.execute_query_using_vector(
+ create_external_kudu_query(
+ unique_database, self.get_kudu_table_base_name(kudu_table.name), table_name),
+ vector)
assert False
except Exception as e:
assert "Table does not exist in Kudu: '%s'" % table_name in str(e)
@SkipIfKudu.no_hybrid_clock()
@SkipIfKudu.hms_integration_enabled()
- def test_table_without_partitioning(self, cursor, kudu_client, unique_database):
+ def test_table_without_partitioning(self, vector, kudu_client, unique_database):
"""Test a Kudu table created without partitioning (i.e. equivalent to a single
unbounded partition). It is not possible to create such a table in Impala, but
it can be created directly in Kudu and then loaded as an external table.
Regression test for IMPALA-5154."""
- cursor.execute("set kudu_read_mode=READ_AT_SNAPSHOT")
+ vector.set_exec_option('kudu_read_mode', 'READ_AT_SNAPSHOT')
schema_builder = SchemaBuilder()
column_spec = schema_builder.add_column("id", INT64)
column_spec.nullable(False)
schema_builder.set_primary_keys(["id"])
schema = schema_builder.build()
partitioning = Partitioning().set_range_partition_columns([])
- name = "%s.one_big_unbounded_partition" % unique_database
+ tbl = 'one_big_unbounded_partition'
+ name = "%s.%s" % (unique_database, tbl)
try:
kudu_client.create_table(name, schema, partitioning=partitioning)
- kudu_table = kudu_client.table(name)
- props = "TBLPROPERTIES('kudu.table_name'='%s')" % name
- cursor.execute("CREATE EXTERNAL TABLE %s STORED AS KUDU %s" % (name, props))
- with self.drop_impala_table_after_context(cursor, name):
- cursor.execute("INSERT INTO %s VALUES (1), (2), (3)" % name)
- cursor.execute("SELECT COUNT(*) FROM %s" % name)
- assert cursor.fetchall() == [(3, )]
- try:
- cursor.execute("SHOW RANGE PARTITIONS %s" % name)
- assert False
- except Exception as e:
- assert "AnalysisException: SHOW RANGE PARTITIONS requested but table does "\
- "not have range partitions" in str(e)
+ self.execute_query_using_vector(
+ create_external_kudu_query(unique_database, tbl, name), vector)
+ self.execute_query_using_vector(
+ "INSERT INTO %s VALUES (1), (2), (3)" % name, vector)
+ result = self.execute_query_using_vector(
+ "SELECT COUNT(*) FROM %s" % name, vector)
+ assert result.data == ['3']
+ try:
+ self.execute_query_using_vector(
+ "SHOW RANGE PARTITIONS %s" % name, vector)
+ assert False
+ except Exception as e:
+ assert "AnalysisException: SHOW RANGE PARTITIONS requested but table does "\
+ "not have range partitions" in str(e)
finally:
if kudu_client.table_exists(name):
kudu_client.delete_table(name)
@SkipIfKudu.no_hybrid_clock()
@SkipIfKudu.hms_integration_enabled()
- def test_column_name_case(self, cursor, kudu_client, unique_database):
+ def test_column_name_case(self, vector, kudu_client, unique_database):
"""IMPALA-5286: Tests that an external Kudu table that was created with a column name
containing upper case letters is handled correctly."""
- cursor.execute("set kudu_read_mode=READ_AT_SNAPSHOT")
- table_name = '%s.kudu_external_test' % unique_database
+ vector.set_exec_option('kudu_read_mode', 'READ_AT_SNAPSHOT')
+ tbl = 'kudu_external_test'
+ table_name = '%s.%s' % (unique_database, tbl)
if kudu_client.table_exists(table_name):
kudu_client.delete_table(table_name)
@@ -892,48 +960,58 @@ class TestCreateExternalTable(KuduTestSuite):
try:
kudu_client.create_table(table_name, schema, partitioning)
- props = "tblproperties('kudu.table_name' = '%s')" % table_name
- cursor.execute("create external table %s stored as kudu %s" % (table_name, props))
+ self.execute_query_using_vector(
+ create_external_kudu_query(unique_database, tbl, table_name), vector)
# Perform a variety of operations on the table.
- cursor.execute("insert into %s (kEy) values (5), (1), (4)" % table_name)
- cursor.execute("select keY from %s where KeY %% 2 = 0" % table_name)
- assert cursor.fetchall() == [(4, )]
- cursor.execute("select * from %s order by kEY" % (table_name))
- assert cursor.fetchall() == [(1, ), (4, ), (5, )]
+ self.execute_query_using_vector(
+ "insert into %s (kEy) values (5), (1), (4)" % table_name, vector)
+ result = self.execute_query_using_vector(
+ "select keY from %s where KeY %% 2 = 0" % table_name, vector)
+ assert result.data == ['4']
+ result = self.execute_query_using_vector(
+ "select * from %s order by kEY" % table_name, vector)
+ assert result.data == ['1', '4', '5']
# Do a join with a runtime filter targeting the column.
- cursor.execute("select count(*) from %s a, %s b where a.key = b.key" %
- (table_name, table_name))
- assert cursor.fetchall() == [(3, )]
+ result = self.execute_query_using_vector(
+ "select count(*) from %s a, %s b where a.key = b.key" % (table_name, table_name),
+ vector)
+ assert result.data == ['3']
- cursor.execute("alter table %s add range partition 11 < values < 20" % table_name)
+ self.execute_query_using_vector(
+ "alter table %s add range partition 11 < values < 20" % table_name, vector)
new_key = "KEY2"
- cursor.execute("alter table %s change KEy %s bigint" % (table_name, new_key))
+ self.execute_query_using_vector(
+ "alter table %s change KEy %s bigint" % (table_name, new_key),
+ vector)
val_col = "vaL"
- cursor.execute("alter table %s add columns (%s bigint)" % (table_name, val_col))
+ self.execute_query_using_vector(
+ "alter table %s add columns (%s bigint)" % (table_name, val_col), vector)
- cursor.execute("describe %s" % table_name)
- results = cursor.fetchall()
+ result = self.execute_query_using_vector("describe %s" % table_name, vector)
# 'describe' should print the column name in lower case.
- assert new_key.lower() in results[0]
- assert val_col.lower() in results[1]
+ assert new_key.lower() in result.data[0]
+ assert val_col.lower() in result.data[1]
- cursor.execute("alter table %s drop column Val" % table_name);
- cursor.execute("describe %s" % table_name)
- assert len(cursor.fetchall()) == 1
+ self.execute_query_using_vector(
+ "alter table %s drop column Val" % table_name, vector)
+ result = self.execute_query_using_vector("describe %s" % table_name, vector)
+ assert len(result.data) == 1
- cursor.execute("alter table %s drop range partition 11 < values < 20" % table_name)
+ self.execute_query_using_vector(
+ "alter table %s drop range partition 11 < values < 20" % table_name, vector)
finally:
if kudu_client.table_exists(table_name):
kudu_client.delete_table(table_name)
@SkipIfKudu.hms_integration_enabled()
- def test_conflicting_column_name(self, cursor, kudu_client, unique_database):
+ def test_conflicting_column_name(self, vector, kudu_client, unique_database):
"""IMPALA-5283: Tests that loading an external Kudu table that was created with column
names that differ only in case results in an error."""
- table_name = '%s.kudu_external_test' % unique_database
+ tbl = 'kudu_external_test'
+ table_name = '%s.%s' % (unique_database, tbl)
if kudu_client.table_exists(table_name):
kudu_client.delete_table(table_name)
@@ -949,8 +1027,8 @@ class TestCreateExternalTable(KuduTestSuite):
try:
kudu_client.create_table(table_name, schema, partitioning)
- props = "tblproperties('kudu.table_name' = '%s')" % table_name
- cursor.execute("create external table %s stored as kudu %s" % (table_name, props))
+ self.execute_query_using_vector(
+ create_external_kudu_query(unique_database, tbl, table_name), vector)
assert False, 'create table should have resulted in an exception'
except Exception as e:
assert 'Error loading Kudu table: Impala does not support column names that ' \
@@ -963,19 +1041,25 @@ class TestCreateExternalTable(KuduTestSuite):
class TestShowCreateTable(KuduTestSuite):
column_properties = "ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION"
- def assert_show_create_equals(self, cursor, create_sql, show_create_sql,
- do_exact_match=False):
+ def assert_show_create_equals(self, db_name, create_sql, show_create_sql,
+ do_exact_match=False, extra_args=None):
"""Executes 'create_sql' to create a table, then runs "SHOW CREATE TABLE" and checks
that the output is the same as 'show_create_sql'. 'create_sql' and
'show_create_sql' can be templates that can be used with str.format(). format()
- will be called with 'table' and 'db' as keyword args. Also, compares HMS-3 specific
- output due to HMS translation. If do_exact_match is True does not manipulate the
- output and compares exactly with the show_create_sql parameter.
+ will be called with 'table', 'db', 'kudu_addr', and 'extra_args' entries as
+ keyword args. Also, compares HMS-3 specific output due to HMS translation.
+ If do_exact_match is True does not manipulate the output and compares exactly
+ with the show_create_sql parameter.
"""
- format_args = {"table": self.random_table_name(), "db": cursor.conn.db_name}
- cursor.execute(create_sql.format(**format_args))
- cursor.execute("SHOW CREATE TABLE {table}".format(**format_args))
- output = cursor.fetchall()[0][0]
+ format_args = {
+ "table": self.random_table_name(),
+ "db": db_name,
+ "kudu_addr": KUDU_MASTER_HOSTS}
+ if extra_args is not None:
+ format_args.update(extra_args)
+ self.execute_query(create_sql.format(**format_args))
+ result = self.execute_query("SHOW CREATE TABLE {db}.{table}".format(**format_args))
+ output = result.data[0]
if not do_exact_match and HIVE_MAJOR_VERSION > 2:
# in case of HMS-3 all Kudu tables are translated to external tables with some
# additional properties. This code below makes sure that we have the expected table
@@ -994,139 +1078,159 @@ class TestShowCreateTable(KuduTestSuite):
textwrap.dedent(show_create_sql.format(**format_args)).strip()
@SkipIfKudu.hms_integration_enabled()
- def test_primary_key_and_distribution(self, cursor):
+ def test_primary_key_and_distribution(self, unique_database):
# TODO: Add case with BLOCK_SIZE
- self.assert_show_create_equals(cursor,
- """
- CREATE TABLE {table} (c INT PRIMARY KEY)
- PARTITION BY HASH (c) PARTITIONS 3 STORED AS KUDU""",
- """
- CREATE TABLE {db}.{{table}} (
- c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- PRIMARY KEY (c)
- )
- PARTITION BY HASH (c) PARTITIONS 3
- STORED AS KUDU
- TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', 'kudu.master_addresses'='{kudu_addr}')"""
- .format(db=cursor.conn.db_name, kudu_addr=KUDU_MASTER_HOSTS))
- self.assert_show_create_equals(cursor,
- """
- CREATE TABLE {table} (c INT PRIMARY KEY, d STRING NULL)
- PARTITION BY HASH (c) PARTITIONS 3, RANGE (c)
- (PARTITION VALUES <= 1, PARTITION 1 < VALUES <= 2,
- PARTITION 2 < VALUES) STORED AS KUDU""",
- """
- CREATE TABLE {db}.{{table}} (
- c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- d STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- PRIMARY KEY (c)
- )
- PARTITION BY HASH (c) PARTITIONS 3, RANGE (c) (...)
- STORED AS KUDU
- TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', 'kudu.master_addresses'='{kudu_addr}')"""
- .format(db=cursor.conn.db_name, kudu_addr=KUDU_MASTER_HOSTS))
- self.assert_show_create_equals(cursor,
- """
- CREATE TABLE {table} (c INT ENCODING PLAIN_ENCODING, PRIMARY KEY (c))
- PARTITION BY HASH (c) PARTITIONS 3 STORED AS KUDU""",
- """
- CREATE TABLE {db}.{{table}} (
- c INT NOT NULL ENCODING PLAIN_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- PRIMARY KEY (c)
- )
- PARTITION BY HASH (c) PARTITIONS 3
- STORED AS KUDU
- TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', 'kudu.master_addresses'='{kudu_addr}')"""
- .format(db=cursor.conn.db_name, kudu_addr=KUDU_MASTER_HOSTS))
- self.assert_show_create_equals(cursor,
- """
- CREATE TABLE {table} (c INT COMPRESSION LZ4, d STRING, PRIMARY KEY(c, d))
- PARTITION BY HASH (c) PARTITIONS 3, HASH (d) PARTITIONS 3,
- RANGE (c, d) (PARTITION VALUE = (1, 'aaa'), PARTITION VALUE = (2, 'bbb'))
- STORED AS KUDU""",
- """
- CREATE TABLE {db}.{{table}} (
- c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION LZ4,
- d STRING NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- PRIMARY KEY (c, d)
- )
- PARTITION BY HASH (c) PARTITIONS 3, HASH (d) PARTITIONS 3, RANGE (c, d) (...)
- STORED AS KUDU
- TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', 'kudu.master_addresses'='{kudu_addr}')"""
- .format(db=cursor.conn.db_name, kudu_addr=KUDU_MASTER_HOSTS))
- self.assert_show_create_equals(cursor,
- """
- CREATE TABLE {table} (c INT, d STRING, e INT NULL DEFAULT 10, PRIMARY KEY(c, d))
- PARTITION BY RANGE (c) (PARTITION VALUES <= 1, PARTITION 1 < VALUES <= 2,
- PARTITION 2 < VALUES <= 3, PARTITION 3 < VALUES) STORED AS KUDU""",
- """
- CREATE TABLE {db}.{{table}} (
- c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- d STRING NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- e INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION DEFAULT 10,
- PRIMARY KEY (c, d)
- )
- PARTITION BY RANGE (c) (...)
- STORED AS KUDU
- TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', 'kudu.master_addresses'='{kudu_addr}')"""
- .format(db=cursor.conn.db_name, kudu_addr=KUDU_MASTER_HOSTS))
- self.assert_show_create_equals(cursor,
- """
- CREATE TABLE {table} (c INT PRIMARY KEY) STORED AS KUDU""",
- """
- CREATE TABLE {db}.{{table}} (
- c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- PRIMARY KEY (c)
- )
- STORED AS KUDU
- TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', 'kudu.master_addresses'='{kudu_addr}')"""
- .format(db=cursor.conn.db_name, kudu_addr=KUDU_MASTER_HOSTS))
- self.assert_show_create_equals(cursor,
- """
- CREATE TABLE {table} (c INT COMMENT 'Ab 1@' PRIMARY KEY) STORED AS KUDU""",
- """
- CREATE TABLE {db}.{{table}} (
- c INT NOT NULL {p} COMMENT 'Ab 1@',
- PRIMARY KEY (c)
- )
- STORED AS KUDU
- TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', 'kudu.master_addresses'='{kudu_addr}')"""
- .format(db=cursor.conn.db_name, p=self.column_properties,
- kudu_addr=KUDU_MASTER_HOSTS))
+ self.assert_show_create_equals(
+ unique_database,
+ """
+ CREATE TABLE {db}.{table} (c INT PRIMARY KEY)
+ PARTITION BY HASH (c) PARTITIONS 3 STORED AS KUDU""",
+ """
+ CREATE TABLE {db}.{table} (
+ c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ PRIMARY KEY (c)
+ )
+ PARTITION BY HASH (c) PARTITIONS 3
+ STORED AS KUDU
+ TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', """
+ """'kudu.master_addresses'='{kudu_addr}')""")
+ self.assert_show_create_equals(
+ unique_database,
+ """
+ CREATE TABLE {db}.{table} (c INT PRIMARY KEY, d STRING NULL)
+ PARTITION BY HASH (c) PARTITIONS 3, RANGE (c)
+ (PARTITION VALUES <= 1, PARTITION 1 < VALUES <= 2,
+ PARTITION 2 < VALUES) STORED AS KUDU""",
+ """
+ CREATE TABLE {db}.{table} (
+ c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ d STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ PRIMARY KEY (c)
+ )
+ PARTITION BY HASH (c) PARTITIONS 3, RANGE (c) (...)
+ STORED AS KUDU
+ TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', """
+ """'kudu.master_addresses'='{kudu_addr}')""")
+ self.assert_show_create_equals(
+ unique_database,
+ """
+ CREATE TABLE {db}.{table} (c INT ENCODING PLAIN_ENCODING, PRIMARY KEY (c))
+ PARTITION BY HASH (c) PARTITIONS 3 STORED AS KUDU""",
+ """
+ CREATE TABLE {db}.{table} (
+ c INT NOT NULL ENCODING PLAIN_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ PRIMARY KEY (c)
+ )
+ PARTITION BY HASH (c) PARTITIONS 3
+ STORED AS KUDU
+ TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', """
+ """'kudu.master_addresses'='{kudu_addr}')""")
+ self.assert_show_create_equals(
+ unique_database,
+ """
+ CREATE TABLE {db}.{table} (c INT COMPRESSION LZ4, d STRING, PRIMARY KEY(c, d))
+ PARTITION BY HASH (c) PARTITIONS 3, HASH (d) PARTITIONS 3,
+ RANGE (c, d) (PARTITION VALUE = (1, 'aaa'), PARTITION VALUE = (2, 'bbb'))
+ STORED AS KUDU""",
+ """
+ CREATE TABLE {db}.{table} (
+ c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION LZ4,
+ d STRING NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ PRIMARY KEY (c, d)
+ )
+ PARTITION BY HASH (c) PARTITIONS 3, HASH (d) PARTITIONS 3, RANGE (c, d) (...)
+ STORED AS KUDU
+ TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', """
+ """'kudu.master_addresses'='{kudu_addr}')""")
+ self.assert_show_create_equals(
+ unique_database,
+ """
+ CREATE TABLE {db}.{table} (c INT, d STRING, e INT NULL DEFAULT 10, """
+ """PRIMARY KEY(c, d))
+ PARTITION BY RANGE (c) (PARTITION VALUES <= 1, PARTITION 1 < VALUES <= 2,
+ PARTITION 2 < VALUES <= 3, PARTITION 3 < VALUES) STORED AS KUDU""",
+ """
+ CREATE TABLE {db}.{table} (
+ c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ d STRING NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ e INT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION DEFAULT 10,
+ PRIMARY KEY (c, d)
+ )
+ PARTITION BY RANGE (c) (...)
+ STORED AS KUDU
+ TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', """
+ """'kudu.master_addresses'='{kudu_addr}')""")
+ self.assert_show_create_equals(
+ unique_database,
+ """
+ CREATE TABLE {db}.{table} (c INT PRIMARY KEY) STORED AS KUDU""",
+ """
+ CREATE TABLE {db}.{table} (
+ c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ PRIMARY KEY (c)
+ )
+ STORED AS KUDU
+ TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', """
+ """'kudu.master_addresses'='{kudu_addr}')""")
+ self.assert_show_create_equals(
+ unique_database,
+ """
+ CREATE TABLE {db}.{table} (c INT COMMENT 'Ab 1@' PRIMARY KEY) STORED AS KUDU""",
+ """
+ CREATE TABLE {db}.{table} (
+ c INT NOT NULL {p} COMMENT 'Ab 1@',
+ PRIMARY KEY (c)
+ )
+ STORED AS KUDU
+ TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', """
+ """'kudu.master_addresses'='{kudu_addr}')""",
+ extra_args={'p': self.column_properties})
@SkipIfKudu.hms_integration_enabled()
- def test_timestamp_default_value(self, cursor):
- create_sql_fmt = """
- CREATE TABLE {table} (c INT, d TIMESTAMP,
- e TIMESTAMP NULL DEFAULT CAST('%s' AS TIMESTAMP),
- PRIMARY KEY(c, d))
- PARTITION BY HASH(c) PARTITIONS 3
- STORED AS KUDU"""
+ def test_timestamp_default_value(self, unique_database):
+ create_sql_fmt = (
+ """
+ CREATE TABLE {db}.{table} (c INT, d TIMESTAMP,
+ e TIMESTAMP NULL DEFAULT CAST('{ts_create}' AS TIMESTAMP),
+ PRIMARY KEY(c, d))
+ PARTITION BY HASH(c) PARTITIONS 3
+ STORED AS KUDU""")
# Long lines are unfortunate, but extra newlines will break the test.
- show_create_sql_fmt = ("""
- CREATE TABLE {db}.{{table}} (
- c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- d TIMESTAMP NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- e TIMESTAMP NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION DEFAULT unix_micros_to_utc_timestamp(%s),
- PRIMARY KEY (c, d)
- )
- PARTITION BY HASH (c) PARTITIONS 3
- STORED AS KUDU
- TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', 'kudu.master_addresses'='{kudu_addr}')"""
- .format(db=cursor.conn.db_name, kudu_addr=KUDU_MASTER_HOSTS))
- self.assert_show_create_equals(cursor,
- create_sql_fmt % ("2009-01-01 00:00:00.000001000"),
- show_create_sql_fmt % ("1230768000000001"))
- self.assert_show_create_equals(cursor,
- create_sql_fmt % ("2009-01-01 00:00:00.000001001"),
- show_create_sql_fmt % ("1230768000000001"))
- self.assert_show_create_equals(cursor,
- create_sql_fmt % ("2009-01-01 00:00:00.000000999"),
- show_create_sql_fmt % ("1230768000000001"))
+ show_create_sql_fmt = (
+ """
+ CREATE TABLE {db}.{table} (
+ c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ d TIMESTAMP NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ e TIMESTAMP NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION """
+ """DEFAULT unix_micros_to_utc_timestamp({ts_show}),
+ PRIMARY KEY (c, d)
+ )
+ PARTITION BY HASH (c) PARTITIONS 3
+ STORED AS KUDU
+ TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', """
+ """'kudu.master_addresses'='{kudu_addr}')""")
+ self.assert_show_create_equals(
+ unique_database,
+ create_sql_fmt, show_create_sql_fmt,
+ extra_args={
+ "ts_create": "2009-01-01 00:00:00.000001000",
+ "ts_show": "1230768000000001"})
+ self.assert_show_create_equals(
+ unique_database,
+ create_sql_fmt, show_create_sql_fmt,
+ extra_args={
+ "ts_create": "2009-01-01 00:00:00.000001001",
+ "ts_show": "1230768000000001"})
+ self.assert_show_create_equals(
+ unique_database,
+ create_sql_fmt, show_create_sql_fmt,
+ extra_args={
+ "ts_create": "2009-01-01 00:00:00.000000999",
+ "ts_show": "1230768000000001"})
@SkipIfKudu.hms_integration_enabled()
- def test_external_kudu_table_name_with_show_create(self, cursor, kudu_client,
- unique_database):
+ def test_external_kudu_table_name_with_show_create(
+ self, kudu_client, unique_database):
"""Check that the generated kudu.table_name tblproperty is present with
show create table with external Kudu tables.
"""
@@ -1143,165 +1247,174 @@ class TestShowCreateTable(KuduTestSuite):
kudu_table = kudu_client.table(kudu_table_name)
table_name_prop = "'kudu.table_name'='%s'" % kudu_table.name
- self.assert_show_create_equals(cursor,
- """
- CREATE EXTERNAL TABLE {{table}} STORED AS KUDU
- TBLPROPERTIES({props})""".format(
- props=table_name_prop),
- """
- CREATE EXTERNAL TABLE {db}.{{table}}
- STORED AS KUDU
- TBLPROPERTIES ('kudu.master_addresses'='{kudu_addr}', {kudu_table})""".format(
- db=cursor.conn.db_name, kudu_addr=KUDU_MASTER_HOSTS,
- kudu_table=table_name_prop), True)
+ self.assert_show_create_equals(
+ unique_database,
+ """
+ CREATE EXTERNAL TABLE {db}.{table} STORED AS KUDU
+ TBLPROPERTIES({props})""",
+ """
+ CREATE EXTERNAL TABLE {db}.{table}
+ STORED AS KUDU
+ TBLPROPERTIES ('kudu.master_addresses'='{kudu_addr}', {kudu_table})""",
+ do_exact_match=True,
+ extra_args={
+ "props": table_name_prop,
+ "kudu_table": table_name_prop})
finally:
if kudu_client.table_exists(kudu_table_name):
kudu_client.delete_table(kudu_table_name)
@SkipIfKudu.hms_integration_enabled()
- def test_managed_kudu_table_name_with_show_create(self, cursor):
+ def test_managed_kudu_table_name_with_show_create(self, unique_database):
"""Check that the generated kudu.table_name tblproperty is not present with
show create table with managed Kudu tables.
"""
- self.assert_show_create_equals(cursor,
- """
- CREATE TABLE {table} (c INT PRIMARY KEY)
- PARTITION BY HASH (c) PARTITIONS 3
- STORED AS KUDU""",
- """
- CREATE TABLE {db}.{{table}} (
- c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- PRIMARY KEY (c)
- )
- PARTITION BY HASH (c) PARTITIONS 3
- STORED AS KUDU
- TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', 'kudu.master_addresses'='{kudu_addr}')"""
- .format(db=cursor.conn.db_name, kudu_addr=KUDU_MASTER_HOSTS))
+ self.assert_show_create_equals(
+ unique_database,
+ """
+ CREATE TABLE {db}.{table} (c INT PRIMARY KEY)
+ PARTITION BY HASH (c) PARTITIONS 3
+ STORED AS KUDU""",
+ """
+ CREATE TABLE {db}.{table} (
+ c INT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ PRIMARY KEY (c)
+ )
+ PARTITION BY HASH (c) PARTITIONS 3
+ STORED AS KUDU
+ TBLPROPERTIES ('TRANSLATED_TO_EXTERNAL'='TRUE', """
+ """'kudu.master_addresses'='{kudu_addr}')""")
- def test_synchronized_kudu_table_with_show_create(self, cursor):
+ def test_synchronized_kudu_table_with_show_create(self, unique_database):
# in this case we do exact match with the provided input since this is specifically
# creating a synchronized table
- self.assert_show_create_equals(cursor,
- """
- CREATE EXTERNAL TABLE {table} (
- id BIGINT,
- name STRING,
- PRIMARY KEY(id))
- PARTITION BY HASH PARTITIONS 16
- STORED AS KUDU
- TBLPROPERTIES('external.table.purge'='true')""",
- """
- CREATE EXTERNAL TABLE {db}.{{table}} (
- id BIGINT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- name STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- PRIMARY KEY (id)
- )
- PARTITION BY HASH (id) PARTITIONS 16
- STORED AS KUDU
- TBLPROPERTIES ('external.table.purge'='true', 'kudu.master_addresses'='{kudu_addr}')"""
- .format(db=cursor.conn.db_name, kudu_addr=KUDU_MASTER_HOSTS), True)
-
- self.assert_show_create_equals(cursor,
- """
- CREATE EXTERNAL TABLE {table} (
- id BIGINT PRIMARY KEY,
- name STRING)
- PARTITION BY HASH(id) PARTITIONS 16
- STORED AS KUDU
- TBLPROPERTIES('external.table.purge'='true')""",
- """
- CREATE EXTERNAL TABLE {db}.{{table}} (
- id BIGINT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- name STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
- PRIMARY KEY (id)
- )
- PARTITION BY HASH (id) PARTITIONS 16
- STORED AS KUDU
- TBLPROPERTIES ('external.table.purge'='true', 'kudu.master_addresses'='{kudu_addr}')"""
- .format(db=cursor.conn.db_name, kudu_addr=KUDU_MASTER_HOSTS), True)
+ self.assert_show_create_equals(
+ unique_database,
+ """
+ CREATE EXTERNAL TABLE {db}.{table} (
+ id BIGINT,
+ name STRING,
+ PRIMARY KEY(id))
+ PARTITION BY HASH PARTITIONS 16
+ STORED AS KUDU
+ TBLPROPERTIES('external.table.purge'='true')""",
+ """
+ CREATE EXTERNAL TABLE {db}.{table} (
+ id BIGINT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ name STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ PRIMARY KEY (id)
+ )
+ PARTITION BY HASH (id) PARTITIONS 16
+ STORED AS KUDU
+ TBLPROPERTIES ('external.table.purge'='true', """
+ """'kudu.master_addresses'='{kudu_addr}')""",
+ do_exact_match=True)
+
+ self.assert_show_create_equals(
+ unique_database,
+ """
+ CREATE EXTERNAL TABLE {db}.{table} (
+ id BIGINT PRIMARY KEY,
+ name STRING)
+ PARTITION BY HASH(id) PARTITIONS 16
+ STORED AS KUDU
+ TBLPROPERTIES('external.table.purge'='true')""",
+ """
+ CREATE EXTERNAL TABLE {db}.{table} (
+ id BIGINT NOT NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ name STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION,
+ PRIMARY KEY (id)
+ )
+ PARTITION BY HASH (id) PARTITIONS 16
+ STORED AS KUDU
+ TBLPROPERTIES ('external.table.purge'='true', """
+ """'kudu.master_addresses'='{kudu_addr}')""",
+ do_exact_match=True)
class TestDropDb(KuduTestSuite):
@SkipIfKudu.hms_integration_enabled()
- def test_drop_non_empty_db(self, unique_cursor, kudu_client):
+ def test_drop_non_empty_db(self, kudu_client, unique_database):
"""Check that an attempt to drop a database will fail if Kudu tables are present
and that the tables remain.
"""
- db_name = unique_cursor.conn.db_name
- with self.temp_kudu_table(kudu_client, [INT32], db_name=db_name) as kudu_table:
+ with self.temp_kudu_table(
+ kudu_client, [INT32], db_name=unique_database) as kudu_table:
impala_table_name = self.get_kudu_table_base_name(kudu_table.name)
- props = "TBLPROPERTIES('kudu.table_name'='%s')" % kudu_table.name
- unique_cursor.execute("CREATE EXTERNAL TABLE %s STORED AS KUDU %s" % (
- impala_table_name, props))
- unique_cursor.execute("USE DEFAULT")
+ self.execute_query(
+ create_external_kudu_query(unique_database, impala_table_name, kudu_table.name))
+ self.execute_query("USE DEFAULT")
try:
- unique_cursor.execute("DROP DATABASE %s" % db_name)
+ self.execute_query("DROP DATABASE %s" % unique_database)
assert False
except Exception as e:
assert "One or more tables exist" in str(e)
- unique_cursor.execute("SELECT COUNT(*) FROM %s.%s" % (db_name, impala_table_name))
- assert unique_cursor.fetchall() == [(0, )]
+ result = self.execute_query("SELECT COUNT(*) FROM %s.%s" % (
+ unique_database, impala_table_name))
+ assert result.data == ['0']
@SkipIfKudu.hms_integration_enabled()
- def test_drop_db_cascade(self, unique_cursor, kudu_client):
+ def test_drop_db_cascade(self, kudu_client, unique_name):
"""Check that an attempt to drop a database will succeed even if Kudu tables are
present and that the managed tables are removed.
"""
- db_name = unique_cursor.conn.db_name
- with self.temp_kudu_table(kudu_client, [INT32], db_name=db_name) as kudu_table:
+ self.execute_query("CREATE DATABASE " + unique_name)
+ with self.temp_kudu_table(kudu_client, [INT32], db_name=unique_name) as kudu_table:
# Create an external Kudu table
impala_table_name = self.get_kudu_table_base_name(kudu_table.name)
- props = "TBLPROPERTIES('kudu.table_name'='%s')" % kudu_table.name
- unique_cursor.execute("CREATE EXTERNAL TABLE %s STORED AS KUDU %s" % (
- impala_table_name, props))
+ self.execute_query(
+ create_external_kudu_query(unique_name, impala_table_name, kudu_table.name))
# Create a managed Kudu table
managed_table_name = self.random_table_name()
- unique_cursor.execute("""
- CREATE TABLE %s (a INT PRIMARY KEY) PARTITION BY HASH (a) PARTITIONS 3
- STORED AS KUDU""" % managed_table_name)
- kudu_table_name = "impala::" + db_name + "." + managed_table_name
+ self.execute_query("""
+ CREATE TABLE %s.%s (a INT PRIMARY KEY) PARTITION BY HASH (a) PARTITIONS 3
+ STORED AS KUDU""" % (unique_name, managed_table_name))
+ kudu_table_name = "impala::" + unique_name + "." + managed_table_name
assert kudu_client.table_exists(kudu_table_name)
# Create a table in HDFS
hdfs_table_name = self.random_table_name()
- unique_cursor.execute("""
- CREATE TABLE %s (a INT) PARTITIONED BY (x INT)""" % (hdfs_table_name))
+ self.execute_query("""CREATE TABLE %s.%s (a INT) PARTITIONED BY (x INT)""" % (
+ unique_name, hdfs_table_name))
- unique_cursor.execute("USE DEFAULT")
- unique_cursor.execute("DROP DATABASE %s CASCADE" % db_name)
- unique_cursor.execute("SHOW DATABASES")
- assert (db_name, '') not in unique_cursor.fetchall()
+ self.execute_query("USE DEFAULT")
+ self.execute_query("DROP DATABASE %s CASCADE" % unique_name)
+ result = self.execute_query("SHOW DATABASES")
+ assert unique_name not in result.data
assert kudu_client.table_exists(kudu_table.name)
assert not kudu_client.table_exists(managed_table_name)
@SkipIfKudu.hms_integration_enabled()
- def test_soft_drop_db_cascade(self, unique_cursor, kudu_client):
+ def test_soft_drop_db_cascade(self, kudu_client, unique_name):
"""Check that an attempt to drop a database will succeed but the managed Kudu tables
are not removed immediately if 'kudu_table_reserve_seconds' is greater than 0.
These Kudu tables are in 'soft_deleted' state and can be recalled during the
reservation period.
"""
- db_name = unique_cursor.conn.db_name
+ options = dict()
+ self.execute_query("CREATE DATABASE " + unique_name, options)
table_name_pattern = "managed_kudu_table_"
for i in range(10):
managed_table_name = table_name_pattern + str(i)
- unique_cursor.execute("""
- CREATE TABLE %s (a INT PRIMARY KEY) PARTITION BY HASH (a) PARTITIONS 3
- STORED AS KUDU""" % managed_table_name)
- kudu_tbl_name = KuduTestSuite.to_kudu_table_name(db_name, managed_table_name)
+ kudu_tbl_name = KuduTestSuite.to_kudu_table_name(unique_name, managed_table_name)
+ if kudu_client.table_exists(kudu_tbl_name):
+ kudu_client.delete_table(kudu_tbl_name)
+ self.execute_query("""
+ CREATE TABLE %s.%s (a INT PRIMARY KEY) PARTITION BY HASH (a) PARTITIONS 3
+ STORED AS KUDU""" % (unique_name, managed_table_name), options)
assert kudu_client.table_exists(kudu_tbl_name)
- unique_cursor.execute("set kudu_table_reserve_seconds=300")
- unique_cursor.execute("USE DEFAULT")
- unique_cursor.execute("DROP DATABASE %s CASCADE" % db_name)
- unique_cursor.execute("SHOW DATABASES")
- assert (db_name, '') not in unique_cursor.fetchall()
+ options['kudu_table_reserve_seconds'] = 300
+ self.execute_query("USE DEFAULT", options)
+ self.execute_query("DROP DATABASE %s CASCADE" % unique_name, options)
+ result = self.execute_query("SHOW DATABASES", options)
+ assert unique_name not in result.data
for i in range(10):
kudu_tbl_name = \
- KuduTestSuite.to_kudu_table_name(db_name, table_name_pattern + str(i))
+ KuduTestSuite.to_kudu_table_name(unique_name, table_name_pattern + str(i))
assert kudu_client.table_exists(kudu_tbl_name)
assert kudu_tbl_name not in kudu_client.list_tables()
assert kudu_tbl_name in kudu_client.list_soft_deleted_tables()
@@ -1310,21 +1423,30 @@ class TestDropDb(KuduTestSuite):
assert kudu_tbl_name in kudu_client.list_tables()
assert kudu_tbl_name not in kudu_client.list_soft_deleted_tables()
+
class TestImpalaKuduIntegration(KuduTestSuite):
+
+ @classmethod
+ def default_test_protocol(cls):
+ # Some tests here inspect result through ImpylaHS2ResultSet.tuples().
+ # Make sure to test only with 'hs2' protocol and client.
+ return HS2
+
@SkipIfKudu.hms_integration_enabled()
- def test_replace_kudu_table(self, cursor, kudu_client):
+ def test_replace_kudu_table(self, kudu_client, unique_database):
"""Check that an external Kudu table is accessible if the underlying Kudu table is
modified using the Kudu client.
"""
# Create an external Kudu table
col_names = ['a']
- with self.temp_kudu_table(kudu_client, [INT32], col_names=col_names) as kudu_table:
+ with self.temp_kudu_table(kudu_client, [INT32], col_names=col_names,
+ db_name=unique_database) as kudu_table:
impala_table_name = self.get_kudu_table_base_name(kudu_table.name)
- props = "TBLPROPERTIES('kudu.table_name'='%s')" % kudu_table.name
- cursor.execute("CREATE EXTERNAL TABLE %s STORED AS KUDU %s" % (
- impala_table_name, props))
- cursor.execute("DESCRIBE %s" % (impala_table_name))
- assert cursor.fetchall() == \
+ self.execute_query(
+ create_external_kudu_query(unique_database, impala_table_name, kudu_table.name))
+ result = self.execute_query("DESCRIBE %s.%s" % (
+ unique_database, impala_table_name))
+ assert result.tuples() == \
[("a", "int", "", "true", "true", "false", "", "AUTO_ENCODING",
"DEFAULT_COMPRESSION", "0")]
@@ -1335,32 +1457,35 @@ class TestImpalaKuduIntegration(KuduTestSuite):
new_col_names = ['b', 'c']
name_parts = kudu_table.name.split(".")
assert len(name_parts) == 2
- with self.temp_kudu_table(kudu_client, [STRING, STRING], col_names=new_col_names,
- db_name=name_parts[0], name= name_parts[1]) as new_kudu_table:
+ with self.temp_kudu_table(
+ kudu_client, [STRING, STRING], col_names=new_col_names, db_name=name_parts[0],
+ name=name_parts[1]) as new_kudu_table:
assert kudu_client.table_exists(new_kudu_table.name)
# Refresh the external table and verify that the new schema is loaded from
# Kudu.
- cursor.execute("REFRESH %s" % (impala_table_name))
- cursor.execute("DESCRIBE %s" % (impala_table_name))
- assert cursor.fetchall() == \
+ self.execute_query("REFRESH %s.%s" % (
+ unique_database, impala_table_name))
+ result = self.execute_query("DESCRIBE %s.%s" % (
+ unique_database, impala_table_name))
+ assert result.tuples() == \
[("b", "string", "", "true", "true", "false", "", "AUTO_ENCODING",
"DEFAULT_COMPRESSION", "0"),
("c", "string", "", "false", "", "true", "", "AUTO_ENCODING",
"DEFAULT_COMPRESSION", "0")]
@SkipIfKudu.hms_integration_enabled()
- def test_delete_external_kudu_table(self, cursor, kudu_client):
+ def test_delete_external_kudu_table(self, kudu_client, unique_database):
"""Check that Impala can recover from the case where the underlying Kudu table of
an external table is dropped using the Kudu client.
"""
with self.temp_kudu_table(kudu_client, [INT32]) as kudu_table:
# Create an external Kudu table
impala_table_name = self.get_kudu_table_base_name(kudu_table.name)
- props = "TBLPROPERTIES('kudu.table_name'='%s')" % kudu_table.name
- cursor.execute("CREATE EXTERNAL TABLE %s STORED AS KUDU %s" % (
- impala_table_name, props))
- cursor.execute("DESCRIBE %s" % (impala_table_name))
- assert cursor.fetchall() == \
+ self.execute_query(
+ create_external_kudu_query(unique_database, impala_table_name, kudu_table.name))
+ result = self.execute_query("DESCRIBE %s.%s" % (
+ unique_database, impala_table_name))
+ assert result.tuples() == \
[("a", "int", "", "true", "true", "false", "", "AUTO_ENCODING",
"DEFAULT_COMPRESSION", "0")]
# Drop the underlying Kudu table
@@ -1368,43 +1493,47 @@ class TestImpalaKuduIntegration(KuduTestSuite):
assert not kudu_client.table_exists(kudu_table.name)
err_msg = 'the table does not exist: table_name: "%s"' % (kudu_table.name)
try:
- cursor.execute("REFRESH %s" % (impala_table_name))
+ self.execute_query("REFRESH %s.%s" % (unique_database, impala_table_name))
except Exception as e:
assert err_msg in str(e)
- cursor.execute("DROP TABLE %s" % (impala_table_name))
- cursor.execute("SHOW TABLES")
- assert (impala_table_name,) not in cursor.fetchall()
+ self.execute_query("DROP TABLE %s.%s" % (unique_database, impala_table_name))
+ result = self.execute_query("SHOW TABLES IN " + unique_database)
+ assert (impala_table_name,) not in result.tuples()
@SkipIfKudu.hms_integration_enabled()
- def test_delete_managed_kudu_table(self, cursor, kudu_client, unique_database):
+ def test_delete_managed_kudu_table(self, kudu_client, unique_database):
"""Check that dropping a managed Kudu table works even if the underlying Kudu table
has been dropped externally."""
impala_tbl_name = "foo"
- cursor.execute("""CREATE TABLE %s.%s (a INT PRIMARY KEY) PARTITION BY HASH (a)
+ self.execute_query("""CREATE TABLE %s.%s (a INT PRIMARY KEY) PARTITION BY HASH (a)
PARTITIONS 3 STORED AS KUDU""" % (unique_database, impala_tbl_name))
kudu_tbl_name = KuduTestSuite.to_kudu_table_name(unique_database, impala_tbl_name)
assert kudu_client.table_exists(kudu_tbl_name)
kudu_client.delete_table(kudu_tbl_name)
assert not kudu_client.table_exists(kudu_tbl_name)
- cursor.execute("DROP TABLE %s.%s" % (unique_database, impala_tbl_name))
- cursor.execute("SHOW TABLES IN %s" % unique_database)
- assert (impala_tbl_name,) not in cursor.fetchall()
+ self.execute_query("DROP TABLE %s.%s" % (unique_database, impala_tbl_name))
+ result = self.execute_query("SHOW TABLES IN %s" % unique_database)
+ assert (impala_tbl_name,) not in result.tuples()
@SkipIfKudu.hms_integration_enabled()
- def test_soft_delete_kudu_table(self, cursor, kudu_client, unique_database):
+ def test_soft_delete_kudu_table(self, kudu_client, unique_database):
"""Check that the query option 'kudu_table_reserve_seconds' works for managed Kudu
table. If it is greater than 0, the underlying Kudu will not be deleted immediately.
During the reservation period, the Kudu table can be recalled."""
- impala_tbl_name = "foo"
- cursor.execute("""CREATE TABLE %s.%s (a INT PRIMARY KEY) PARTITION BY HASH (a)
- PARTITIONS 3 STORED AS KUDU""" % (unique_database, impala_tbl_name))
- kudu_tbl_name = KuduTestSuite.to_kudu_table_name(unique_database, impala_tbl_name)
+ options = dict()
+ tbl = "foo"
+ db_tbl = unique_database + "." + tbl
+ kudu_tbl_name = KuduTestSuite.to_kudu_table_name(unique_database, tbl)
+ if kudu_client.table_exists(kudu_tbl_name):
+ kudu_client.delete_table(kudu_tbl_name)
+ self.execute_query("""CREATE TABLE %s (a INT PRIMARY KEY) PARTITION BY HASH (a)
+ PARTITIONS 3 STORED AS KUDU""" % db_tbl, options)
assert kudu_client.table_exists(kudu_tbl_name)
- cursor.execute("set kudu_table_reserve_seconds=300")
- cursor.execute("DROP TABLE %s.%s" % (unique_database, impala_tbl_name))
- cursor.execute("SHOW TABLES IN %s" % unique_database)
- assert (impala_tbl_name,) not in cursor.fetchall()
+ options["kudu_table_reserve_seconds"] = 300
+ self.execute_query("DROP TABLE %s" % db_tbl, options)
+ result = self.execute_query("SHOW TABLES IN %s" % unique_database, options)
+ assert (db_tbl,) not in result.tuples()
assert kudu_client.table_exists(kudu_tbl_name)
assert kudu_tbl_name not in kudu_client.list_tables()
@@ -1415,6 +1544,7 @@ class TestImpalaKuduIntegration(KuduTestSuite):
assert kudu_tbl_name in kudu_client.list_tables()
assert kudu_tbl_name not in kudu_client.list_soft_deleted_tables()
+
@SkipIfNotHdfsMinicluster.tuned_for_minicluster
class TestKuduMemLimits(KuduTestSuite):
@@ -1424,25 +1554,28 @@ class TestKuduMemLimits(KuduTestSuite):
# The value indicates the minimum memory requirements for the queries above, the first
# memory limit corresponds to the first query
- QUERY_MEM_LIMITS = [1, 1, 10]
+ MB = 1024 * 1024
+ QUERY_MEM_LIMITS = [MB, MB, 10 * MB]
+
+ @classmethod
+ def add_test_dimensions(cls):
+ super(TestKuduMemLimits, cls).add_test_dimensions()
+ # The default read mode of READ_LATEST does not provide high enough consistency for
+ # these tests.
+ add_exec_option_dimension(cls, "mem_limit", [cls.MB, 10 * cls.MB, 0])
+ # IMPALA-9856: We disable query result spooling so that this test can run queries
+ # with low mem_limit.
+ add_mandatory_exec_option(cls, 'spool_query_results', False)
@pytest.mark.execute_serially
- @pytest.mark.parametrize("mem_limit", [1, 10, 0])
- def test_low_mem_limit_low_selectivity_scan(self, cursor, mem_limit, vector):
+ def test_low_mem_limit_low_selectivity_scan(self, vector):
"""Tests that the queries specified in this test suite run under the given
memory limits."""
- exec_options = dict((k, str(v)) for k, v
- in vector.get_value('exec_option').items())
- exec_options['mem_limit'] = "{0}m".format(mem_limit)
- # IMPALA-9856: We disable query result spooling so that this test can run queries
- # with low mem_limit.
- exec_options['spool_query_results'] = "0"
for i, q in enumerate(self.QUERIES):
try:
- cursor.execute(q, configuration=exec_options)
- cursor.fetchall()
+ self.execute_query(q, vector.get_exec_option_dict())
except Exception as e:
- if (mem_limit > self.QUERY_MEM_LIMITS[i]):
+ if (vector.get_exec_option('mem_limit') > self.QUERY_MEM_LIMITS[i]):
raise
assert "Memory limit exceeded" in str(e)
@@ -1460,65 +1593,68 @@ class TestKuduMemLimits(KuduTestSuite):
@SkipIfHive2.create_external_kudu_table
class TestCreateSynchronizedTable(KuduTestSuite):
- def test_create_synchronized_table(self, cursor, kudu_client, unique_database):
+ def test_create_synchronized_table(self, kudu_client, unique_database):
"""
Creates a synchronized Kudu table and makes sure that the statement does not fail.
"""
table_name = self.random_table_name()
# create a external kudu table with external.table.purge=true
- cursor.execute("""
- CREATE EXTERNAL TABLE %s.%s (
+ self.execute_query("""
+ CREATE EXTERNAL TABLE {}.{} (
id int PRIMARY KEY,
name string)
PARTITION BY HASH PARTITIONS 8
STORED AS KUDU
TBLPROPERTIES ('external.table.purge'='true')
- """ % (unique_database, table_name))
+ """.format(unique_database, table_name))
# make sure that the table was created
- cursor.execute("SHOW TABLES IN %s" % unique_database)
- assert (table_name,) in cursor.fetchall()
+ result = self.execute_query("SHOW TABLES IN {}".format(unique_database))
+ assert table_name in result.data
# make sure that the kudu table was created with default name
assert kudu_client.table_exists(self.to_kudu_table_name(unique_database, table_name))
# make sure that the external.table.purge property can be changed
- cursor.execute("ALTER TABLE %s.%s set TBLPROPERTIES ("
- "'external.table.purge'='FALSE')" % (unique_database, table_name))
- cursor.execute("SHOW TABLES IN %s" % unique_database)
- assert (table_name,) in cursor.fetchall()
- cursor.execute("ALTER TABLE %s.%s set TBLPROPERTIES ("
- "'external.table.purge'='TRUE')" % (unique_database, table_name))
- cursor.execute("SHOW TABLES IN %s" % unique_database)
- assert (table_name,) in cursor.fetchall()
+ self.execute_query(
+ "ALTER TABLE {}.{} set TBLPROPERTIES ('external.table.purge'='FALSE')".format(
+ unique_database, table_name))
+ result = self.execute_query("SHOW TABLES IN {}".format(unique_database))
+ assert table_name in result.data
+ self.execute_query(
+ "ALTER TABLE {}.{} set TBLPROPERTIES ('external.table.purge'='TRUE')".format(
+ unique_database, table_name))
+ result = self.execute_query("SHOW TABLES IN {}".format(unique_database))
+ assert table_name in result.data
# make sure that table can be renamed
new_table_name = self.random_table_name()
- cursor.execute("ALTER TABLE %s.%s rename to %s.%s" %
- (unique_database, table_name, unique_database, new_table_name))
- cursor.execute("SHOW TABLES IN %s" % unique_database)
- assert (new_table_name,) in cursor.fetchall()
+ self.execute_query(
+ "ALTER TABLE {}.{} rename to {}.{}".format(
+ unique_database, table_name, unique_database, new_table_name))
+ result = self.execute_query("SHOW TABLES IN {}".format(unique_database))
+ assert new_table_name in result.data
# make sure that the kudu table was created with default name
assert kudu_client.table_exists(
self.to_kudu_table_name(unique_database, new_table_name))
# now make sure that table disappears after we remove it
- cursor.execute("DROP TABLE %s.%s" % (unique_database, new_table_name))
- cursor.execute("SHOW TABLES IN %s" % unique_database)
- assert (new_table_name,) not in cursor.fetchall()
+ self.execute_query("DROP TABLE {}.{}".format(unique_database, new_table_name))
+ result = self.execute_query("SHOW TABLES IN {}".format(unique_database))
+ assert new_table_name not in result.data
assert not kudu_client.table_exists(
self.to_kudu_table_name(unique_database, new_table_name))
- def test_invalid_sync_table_stmts(self, cursor, kudu_client, unique_database):
+ def test_invalid_sync_table_stmts(self, unique_database):
"""
Test makes sure that a invalid way to create a synchronized table is erroring out
"""
table_name = self.random_table_name()
try:
- cursor.execute("""
+ self.execute_query("""
CREATE EXTERNAL TABLE %s.%s (
a int PRIMARY KEY)
PARTITION BY HASH PARTITIONS 8
STORED AS KUDU
TBLPROPERTIES ('external.table.purge'='false')
""" % (unique_database, table_name))
- assert False,\
- "Create table statement with external.table.purge=False should error out"
+ assert False, \
+ "Create table statement with external.table.purge=False should error out"
except Exception as e:
# We throw this exception since the analyzer checks for properties one by one.
# This is the first property that it checks for an external table
@@ -1527,7 +1663,7 @@ class TestCreateSynchronizedTable(KuduTestSuite):
try:
# missing external.table.purge in TBLPROPERTIES
- cursor.execute("""
+ self.execute_query("""
CREATE EXTERNAL TABLE %s.%s (
a int PRIMARY KEY)
PARTITION BY HASH PARTITIONS 8
@@ -1535,7 +1671,7 @@ class TestCreateSynchronizedTable(KuduTestSuite):
TBLPROPERTIES ('FOO'='BAR')
""" % (unique_database, table_name))
assert False, \
- "Create external table statement must include external.table.purge property"
+ "Create external table statement must include external.table.purge property"
except Exception as e:
# We throw this exception since the analyzer checks for properties one by one.
# This is the first property that it checks for an external table
@@ -1544,7 +1680,7 @@ class TestCreateSynchronizedTable(KuduTestSuite):
try:
# Trying to create a managed table with external.purge.table property in it
- cursor.execute("""
+ self.execute_query("""
CREATE TABLE %s.%s (
a int PRIMARY KEY)
PARTITION BY HASH PARTITIONS 8
@@ -1552,58 +1688,60 @@ class TestCreateSynchronizedTable(KuduTestSuite):
TBLPROPERTIES ('external.table.purge'='true')
""" % (unique_database, table_name))
assert False, \
- "Managed table creation with external.table.purge property must be disallowed"
+ "Managed table creation with external.table.purge property must be disallowed"
except Exception as e:
assert "Table property 'external.table.purge' cannot be set to true " \
"with an managed Kudu table." in str(e)
# TODO should we block this?
- cursor.execute("""
+ self.execute_query("""
CREATE TABLE %s.%s (
a int PRIMARY KEY)
PARTITION BY HASH PARTITIONS 8
STORED AS KUDU
TBLPROPERTIES ('external.table.purge'='False')""" % (unique_database, table_name))
- cursor.execute("SHOW TABLES IN %s" % unique_database)
- assert (table_name,) in cursor.fetchall()
+ result = self.execute_query("SHOW TABLES IN %s" % unique_database)
+ assert table_name in result.data
- def test_sync_tbl_with_kudu_table(self, cursor, kudu_client, unique_database):
+ def test_sync_tbl_with_kudu_table(self, kudu_client, unique_database):
"""
Test tries to create a synchronized table with an existing Kudu table name and
makes sure it fails.
"""
+ create_query = (
+ """
+ CREATE EXTERNAL TABLE {}.{} (
+ a int PRIMARY KEY)
+ PARTITION BY HASH PARTITIONS 8
+ STORED AS KUDU
+ TBLPROPERTIES('external.table.purge'='true', 'kudu.table_name' = '{}')"""
+ )
with self.temp_kudu_table(kudu_client, [INT32]) as kudu_table:
table_name = self.random_table_name()
try:
- cursor.execute("""
- CREATE EXTERNAL TABLE %s.%s (
- a int PRIMARY KEY)
- PARTITION BY HASH PARTITIONS 8
- STORED AS KUDU
- TBLPROPERTIES('external.table.purge'='true', 'kudu.table_name' = '%s')"""
- % (unique_database, table_name,
- self.get_kudu_table_base_name(kudu_table.name)))
- assert False, "External tables with external.purge.table property must fail " \
- "if the kudu table already exists"
+ self.execute_query(
+ create_query.format(
+ unique_database, table_name, self.get_kudu_table_base_name(kudu_table.name)))
+ assert False, ("External tables with external.purge.table property must fail "
+ "if the kudu table already exists")
except Exception as e:
- assert "Not allowed to set 'kudu.table_name' manually for" \
- " synchronized Kudu tables" in str(e)
+ assert ("Not allowed to set 'kudu.table_name' manually for"
+ " synchronized Kudu tables") in str(e)
class TestKuduReadTokenSplit(KuduTestSuite):
"""
This suite verifies impala's integration of Kudu's split token API.
"""
+
@classmethod
def add_test_dimensions(cls):
super(TestKuduReadTokenSplit, cls).add_test_dimensions()
- # The default read mode of READ_LATEST does not provide high enough consistency for
- # these tests.
- add_mandatory_exec_option(cls, "kudu_read_mode", "READ_AT_SNAPSHOT")
+ cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
@SkipIfKudu.no_hybrid_clock()
@SkipIfNotHdfsMinicluster.tuned_for_minicluster
- def test_kudu_scanner(self, vector, unique_database):
+ def test_kudu_scanner(self, vector):
"""This runs explain query with variations of mt_dop and
targeted_kudu_scan_range_length to verify targeted_kudu_scan_range_length's
functionality.
@@ -1612,43 +1750,51 @@ class TestKuduReadTokenSplit(KuduTestSuite):
explain_query = "explain select * from tpch_kudu.lineitem "
plans = []
- regular_num_inst = self.__get_num_scanner_instances(explain_query, mt_dop=None,
- targeted_kudu_scan_range_length=None, plans=plans)
+ regular_num_inst = self.__get_num_scanner_instances(
+ explain_query, vector, mt_dop=None, targeted_kudu_scan_range_length=None,
+ plans=plans)
- mt_dop_1_num_inst = self.__get_num_scanner_instances(explain_query, mt_dop=1,
- targeted_kudu_scan_range_length=None, plans=plans)
+ mt_dop_1_num_inst = self.__get_num_scanner_instances(
+ explain_query, vector, mt_dop=1, targeted_kudu_scan_range_length=None,
+ plans=plans)
# targeted_kudu_scan_range_length should be disabled by default and num instances
# will be equal to the number of partitions
- with_mt_dop_num_inst = self.__get_num_scanner_instances(explain_query, mt_dop=10,
- targeted_kudu_scan_range_length=None, plans=plans)
+ with_mt_dop_num_inst = self.__get_num_scanner_instances(
+ explain_query, vector, mt_dop=10, targeted_kudu_scan_range_length=None,
+ plans=plans)
# This will result is more splits
with_mt_dop_and_low_range_len_num_inst = self.__get_num_scanner_instances(
- explain_query, mt_dop=10, targeted_kudu_scan_range_length="8mb", plans=plans)
+ explain_query, vector, mt_dop=10, targeted_kudu_scan_range_length="8mb",
+ plans=plans)
assert mt_dop_1_num_inst == regular_num_inst, str(plans)
assert regular_num_inst < with_mt_dop_num_inst, str(plans)
assert with_mt_dop_num_inst < with_mt_dop_and_low_range_len_num_inst, str(plans)
- def __get_num_scanner_instances(self, explain_query, mt_dop,
+ def __get_num_scanner_instances(self, explain_query, vector, mt_dop,
targeted_kudu_scan_range_length, plans):
"""This is a helper method that runs the explain query with the provided query
options (mt_dop and targeted_kudu_scan_range_length). Appends the generated plan to
'plans' and returns the num of kudu scanner instances """
regex = r'F00:PLAN FRAGMENT \[RANDOM\] hosts=3 instances=([0-9]+)'
- self.client.set_configuration_option("explain_level", 3)
+ # The default read mode of READ_LATEST does not provide high enough consistency for
+ # these tests.
+ client = self.create_impala_client(protocol=vector.get_value('protocol'))
+ client.set_configuration_option("kudu_read_mode", "READ_AT_SNAPSHOT")
+ client.set_configuration_option("explain_level", 3)
if targeted_kudu_scan_range_length:
- self.client.set_configuration_option("targeted_kudu_scan_range_length",
- targeted_kudu_scan_range_length)
+ client.set_configuration_option(
+ "targeted_kudu_scan_range_length", targeted_kudu_scan_range_length)
if mt_dop:
- self.client.set_configuration_option("mt_dop", mt_dop)
- result = self.client.execute(explain_query)
+ client.set_configuration_option("mt_dop", mt_dop)
+ result = client.execute(explain_query)
plan = "\n".join(result.data)
plans.append(plan)
matches = re.search(regex, plan)
assert len(matches.groups()) == 1, plan
- self.client.clear_configuration()
+ client.clear_configuration()
return int(matches.group(1))
@@ -1679,23 +1825,22 @@ class TestKuduInsertWithBufferedTupleDesc(KuduTestSuite):
"where u.big_id is null"
@SkipIfKudu.no_hybrid_clock()
- def test_kudu_insert_with_buffered_tuple_desc(self, cursor, kudu_client,
- unique_database):
+ def test_kudu_insert_with_buffered_tuple_desc(self, kudu_client, unique_database):
# Create Kudu tables.
table_1_name = "%s.tab1" % unique_database
- cursor.execute(self._create_kudu_table_1_query.format(table_1_name))
+ self.execute_query(self._create_kudu_table_1_query.format(table_1_name))
assert kudu_client.table_exists(
KuduTestSuite.to_kudu_table_name(unique_database, "tab1"))
table_2_name = "%s.tab2" % unique_database
- cursor.execute(self._create_kudu_table_2_query.format(table_2_name))
+ self.execute_query(self._create_kudu_table_2_query.format(table_2_name))
assert kudu_client.table_exists(
KuduTestSuite.to_kudu_table_name(unique_database, "tab2"))
# Insert rows
try:
- cursor.execute(self._insert_query.format(table_1_name, table_2_name))
- cursor.execute("SELECT * FROM %s" % table_1_name)
- assert len(cursor.fetchall()) == 0
+ self.execute_query(self._insert_query.format(table_1_name, table_2_name))
+ result = self.execute_query("SELECT * FROM %s" % table_1_name)
+ assert len(result.data) == 0
except Exception as e:
# Not expect to throw exception like "IllegalStateException: null"
assert False, str(e)
diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py
index 86a1e559a..251032200 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -752,7 +752,7 @@ class TestObservability(ImpalaTestSuite):
self.hs2_client.set_configuration(query_opts)
result = self.hs2_client.execute("select sleep(2000)",
profile_format=TRuntimeProfileFormat.THRIFT)
- thrift_profile = result.profile
+ thrift_profile = result.runtime_profile
expected_keys = ["HostCpuUserPercentage", "HostNetworkRx", "HostDiskReadThroughput"]
for key in expected_keys:
@@ -769,7 +769,7 @@ class TestObservability(ImpalaTestSuite):
that consume Impala debug webpages."""
query = "select sleep(5)"
result = self.hs2_client.execute(query, profile_format=TRuntimeProfileFormat.THRIFT)
- tree = result.profile
+ tree = result.runtime_profile
# tree.nodes[1] corresponds to ClientRequestState::summary_profile_
# See be/src/service/client-request-state.[h|cc].
diff --git a/tests/statestore/test_statestore.py b/tests/statestore/test_statestore.py
index 94d7f60f1..b7a6b28a3 100644
--- a/tests/statestore/test_statestore.py
+++ b/tests/statestore/test_statestore.py
@@ -20,7 +20,6 @@ from builtins import range
from collections import defaultdict
import json
import logging
-from random import randint
import socket
import threading
import traceback
@@ -45,6 +44,7 @@ from StatestoreService.StatestoreSubscriber import TTopicRegistration
from ErrorCodes.ttypes import TErrorCode
from Status.ttypes import TStatus
+from tests.common.base_test_suite import BaseTestSuite
from tests.common.environ import build_flavor_timeout
from tests.common.skip import SkipIfDockerizedCluster
@@ -66,11 +66,13 @@ LOG = logging.getLogger('test_statestore')
# Test that only the subscribed-to topics are sent
# Test that topic deletions take effect correctly.
+
def get_statestore_subscribers(host='localhost', port=25010):
response = urlopen("http://{0}:{1}/subscribers?json".format(host, port))
page = response.read()
return json.loads(page)
+
STATUS_OK = TStatus(TErrorCode.OK)
DEFAULT_UPDATE_STATE_RESPONSE = TUpdateStateResponse(status=STATUS_OK, topic_updates=[],
skipped=False)
@@ -81,6 +83,7 @@ WAIT_FOR_HEARTBEAT_TIMEOUT = build_flavor_timeout(
40, code_coverage_build_timeout=60)
WAIT_FOR_UPDATE_TIMEOUT = build_flavor_timeout(40, code_coverage_build_timeout=60)
+
class WildcardServerSocket(TSocket.TSocketBase, TTransport.TServerTransportBase):
"""Specialised server socket that binds to a random port at construction"""
def __init__(self, host=None, port=0):
@@ -125,7 +128,7 @@ class KillableThreadedServer(TServer):
try:
cnxn.open()
return
- except Exception as e:
+ except Exception:
if i == num_tries - 1: raise
time.sleep(0.1)
@@ -135,7 +138,7 @@ class KillableThreadedServer(TServer):
try:
cnxn.open()
time.sleep(0.1)
- except Exception as e:
+ except Exception:
return
raise Exception("Server did not stop")
@@ -158,7 +161,7 @@ class KillableThreadedServer(TServer):
try:
while not self.is_shutdown:
self.processor.process(iprot, oprot)
- except TTransport.TTransportException as tx:
+ except TTransport.TTransportException:
pass
except Exception as x:
print(x)
@@ -166,6 +169,7 @@ class KillableThreadedServer(TServer):
itrans.close()
otrans.close()
+
class StatestoreSubscriber(object):
"""A bare-bones subscriber skeleton. Tests should create a new StatestoreSubscriber(),
call start() and then register(). The subscriber will run a Thrift server on an unused
@@ -186,7 +190,7 @@ class StatestoreSubscriber(object):
def __init__(self, heartbeat_cb=None, update_cb=None):
self.heartbeat_event, self.heartbeat_count = threading.Condition(), 0
# Track the number of updates received per topic.
- self.update_counts = defaultdict(lambda : 0)
+ self.update_counts = defaultdict(lambda: 0)
# Variables to notify for updates on each topic.
self.update_event = threading.Condition()
self.heartbeat_cb, self.update_cb = heartbeat_cb, update_cb
@@ -325,8 +329,8 @@ class StatestoreSubscriber(object):
self.check_thread_exceptions()
last_count = self.update_counts[topic_name]
self.update_event.wait(WAIT_FOR_UPDATE_TIMEOUT)
- if (time.time() > start_time + WAIT_FOR_UPDATE_TIMEOUT and
- last_count == self.update_counts[topic_name]):
+ if (time.time() > start_time + WAIT_FOR_UPDATE_TIMEOUT
+ and last_count == self.update_counts[topic_name]):
raise Exception(
"Update not received for {0} within {1} (update count: {2})".format(
topic_name, WAIT_FOR_UPDATE_TIMEOUT, last_count))
@@ -335,7 +339,6 @@ class StatestoreSubscriber(object):
finally:
self.update_event.release()
-
def wait_for_failure(self, timeout=WAIT_FOR_FAILURE_TIMEOUT):
"""Waits until this subscriber no longer appears in the statestore's subscriber
list. If 'timeout' seconds pass, throws an exception."""
@@ -350,7 +353,7 @@ class StatestoreSubscriber(object):
@SkipIfDockerizedCluster.statestore_not_exposed
-class TestStatestore():
+class TestStatestore(BaseTestSuite):
def make_topic_update(self, topic_name, key_template="foo", value_template="bar",
num_updates=1, clear_topic_entries=False):
topic_entries = [
@@ -468,14 +471,14 @@ class TestStatestore():
# The update doesn't contain our topic.
pass
elif update_count == 1:
- assert args.topic_deltas[topic_name].is_delta == False
+ assert args.topic_deltas[topic_name].is_delta is False
delta = self.make_topic_update(topic_name)
return TUpdateStateResponse(status=STATUS_OK, topic_updates=[delta],
skipped=False)
elif update_count == 2:
- assert args.topic_deltas[topic_name].is_delta == False
+ assert args.topic_deltas[topic_name].is_delta is False
elif update_count == 3:
- assert args.topic_deltas[topic_name].is_delta == True
+ assert args.topic_deltas[topic_name].is_delta is True
assert len(args.topic_deltas[topic_name].topic_entries) == 0
assert args.topic_deltas[topic_name].to_version == 1
@@ -503,7 +506,7 @@ class TestStatestore():
skipped=False)
# All subsequent updates: set skipped=True and expected the full topic to be resent
# every time
- assert args.topic_deltas[topic_name].is_delta == False
+ assert args.topic_deltas[topic_name].is_delta is False
assert len(args.topic_deltas[topic_name].topic_entries) == 1
return TUpdateStateResponse(status=STATUS_OK, skipped=True)
@@ -602,12 +605,12 @@ class TestStatestore():
def add_entries(sub, args):
# None of, one or both of the topics may be in the update.
updates = []
- if (persistent_topic_name in args.topic_deltas and
- sub.update_counts[persistent_topic_name] == 1):
+ if (persistent_topic_name in args.topic_deltas
+ and sub.update_counts[persistent_topic_name] == 1):
updates.append(self.make_topic_update(persistent_topic_name))
- if (transient_topic_name in args.topic_deltas and
- sub.update_counts[transient_topic_name] == 1):
+ if (transient_topic_name in args.topic_deltas
+ and sub.update_counts[transient_topic_name] == 1):
updates.append(self.make_topic_update(transient_topic_name))
if len(updates) > 0:
@@ -617,14 +620,14 @@ class TestStatestore():
def check_entries(sub, args):
# None of, one or both of the topics may be in the update.
- if (persistent_topic_name in args.topic_deltas and
- sub.update_counts[persistent_topic_name] == 1):
+ if (persistent_topic_name in args.topic_deltas
+ and sub.update_counts[persistent_topic_name] == 1):
assert len(args.topic_deltas[persistent_topic_name].topic_entries) == 1
# Statestore should not send deletions when the update is not a delta, see
# IMPALA-1891
- assert args.topic_deltas[persistent_topic_name].topic_entries[0].deleted == False
- if (transient_topic_name in args.topic_deltas and
- sub.update_counts[persistent_topic_name] == 1):
+ assert args.topic_deltas[persistent_topic_name].topic_entries[0].deleted is False
+ if (transient_topic_name in args.topic_deltas
+ and sub.update_counts[persistent_topic_name] == 1):
assert len(args.topic_deltas[transient_topic_name].topic_entries) == 0
return DEFAULT_UPDATE_STATE_RESPONSE
@@ -737,6 +740,7 @@ class TestStatestore():
update_lock = threading.Lock()
last_to_versions = {}
TOTAL_SUBSCRIBERS = 2
+
def callback(sub, args, is_producer, sub_name):
"""Callback for subscriber to verify min_subscriber_topic_version behaviour.
If 'is_producer' is true, this acts as the producer, otherwise it acts as the
@@ -794,8 +798,12 @@ class TestStatestore():
# Two concurrent subscribers, which pushes out updates and checks the minimum
# version, the other which just consumes the updates.
- def producer_callback(sub, args): return callback(sub, args, True, "producer")
- def consumer_callback(sub, args): return callback(sub, args, False, "consumer")
+ def producer_callback(sub, args):
+ return callback(sub, args, True, "producer")
+
+ def consumer_callback(sub, args):
+ return callback(sub, args, False, "consumer")
+
with StatestoreSubscriber(update_cb=consumer_callback) as consumer_sub:
with StatestoreSubscriber(update_cb=producer_callback) as producer_sub:
consumer_reg = TTopicRegistration(topic_name=topic_name, is_transient=True)
diff --git a/tests/unittests/test_command.py b/tests/unittests/test_command.py
index 2ab58f92d..262c53e2c 100644
--- a/tests/unittests/test_command.py
+++ b/tests/unittests/test_command.py
@@ -19,15 +19,16 @@
from __future__ import absolute_import, division, print_function
import os
-import pytest
import sys
+from tests.common.base_test_suite import BaseTestSuite
# Update the sys.path to include the modules from bin/diagnostics.
sys.path.insert(0,
os.path.abspath(os.path.join(os.path.dirname(__file__), '../../bin/diagnostics')))
from collect_diagnostics import Command
-class TestCommand(object):
+
+class TestCommand(BaseTestSuite):
""" Unit tests for the Command class"""
def test_simple_commands(self):
@@ -46,5 +47,3 @@ class TestCommand(object):
c = Command(["sleep", "1000"], 1)
assert c.run() != 0, "Command expected to timeout but succeeded."
assert c.child_killed_by_timeout, "Command didn't timeout as expected."
-
-
diff --git a/tests/verifiers/test_banned_log_messages.py b/tests/verifiers/test_banned_log_messages.py
index 2b70f45a0..e7817b8c9 100644
--- a/tests/verifiers/test_banned_log_messages.py
+++ b/tests/verifiers/test_banned_log_messages.py
@@ -21,8 +21,10 @@ from __future__ import absolute_import, division, print_function
import os
import subprocess
+from tests.common.base_test_suite import BaseTestSuite
-class TestBannedLogMessages:
+
+class TestBannedLogMessages(BaseTestSuite):
"""Verify that specific log messages are banned from Impala logs.
This test suite should be run after all the tests have been run.