You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/06/03 16:35:58 UTC
[impala] 06/06: IMPALA-8369: Add HIVE_MAJOR_VERSION section to EE
tests + some fixes
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit e32a4967c8094c334f4f6415019b52ef96a76e6d
Author: Csaba Ringhofer <cs...@cloudera.com>
AuthorDate: Thu May 30 16:24:15 2019 +0200
IMPALA-8369: Add HIVE_MAJOR_VERSION section to EE tests + some fixes
Fixed tests with Hive3:
test_scanners.py - test_scan_truncated_file_empty (exhaustive):
Added REFRESH after Hive INSERT OVERWRITE. The test worked in Hive2
only because there was an empty file with the same name as before
the overwrite.
test_ddl.py - test_alter_table:
A Hive3 regression broke some tests + caused the dropping of
the test database to hang. These tests are skipped for now,
the Hive side fix is tracked in HIVE-21806.
Change-Id: I4c3cff05ed7080b655b6af64ea09c0691e7dd931
Reviewed-on: http://gerrit.cloudera.org:8080/13472
Reviewed-by: Zoltan Borok-Nagy <bo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../functional-query/queries/QueryTest/alter-table.test | 16 ++++++++++++++++
tests/common/impala_test_suite.py | 8 ++++++++
tests/query_test/test_scanners.py | 4 ++++
tests/util/test_file_parser.py | 2 +-
4 files changed, 29 insertions(+), 1 deletion(-)
diff --git a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
index 5d9e6cf..ac65d29 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
@@ -900,10 +900,14 @@ show tables in $DATABASE2 like '*mv*'
---- TYPES
STRING
====
+---- HIVE_MAJOR_VERSION
+2
---- QUERY
# Tests that renaming a partitioned table with column stats across databases
# succeeds and preserves table and column stats, and allows the renamed table
# to be dropped (IMPALA-2810).
+#
+# Skip in Hive 3, see HIVE-21806 for details.
create table $DATABASE.mv (x int) partitioned by (y string);
insert into $DATABASE.mv partition(y='a') values(1);
insert into $DATABASE.mv partition(y='b') values(2);
@@ -913,12 +917,16 @@ alter table $DATABASE.mv rename to $DATABASE2.mv2;
invalidate metadata $DATABASE2.mv2
---- RESULTS
====
+---- HIVE_MAJOR_VERSION
+2
---- QUERY
show tables in $DATABASE like '*mv*'
---- RESULTS
---- TYPES
STRING
====
+---- HIVE_MAJOR_VERSION
+2
---- QUERY
show tables in $DATABASE2 like '*mv*'
---- RESULTS
@@ -926,6 +934,8 @@ show tables in $DATABASE2 like '*mv*'
---- TYPES
STRING
====
+---- HIVE_MAJOR_VERSION
+2
---- QUERY
show table stats $DATABASE2.mv2
---- RESULTS: VERIFY_IS_EQUAL_SORTED
@@ -936,6 +946,8 @@ show table stats $DATABASE2.mv2
---- TYPES
STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
====
+---- HIVE_MAJOR_VERSION
+2
---- QUERY
show column stats $DATABASE2.mv2
---- RESULTS: VERIFY_IS_EQUAL_SORTED
@@ -944,11 +956,15 @@ show column stats $DATABASE2.mv2
---- TYPES
STRING, STRING, BIGINT, BIGINT, BIGINT, DOUBLE
====
+---- HIVE_MAJOR_VERSION
+2
---- QUERY
drop table $DATABASE2.mv2
---- RESULTS
'Table has been dropped.'
====
+---- HIVE_MAJOR_VERSION
+2
---- QUERY
show tables in $DATABASE2 like '*mv*'
---- RESULTS
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index 1be55f7..44e9124 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -36,6 +36,7 @@ from getpass import getuser
from random import choice
from subprocess import check_call
from tests.common.base_test_suite import BaseTestSuite
+from tests.common.environ import HIVE_MAJOR_VERSION
from tests.common.errors import Timeout
from tests.common.impala_connection import create_connection
from tests.common.impala_service import ImpaladService
@@ -515,6 +516,13 @@ class ImpalaTestSuite(BaseTestSuite):
sections = self.load_query_test_file(self.get_workload(), test_file_name,
encoding=encoding)
for test_section in sections:
+ if 'HIVE_MAJOR_VERSION' in test_section:
+ needed_hive_major_version = int(test_section['HIVE_MAJOR_VERSION'])
+ assert needed_hive_major_version in [2, 3]
+ assert HIVE_MAJOR_VERSION in [2, 3]
+ if needed_hive_major_version != HIVE_MAJOR_VERSION:
+ continue
+
if 'SHELL' in test_section:
assert len(test_section) == 1, \
"SHELL test sections can't contain other sections"
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 5d6e8c1..15686de 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -1145,6 +1145,10 @@ class TestScanTruncatedFiles(ImpalaTestSuite):
self.run_stmt_in_hive("insert overwrite table %s select string_col from "
"functional.alltypes limit %s" % (fq_tbl_name, num_rows))
+ # The file will not exist if the table is empty and the insert is done by Hive 3, so
+ # another refresh is needed.
+ self.execute_query("refresh %s" % fq_tbl_name)
+
result = self.execute_query("select count(*) from %s" % fq_tbl_name)
assert(len(result.data) == 1)
assert(result.data[0] == str(num_rows))
diff --git a/tests/util/test_file_parser.py b/tests/util/test_file_parser.py
index 789fb9a..9d90a99 100644
--- a/tests/util/test_file_parser.py
+++ b/tests/util/test_file_parser.py
@@ -96,7 +96,7 @@ def parse_query_test_file(file_name, valid_section_names=None, encoding=None):
if section_names is None:
section_names = ['QUERY', 'HIVE_QUERY', 'RESULTS', 'TYPES', 'LABELS', 'SETUP',
'CATCH', 'ERRORS', 'USER', 'RUNTIME_PROFILE', 'SHELL', 'DML_RESULTS',
- 'DBAPI_RESULTS', 'HS2_TYPES']
+ 'DBAPI_RESULTS', 'HS2_TYPES', 'HIVE_MAJOR_VERSION']
return parse_test_file(file_name, section_names, encoding=encoding,
skip_unknown_sections=False)