You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bo...@apache.org on 2019/05/28 10:26:11 UTC
[impala] 02/08: IMPALA-8524: part2: Avoid calling "hive" via
command line in EE tests
This is an automated email from the ASF dual-hosted git repository.
boroknagyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 6839d9738b0d5c3c176fea4345ab623783915454
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Wed May 22 17:05:25 2019 +0200
IMPALA-8524: part2: Avoid calling "hive" via command line in EE tests
"hive -e SQL..." without further parameters no longer works
when USE_CDP_HIVE=true (it doesn't establish a connection).
Some tests used this to load data.
part2: there were some places that still called hive without
a beeline connection.
I had to break up views-compatibility.test for the different versions
of Hive.
Change-Id: Ia45b64cc1da78190e6f239a5f462308d7fa56f4b
Reviewed-on: http://gerrit.cloudera.org:8080/13402
Reviewed-by: Zoltan Borok-Nagy <bo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../QueryTest/views-compatibility-hive2-only.test | 30 ++++++++++++++++++++++
.../QueryTest/views-compatibility-hive3-only.test | 15 +++++++++++
.../queries/QueryTest/views-compatibility.test | 27 -------------------
tests/metadata/test_views_compatibility.py | 15 +++++++++--
tests/query_test/test_scanners.py | 4 +--
5 files changed, 60 insertions(+), 31 deletions(-)
diff --git a/testdata/workloads/functional-query/queries/QueryTest/views-compatibility-hive2-only.test b/testdata/workloads/functional-query/queries/QueryTest/views-compatibility-hive2-only.test
new file mode 100644
index 0000000..f28e9d7
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/views-compatibility-hive2-only.test
@@ -0,0 +1,30 @@
+====
+---- CREATE_VIEW
+# Create a view in Impala with plan hints. Hive should recognize the hints as
+# comments and ignore them.
+create view test as
+select /* +straight_join */ a.* from functional.alltypestiny a
+inner join /* +broadcast */ functional.alltypes b on a.id = b.id
+inner join /* +shuffle */ functional.alltypessmall c on b.id = c.id;
+---- CREATE_VIEW_RESULTS
+IMPALA=SUCCESS
+HIVE=FAILURE
+---- QUERY_IMPALA_VIEW_RESULTS
+IMPALA=SUCCESS
+HIVE=SUCCESS
+====
+---- CREATE_VIEW
+# Create a view in Hive with plan hints. Impala should ignore the unknown hints.
+# TODO: move this to the common .test file once "HIVE-21782: Cannot use query hints
+# in views" is resolved.
+create view test as
+select /*+ MAPJOIN(alltypestiny) */ count(*) from
+functional.alltypes a inner join functional.alltypestiny b
+on (a.id = b.id);
+---- CREATE_VIEW_RESULTS
+IMPALA=SUCCESS
+HIVE=SUCCESS
+---- QUERY_HIVE_VIEW_RESULTS
+IMPALA=SUCCESS
+HIVE=SUCCESS
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/views-compatibility-hive3-only.test b/testdata/workloads/functional-query/queries/QueryTest/views-compatibility-hive3-only.test
new file mode 100644
index 0000000..eb9abbf
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/views-compatibility-hive3-only.test
@@ -0,0 +1,15 @@
+====
+---- CREATE_VIEW
+# Create a view in Impala with plan hints. Hive should recognize the hints as
+# comments and ignore them.
+create view test as
+select /* +straight_join */ a.* from functional.alltypestiny a
+inner join /* +broadcast */ functional.alltypes b on a.id = b.id
+inner join /* +shuffle */ functional.alltypessmall c on b.id = c.id;
+---- CREATE_VIEW_RESULTS
+IMPALA=SUCCESS
+HIVE=SUCCESS
+---- QUERY_IMPALA_VIEW_RESULTS
+IMPALA=SUCCESS
+HIVE=SUCCESS
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/views-compatibility.test b/testdata/workloads/functional-query/queries/QueryTest/views-compatibility.test
index f90d9b4..2ca7a2d 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/views-compatibility.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/views-compatibility.test
@@ -232,30 +232,3 @@ IMPALA=SUCCESS
---- QUERY_IMPALA_VIEW_RESULTS
HIVE=SUCCESS
====
----- CREATE_VIEW
-# Create a view in Impala with plan hints. Hive should recognize the hints as
-# comments and ignore them.
-create view test as
-select /* +straight_join */ a.* from functional.alltypestiny a
-inner join /* +broadcast */ functional.alltypes b on a.id = b.id
-inner join /* +shuffle */ functional.alltypessmall c on b.id = c.id;
----- CREATE_VIEW_RESULTS
-IMPALA=SUCCESS
-HIVE=FAILURE
----- QUERY_IMPALA_VIEW_RESULTS
-IMPALA=SUCCESS
-HIVE=SUCCESS
-====
----- CREATE_VIEW
-# Create a view in Hive with plan hints. Impala should ignore the unknown hints.
-create view test as
-select /*+ MAPJOIN(alltypestiny) */ count(*) from
-functional.alltypes a inner join functional.alltypestiny b
-on (a.id = b.id);
----- CREATE_VIEW_RESULTS
-IMPALA=SUCCESS
-HIVE=SUCCESS
----- QUERY_HIVE_VIEW_RESULTS
-IMPALA=SUCCESS
-HIVE=SUCCESS
-====
diff --git a/tests/metadata/test_views_compatibility.py b/tests/metadata/test_views_compatibility.py
index a0103e2..c0716a5 100644
--- a/tests/metadata/test_views_compatibility.py
+++ b/tests/metadata/test_views_compatibility.py
@@ -22,6 +22,7 @@ import shlex
from subprocess import call
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
+from tests.common.environ import HIVE_MAJOR_VERSION
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfS3, SkipIfABFS, SkipIfADLS, SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import create_uncompressed_text_dimension
@@ -75,6 +76,12 @@ class TestViewCompatibility(ImpalaTestSuite):
def test_view_compatibility(self, vector, unique_database):
self._run_view_compat_test_case('QueryTest/views-compatibility', vector,
unique_database)
+ if HIVE_MAJOR_VERSION == 2:
+ self._run_view_compat_test_case('QueryTest/views-compatibility-hive2-only', vector,
+ unique_database)
+ if HIVE_MAJOR_VERSION >= 3:
+ self._run_view_compat_test_case('QueryTest/views-compatibility-hive3-only', vector,
+ unique_database)
def _run_view_compat_test_case(self, test_file_name, vector, test_db_name):
"""
@@ -144,8 +151,12 @@ class TestViewCompatibility(ImpalaTestSuite):
test_case.get_create_view_sql('IMPALA'), None)
def _exec_in_hive(self, sql_str, create_view_sql, exp_res):
- hive_ret = call(['hive', '-e', sql_str])
- self._cmp_expected(sql_str, create_view_sql, exp_res, "HIVE", hive_ret == 0)
+ try:
+ self.run_stmt_in_hive(sql_str)
+ success = True
+ except: # consider any exception a failure
+ success = False
+ self._cmp_expected(sql_str, create_view_sql, exp_res, "HIVE", success)
def _exec_in_impala(self, sql_str, create_view_sql, exp_res):
success = True
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 4c4a00e..5d6e8c1 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -410,8 +410,8 @@ class TestParquet(ImpalaTestSuite):
" float_col, double_col,date_string_col,string_col,timestamp_col" \
" from functional_parquet.alltypes" \
" where year = {year} and month = {month}" % unique_database
- check_call(['hive', '-e', hql_format.format(codec="snappy", year=2010, month=1)])
- check_call(['hive', '-e', hql_format.format(codec="gzip", year=2010, month=2)])
+ self.run_stmt_in_hive(hql_format.format(codec="snappy", year=2010, month=1))
+ self.run_stmt_in_hive(hql_format.format(codec="gzip", year=2010, month=2))
test_files = ["testdata/multi_compression_parquet_data/tinytable_0_gzip_snappy.parq",
"testdata/multi_compression_parquet_data/tinytable_1_snappy_gzip.parq"]