You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2022/08/25 12:23:48 UTC
[impala] 01/05: IMPALA-11249: Fix add_test_dimensions() locations to call super()
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch branch-4.1.1
in repository https://gitbox.apache.org/repos/asf/impala.git
commit b28e133f008e5196bf2bc7f2114cc45d0dfbe963
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Sat May 21 19:10:11 2022 -0700
IMPALA-11249: Fix add_test_dimensions() locations to call super()
The original issue is that the strict HS2 shell tests
are not running in precommit or nightly jobs, but they
do run in local developer environments. Investigating
this showed that the shell tests were running with a
weird set of test dimensions that includes
table_format_and_file_extension. That dimension is only
used in test_insert.py::TestInsertFileExtension.
What is happening is that the shell tests and other
locations are running add_test_dimensions() without
calling super(..., cls).add_test_dimensions(). The
behavior is unclear, but there is clearly cross-talk
between the different tests that do this.
This changes all add_test_dimensions() locations to
call super(..., cls).add_test_dimensions() if they
don't already. Each location has been tuned to run
the same set of tests as before (except the shell
tests which now run the strict HS2 tests).
As part of this, several shell tests need to be
skipped or fixed for strict HS2.
Testing:
- Ran core job
- Ran tests locally to verify the set of tests
didn't change.
Change-Id: Ib20fd479d3b91ed0ed89a0bc5623cd2a5a458614
Reviewed-on: http://gerrit.cloudera.org:8080/18557
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-on: http://gerrit.cloudera.org:8080/18905
Reviewed-by: Wenzhe Zhou <wz...@cloudera.com>
Tested-by: Quanlong Huang <hu...@gmail.com>
---
tests/authorization/test_ranger.py | 8 ++++++++
tests/custom_cluster/test_client_ssl.py | 1 +
tests/query_test/test_insert.py | 22 +++++++++++++++-------
tests/query_test/test_nested_types.py | 5 ++++-
tests/shell/test_shell_client.py | 11 +++++++++--
tests/shell/test_shell_commandline.py | 10 ++++++++--
tests/shell/test_shell_interactive.py | 16 +++++++++++++---
7 files changed, 58 insertions(+), 15 deletions(-)
diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py
index 875886b12..d0caa7b17 100644
--- a/tests/authorization/test_ranger.py
+++ b/tests/authorization/test_ranger.py
@@ -1591,6 +1591,7 @@ class TestRangerColumnMaskingComplexTypesInSelectList(CustomClusterTestSuite):
@classmethod
def add_test_dimensions(cls):
+ super(TestRangerColumnMaskingComplexTypesInSelectList, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_dimension(create_orc_dimension(cls.get_workload()))
cls.ImpalaTestMatrix.add_constraint(lambda v:
@@ -1598,6 +1599,13 @@ class TestRangerColumnMaskingComplexTypesInSelectList(CustomClusterTestSuite):
cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
disable_codegen_options=[True]))
+ @classmethod
+ def add_custom_cluster_constraints(cls):
+ # Do not call the super() implementation, because this class needs to relax
+ # the set of constraints. The usual constraints only run on uncompressed text.
+ # This disables that constraint to let us run against only ORC.
+ return
+
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
impalad_args=IMPALAD_ARGS, catalogd_args=CATALOGD_ARGS)
diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py
index 66a3fff78..4add40501 100644
--- a/tests/custom_cluster/test_client_ssl.py
+++ b/tests/custom_cluster/test_client_ssl.py
@@ -140,6 +140,7 @@ class TestClientSsl(CustomClusterTestSuite):
@classmethod
def add_test_dimensions(cls):
+ super(TestClientSsl, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
@pytest.mark.execute_serially
diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py
index 401d9b960..30d75fdc3 100644
--- a/tests/query_test/test_insert.py
+++ b/tests/query_test/test_insert.py
@@ -29,7 +29,9 @@ from tests.common.skip import SkipIfABFS, SkipIfEC, SkipIfLocal, \
SkipIfHive2, SkipIfNotHdfsMinicluster, SkipIfS3, SkipIfDockerizedCluster
from tests.common.test_dimensions import (
create_exec_option_dimension,
- create_uncompressed_text_dimension)
+ create_uncompressed_text_dimension,
+ create_single_exec_option_dimension,
+ is_supported_insert_format)
from tests.common.test_result_verifier import (
QueryTestResult,
parse_result_rows)
@@ -335,20 +337,26 @@ class TestInsertFileExtension(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
- cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension(
- 'table_format_and_file_extension',
- *[('parquet', '.parq'), ('textfile', '.txt')]))
+ super(TestInsertFileExtension, cls).add_test_dimensions()
+ cls.ImpalaTestMatrix.add_constraint(lambda v:
+ is_supported_insert_format(v.get_value('table_format')))
+ cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
@classmethod
def setup_class(cls):
super(TestInsertFileExtension, cls).setup_class()
def test_file_extension(self, vector, unique_database):
- table_format = vector.get_value('table_format_and_file_extension')[0]
- file_extension = vector.get_value('table_format_and_file_extension')[1]
+ table_format = vector.get_value('table_format').file_format
+ if table_format == 'parquet':
+ file_extension = '.parq'
+ stored_as_format = 'parquet'
+ else:
+ file_extension = '.txt'
+ stored_as_format = 'textfile'
table_name = "{0}_table".format(table_format)
ctas_query = "create table {0}.{1} stored as {2} as select 1".format(
- unique_database, table_name, table_format)
+ unique_database, table_name, stored_as_format)
self.execute_query_expect_success(self.client, ctas_query)
for path in self.filesystem_client.ls("test-warehouse/{0}.db/{1}".format(
unique_database, table_name)):
diff --git a/tests/query_test/test_nested_types.py b/tests/query_test/test_nested_types.py
index 3ca64f885..ef64e1181 100644
--- a/tests/query_test/test_nested_types.py
+++ b/tests/query_test/test_nested_types.py
@@ -36,7 +36,8 @@ from tests.common.skip import (
SkipIfNotHdfsMinicluster
)
from tests.common.test_dimensions import (create_exec_option_dimension,
- create_exec_option_dimension_from_dict, create_client_protocol_dimension)
+ create_exec_option_dimension_from_dict, create_client_protocol_dimension,
+ create_orc_dimension)
from tests.common.test_vector import ImpalaTestDimension
from tests.util.filesystem_utils import WAREHOUSE, get_fs_path, IS_HDFS
@@ -185,9 +186,11 @@ class TestNestedTypesInSelectListWithBeeswax(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
+ super(TestNestedTypesInSelectListWithBeeswax, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_constraint(lambda v:
v.get_value('protocol') == 'beeswax')
+ cls.ImpalaTestMatrix.add_dimension(create_orc_dimension(cls.get_workload()))
cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
disable_codegen_options=[True]))
diff --git a/tests/shell/test_shell_client.py b/tests/shell/test_shell_client.py
index 5fa1d5caa..1388b527f 100644
--- a/tests/shell/test_shell_client.py
+++ b/tests/shell/test_shell_client.py
@@ -20,8 +20,9 @@
from shell.impala_client import ImpalaBeeswaxClient, ImpalaHS2Client
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.test_dimensions import create_client_protocol_dimension
-from tests.common.test_dimensions import create_client_protocol_no_strict_dimension
+from tests.common.test_dimensions import (
+ create_client_protocol_dimension, create_client_protocol_no_strict_dimension,
+ create_uncompressed_text_dimension, create_single_exec_option_dimension)
from util import get_impalad_host_port
@@ -34,6 +35,12 @@ class TestShellClient(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
+ super(TestShellClient, cls).add_test_dimensions()
+ # Limit to uncompressed text with default exec options
+ cls.ImpalaTestMatrix.add_dimension(
+ create_uncompressed_text_dimension(cls.get_workload()))
+ cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
+ # Run with beeswax and HS2
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_no_strict_dimension())
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index cd53404c2..de81c9761 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -34,8 +34,9 @@ from tests.common.environ import ImpalaTestClusterProperties
from tests.common.impala_service import ImpaladService
from tests.common.impala_test_suite import ImpalaTestSuite, IMPALAD_HS2_HOST_PORT
from tests.common.skip import SkipIf
-from tests.common.test_dimensions import create_client_protocol_dimension
-from tests.common.test_dimensions import create_client_protocol_strict_dimension
+from tests.common.test_dimensions import (
+ create_client_protocol_dimension, create_client_protocol_strict_dimension,
+ create_uncompressed_text_dimension, create_single_exec_option_dimension)
from time import sleep, time
from util import (get_impalad_host_port, assert_var_substitution, run_impala_shell_cmd,
ImpalaShell, IMPALA_SHELL_EXECUTABLE, SHELL_IS_PYTHON_2,
@@ -134,6 +135,11 @@ class TestImpalaShell(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
+ super(TestImpalaShell, cls).add_test_dimensions()
+ # Limit to uncompressed text with default exec options
+ cls.ImpalaTestMatrix.add_dimension(
+ create_uncompressed_text_dimension(cls.get_workload()))
+ cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
# Run with both beeswax and HS2 to ensure that behaviour is the same.
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_strict_dimension())
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index 2d1a4aa7a..0a6ad20b4 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -40,8 +40,9 @@ from tempfile import NamedTemporaryFile
from tests.common.impala_service import ImpaladService
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfLocal
-from tests.common.test_dimensions import create_client_protocol_dimension
-from tests.common.test_dimensions import create_client_protocol_strict_dimension
+from tests.common.test_dimensions import (
+ create_client_protocol_dimension, create_client_protocol_strict_dimension,
+ create_uncompressed_text_dimension, create_single_exec_option_dimension)
from tests.shell.util import get_unused_port
from util import (assert_var_substitution, ImpalaShell, get_impalad_port, get_shell_cmd,
get_open_sessions_metric, IMPALA_SHELL_EXECUTABLE, spawn_shell)
@@ -165,6 +166,11 @@ class TestImpalaShellInteractive(ImpalaTestSuite):
@classmethod
def add_test_dimensions(cls):
+ super(TestImpalaShellInteractive, cls).add_test_dimensions()
+ # Limit to uncompressed text with default exec options
+ cls.ImpalaTestMatrix.add_dimension(
+ create_uncompressed_text_dimension(cls.get_workload()))
+ cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
# Run with both beeswax and HS2 to ensure that behaviour is the same.
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
cls.ImpalaTestMatrix.add_dimension(create_client_protocol_strict_dimension())
@@ -681,6 +687,8 @@ class TestImpalaShellInteractive(ImpalaTestSuite):
def test_commandline_flag_disable_live_progress(self, vector):
"""Test the command line flag disable_live_progress with live_progress."""
+ if vector.get_value('strict_hs2_protocol'):
+ pytest.skip("Live option not supported in strict hs2 mode.")
# By default, shell option live_progress is set to True in the interactive mode.
cmds = "set all;"
result = run_impala_shell_interactive(vector, cmds)
@@ -704,13 +712,15 @@ class TestImpalaShellInteractive(ImpalaTestSuite):
cmds = "set all;"
# override the default option through command line argument.
- args = ['--strict_h2_protocol']
+ args = ['--strict_hs2_protocol']
result = run_impala_shell_interactive(vector, cmds, shell_args=args)
assert "\tLIVE_PROGRESS: False" in result.stdout
assert "\tLIVE_SUMMARY: False" in result.stdout
def test_live_option_configuration(self, vector):
"""Test the optional configuration file with live_progress and live_summary."""
+ if vector.get_value('strict_hs2_protocol'):
+ pytest.skip("Live option not supported in strict hs2 mode.")
# Positive tests
# set live_summary and live_progress as True with config file
rcfile_path = os.path.join(QUERY_FILE_PATH, 'good_impalarc3')