[impala] 02/03: IMPALA-11249: Fix add_test_dimensions() locations to call super()

[bo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

boroknagyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 6a199be8544f5d76e00829f7493fcaac2f9c274c
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Sat May 21 19:10:11 2022 -0700

    IMPALA-11249: Fix add_test_dimensions() locations to call super()
    
    The original issue is that the strict HS2 shell tests
    are not running in precommit or nightly jobs, but they
    do run in local developer environments. Investigating
    this showed that the shell tests were running with a
    weird set of test dimensions that includes
    table_format_and_file_extension. That dimension is only
    used in test_insert.py::TestInsertFileExtension.
    
    What is happening is that the shell tests and other
    locations are running add_test_dimensions() without
    calling super(..., cls).add_test_dimensions(). The
    behavior is unclear, but there is clearly cross-talk
    between the different tests that do this.
    
    This changes all add_test_dimensions() locations to
    call super(..., cls).add_test_dimensions() if they
    don't already. Each location has been tuned to run
    the same set of tests as before (except the shell
    tests which now run the strict HS2 tests).
    
    As part of this, several shell tests need to be
    skipped or fixed for strict HS2.
    
    Testing:
     - Ran core job
     - Ran tests locally to verify the set of tests
       didn't change.
    
    Change-Id: Ib20fd479d3b91ed0ed89a0bc5623cd2a5a458614
    Reviewed-on: http://gerrit.cloudera.org:8080/18557
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 tests/authorization/test_ranger.py      |  8 ++++++++
 tests/custom_cluster/test_client_ssl.py |  1 +
 tests/query_test/test_insert.py         | 22 +++++++++++++++-------
 tests/query_test/test_nested_types.py   |  5 ++++-
 tests/shell/test_shell_client.py        | 11 +++++++++--
 tests/shell/test_shell_commandline.py   | 10 ++++++++--
 tests/shell/test_shell_interactive.py   | 16 +++++++++++++---
 7 files changed, 58 insertions(+), 15 deletions(-)

diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py
index 875886b12..d0caa7b17 100644
--- a/tests/authorization/test_ranger.py
+++ b/tests/authorization/test_ranger.py
@@ -1591,6 +1591,7 @@ class TestRangerColumnMaskingComplexTypesInSelectList(CustomClusterTestSuite):
 
   @classmethod
   def add_test_dimensions(cls):
+    super(TestRangerColumnMaskingComplexTypesInSelectList, cls).add_test_dimensions()
     cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
     cls.ImpalaTestMatrix.add_dimension(create_orc_dimension(cls.get_workload()))
     cls.ImpalaTestMatrix.add_constraint(lambda v:
@@ -1598,6 +1599,13 @@ class TestRangerColumnMaskingComplexTypesInSelectList(CustomClusterTestSuite):
     cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
         disable_codegen_options=[True]))
 
+  @classmethod
+  def add_custom_cluster_constraints(cls):
+    # Do not call the super() implementation, because this class needs to relax
+    # the set of constraints. The usual constraints only run on uncompressed text.
+    # This disables that constraint to let us run against only ORC.
+    return
+
   @pytest.mark.execute_serially
   @CustomClusterTestSuite.with_args(
     impalad_args=IMPALAD_ARGS, catalogd_args=CATALOGD_ARGS)
diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py
index 66a3fff78..4add40501 100644
--- a/tests/custom_cluster/test_client_ssl.py
+++ b/tests/custom_cluster/test_client_ssl.py
@@ -140,6 +140,7 @@ class TestClientSsl(CustomClusterTestSuite):
 
   @classmethod
   def add_test_dimensions(cls):
+    super(TestClientSsl, cls).add_test_dimensions()
     cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
 
   @pytest.mark.execute_serially
diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py
index 401d9b960..30d75fdc3 100644
--- a/tests/query_test/test_insert.py
+++ b/tests/query_test/test_insert.py
@@ -29,7 +29,9 @@ from tests.common.skip import SkipIfABFS, SkipIfEC, SkipIfLocal, \
     SkipIfHive2, SkipIfNotHdfsMinicluster, SkipIfS3, SkipIfDockerizedCluster
 from tests.common.test_dimensions import (
     create_exec_option_dimension,
-    create_uncompressed_text_dimension)
+    create_uncompressed_text_dimension,
+    create_single_exec_option_dimension,
+    is_supported_insert_format)
 from tests.common.test_result_verifier import (
     QueryTestResult,
     parse_result_rows)
@@ -335,20 +337,26 @@ class TestInsertFileExtension(ImpalaTestSuite):
 
   @classmethod
   def add_test_dimensions(cls):
-    cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension(
-        'table_format_and_file_extension',
-        *[('parquet', '.parq'), ('textfile', '.txt')]))
+    super(TestInsertFileExtension, cls).add_test_dimensions()
+    cls.ImpalaTestMatrix.add_constraint(lambda v:
+        is_supported_insert_format(v.get_value('table_format')))
+    cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
 
   @classmethod
   def setup_class(cls):
     super(TestInsertFileExtension, cls).setup_class()
 
   def test_file_extension(self, vector, unique_database):
-    table_format = vector.get_value('table_format_and_file_extension')[0]
-    file_extension = vector.get_value('table_format_and_file_extension')[1]
+    table_format = vector.get_value('table_format').file_format
+    if table_format == 'parquet':
+      file_extension = '.parq'
+      stored_as_format = 'parquet'
+    else:
+      file_extension = '.txt'
+      stored_as_format = 'textfile'
     table_name = "{0}_table".format(table_format)
     ctas_query = "create table {0}.{1} stored as {2} as select 1".format(
-        unique_database, table_name, table_format)
+        unique_database, table_name, stored_as_format)
     self.execute_query_expect_success(self.client, ctas_query)
     for path in self.filesystem_client.ls("test-warehouse/{0}.db/{1}".format(
         unique_database, table_name)):
diff --git a/tests/query_test/test_nested_types.py b/tests/query_test/test_nested_types.py
index 3ca64f885..ef64e1181 100644
--- a/tests/query_test/test_nested_types.py
+++ b/tests/query_test/test_nested_types.py
@@ -36,7 +36,8 @@ from tests.common.skip import (
     SkipIfNotHdfsMinicluster
     )
 from tests.common.test_dimensions import (create_exec_option_dimension,
-    create_exec_option_dimension_from_dict, create_client_protocol_dimension)
+    create_exec_option_dimension_from_dict, create_client_protocol_dimension,
+    create_orc_dimension)
 from tests.common.test_vector import ImpalaTestDimension
 from tests.util.filesystem_utils import WAREHOUSE, get_fs_path, IS_HDFS
 
@@ -185,9 +186,11 @@ class TestNestedTypesInSelectListWithBeeswax(ImpalaTestSuite):
 
   @classmethod
   def add_test_dimensions(cls):
+    super(TestNestedTypesInSelectListWithBeeswax, cls).add_test_dimensions()
     cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
     cls.ImpalaTestMatrix.add_constraint(lambda v:
         v.get_value('protocol') == 'beeswax')
+    cls.ImpalaTestMatrix.add_dimension(create_orc_dimension(cls.get_workload()))
     cls.ImpalaTestMatrix.add_dimension(create_exec_option_dimension(
         disable_codegen_options=[True]))
 
diff --git a/tests/shell/test_shell_client.py b/tests/shell/test_shell_client.py
index 5fa1d5caa..1388b527f 100644
--- a/tests/shell/test_shell_client.py
+++ b/tests/shell/test_shell_client.py
@@ -20,8 +20,9 @@
 
 from shell.impala_client import ImpalaBeeswaxClient, ImpalaHS2Client
 from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.test_dimensions import create_client_protocol_dimension
-from tests.common.test_dimensions import create_client_protocol_no_strict_dimension
+from tests.common.test_dimensions import (
+  create_client_protocol_dimension, create_client_protocol_no_strict_dimension,
+  create_uncompressed_text_dimension, create_single_exec_option_dimension)
 from util import get_impalad_host_port
 
 
@@ -34,6 +35,12 @@ class TestShellClient(ImpalaTestSuite):
 
   @classmethod
   def add_test_dimensions(cls):
+    super(TestShellClient, cls).add_test_dimensions()
+    # Limit to uncompressed text with default exec options
+    cls.ImpalaTestMatrix.add_dimension(
+        create_uncompressed_text_dimension(cls.get_workload()))
+    cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
+    # Run with beeswax and HS2
     cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
     cls.ImpalaTestMatrix.add_dimension(create_client_protocol_no_strict_dimension())
 
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index 3d698fb8a..3c6454d5a 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -34,8 +34,9 @@ from tests.common.environ import ImpalaTestClusterProperties
 from tests.common.impala_service import ImpaladService
 from tests.common.impala_test_suite import ImpalaTestSuite, IMPALAD_HS2_HOST_PORT
 from tests.common.skip import SkipIf
-from tests.common.test_dimensions import create_client_protocol_dimension
-from tests.common.test_dimensions import create_client_protocol_strict_dimension
+from tests.common.test_dimensions import (
+  create_client_protocol_dimension, create_client_protocol_strict_dimension,
+  create_uncompressed_text_dimension, create_single_exec_option_dimension)
 from time import sleep, time
 from util import (get_impalad_host_port, assert_var_substitution, run_impala_shell_cmd,
                   ImpalaShell, IMPALA_SHELL_EXECUTABLE, SHELL_IS_PYTHON_2,
@@ -134,6 +135,11 @@ class TestImpalaShell(ImpalaTestSuite):
 
   @classmethod
   def add_test_dimensions(cls):
+    super(TestImpalaShell, cls).add_test_dimensions()
+    # Limit to uncompressed text with default exec options
+    cls.ImpalaTestMatrix.add_dimension(
+        create_uncompressed_text_dimension(cls.get_workload()))
+    cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
     # Run with both beeswax and HS2 to ensure that behaviour is the same.
     cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
     cls.ImpalaTestMatrix.add_dimension(create_client_protocol_strict_dimension())
diff --git a/tests/shell/test_shell_interactive.py b/tests/shell/test_shell_interactive.py
index c1411f86b..cd2db996b 100755
--- a/tests/shell/test_shell_interactive.py
+++ b/tests/shell/test_shell_interactive.py
@@ -40,8 +40,9 @@ from tempfile import NamedTemporaryFile
 from tests.common.impala_service import ImpaladService
 from tests.common.impala_test_suite import ImpalaTestSuite
 from tests.common.skip import SkipIfLocal
-from tests.common.test_dimensions import create_client_protocol_dimension
-from tests.common.test_dimensions import create_client_protocol_strict_dimension
+from tests.common.test_dimensions import (
+  create_client_protocol_dimension, create_client_protocol_strict_dimension,
+  create_uncompressed_text_dimension, create_single_exec_option_dimension)
 from tests.shell.util import get_unused_port
 from util import (assert_var_substitution, ImpalaShell, get_impalad_port, get_shell_cmd,
                   get_open_sessions_metric, IMPALA_SHELL_EXECUTABLE, spawn_shell)
@@ -165,6 +166,11 @@ class TestImpalaShellInteractive(ImpalaTestSuite):
 
   @classmethod
   def add_test_dimensions(cls):
+    super(TestImpalaShellInteractive, cls).add_test_dimensions()
+    # Limit to uncompressed text with default exec options
+    cls.ImpalaTestMatrix.add_dimension(
+        create_uncompressed_text_dimension(cls.get_workload()))
+    cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
     # Run with both beeswax and HS2 to ensure that behaviour is the same.
     cls.ImpalaTestMatrix.add_dimension(create_client_protocol_dimension())
     cls.ImpalaTestMatrix.add_dimension(create_client_protocol_strict_dimension())
@@ -681,6 +687,8 @@ class TestImpalaShellInteractive(ImpalaTestSuite):
 
   def test_commandline_flag_disable_live_progress(self, vector):
     """Test the command line flag disable_live_progress with live_progress."""
+    if vector.get_value('strict_hs2_protocol'):
+      pytest.skip("Live option not supported in strict hs2 mode.")
     # By default, shell option live_progress is set to True in the interactive mode.
     cmds = "set all;"
     result = run_impala_shell_interactive(vector, cmds)
@@ -704,13 +712,15 @@ class TestImpalaShellInteractive(ImpalaTestSuite):
 
     cmds = "set all;"
     # override the default option through command line argument.
-    args = ['--strict_h2_protocol']
+    args = ['--strict_hs2_protocol']
     result = run_impala_shell_interactive(vector, cmds, shell_args=args)
     assert "\tLIVE_PROGRESS: False" in result.stdout
     assert "\tLIVE_SUMMARY: False" in result.stdout
 
   def test_live_option_configuration(self, vector):
     """Test the optional configuration file with live_progress and live_summary."""
+    if vector.get_value('strict_hs2_protocol'):
+      pytest.skip("Live option not supported in strict hs2 mode.")
     # Positive tests
     # set live_summary and live_progress as True with config file
     rcfile_path = os.path.join(QUERY_FILE_PATH, 'good_impalarc3')