You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@impala.apache.org by "Michael Ho (JIRA)" <ji...@apache.org> on 2017/08/07 01:59:01 UTC
[jira] [Created] (IMPALA-5772) Expected failure in
test_scratch_disk.TestScratchDir didn't occur
Michael Ho created IMPALA-5772:
----------------------------------
Summary: Expected failure in test_scratch_disk.TestScratchDir didn't occur
Key: IMPALA-5772
URL: https://issues.apache.org/jira/browse/IMPALA-5772
Project: IMPALA
Issue Type: Bug
Components: Backend
Affects Versions: Impala 2.10.0
Reporter: Michael Ho
Priority: Blocker
Expected failure in test_non_writable_dirs and test_non_existing_dirs didn't occur on S3 builds. Not sure if there were some changes to S3 related code recently.
cc'ing [~sailesh] and [~tarmstrong]
{noformat}
09:04:19 =================================== FAILURES ===================================
09:04:19 TestScratchDir.test_no_dirs[exec_option: {'batch_size': 0, 'num_nodes': 0, 'disable_codegen_rows_threshold': 0, 'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format: text/none]
09:04:19
09:04:19 self = <test_scratch_disk.TestScratchDir object at 0x36dfad0>
09:04:19 vector = <tests.common.test_vector.ImpalaTestVector object at 0x56b4950>
09:04:19
09:04:19 @pytest.mark.execute_serially
09:04:19 def test_no_dirs(self, vector):
09:04:19 """ Test we can execute a query with no scratch dirs """
09:04:19 self._start_impala_cluster(['--impalad_args="-logbuflevel=-1 -scratch_dirs="'])
09:04:19 self.assert_impalad_log_contains("WARNING",
09:04:19 "Running without spill to disk: no scratch directories provided\.")
09:04:19 exec_option = vector.get_value('exec_option')
09:04:19 exec_option['buffer_pool_limit'] = self.buffer_pool_limit
09:04:19 impalad = self.cluster.get_any_impalad()
09:04:19 client = impalad.service.create_beeswax_client()
09:04:19 # Expect spill to disk to fail
09:04:19 > self.execute_query_expect_failure(client, self.spill_query, exec_option)
09:04:19
09:04:19 custom_cluster/test_scratch_disk.py:107:
09:04:19 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
09:04:19 common/impala_test_suite.py:506: in wrapper
09:04:19 return function(*args, **kwargs)
09:04:19 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
09:04:19
09:04:19 self = <test_scratch_disk.TestScratchDir object at 0x36dfad0>
09:04:19 impalad_client = <tests.common.impala_connection.BeeswaxConnection object at 0x36dfb90>
09:04:19 query = '\n select o_orderdate, o_custkey, o_comment\n from tpch.orders\n order by o_orderdate\n '
09:04:19 query_options = {'abort_on_error': 1, 'batch_size': 0, 'buffer_pool_limit': '64m', 'disable_codegen': False, ...}
09:04:19
09:04:19 @execute_wrapper
09:04:19 def execute_query_expect_failure(self, impalad_client, query, query_options=None):
09:04:19 """Executes a query and asserts if the query succeeds"""
09:04:19 result = None
09:04:19 try:
09:04:19 result = self.__execute_query(impalad_client, query, query_options)
09:04:19 except Exception, e:
09:04:19 return e
09:04:19
09:04:19 > assert not result.success, "No failure encountered for query %s" % query
09:04:19 E AssertionError: No failure encountered for query
09:04:19 E select o_orderdate, o_custkey, o_comment
09:04:19 E from tpch.orders
09:04:19 E order by o_orderdate
09:04:19
09:04:19 common/impala_test_suite.py:526: AssertionError
09:04:19 ----------------------------- Captured stdout call -----------------------------
{noformat}
{noformat}
09:04:19 -- connecting to: impala-boost-static-burst-slave-1780.vpc.cloudera.com:21002
09:04:19 SET batch_size=0;
09:04:19 SET num_nodes=0;
09:04:19 SET disable_codegen_rows_threshold=0;
09:04:19 SET disable_codegen=False;
09:04:19 SET abort_on_error=1;
09:04:19 SET exec_single_node_rows_threshold=0;
09:04:19 SET buffer_pool_limit=64m;
09:04:19 -- executing against impala-boost-static-burst-slave-1780.vpc.cloudera.com:21002
09:04:19
09:04:19 select o_orderdate, o_custkey, o_comment
09:04:19 from tpch.orders
09:04:19 order by o_orderdate
09:04:19 ;
09:04:19
09:04:19 TestScratchDir.test_non_writable_dirs[exec_option: {'batch_size': 0, 'num_nodes': 0, 'disable_codegen_rows_threshold': 0, 'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format: text/none]
09:04:19
09:04:19 self = <test_scratch_disk.TestScratchDir object at 0x4d1b450>
09:04:19 vector = <tests.common.test_vector.ImpalaTestVector object at 0x56b49d0>
09:04:19
09:04:19 @pytest.mark.execute_serially
09:04:19 def test_non_writable_dirs(self, vector):
09:04:19 """ Test we can execute a query with only bad non-writable scratch """
09:04:19 non_writable_dirs = self.generate_dirs(5, writable=False)
09:04:19 self._start_impala_cluster([
09:04:19 '--impalad_args="-logbuflevel=-1 -scratch_dirs={0}"'.format(
09:04:19 ','.join(non_writable_dirs))])
09:04:19 self.assert_impalad_log_contains("ERROR", "Running without spill to disk: could "
09:04:19 + "not use any scratch directories in list:.*. See previous "
09:04:19 + "warnings for information on causes.")
09:04:19 self.assert_impalad_log_contains("WARNING", "Could not remove and recreate directory "
09:04:19 + ".*: cannot use it for scratch\. Error was: .*", expected_count=5)
09:04:19 exec_option = vector.get_value('exec_option')
09:04:19 exec_option['buffer_pool_limit'] = self.buffer_pool_limit
09:04:19 impalad = self.cluster.get_any_impalad()
09:04:19 client = impalad.service.create_beeswax_client()
09:04:19 # Expect spill to disk to fail
09:04:19 > self.execute_query_expect_failure(client, self.spill_query, exec_option)
09:04:19
09:04:19 custom_cluster/test_scratch_disk.py:128:
09:04:19 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
09:04:19 common/impala_test_suite.py:506: in wrapper
09:04:19 return function(*args, **kwargs)
09:04:19 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
09:04:19
09:04:19 self = <test_scratch_disk.TestScratchDir object at 0x4d1b450>
09:04:19 impalad_client = <tests.common.impala_connection.BeeswaxConnection object at 0x4d1bed0>
09:04:19 query = '\n select o_orderdate, o_custkey, o_comment\n from tpch.orders\n order by o_orderdate\n '
09:04:19 query_options = {'abort_on_error': 1, 'batch_size': 0, 'buffer_pool_limit': '64m', 'disable_codegen': False, ...}
09:04:19
09:04:19 @execute_wrapper
09:04:19 def execute_query_expect_failure(self, impalad_client, query, query_options=None):
09:04:19 """Executes a query and asserts if the query succeeds"""
09:04:19 result = None
09:04:19 try:
09:04:19 result = self.__execute_query(impalad_client, query, query_options)
09:04:19 except Exception, e:
09:04:19 return e
09:04:19
09:04:19 > assert not result.success, "No failure encountered for query %s" % query
09:04:19 E AssertionError: No failure encountered for query
09:04:19 E select o_orderdate, o_custkey, o_comment
09:04:19 E from tpch.orders
09:04:19 E order by o_orderdate
09:04:19
09:04:19 common/impala_test_suite.py:526: AssertionError
09:04:19 ----------------------------- Captured stdout call -----------------------------
{noformat}
{noformat}
09:04:19 -- connecting to: impala-boost-static-burst-slave-1780.vpc.cloudera.com:21000
09:04:19 SET batch_size=0;
09:04:19 SET num_nodes=0;
09:04:19 SET disable_codegen_rows_threshold=0;
09:04:19 SET disable_codegen=False;
09:04:19 SET abort_on_error=1;
09:04:19 SET exec_single_node_rows_threshold=0;
09:04:19 SET buffer_pool_limit=64m;
09:04:19 -- executing against impala-boost-static-burst-slave-1780.vpc.cloudera.com:21000
09:04:19
09:04:19 select o_orderdate, o_custkey, o_comment
09:04:19 from tpch.orders
09:04:19 order by o_orderdate
09:04:19 ;
09:04:19
09:04:19 TestScratchDir.test_non_existing_dirs[exec_option: {'batch_size': 0, 'num_nodes': 0, 'disable_codegen_rows_threshold': 0, 'disable_codegen': False, 'abort_on_error': 1, 'exec_single_node_rows_threshold': 0} | table_format: text/none]
09:04:19
09:04:19 self = <test_scratch_disk.TestScratchDir object at 0x4d2bad0>
09:04:19 vector = <tests.common.test_vector.ImpalaTestVector object at 0x56b4410>
09:04:19
09:04:19 @pytest.mark.execute_serially
09:04:19 def test_non_existing_dirs(self, vector):
09:04:19 """ Test that non-existing directories are not created or used """
09:04:19 non_existing_dirs = self.generate_dirs(5, non_existing=True)
09:04:19 self._start_impala_cluster([
09:04:19 '--impalad_args="-logbuflevel=-1 -scratch_dirs={0}"'.format(
09:04:19 ','.join(non_existing_dirs))])
09:04:19 self.assert_impalad_log_contains("ERROR", "Running without spill to disk: could "
09:04:19 + "not use any scratch directories in list:.*. See previous "
09:04:19 + "warnings for information on causes.")
09:04:19 self.assert_impalad_log_contains("WARNING", "Cannot use directory .* for scratch: "
09:04:19 + "Encountered exception while verifying existence of directory path",
09:04:19 expected_count=5)
09:04:19 exec_option = vector.get_value('exec_option')
09:04:19 exec_option['buffer_pool_limit'] = self.buffer_pool_limit
09:04:19 impalad = self.cluster.get_any_impalad()
09:04:19 client = impalad.service.create_beeswax_client()
09:04:19 # Expect spill to disk to fail
09:04:19 > self.execute_query_expect_failure(client, self.spill_query, exec_option)
09:04:19
09:04:19 custom_cluster/test_scratch_disk.py:151:
09:04:19 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
09:04:19 common/impala_test_suite.py:506: in wrapper
09:04:19 return function(*args, **kwargs)
09:04:19 _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
09:04:19
09:04:19 self = <test_scratch_disk.TestScratchDir object at 0x4d2bad0>
09:04:19 impalad_client = <tests.common.impala_connection.BeeswaxConnection object at 0x4d2b290>
09:04:19 query = '\n select o_orderdate, o_custkey, o_comment\n from tpch.orders\n order by o_orderdate\n '
09:04:19 query_options = {'abort_on_error': 1, 'batch_size': 0, 'buffer_pool_limit': '64m', 'disable_codegen': False, ...}
09:04:19
09:04:19 @execute_wrapper
09:04:19 def execute_query_expect_failure(self, impalad_client, query, query_options=None):
09:04:19 """Executes a query and asserts if the query succeeds"""
09:04:19 result = None
09:04:19 try:
09:04:19 result = self.__execute_query(impalad_client, query, query_options)
09:04:19 except Exception, e:
09:04:19 return e
09:04:19
09:04:19 > assert not result.success, "No failure encountered for query %s" % query
09:04:19 E AssertionError: No failure encountered for query
09:04:19 E select o_orderdate, o_custkey, o_comment
09:04:19 E from tpch.orders
09:04:19 E order by o_orderdate
09:04:19
09:04:19 common/impala_test_suite.py:526: AssertionError
09:04:19 ----------------------------- Captured stdout call -----------------------------
{noformat}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)