You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/08/24 03:26:48 UTC
incubator-impala git commit: IMPALA-3832: test invalid data handling
in lzo text scanner
Repository: incubator-impala
Updated Branches:
refs/heads/master 9e438a08f -> 310edd5d0
IMPALA-3832: test invalid data handling in lzo text scanner
This adds the lzo text scanner to the fuzz testing.
I ran the test in a loop under ASAN overnight and didn't see any
failures.
Change-Id: Ib707014c1fcfb80cb8076f644fc2b62a5ae758d7
Reviewed-on: http://gerrit.cloudera.org:8080/4096
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/310edd5d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/310edd5d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/310edd5d
Branch: refs/heads/master
Commit: 310edd5d0040b2848f68354091d311fe9ee1c822
Parents: 9e438a0
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Mon Aug 15 17:58:57 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Wed Aug 24 03:09:32 2016 +0000
----------------------------------------------------------------------
tests/query_test/test_scanners_fuzz.py | 25 +++++++++++++++++--------
1 file changed, 17 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/310edd5d/tests/query_test/test_scanners_fuzz.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_scanners_fuzz.py b/tests/query_test/test_scanners_fuzz.py
index ae17572..76cc62a 100644
--- a/tests/query_test/test_scanners_fuzz.py
+++ b/tests/query_test/test_scanners_fuzz.py
@@ -39,10 +39,11 @@ class TestScannersFuzzing(ImpalaTestSuite):
def add_test_dimensions(cls):
super(TestScannersFuzzing, cls).add_test_dimensions()
# TODO: enable for more table formats once they consistently pass the fuzz test.
- cls.TestMatrix.add_constraint(lambda v:\
+ cls.TestMatrix.add_constraint(lambda v:
v.get_value('table_format').file_format in ('avro', 'parquet') or
- (v.get_value('table_format').file_format == 'text'
- and v.get_value('table_format').compression_type == 'none'))
+ (v.get_value('table_format').file_format == 'text' and
+ v.get_value('table_format').compression_codec in ('none', 'lzo')))
+
def test_fuzz_alltypes(self, vector, unique_database):
self.run_fuzz_test(vector, unique_database, "alltypes")
@@ -50,11 +51,15 @@ class TestScannersFuzzing(ImpalaTestSuite):
def test_fuzz_decimal_tbl(self, vector, unique_database):
table_format = vector.get_value('table_format')
table_name = "decimal_tbl"
- if table_format.file_format in ('avro'):
+ if table_format.file_format == 'avro':
table_name = "avro_decimal_tbl"
- if table_format.compression_codec != 'block' or \
- table_format.compression_type != 'snap':
+ if table_format.compression_codec != 'snap' or \
+ table_format.compression_type != 'block':
pytest.skip()
+ elif table_format.file_format == 'text' and \
+ table_format.compression_codec != 'none':
+ # decimal_tbl is not present for these file formats
+ pytest.skip()
self.run_fuzz_test(vector, unique_database, table_name, 10)
@@ -134,8 +139,12 @@ class TestScannersFuzzing(ImpalaTestSuite):
continue
msg = "Should not throw error when abort_on_error=0: '{0}'".format(e)
LOG.error(msg)
- # Parquet fails the query for some parse errors.
- if table_format.file_format == 'parquet':
+ # Parquet and compressed text can fail the query for some parse errors.
+ # E.g. corrupt Parquet footer (IMPALA-3773) or a corrupt LZO index file
+ # (IMPALA-4013).
+ if table_format.file_format == 'parquet' or \
+ (table_format.file_format == 'text' and
+ table_format.compression_codec != 'none'):
xfail_msgs.append(msg)
else:
raise