You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2020/10/30 04:32:26 UTC
[impala] branch master updated (1bd27a3 -> 047906b)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from 1bd27a3 IMPALA-7097 Print EC info in the query plan and profile
new 193c2e7 IMPALA-10132: Implement ds_hll_estimate_bounds_as_string() function.
new 1682afc IMPALA-10298: Change column mask hash as SHA512 in FIPS mode
new 01e1b4d IMPALA-10303: Fix warnings from impala-shell with --quiet
new 047906b IMPALA-10302: Enable logging at the INFO level for test_scanners_fuzz.py
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exprs/datasketches-common.h | 3 ++
be/src/exprs/datasketches-functions-ir.cc | 24 ++++++++++
be/src/exprs/datasketches-functions.h | 16 +++++++
be/src/exprs/expr-test.cc | 26 +++++++---
be/src/exprs/mask-functions-ir.cc | 19 ++++++--
common/function-registry/impala_functions.py | 4 ++
shell/impala_client.py | 2 +-
shell/impala_shell.py | 8 ++--
.../queries/QueryTest/datasketches-hll.test | 56 ++++++++++++++++++++++
tests/query_test/test_scanners_fuzz.py | 8 +++-
tests/shell/test_shell_commandline.py | 8 ++++
11 files changed, 157 insertions(+), 17 deletions(-)
[impala] 02/04: IMPALA-10298: Change column mask hash as SHA512 in
FIPS mode
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 1682afcda6e28571c83200a89f0e334a5ae49aa7
Author: wzhou-code <wz...@cloudera.com>
AuthorDate: Tue Oct 27 14:57:53 2020 -0700
IMPALA-10298: Change column mask hash as SHA512 in FIPS mode
Column masking API is called by Ranger during policy evaluation.
Ranger team requires to change the column mask hash as SHA-512 in
FIPS mode without changing API.
This patch changes the MaskFunctions::MaskHash() for string type
to use SHA-512 in FIPS mode.
Testing:
- Passed exhaustive tests.
- Manually test the API.
Change-Id: I422d4b11b31c3e6eb7963260a1da730579c4ca74
Reviewed-on: http://gerrit.cloudera.org:8080/16671
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/exprs/expr-test.cc | 26 ++++++++++++++++++++------
be/src/exprs/mask-functions-ir.cc | 19 ++++++++++++++-----
2 files changed, 34 insertions(+), 11 deletions(-)
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index 72e7aac..024799b 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -29,6 +29,8 @@
#include <boost/scoped_ptr.hpp>
#include <boost/unordered_map.hpp>
+#include <openssl/crypto.h>
+
#include "codegen/llvm-codegen.h"
#include "common/init.h"
#include "common/object-pool.h"
@@ -10481,12 +10483,24 @@ TEST_P(ExprTest, MaskTest) {
}
TEST_P(ExprTest, MaskHashTest) {
- TestStringValue("mask_hash('TestString-123')",
- "8b44d559dc5d60e4453c9b4edf2a455fbce054bb8504cd3eb9b5f391bd239c90");
- TestStringValue("mask_hash(cast('TestString-123' as varchar(24)))",
- "8b44d559dc5d60e4453c9b4edf2a455fbce054bb8504cd3eb9b5f391bd239c90");
- TestStringValue("mask_hash(cast('TestString-123' as char(24)))",
- "30a88603135d3a6f7a66b4f9193da1ab4423aed45fb8fe736c2f2a08977f2bdd");
+ if (FIPS_mode()) {
+ TestStringValue("mask_hash('TestString-123')",
+ "f3a58111be6ecec11449ac44654e72376b7759883ea11723b6e51354d50436de"
+ "645bd061cb5c2b07b68e15b7a7c342cac41f69b9c4efe19e810bbd7abf639a1c");
+ TestStringValue("mask_hash(cast('TestString-123' as varchar(24)))",
+ "f3a58111be6ecec11449ac44654e72376b7759883ea11723b6e51354d50436de"
+ "645bd061cb5c2b07b68e15b7a7c342cac41f69b9c4efe19e810bbd7abf639a1c");
+ TestStringValue("mask_hash(cast('TestString-123' as char(24)))",
+ "8eb5cfb29df20ccb1142aab8700ef4649c3b26304c35263c9bbc7db0d20e1098"
+ "47a728afe0dccdfbbb3d876a5cb3ceb0bd34b5104dd62af1feb234d705bfb193");
+ } else {
+ TestStringValue("mask_hash('TestString-123')",
+ "8b44d559dc5d60e4453c9b4edf2a455fbce054bb8504cd3eb9b5f391bd239c90");
+ TestStringValue("mask_hash(cast('TestString-123' as varchar(24)))",
+ "8b44d559dc5d60e4453c9b4edf2a455fbce054bb8504cd3eb9b5f391bd239c90");
+ TestStringValue("mask_hash(cast('TestString-123' as char(24)))",
+ "30a88603135d3a6f7a66b4f9193da1ab4423aed45fb8fe736c2f2a08977f2bdd");
+ }
TestIsNull("mask_hash(cast(123 as tinyint))", TYPE_BIGINT);
TestIsNull("mask_hash(cast(12345 as smallint))", TYPE_BIGINT);
TestIsNull("mask_hash(cast(12345 as int))", TYPE_BIGINT);
diff --git a/be/src/exprs/mask-functions-ir.cc b/be/src/exprs/mask-functions-ir.cc
index 15177bf..6034501 100644
--- a/be/src/exprs/mask-functions-ir.cc
+++ b/be/src/exprs/mask-functions-ir.cc
@@ -18,6 +18,7 @@
#include "exprs/mask-functions.h"
#include <gutil/strings/substitute.h>
+#include <openssl/crypto.h>
#include <openssl/err.h>
#include <openssl/sha.h>
@@ -702,11 +703,19 @@ BigIntVal MaskFunctions::Mask(FunctionContext* ctx, const BigIntVal& val,
}
StringVal MaskFunctions::MaskHash(FunctionContext* ctx, const StringVal& val) {
- // Hive hash the value by sha256 and encoding it into a lower case hex string.
- StringVal sha256_hash(ctx, SHA256_DIGEST_LENGTH);
- if (UNLIKELY(sha256_hash.is_null)) return StringVal::null();
- discard_result(SHA256(val.ptr, val.len, sha256_hash.ptr));
- return StringFunctions::Lower(ctx, MathFunctions::HexString(ctx, sha256_hash));
+ // Hive hash the value by sha256 and encoding it into a lower case hex string in
+ // non FIPS mode. In FIPS enabled mode, it's required to use sha512 for mask hash.
+ if (FIPS_mode()) {
+ StringVal sha512_hash(ctx, SHA512_DIGEST_LENGTH);
+ if (UNLIKELY(sha512_hash.is_null)) return StringVal::null();
+ discard_result(SHA512(val.ptr, val.len, sha512_hash.ptr));
+ return StringFunctions::Lower(ctx, MathFunctions::HexString(ctx, sha512_hash));
+ } else {
+ StringVal sha256_hash(ctx, SHA256_DIGEST_LENGTH);
+ if (UNLIKELY(sha256_hash.is_null)) return StringVal::null();
+ discard_result(SHA256(val.ptr, val.len, sha256_hash.ptr));
+ return StringFunctions::Lower(ctx, MathFunctions::HexString(ctx, sha256_hash));
+ }
}
// For other types, the hash values are always NULL.
BigIntVal MaskFunctions::MaskHash(FunctionContext* ctx, const BigIntVal& val) {
[impala] 01/04: IMPALA-10132: Implement
ds_hll_estimate_bounds_as_string() function.
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 193c2e773fa9f6772e4a7c30ed3a4f75029863f1
Author: Fucun Chu <ch...@hotmail.com>
AuthorDate: Thu Oct 22 15:33:48 2020 +0800
IMPALA-10132: Implement ds_hll_estimate_bounds_as_string() function.
This function receives a string that is a serialized Apache DataSketches
HLL sketch and optional kappa that is a number of standard deviations
from the mean: 1, 2 or 3 (default 2). Returns estimate and bounds with
the values separated with commas.
The result is three values: estimate, lower bound and upper bound.
ds_hll_estimate_bounds_as_string(sketch [, kappa])
Kappa:
1 represent the 68.3% confidence bounds
2 represent the 95.4% confidence bounds
3 represent the 99.7% confidence bounds
Note, ds_hll_estimate_bounds() should return an Array of doubles as
the result but with that we have to wait for the complex type support.
Until, we provide ds_hll_estimate_bounds_as_string() that can be
deprecated once we have array support. Tracking Jira for returning
complex types from functions is IMPALA-9520.
Example:
select ds_hll_estimate_bounds_as_string(ds_hll_sketch(int_col)) from
functional_parquet.alltypestiny;
+----------------------------------------------------------+
| ds_hll_estimate_bounds_as_string(ds_hll_sketch(int_col)) |
+----------------------------------------------------------+
| 2,2,2.0002 |
+----------------------------------------------------------+
Change-Id: I46bf8263e8fd3877a087b9cb6f0d1a2392bb9153
Reviewed-on: http://gerrit.cloudera.org:8080/16626
Reviewed-by: Gabor Kaszab <ga...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/exprs/datasketches-common.h | 3 ++
be/src/exprs/datasketches-functions-ir.cc | 24 ++++++++++
be/src/exprs/datasketches-functions.h | 16 +++++++
common/function-registry/impala_functions.py | 4 ++
.../queries/QueryTest/datasketches-hll.test | 56 ++++++++++++++++++++++
5 files changed, 103 insertions(+)
diff --git a/be/src/exprs/datasketches-common.h b/be/src/exprs/datasketches-common.h
index cb3d7dc..2f2b4d5 100644
--- a/be/src/exprs/datasketches-common.h
+++ b/be/src/exprs/datasketches-common.h
@@ -39,6 +39,9 @@ const datasketches::target_hll_type DS_HLL_TYPE = datasketches::target_hll_type:
/// buckets equals 2^DS_SKETCH_CONFIG.
const int DS_SKETCH_CONFIG = 12;
+/// 'kappa' is a number of standard deviations from the mean: 1, 2 or 3 (default 2).
+const int DS_DEFAULT_KAPPA = 2;
+
/// Logs a common error message saying that sketch deserialization failed.
void LogSketchDeserializationError(FunctionContext* ctx);
diff --git a/be/src/exprs/datasketches-functions-ir.cc b/be/src/exprs/datasketches-functions-ir.cc
index 1cef6c9..ceef4ae 100644
--- a/be/src/exprs/datasketches-functions-ir.cc
+++ b/be/src/exprs/datasketches-functions-ir.cc
@@ -38,6 +38,30 @@ BigIntVal DataSketchesFunctions::DsHllEstimate(FunctionContext* ctx,
return sketch.get_estimate();
}
+StringVal DataSketchesFunctions::DsHllEstimateBoundsAsString(
+ FunctionContext* ctx, const StringVal& serialized_sketch) {
+ return DsHllEstimateBoundsAsString(ctx, serialized_sketch, DS_DEFAULT_KAPPA);
+}
+
+StringVal DataSketchesFunctions::DsHllEstimateBoundsAsString(
+ FunctionContext* ctx, const StringVal& serialized_sketch, const IntVal& kappa) {
+ if (serialized_sketch.is_null || serialized_sketch.len == 0 || kappa.is_null)
+ return StringVal::null();
+ if (UNLIKELY(kappa.val < 1 || kappa.val > 3)) {
+ ctx->SetError("Kappa must be 1, 2 or 3");
+ return StringVal::null();
+ }
+ datasketches::hll_sketch sketch(DS_SKETCH_CONFIG, DS_HLL_TYPE);
+ if (!DeserializeDsSketch(serialized_sketch, &sketch)) {
+ LogSketchDeserializationError(ctx);
+ return StringVal::null();
+ }
+ std::stringstream buffer;
+ buffer << sketch.get_estimate() << "," << sketch.get_lower_bound(kappa.val) << ","
+ << sketch.get_upper_bound(kappa.val);
+ return StringStreamToStringVal(ctx, buffer);
+}
+
StringVal DataSketchesFunctions::DsHllStringify(FunctionContext* ctx,
const StringVal& serialized_sketch) {
if (serialized_sketch.is_null || serialized_sketch.len == 0) return StringVal::null();
diff --git a/be/src/exprs/datasketches-functions.h b/be/src/exprs/datasketches-functions.h
index 91d9313..8d1aaa6 100644
--- a/be/src/exprs/datasketches-functions.h
+++ b/be/src/exprs/datasketches-functions.h
@@ -21,6 +21,7 @@
namespace impala {
+using impala_udf::IntVal;
using impala_udf::BigIntVal;
using impala_udf::DoubleVal;
using impala_udf::FloatVal;
@@ -36,6 +37,21 @@ public:
const StringVal& serialized_sketch);
/// 'serialized_sketch' is expected as a serialized Apache DataSketches HLL sketch. If
+ /// it is not, then the query fails. Otherwise, returns an estimate of distinct count
+ /// and bounds from the sketch.
+ /// The result is three values: estimate, lower bound and upper bound.
+ /// Note, this function is meant to return an Array of doubles as the result but with
+ /// that we have to wait for the complex type support. Tracking Jira is IMPALA-9520.
+ static StringVal DsHllEstimateBoundsAsString(
+ FunctionContext* ctx, const StringVal& serialized_sketch);
+
+ /// utilizing the kappa value indirectly specified through the 2nd argument in
+ /// ds_hll_estimate_bounds_as_string()
+ ///'kappa' is a number of standard deviations from the mean: 1, 2 or 3 (default 2).
+ static StringVal DsHllEstimateBoundsAsString(FunctionContext* ctx,
+ const StringVal& serialized_sketch, const IntVal& kappa);
+
+ /// 'serialized_sketch' is expected as a serialized Apache DataSketches HLL sketch. If
/// it is not, then the query fails. This function returns the stringified format of
/// an Apache DataSketches HLL sketch.
static StringVal DsHllStringify(FunctionContext* ctx,
diff --git a/common/function-registry/impala_functions.py b/common/function-registry/impala_functions.py
index 6a644fe..16ee0ce 100644
--- a/common/function-registry/impala_functions.py
+++ b/common/function-registry/impala_functions.py
@@ -933,6 +933,10 @@ visible_functions = [
# Functions to use Apache DataSketches functionality
[['ds_hll_estimate'], 'BIGINT', ['STRING'],
'_ZN6impala21DataSketchesFunctions13DsHllEstimateEPN10impala_udf15FunctionContextERKNS1_9StringValE'],
+ [['ds_hll_estimate_bounds_as_string'], 'STRING', ['STRING'],
+ '_ZN6impala21DataSketchesFunctions27DsHllEstimateBoundsAsStringEPN10impala_udf15FunctionContextERKNS1_9StringValE'],
+ [['ds_hll_estimate_bounds_as_string'], 'STRING', ['STRING', 'INT'],
+ '_ZN6impala21DataSketchesFunctions27DsHllEstimateBoundsAsStringEPN10impala_udf15FunctionContextERKNS1_9StringValERKNS1_6IntValE'],
[['ds_hll_stringify'], 'STRING', ['STRING'],
'_ZN6impala21DataSketchesFunctions14DsHllStringifyEPN10impala_udf15FunctionContextERKNS1_9StringValE'],
[['ds_kll_quantile'], 'FLOAT', ['STRING', 'DOUBLE'],
diff --git a/testdata/workloads/functional-query/queries/QueryTest/datasketches-hll.test b/testdata/workloads/functional-query/queries/QueryTest/datasketches-hll.test
index 3327ed3..871f20c 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/datasketches-hll.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/datasketches-hll.test
@@ -286,3 +286,59 @@ row_regex: .*### HLL sketch summary:.*Log Config K.*Hll Target.*Current Mode.*LB
---- TYPES
STRING
====
+---- QUERY
+# Check that ds_hll_estimate_bounds_as_string() returns error for strings that are not
+# serialized sketches.
+select ds_hll_estimate_bounds_as_string(date_string_col) from functional_parquet.alltypestiny;
+---- CATCH
+UDF ERROR: Unable to deserialize sketch.
+====
+---- QUERY
+# Check that ds_hll_estimate_bounds_as_string() returns error for kappa is not 1, 2, 3.
+select ds_hll_estimate_bounds_as_string(date_string_col, 4) from functional_parquet.alltypestiny;
+---- CATCH
+UDF ERROR: Kappa must be 1, 2 or 3
+====
+---- QUERY
+select ds_hll_estimate_bounds_as_string(date_string_col, 1.5) from functional_parquet.alltypestiny;
+---- CATCH
+AnalysisException: No matching function with signature: ds_hll_estimate_bounds_as_string(STRING, DECIMAL(2,1)).
+====
+---- QUERY
+# Check that ds_hll_estimate_bounds_as_string() works on sketches created by Hive.
+select ds_hll_estimate_bounds_as_string(f) from hll_sketches_from_hive;
+---- RESULTS
+'5,5,5.0005'
+---- TYPES
+STRING
+====
+---- QUERY
+select ds_hll_estimate_bounds_as_string(f, 2) from hll_sketches_from_hive;
+---- RESULTS
+'5,5,5.0005'
+---- TYPES
+STRING
+====
+---- QUERY
+select ds_hll_estimate_bounds_as_string(f, 1) from hll_sketches_from_hive;
+---- RESULTS
+'5,5,5.00025'
+---- TYPES
+STRING
+====
+---- QUERY
+select ds_hll_estimate_bounds_as_string(ds_hll_sketch(float_col))
+from functional_parquet.alltypestiny;
+---- RESULTS
+'2,2,2.0002'
+---- TYPES
+STRING
+====
+---- QUERY
+select ds_hll_estimate_bounds_as_string(ds_hll_sketch(float_col), 3)
+from functional_parquet.alltypestiny;
+---- RESULTS
+'2,2,2.0003'
+---- TYPES
+STRING
+====
\ No newline at end of file
[impala] 03/04: IMPALA-10303: Fix warnings from impala-shell with
--quiet
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 01e1b4df805bbaa92cd15fdfc579e5abc3898fc1
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
AuthorDate: Wed Oct 28 18:32:21 2020 -0700
IMPALA-10303: Fix warnings from impala-shell with --quiet
When the --quiet flag is used with impala-shell, the intention is that
if the query is successful then only the query results should be
printed.
This patch fixes two cases where --quiet was not being respected:
- When using the HTTP transport and --client_connect_timeout_ms is
set, a warning is printed that the timeout is not applied.
- When running in non-interactive mode, a warning is printed that
--live_progress is automatically disabled. This warning is now also
only printed if --live_progress is actually set.
Testing:
- Added a test that runs a simple query with --quiet and confirms the
output is as expected.
Change-Id: I1e94c9445ffba159725bacd6f6bc36f7c91b88fe
Reviewed-on: http://gerrit.cloudera.org:8080/16673
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
shell/impala_client.py | 2 +-
shell/impala_shell.py | 8 +++++---
tests/shell/test_shell_commandline.py | 8 ++++++++
3 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/shell/impala_client.py b/shell/impala_client.py
index f8a7e4f..ec60a1b 100755
--- a/shell/impala_client.py
+++ b/shell/impala_client.py
@@ -360,7 +360,7 @@ class ImpalaClient(object):
# connect timeout does not achieve the desirable result as the subsequent open() could
# block similary in case of problematic remote end points.
# TODO: Investigate connection reuse in ImpalaHttpClient and revisit this.
- if connect_timeout_ms > 0:
+ if connect_timeout_ms > 0 and self.verbose:
print("Warning: --connect_timeout_ms is currently ignored with HTTP transport.",
file=sys.stderr)
diff --git a/shell/impala_shell.py b/shell/impala_shell.py
index cf06caf..1ef9994 100755
--- a/shell/impala_shell.py
+++ b/shell/impala_shell.py
@@ -2001,9 +2001,11 @@ def impala_shell_main():
# Non-interactive mode
if options.query or options.query_file:
# Impala shell will disable live_progress if non-interactive mode is detected.
- options.live_progress = False
- print("Warning: live_progress only applies to interactive shell sessions, "
- "and is being skipped for now.", file=sys.stderr)
+ if options.live_progress:
+ if options.verbose:
+ print("Warning: live_progress only applies to interactive shell sessions, "
+ "and is being skipped for now.", file=sys.stderr)
+ options.live_progress = False
if options.live_summary:
print("Error: live_summary is available for interactive mode only.",
file=sys.stderr)
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index 176530f..bf798c9 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -1019,3 +1019,11 @@ class TestImpalaShell(ImpalaTestSuite):
result = run_impala_shell_cmd(vector, ['-q', query_options + query, '-B'])
result_rows = result.stdout.strip().split('\n')
assert len(result_rows) == 2
+
+ def test_quiet_mode(self, vector):
+ """Checks that extraneous output isn't included when --quiet is set."""
+ args = ['--quiet', '-q', 'select 1']
+ result = run_impala_shell_cmd(vector, args)
+ expected_result = """+---+\n| 1 |\n+---+\n| 1 |\n+---+\n"""
+ assert result.stdout == expected_result
+ assert result.stderr == ""
[impala] 04/04: IMPALA-10302: Enable logging at the INFO level for
test_scanners_fuzz.py
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 047906bd6b91cb66a6984639bfda3a25fc3724ac
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Thu Oct 29 09:47:08 2020 -0700
IMPALA-10302: Enable logging at the INFO level for test_scanners_fuzz.py
This changes test_scanners_fuzz.py to set the logging level
to INFO. By default, it is WARNING, so it was missing some useful
INFO log messages like the random seed used. This also fixes formatting
on one of the log lines.
Testing:
- Ran test_scanners_fuzz.py locally and checked to make
sure the INFO messages were present
Change-Id: Ida4a9cbed6572520998def9618a8b4189c1ba799
Reviewed-on: http://gerrit.cloudera.org:8080/16677
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
tests/query_test/test_scanners_fuzz.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/tests/query_test/test_scanners_fuzz.py b/tests/query_test/test_scanners_fuzz.py
index c25dd39..9c4b48a 100644
--- a/tests/query_test/test_scanners_fuzz.py
+++ b/tests/query_test/test_scanners_fuzz.py
@@ -17,6 +17,7 @@
from copy import copy
import itertools
+import logging
import math
import os
import pytest
@@ -27,10 +28,13 @@ import time
from subprocess import check_call
from tests.common.environ import HIVE_MAJOR_VERSION
from tests.common.test_dimensions import create_exec_option_dimension_from_dict
-from tests.common.impala_test_suite import ImpalaTestSuite, LOG
+from tests.common.impala_test_suite import ImpalaTestSuite
from tests.util.filesystem_utils import IS_HDFS, WAREHOUSE, get_fs_path
from tests.util.test_file_parser import QueryTestSectionReader
+LOG = logging.getLogger(__name__)
+LOG.setLevel(level=logging.INFO)
+
# Random fuzz testing of HDFS scanners. Existing tables for any HDFS file format
# are corrupted in random ways to flush out bugs with handling of corrupted data.
class TestScannersFuzzing(ImpalaTestSuite):
@@ -236,7 +240,7 @@ class TestScannersFuzzing(ImpalaTestSuite):
query_options['disable_codegen_rows_threshold'] = 0
try:
result = self.execute_query(query, query_options = query_options)
- LOG.info('\n'.join(result.log))
+ LOG.info(result.log)
except Exception as e:
# We should only test queries that parse succesfully.
assert "AnalysisException" not in str(e)