You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by la...@apache.org on 2022/06/28 19:01:32 UTC
[impala] 02/02: IMPALA-9615: re2's max_mem opt configurable via an Impala startup flag
This is an automated email from the ASF dual-hosted git repository.
laszlog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit a625a95dbd347d5a5e64566c77bcb27e991ce352
Author: Omid Shahidi <os...@cloudera.com>
AuthorDate: Thu Jun 2 16:39:40 2022 -0700
IMPALA-9615: re2's max_mem opt configurable via an Impala startup flag
Some regex patterns require more memory to be compiled and pattern matched
using different string functions and like predicate available.
For more memory consuming patterns this can cause the following error:
"re2/re2.cc:667: DFA out of memory:
size xxxxx, bytemap range xx, list count xxxxx".
To avoid such errors in Impalad's ERROR log, a global flag can
be added to impala cluster startup. The re2_mem_limit flag will
accept a memory specification string to set the re2 max_mem parameter for
memory used to store regexps in Bytes.
Testing:
- Use a long regex pattern to use up all the memory in the
case of allocating less or the same amount of memory as default for re2.
By using a greater value for re2_mem_limit flag, the regexp can be
consumed with no error.
Change-Id: Idf28d2f7217b1322ab8fdfb2c02fff0608078571
Reviewed-on: http://gerrit.cloudera.org:8080/18602
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/common/global-flags.cc | 9 +++++
be/src/common/init.cc | 13 ++++++
be/src/exprs/like-predicate.cc | 5 +++
be/src/exprs/string-functions-ir.cc | 19 ++++++++-
be/src/exprs/string-functions.h | 4 ++
tests/custom_cluster/test_re2_max_mem.py | 69 ++++++++++++++++++++++++++++++++
6 files changed, 118 insertions(+), 1 deletion(-)
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index 563ed05be..c427ed595 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -228,6 +228,15 @@ DEFINE_int32(max_log_files, 10, "Maximum number of log files to retain per sever
"level. The most recent log files are retained. If set to 0, all log files are "
"retained.");
+static const string re2_mem_limit_help_msg =
+ "Maximum bytes of memory to be used by re2's regex engine "
+ "to hold the compiled form of the regexp. For more memory-consuming patterns, "
+ "this can be set to be a higher number."
+ + Substitute(MEM_UNITS_HELP_MSG, "memory limit for RE2 max_mem opt")
+ + "Default to 8MB. Using percentage is discouraged.";
+
+DEFINE_string(re2_mem_limit, "8MB", re2_mem_limit_help_msg.c_str());
+
// The read size is the preferred size of the reads issued to HDFS or the local FS.
// There is a trade off of latency and throughput, trying to keep disks busy but
// not introduce seeks. The literature seems to agree that with 8 MB reads, random
diff --git a/be/src/common/init.cc b/be/src/common/init.cc
index 2befa4807..fdeb2a835 100644
--- a/be/src/common/init.cc
+++ b/be/src/common/init.cc
@@ -26,6 +26,7 @@
#include "common/status.h"
#include "exec/kudu-util.h"
#include "exprs/scalar-expr-evaluator.h"
+#include "exprs/string-functions.h"
#include "exprs/timezone_db.h"
#include "gutil/atomicops.h"
#include "gutil/strings/substitute.h"
@@ -50,6 +51,7 @@
#include "util/network-util.h"
#include "util/openssl-util.h"
#include "util/os-info.h"
+#include "util/parse-util.h"
#include "util/periodic-counter-updater.h"
#include "util/pretty-printer.h"
#include "util/redactor.h"
@@ -72,6 +74,7 @@ DECLARE_int32(max_log_files);
DECLARE_int32(max_minidumps);
DECLARE_string(redaction_rules_file);
DECLARE_bool(redirect_stdout_stderr);
+DECLARE_string(re2_mem_limit);
DECLARE_string(reserved_words_version);
DECLARE_bool(symbolize_stacktrace);
DECLARE_string(debug_actions);
@@ -338,6 +341,16 @@ void impala::InitCommonRuntime(int argc, char** argv, bool init_jvm,
CLEAN_EXIT_WITH_ERROR(Substitute("read_size can not be lower than $0",
READ_SIZE_MIN_VALUE));
}
+
+ bool is_percent = false; // not used
+ int64_t re2_mem_limit = ParseUtil::ParseMemSpec(FLAGS_re2_mem_limit, &is_percent, 0);
+ if (re2_mem_limit <= 0) {
+ CLEAN_EXIT_WITH_ERROR(
+ Substitute("Invalid mem limit for re2's regex engine: $0", FLAGS_re2_mem_limit));
+ } else {
+ StringFunctions::SetRE2MemLimit(re2_mem_limit);
+ }
+
if (FLAGS_reserved_words_version != "2.11.0" && FLAGS_reserved_words_version != "3.0.0")
{
CLEAN_EXIT_WITH_ERROR(Substitute("Invalid flag reserved_words_version. The value must"
diff --git a/be/src/exprs/like-predicate.cc b/be/src/exprs/like-predicate.cc
index 4eb1110a3..899251920 100644
--- a/be/src/exprs/like-predicate.cc
+++ b/be/src/exprs/like-predicate.cc
@@ -101,6 +101,7 @@ void LikePredicate::LikePrepareInternal(FunctionContext* context,
opts.set_never_nl(false);
opts.set_dot_nl(true);
opts.set_case_sensitive(case_sensitive);
+ StringFunctions::SetRE2MemOpt(&opts);
state->regex_.reset(new RE2(re_pattern, opts));
if (!state->regex_->ok()) {
context->SetError(Substitute("Invalid regex: $0", pattern_str).c_str());
@@ -164,6 +165,7 @@ void LikePredicate::RegexPrepareInternal(FunctionContext* context,
} else {
RE2::Options opts;
opts.set_case_sensitive(case_sensitive);
+ StringFunctions::SetRE2MemOpt(&opts);
state->regex_.reset(new RE2(pattern_str, opts));
if (!state->regex_->ok()) {
context->SetError(
@@ -196,6 +198,7 @@ void LikePredicate::RegexpLikePrepare(FunctionContext* context,
}
RE2::Options opts;
string error_str;
+ StringFunctions::SetRE2MemOpt(&opts);
if (!StringFunctions::SetRE2Options(*match_parameter, &error_str, &opts)) {
context->SetError(error_str.c_str());
return;
@@ -220,6 +223,7 @@ BooleanVal LikePredicate::RegexpLikeInternal(FunctionContext* context,
if (match_parameter.is_null) return BooleanVal::null();
RE2::Options opts;
string error_str;
+ StringFunctions::SetRE2MemOpt(&opts);
if (!StringFunctions::SetRE2Options(match_parameter, &error_str, &opts)) {
context->SetError(error_str.c_str());
return BooleanVal(false);
@@ -340,6 +344,7 @@ BooleanVal LikePredicate::RegexMatch(FunctionContext* context,
} else {
string re_pattern;
RE2::Options opts;
+ StringFunctions::SetRE2MemOpt(&opts);
if (is_like_pattern) {
ConvertLikePattern(context, pattern_value, &re_pattern);
opts.set_never_nl(false);
diff --git a/be/src/exprs/string-functions-ir.cc b/be/src/exprs/string-functions-ir.cc
index 96724ad9b..ee09f13e1 100644
--- a/be/src/exprs/string-functions-ir.cc
+++ b/be/src/exprs/string-functions-ir.cc
@@ -49,6 +49,8 @@ namespace impala {
const char* ERROR_CHARACTER_LIMIT_EXCEEDED =
"$0 is larger than allowed limit of $1 character data.";
+uint64_t StringFunctions::re2_mem_limit_ = 8 << 20;
+
// This behaves identically to the mysql implementation, namely:
// - 1-indexed positions
// - supported negative positions (count from the end of the string)
@@ -880,6 +882,8 @@ re2::RE2* CompileRegex(const StringVal& pattern, string* error_str,
options.set_log_errors(false);
// Return the leftmost longest match (rather than the first match).
options.set_longest_match(true);
+ // Set the maximum memory used by re2's regex engine for storage
+ StringFunctions::SetRE2MemOpt(&options);
if (!match_parameter.is_null &&
!StringFunctions::SetRE2Options(match_parameter, error_str, &options)) {
return NULL;
@@ -926,6 +930,19 @@ bool StringFunctions::SetRE2Options(const StringVal& match_parameter,
return true;
}
+void StringFunctions::SetRE2MemLimit(int64_t re2_mem_limit) {
+ // TODO: include the memory requirement for re2 in the memory planner estimates
+ DCHECK(re2_mem_limit > 0);
+ StringFunctions::re2_mem_limit_ = re2_mem_limit;
+}
+
+// Set the maximum memory used by re2's regex engine for a compiled regex expression's
+// storage. By default, it uses 8 MiB. This can be used to avoid DFA state cache flush
+// resulting in slower execution
+void StringFunctions::SetRE2MemOpt(re2::RE2::Options* opts) {
+ opts->set_max_mem(StringFunctions::re2_mem_limit_);
+}
+
void StringFunctions::RegexpPrepare(
FunctionContext* context, FunctionContext::FunctionStateScope scope) {
if (scope != FunctionContext::THREAD_LOCAL) return;
@@ -1778,4 +1795,4 @@ IntVal StringFunctions::DamerauLevenshtein(
ctx->Free(reinterpret_cast<uint8_t*>(rows));
return IntVal(result);
}
-}
+}
\ No newline at end of file
diff --git a/be/src/exprs/string-functions.h b/be/src/exprs/string-functions.h
index e2b7dea9a..1deb8b9fa 100644
--- a/be/src/exprs/string-functions.h
+++ b/be/src/exprs/string-functions.h
@@ -128,6 +128,7 @@ class StringFunctions {
static bool SetRE2Options(const StringVal& match_parameter, std::string* error_str,
re2::RE2::Options* opts);
+ static void SetRE2MemOpt(re2::RE2::Options* opts);
static void RegexpPrepare(FunctionContext*, FunctionContext::FunctionStateScope);
static void RegexpClose(FunctionContext*, FunctionContext::FunctionStateScope);
static StringVal RegexpEscape(FunctionContext*, const StringVal& str);
@@ -155,6 +156,8 @@ class StringFunctions {
const StringVal& key, const StringVal& part);
static void ParseUrlClose(FunctionContext*, FunctionContext::FunctionStateScope);
+ static void SetRE2MemLimit(int64_t re2_mem_limit);
+
/// Converts ASCII 'val' to corresponding character.
static StringVal Chr(FunctionContext* context, const IntVal& val);
@@ -201,6 +204,7 @@ class StringFunctions {
const DoubleVal& boost_threshold);
private:
+ static uint64_t re2_mem_limit_;
/// Templatized implementation of the actual string trimming function.
/// The first parameter, 'D', is one of StringFunctions::TrimPosition values.
/// The second parameter, 'IS_IMPLICIT_WHITESPACE', is true when the set of characters
diff --git a/tests/custom_cluster/test_re2_max_mem.py b/tests/custom_cluster/test_re2_max_mem.py
new file mode 100755
index 000000000..9348bcb1a
--- /dev/null
+++ b/tests/custom_cluster/test_re2_max_mem.py
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pytest
+
+from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+
+
+class TestRE2MaxMem(CustomClusterTestSuite):
+ """test if re2 max_mem parameter is set using the global flag in imapalad"""
+ @classmethod
+ def get_workload(cls):
+ return 'tpch'
+
+ def _test_re2_max_mem(self, expect_fail, dfa_out_of_mem):
+ """"test to see given an amount of memory (in Bytes) does the re2 print an error for
+ DFA run out of memory when compiling and pattern matching a long regexp"""
+
+ client = self.create_impala_client()
+
+ query = (
+ "select c_comment from tpch.customer where regexp_like(c_comment,"
+ "repeat('([a-c].*[t-w]|[t].?[h]|[^xyz]|.*?(\\\\d+))\\\\w', 1000),'i');")
+
+ # RE2 regex compilation storage is also dependent on max_mem parameter, for small
+ # values the regex pattern will fail as invalid pattern although it can be valid
+ # given a higher max_mem for RE2.
+ # See: https://github.com/google/re2/blob/3e9622e/re2/re2.h#L619-L648
+ if expect_fail:
+ # if we expect the regex compilation to fail we can ignore
+ # DFA out of memory issue at that will be brought up when
+ # RE2 does not having enough storage to store compiled regex
+ self.execute_query_expect_failure(client, query)
+ else:
+ self.execute_query_expect_success(client, query)
+ self.assert_impalad_log_contains(
+ "ERROR", "DFA out of memory", -1 if dfa_out_of_mem else 0)
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(cluster_size=1,
+ impalad_args="--re2_mem_limit=1KB")
+ def test_dfa_out_of_mem(self):
+ self._test_re2_max_mem(True, True)
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(cluster_size=1)
+ def test_re2_max_mem_not_specified(self):
+ # default max_mem set by re2's regex engine is 8 MiB
+ self._test_re2_max_mem(False, True)
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(cluster_size=1,
+ impalad_args="--re2_mem_limit=200MB")
+ def test_dfa_not_out_of_mem(self):
+ self._test_re2_max_mem(False, False)