You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/07/06 12:01:25 UTC
[doris] 20/20: [fix](executor) make elt / repeat smooth upgrade. (#21493)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 726d7c326f8c26094acd6aabe9bb836ec6e1557f
Author: Mryange <59...@users.noreply.github.com>
AuthorDate: Thu Jul 6 19:15:06 2023 +0800
[fix](executor) make elt / repeat smooth upgrade. (#21493)
BE : 2.0，FE : 1.2
before
mysql [(none)]>select elt(1, 'aaa', 'bbb');
ERROR 1105 (HY000): errCode = 2, detailMessage = (127.0.0.1)[INTERNAL_ERROR]Function elt get failed, expr is VectorizedFnCall[elt](arguments=,return=String) and return type is String.
mysql [test]> INSERT INTO tbb VALUES (1, repeat("test1111", 8192))(2, repeat("test1111", 131072));
mysql [test]>select k1, md5(v1), length(v1) from tbb;
+------+----------------------------------+--------------+
| k1 | md5(`v1`) | length(`v1`) |
+------+----------------------------------+--------------+
| 1 | d41d8cd98f00b204e9800998ecf8427e | 0 |
| 2 | d41d8cd98f00b204e9800998ecf8427e | 0 |
+------+----------------------------------+--------------+
now
mysql [test]>select elt(1, 'aaa', 'bbb');
+----------------------+
| elt(1, 'aaa', 'bbb') |
+----------------------+
| aaa |
+----------------------+
mysql [test]>select k1, md5(v1), length(v1) from tbb;
+------+----------------------------------+--------------+
| k1 | md5(`v1`) | length(`v1`) |
+------+----------------------------------+--------------+
| 1 | 1f44fb91f47cab16f711973af06294a0 | 65536 |
| 2 | 3c514d3b89e26e2f983b7bd4cbb82055 | 1048576 |
+------+----------------------------------+--------------+
---
be/src/agent/be_exec_version_manager.h | 3 +-
be/src/vec/functions/function_string.cpp | 6 +++-
be/src/vec/functions/function_string.h | 61 +++++++++++++++++++++++++++++---
3 files changed, 63 insertions(+), 7 deletions(-)
diff --git a/be/src/agent/be_exec_version_manager.h b/be/src/agent/be_exec_version_manager.h
index 657ebab02d..0ecc868651 100644
--- a/be/src/agent/be_exec_version_manager.h
+++ b/be/src/agent/be_exec_version_manager.h
@@ -56,7 +56,8 @@ private:
* a. function month/day/hour/minute/second's return type is changed to smaller type.
* b. in order to solve agg of sum/count is not compatibility during the upgrade process
* c. change the string hash method in runtime filter
- *
+ * d. elt funciton return type change to nullable(string)
+ * e. add repeat_max_num in repeat function
*/
inline const int BeExecVersionManager::max_be_exec_version = 2;
inline const int BeExecVersionManager::min_be_exec_version = 0;
diff --git a/be/src/vec/functions/function_string.cpp b/be/src/vec/functions/function_string.cpp
index dbfbff8800..5812c7badb 100644
--- a/be/src/vec/functions/function_string.cpp
+++ b/be/src/vec/functions/function_string.cpp
@@ -965,7 +965,7 @@ void register_function_string(SimpleFunctionFactory& factory) {
factory.register_function<FunctionStringElt>();
factory.register_function<FunctionStringConcatWs>();
factory.register_function<FunctionStringAppendTrailingCharIfAbsent>();
- factory.register_function<FunctionStringRepeat>();
+ factory.register_function<FunctionStringRepeat<false>>();
factory.register_function<FunctionStringLPad>();
factory.register_function<FunctionStringRPad>();
factory.register_function<FunctionToBase64>();
@@ -996,6 +996,10 @@ void register_function_string(SimpleFunctionFactory& factory) {
factory.register_alias(FunctionStringMd5AndSM3<MD5Sum>::name, "md5");
factory.register_alias(FunctionStringUTF8Length::name, "character_length");
factory.register_alias(FunctionStringMd5AndSM3<SM3Sum>::name, "sm3");
+
+ /// @TEMPORARY: for be_exec_version=2
+ factory.register_alternative_function<FunctionStringEltOld>();
+ factory.register_alternative_function<FunctionStringRepeat<true>>();
}
} // namespace doris::vectorized
diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h
index ff48e01e26..83f98d726a 100644
--- a/be/src/vec/functions/function_string.h
+++ b/be/src/vec/functions/function_string.h
@@ -914,6 +914,48 @@ public:
}
};
+class FunctionStringEltOld : public IFunction {
+public:
+ static constexpr auto name = "elt";
+ static FunctionPtr create() { return std::make_shared<FunctionStringEltOld>(); }
+ String get_name() const override { return name; }
+ size_t get_number_of_arguments() const override { return 0; }
+ bool is_variadic() const override { return true; }
+
+ DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
+ return std::make_shared<DataTypeString>();
+ }
+ bool use_default_implementation_for_nulls() const override { return true; }
+ bool use_default_implementation_for_constants() const override { return true; }
+
+ Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
+ size_t result, size_t input_rows_count) override {
+ int arguent_size = arguments.size();
+ auto pos_col =
+ block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();
+ if (auto* nullable = check_and_get_column<const ColumnNullable>(*pos_col)) {
+ pos_col = nullable->get_nested_column_ptr();
+ }
+ auto& pos_data = assert_cast<const ColumnInt32*>(pos_col.get())->get_data();
+ auto pos = pos_data[0];
+ int num_children = arguent_size - 1;
+ if (pos < 1 || num_children == 0 || pos > num_children) {
+ auto null_map = ColumnUInt8::create(input_rows_count, 1);
+ auto res = ColumnString::create();
+ auto& res_data = res->get_chars();
+ auto& res_offset = res->get_offsets();
+ res_offset.resize(input_rows_count);
+ for (size_t i = 0; i < input_rows_count; ++i) {
+ res_offset[i] = res_data.size();
+ }
+ block.get_by_position(result).column =
+ ColumnNullable::create(std::move(res), std::move(null_map));
+ return Status::OK();
+ }
+ block.get_by_position(result).column = block.get_by_position(arguments[pos]).column;
+ return Status::OK();
+ }
+};
// concat_ws (string,string....) or (string, Array)
// TODO: avoid use fmtlib
class FunctionStringConcatWs : public IFunction {
@@ -1123,7 +1165,7 @@ private:
}
}
};
-
+template <bool use_old_function>
class FunctionStringRepeat : public IFunction {
public:
static constexpr auto name = "repeat";
@@ -1155,8 +1197,13 @@ public:
return Status::OK();
} else if (auto* col2_const = check_and_get_column<ColumnConst>(*argument_ptr[1])) {
DCHECK(check_and_get_column<ColumnInt32>(col2_const->get_data_column()));
- int repeat =
- std::min<int>(col2_const->get_int(0), context->state()->repeat_max_num());
+ int repeat = 0;
+ if constexpr (use_old_function) {
+ repeat = col2_const->get_int(0);
+ } else {
+ repeat = std::min<int>(col2_const->get_int(0),
+ context->state()->repeat_max_num());
+ }
if (repeat <= 0) {
null_map->get_data().resize_fill(input_rows_count, 0);
res->insert_many_defaults(input_rows_count);
@@ -1187,8 +1234,12 @@ public:
buffer.clear();
const char* raw_str = reinterpret_cast<const char*>(&data[offsets[i - 1]]);
size_t size = offsets[i] - offsets[i - 1];
- int repeat = std::min<int>(repeats[i], repeat_max_num);
-
+ int repeat = 0;
+ if constexpr (use_old_function) {
+ repeat = repeats[i];
+ } else {
+ repeat = std::min<int>(repeats[i], repeat_max_num);
+ }
if (repeat <= 0) {
StringOP::push_empty_string(i, res_data, res_offsets);
} else if (repeat * size > DEFAULT_MAX_STRING_SIZE) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org