You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/04/26 07:10:20 UTC
[doris] branch master updated: [bugfix](jsonb) fix jsonb parser crash on noavx2 host (#18977)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1dfc5ea34c [bugfix](jsonb) fix jsonb parser crash on noavx2 host (#18977)
1dfc5ea34c is described below
commit 1dfc5ea34c736c7a4c44fce9ff1f22bccb7204e9
Author: Kang <kx...@gmail.com>
AuthorDate: Wed Apr 26 15:10:12 2023 +0800
[bugfix](jsonb) fix jsonb parser crash on noavx2 host (#18977)
support avx2 and noavx2 for jsonb parser using __AVX2__ macro.
---
be/src/runtime/jsonb_value.h | 6 +++++-
be/src/util/jsonb_parser.h | 3 +++
be/src/util/jsonb_parser_simd.h | 2 +-
be/src/vec/functions/function_json.cpp | 6 +++++-
be/src/vec/functions/function_jsonb.cpp | 8 ++++++--
5 files changed, 20 insertions(+), 5 deletions(-)
diff --git a/be/src/runtime/jsonb_value.h b/be/src/runtime/jsonb_value.h
index 88abb379d9..915094c330 100644
--- a/be/src/runtime/jsonb_value.h
+++ b/be/src/runtime/jsonb_value.h
@@ -26,7 +26,11 @@
#include "common/status.h"
#include "util/hash_util.hpp"
+#ifdef __AVX2__
#include "util/jsonb_parser_simd.h"
+#else
+#include "util/jsonb_parser.h"
+#endif
namespace doris {
@@ -36,7 +40,7 @@ struct JsonBinaryValue {
// default nullprt and size 0 for invalid or NULL value
const char* ptr = nullptr;
size_t len = 0;
- JsonbParserSIMD parser;
+ JsonbParser parser;
JsonBinaryValue() : ptr(nullptr), len(0) {}
JsonBinaryValue(char* ptr, int len) { from_json_string(const_cast<const char*>(ptr), len); }
diff --git a/be/src/util/jsonb_parser.h b/be/src/util/jsonb_parser.h
index f4711f9a62..8d29b52faf 100644
--- a/be/src/util/jsonb_parser.h
+++ b/be/src/util/jsonb_parser.h
@@ -665,12 +665,15 @@ private:
case 4:
*--out = ((uc | 0x80) & 0xBF);
uc >>= 6;
+ [[fallthrough]];
case 3:
*--out = ((uc | 0x80) & 0xBF);
uc >>= 6;
+ [[fallthrough]];
case 2:
*--out = ((uc | 0x80) & 0xBF);
uc >>= 6;
+ [[fallthrough]];
case 1:
// Mask the first byte according to the standard.
*--out = (uc | firstByteMark[len - 1]);
diff --git a/be/src/util/jsonb_parser_simd.h b/be/src/util/jsonb_parser_simd.h
index 10d19a3f57..971d06d46e 100644
--- a/be/src/util/jsonb_parser_simd.h
+++ b/be/src/util/jsonb_parser_simd.h
@@ -343,7 +343,7 @@ private:
JsonbErrType err_;
};
-using JsonbParserSIMD = JsonbParserTSIMD<JsonbOutStream>;
+using JsonbParser = JsonbParserTSIMD<JsonbOutStream>;
} // namespace doris
diff --git a/be/src/vec/functions/function_json.cpp b/be/src/vec/functions/function_json.cpp
index ccaad26a9e..9c256483c6 100644
--- a/be/src/vec/functions/function_json.cpp
+++ b/be/src/vec/functions/function_json.cpp
@@ -41,7 +41,11 @@
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/status.h"
#include "exprs/json_functions.h"
+#ifdef __AVX2__
#include "util/jsonb_parser_simd.h"
+#else
+#include "util/jsonb_parser.h"
+#endif
#include "util/string_parser.hpp"
#include "util/string_util.h"
#include "vec/aggregate_functions/aggregate_function.h"
@@ -910,7 +914,7 @@ public:
vec_to.resize(size);
// parser can be reused for performance
- JsonbParserSIMD parser;
+ JsonbParser parser;
for (size_t i = 0; i < input_rows_count; ++i) {
if (col_from.is_null_at(i)) {
null_map->get_data()[i] = 1;
diff --git a/be/src/vec/functions/function_jsonb.cpp b/be/src/vec/functions/function_jsonb.cpp
index 8be6536b56..a0af1e13a2 100644
--- a/be/src/vec/functions/function_jsonb.cpp
+++ b/be/src/vec/functions/function_jsonb.cpp
@@ -33,7 +33,11 @@
#include "udf/udf.h"
#include "util/jsonb_document.h"
#include "util/jsonb_error.h"
+#ifdef __AVX2__
#include "util/jsonb_parser_simd.h"
+#else
+#include "util/jsonb_parser.h"
+#endif
#include "util/jsonb_stream.h"
#include "util/jsonb_utils.h"
#include "util/jsonb_writer.h"
@@ -74,7 +78,7 @@ enum class JsonbParseErrorMode { FAIL = 0, RETURN_NULL, RETURN_VALUE, RETURN_INV
template <NullalbeMode nullable_mode, JsonbParseErrorMode parse_error_handle_mode>
class FunctionJsonbParseBase : public IFunction {
private:
- JsonbParserSIMD default_value_parser;
+ JsonbParser default_value_parser;
bool has_const_default_value = false;
public:
@@ -221,7 +225,7 @@ public:
col_to->reserve(size);
// parser can be reused for performance
- JsonbParserSIMD parser;
+ JsonbParser parser;
JsonbErrType error = JsonbErrType::E_NONE;
for (size_t i = 0; i < input_rows_count; ++i) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org