You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by "lgbo-ustc (via GitHub)" <gi...@apache.org> on 2024/08/27 12:19:50 UTC
[PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
lgbo-ustc opened a new pull request, #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034
## What changes were proposed in this pull request?
(Please fill in changes proposed in this fix)
Fixes: #7014
## How was this patch tested?
(Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests)
unit tests
(If this patch involves UI changes, please attach a screenshot; otherwise, remove this)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2314386612
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "lgbo-ustc (via GitHub)" <gi...@apache.org>.
lgbo-ustc commented on code in PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#discussion_r1734245954
##########
cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.h:
##########
@@ -66,6 +68,261 @@ struct GetJsonObject
static constexpr auto name{"get_json_object"};
};
+class JSONTextNormalizer
+{
+public:
+ // simd json will fail to parse the json text on some cases, see #7014, #3750, #3337, #5303
+ // To keep the result same with vanilla, we normalize the json string when simd json fails.
+ // It returns null when normalize the json text fail, otherwise returns a position among `pos`
+ // and `end` which points to the whole json object end.
+ // `dst` refer to a memory buffer that is used to store the normalization result.
+ static const char * normalize(const char * pos, const char * end, char *& dst)
+ {
+ pos = normalizeWhitespace(pos, end, dst);
+ if (!pos || pos >= end)
+ return nullptr;
+ if (*pos == '[')
+ return normalizeArray(pos, end, dst);
+ else if (*pos == '{')
+ return normalizeObject(pos, end, dst);
+ return nullptr;
+ }
+
+private:
+ inline static void copyToDst(char *& p, char c)
+ {
+ *p = c;
+ p++;
+ }
+
+ inline static void copyToDst(char *& p, const char * src, size_t len)
+ {
+ memcpy(p, src, len);
+ p += len;
+ }
+
+ inline static bool isExpectedChar(char c, const char * pos, const char * end) { return pos && pos < end && *pos == c; }
+
+ inline static const char * normalizeWhitespace(const char * pos, const char * end, char *& dst)
+ {
+ const auto * start_pos = pos;
+ while (pos && pos < end)
+ {
+ if (isWhitespaceASCII(*pos))
+ pos++;
+ else
+ break;
+ }
+ if (pos != start_pos)
+ copyToDst(dst, start_pos, pos - start_pos);
+ return pos;
+ }
+
+ inline static const char * normalizeComma(const char * pos, const char * end, char *& dst)
+ {
+ pos = normalizeWhitespace(pos, end, dst);
+ if (!isExpectedChar(',', pos, end)) [[unlikely]]
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeComma. not ,");
+ return nullptr;
+ }
+ pos += 1;
+ copyToDst(dst, ',');
+ return normalizeWhitespace(pos, end, dst);
+ }
+
+ inline static const char * normalizeColon(const char * pos, const char * end, char *& dst)
+ {
+ pos = normalizeWhitespace(pos, end, dst);
+ if (!isExpectedChar(':', pos, end))
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeColon. not :");
+ return nullptr;
+ }
+ pos += 1;
+ copyToDst(dst, ':');
+ return normalizeWhitespace(pos, end, dst);
+ }
+
+ inline static const char * normalizeField(const char * pos, const char * end, char *& dst)
+ {
+ const auto * start_pos = pos;
+ pos = find_first_symbols<',', '}', ']'>(pos, end);
+ if (pos >= end) [[unlikely]]
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeField. not field");
+ return nullptr;
+ }
+ copyToDst(dst, start_pos, pos - start_pos);
+ return pos;
+ ;
+ }
+
+ inline static const char * normalizeString(const char * pos, const char * end, char *& dst)
+ {
+ const auto * start_pos = pos;
+ if (!isExpectedChar('"', pos, end)) [[unlikely]]
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeString. not \"");
+ return nullptr;
+ }
+ pos += 1;
+
+ do
+ {
+ pos = find_first_symbols<'\\', '"'>(pos, end);
+ if (pos != end && *pos == '\\')
+ {
+ // escape charaters. e.g. '\"', '\\'
+ pos += 2;
+ if (pos >= end)
+ return nullptr;
+ }
+ else
+ break;
+ } while (pos != end);
+
+ pos = find_first_symbols<'"'>(pos, end);
+ if (!isExpectedChar('"', pos, end))
+ return nullptr;
+ pos += 1;
+
+ size_t n = 0;
+ for (; start_pos != pos; ++start_pos)
+ {
+ if ((*start_pos >= 0x00 && *start_pos <= 0x1f) || *start_pos == 0x7f)
+ {
+ if (n)
+ {
+ copyToDst(dst, start_pos - n, n);
+ n = 0;
+ }
+ continue;
+ }
+ else
+ {
+ n += 1;
+ }
+ }
+ if (n)
+ copyToDst(dst, start_pos - n, n);
+
+ return normalizeWhitespace(pos, end, dst);
+ }
+
+ inline static const char * normalizeArray(const char * pos, const char * end, char *& dst)
+ {
+ if (!isExpectedChar('[', pos, end)) [[unlikely]]
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeArray. not [");
+ return nullptr;
+ }
+ pos += 1;
+ copyToDst(dst, '[');
+
+ pos = normalizeWhitespace(pos, end, dst);
+
+ bool has_more = false;
+ while (pos && pos < end && *pos != ']')
+ {
+ has_more = false;
+ switch (*pos)
+ {
+ case '{': {
+ pos = normalizeObject(pos, end, dst);
+ break;
+ }
+ case '"': {
+ pos = normalizeString(pos, end, dst);
+ break;
+ }
+ case '[': {
+ pos = normalizeArray(pos, end, dst);
+ break;
+ }
+ default: {
+ pos = normalizeField(pos, end, dst);
+ break;
+ }
+ }
+ if (!isExpectedChar(',', pos, end))
+ break;
+ pos = normalizeComma(pos, end, dst);
+ has_more = true;
+ }
+
+ if (!isExpectedChar(']', pos, end) || has_more)
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeArray. not ]");
+ return nullptr;
+ }
+ pos += 1;
+ copyToDst(dst, ']');
+ return normalizeWhitespace(pos, end, dst);
+ }
+
+ inline static const char * normalizeObject(const char * pos, const char * end, char *& dst)
Review Comment:
只是个提示，倒没什么影响
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "liuneng1994 (via GitHub)" <gi...@apache.org>.
liuneng1994 commented on code in PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#discussion_r1734225240
##########
cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.h:
##########
@@ -66,6 +68,263 @@ struct GetJsonObject
static constexpr auto name{"get_json_object"};
};
+class JSONTextNormalizer
+{
+public:
+ // simd json will fail to parse the json text on some cases, see #7014, #3750, #3337, #5303
+ // To keep the result same with vanilla, we normalize the json string when simd json fails.
+ // It returns null when normalize the json text fail, otherwise returns a position among `pos`
+ // and `end` which points to the whole json object end.
+ // `dst` refer to a memory buffer that is used to store the normalization result.
+ static const char * normalize(const char * pos, const char * end, char * & dst)
+ {
+ pos = normalizeWhitespace(pos, end, dst);
+ if (!pos || pos >= end)
+ return nullptr;
+ if (*pos == '[')
+ return normalizeArray(pos, end, dst);
+ else if (*pos == '{')
+ return normalizeObject(pos, end, dst);
+ return nullptr;
+ }
+private:
+ inline static void copyToDst(char * & p, char c)
+ {
+ *p = c;
+ p++;
+ }
+
+ inline static void copyToDst(char * & p, const char * src, size_t len)
+ {
+ memcpy(p, src, len);
+ p += len;
+ }
+
+ inline static bool isExpectedChar(char c, const char * pos, const char * end)
+ {
+ return pos && pos < end && *pos == c;
+ }
+
+ inline static const char * normalizeWhitespace(const char * pos, const char * end, char * & dst)
+ {
+ const auto * start_pos = pos;
+ while(pos && pos < end)
+ {
+ if (isWhitespaceASCII(*pos))
+ pos++;
+ else
+ break;
+ }
+ if (pos != start_pos)
+ copyToDst(dst, start_pos, pos - start_pos);
+ return pos;
+ }
+
+ inline static const char * normalizeComma(const char * pos, const char * end, char * & dst)
+ {
+ pos = normalizeWhitespace(pos, end, dst);
+ if (!isExpectedChar(',', pos, end)) [[unlikely]]
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeComma. not ,");
+ return nullptr;
+ }
+ pos += 1;
+ copyToDst(dst, ',');
+ return normalizeWhitespace(pos, end, dst);
+ }
+
+ inline static const char * normalizeColon(const char * pos, const char * end, char * & dst)
+ {
+ pos = normalizeWhitespace(pos, end, dst);
+ if (!isExpectedChar(':', pos, end))
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeColon. not :");
+ return nullptr;
+ }
+ pos += 1;
+ copyToDst(dst, ':');
+ return normalizeWhitespace(pos, end, dst);
+ }
+
+ inline static const char * normalizeField(const char * pos, const char * end, char * & dst)
+ {
+ const auto * start_pos = pos;
+ pos = find_first_symbols<',', '}', ']'>(pos, end);
+ if (pos >= end) [[unlikely]]
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeField. not field");
+ return nullptr;
+ }
+ copyToDst(dst, start_pos, pos - start_pos);
+ return pos;;
+ }
+
+ inline static const char * normalizeString(const char * pos, const char * end, char * & dst)
+ {
+ const auto * start_pos = pos;
+ if (!isExpectedChar('"', pos, end)) [[unlikely]]
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeString. not \"");
+ return nullptr;
+ }
+ pos += 1;
+
+ do
+ {
+ pos = find_first_symbols<'\\', '"'>(pos, end);
+ if (pos != end && *pos == '\\')
+ {
+ // escape charaters. e.g. '\"', '\\'
+ pos += 2;
+ if (pos >= end)
+ return nullptr;
+ }
+ else
+ break;
+ } while(pos != end);
+
+ pos = find_first_symbols<'"'>(pos, end);
+ if (!isExpectedChar('"', pos, end))
+ return nullptr;
+ pos += 1;
+
+ size_t n = 0;
+ for (; start_pos != pos; ++start_pos)
+ {
+ if ((*start_pos >= 0x00 && *start_pos <= 0x1f) || *start_pos == 0x7f)
+ {
+ if (n)
+ {
+ copyToDst(dst, start_pos - n, n);
+ n = 0;
+ }
+ continue;
+ }
+ else
+ {
+ n += 1;
+ }
+ }
+ if (n)
+ copyToDst(dst, start_pos - n, n);
+
+ return normalizeWhitespace(pos, end, dst);
+ }
+
+ inline static const char * normalizeArray(const char * pos, const char * end, char * & dst)
+ {
+ if (!isExpectedChar('[', pos, end)) [[unlikely]]
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeArray. not [");
+ return nullptr;
+ }
+ pos += 1;
+ copyToDst(dst, '[');
+
+ pos = normalizeWhitespace(pos, end, dst);
+
+ bool has_more = false;
+ while (pos && pos < end && *pos != ']')
+ {
+ has_more = false;
+ switch(*pos)
+ {
+ case '{': {
Review Comment:
need format code
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2316578605
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "KevinyhZou (via GitHub)" <gi...@apache.org>.
KevinyhZou commented on code in PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#discussion_r1734285642
##########
backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala:
##########
@@ -877,12 +877,15 @@ class GlutenClickHouseHiveTableSuite
val select_sql_5 = "select id, get_json_object(data, 'v112') from test_tbl_3337"
val select_sql_6 =
"select id, get_json_object(data, '$.id') from test_tbl_3337 where id = 123"
+ val select_sql_7 =
+ "select id, get_json_object(data, '$.id') from test_tbl_3337"
compareResultsAgainstVanillaSpark(select_sql_1, compareResult = true, _ => {})
compareResultsAgainstVanillaSpark(select_sql_2, compareResult = true, _ => {})
compareResultsAgainstVanillaSpark(select_sql_3, compareResult = true, _ => {})
compareResultsAgainstVanillaSpark(select_sql_4, compareResult = true, _ => {})
compareResultsAgainstVanillaSpark(select_sql_5, compareResult = true, _ => {})
compareResultsAgainstVanillaSpark(select_sql_6, compareResult = true, _ => {})
+ compareResultsAgainstVanillaSpark(select_sql_7, compareResult = true, _ => {})
Review Comment:
maybe we should open the `get_json_object` test in spark-ut ，to ensure no other corner case problems?
##########
cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.h:
##########
@@ -16,8 +16,8 @@
*/
#pragma once
#include <memory>
-#include <string_view>
#include <stack>
Review Comment:
remove `#include <stack>`, not used now
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "exmy (via GitHub)" <gi...@apache.org>.
exmy commented on code in PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#discussion_r1735463547
##########
gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/clickhouse/ClickHouseTestSettings.scala:
##########
@@ -371,8 +371,6 @@ class ClickHouseTestSettings extends BackendTestSettings {
enableSuite[GlutenIntervalFunctionsSuite]
enableSuite[GlutenJoinSuite]
enableSuite[GlutenJsonFunctionsSuite]
Review Comment:
The `JsonExpressionsSuite` of vanilla Spark ut seems missing in spark33 ut module, please add this suite to ensure cover more cases.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "lgbo-ustc (via GitHub)" <gi...@apache.org>.
lgbo-ustc commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2312413043
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "liuneng1994 (via GitHub)" <gi...@apache.org>.
liuneng1994 commented on code in PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#discussion_r1734244719
##########
cpp-ch/local-engine/Functions/SparkFunctionGetJsonObject.h:
##########
@@ -66,6 +68,261 @@ struct GetJsonObject
static constexpr auto name{"get_json_object"};
};
+class JSONTextNormalizer
+{
+public:
+ // simd json will fail to parse the json text on some cases, see #7014, #3750, #3337, #5303
+ // To keep the result same with vanilla, we normalize the json string when simd json fails.
+ // It returns null when normalize the json text fail, otherwise returns a position among `pos`
+ // and `end` which points to the whole json object end.
+ // `dst` refer to a memory buffer that is used to store the normalization result.
+ static const char * normalize(const char * pos, const char * end, char *& dst)
+ {
+ pos = normalizeWhitespace(pos, end, dst);
+ if (!pos || pos >= end)
+ return nullptr;
+ if (*pos == '[')
+ return normalizeArray(pos, end, dst);
+ else if (*pos == '{')
+ return normalizeObject(pos, end, dst);
+ return nullptr;
+ }
+
+private:
+ inline static void copyToDst(char *& p, char c)
+ {
+ *p = c;
+ p++;
+ }
+
+ inline static void copyToDst(char *& p, const char * src, size_t len)
+ {
+ memcpy(p, src, len);
+ p += len;
+ }
+
+ inline static bool isExpectedChar(char c, const char * pos, const char * end) { return pos && pos < end && *pos == c; }
+
+ inline static const char * normalizeWhitespace(const char * pos, const char * end, char *& dst)
+ {
+ const auto * start_pos = pos;
+ while (pos && pos < end)
+ {
+ if (isWhitespaceASCII(*pos))
+ pos++;
+ else
+ break;
+ }
+ if (pos != start_pos)
+ copyToDst(dst, start_pos, pos - start_pos);
+ return pos;
+ }
+
+ inline static const char * normalizeComma(const char * pos, const char * end, char *& dst)
+ {
+ pos = normalizeWhitespace(pos, end, dst);
+ if (!isExpectedChar(',', pos, end)) [[unlikely]]
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeComma. not ,");
+ return nullptr;
+ }
+ pos += 1;
+ copyToDst(dst, ',');
+ return normalizeWhitespace(pos, end, dst);
+ }
+
+ inline static const char * normalizeColon(const char * pos, const char * end, char *& dst)
+ {
+ pos = normalizeWhitespace(pos, end, dst);
+ if (!isExpectedChar(':', pos, end))
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeColon. not :");
+ return nullptr;
+ }
+ pos += 1;
+ copyToDst(dst, ':');
+ return normalizeWhitespace(pos, end, dst);
+ }
+
+ inline static const char * normalizeField(const char * pos, const char * end, char *& dst)
+ {
+ const auto * start_pos = pos;
+ pos = find_first_symbols<',', '}', ']'>(pos, end);
+ if (pos >= end) [[unlikely]]
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeField. not field");
+ return nullptr;
+ }
+ copyToDst(dst, start_pos, pos - start_pos);
+ return pos;
+ ;
+ }
+
+ inline static const char * normalizeString(const char * pos, const char * end, char *& dst)
+ {
+ const auto * start_pos = pos;
+ if (!isExpectedChar('"', pos, end)) [[unlikely]]
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeString. not \"");
+ return nullptr;
+ }
+ pos += 1;
+
+ do
+ {
+ pos = find_first_symbols<'\\', '"'>(pos, end);
+ if (pos != end && *pos == '\\')
+ {
+ // escape charaters. e.g. '\"', '\\'
+ pos += 2;
+ if (pos >= end)
+ return nullptr;
+ }
+ else
+ break;
+ } while (pos != end);
+
+ pos = find_first_symbols<'"'>(pos, end);
+ if (!isExpectedChar('"', pos, end))
+ return nullptr;
+ pos += 1;
+
+ size_t n = 0;
+ for (; start_pos != pos; ++start_pos)
+ {
+ if ((*start_pos >= 0x00 && *start_pos <= 0x1f) || *start_pos == 0x7f)
+ {
+ if (n)
+ {
+ copyToDst(dst, start_pos - n, n);
+ n = 0;
+ }
+ continue;
+ }
+ else
+ {
+ n += 1;
+ }
+ }
+ if (n)
+ copyToDst(dst, start_pos - n, n);
+
+ return normalizeWhitespace(pos, end, dst);
+ }
+
+ inline static const char * normalizeArray(const char * pos, const char * end, char *& dst)
+ {
+ if (!isExpectedChar('[', pos, end)) [[unlikely]]
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeArray. not [");
+ return nullptr;
+ }
+ pos += 1;
+ copyToDst(dst, '[');
+
+ pos = normalizeWhitespace(pos, end, dst);
+
+ bool has_more = false;
+ while (pos && pos < end && *pos != ']')
+ {
+ has_more = false;
+ switch (*pos)
+ {
+ case '{': {
+ pos = normalizeObject(pos, end, dst);
+ break;
+ }
+ case '"': {
+ pos = normalizeString(pos, end, dst);
+ break;
+ }
+ case '[': {
+ pos = normalizeArray(pos, end, dst);
+ break;
+ }
+ default: {
+ pos = normalizeField(pos, end, dst);
+ break;
+ }
+ }
+ if (!isExpectedChar(',', pos, end))
+ break;
+ pos = normalizeComma(pos, end, dst);
+ has_more = true;
+ }
+
+ if (!isExpectedChar(']', pos, end) || has_more)
+ {
+ // LOG_DEBUG(getLogger("GetJsonObject"), "xxx normalizeArray. not ]");
+ return nullptr;
+ }
+ pos += 1;
+ copyToDst(dst, ']');
+ return normalizeWhitespace(pos, end, dst);
+ }
+
+ inline static const char * normalizeObject(const char * pos, const char * end, char *& dst)
Review Comment:
这里inline感觉没啥作用，有递归调用
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2314699727
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2314393411
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2314694555
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2314787671
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2316585027
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2312412345
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2313937006
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2314674580
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "lgbo-ustc (via GitHub)" <gi...@apache.org>.
lgbo-ustc merged PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2314390459
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2312411777
https://github.com/apache/incubator-gluten/issues/7014
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2314229010
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org
Re: [PR] [GLUTEN-7014][CH] Fix: different results from `get_json_object` [incubator-gluten]
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #7034:
URL: https://github.com/apache/incubator-gluten/pull/7034#issuecomment-2316581652
Run Gluten Clickhouse CI
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org