You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by el...@apache.org on 2023/07/20 05:10:07 UTC
[doris] branch fix-json created (now fc2ac5f6ae)
This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a change to branch fix-json
in repository https://gitbox.apache.org/repos/asf/doris.git
at fc2ac5f6ae rebase
This branch includes the following new commits:
new 1f3f3b0f61 [Fix](json reader) fix rapidjson `array->PushBack` may take ownership original document
new fc2ac5f6ae rebase
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/02: rebase
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch fix-json
in repository https://gitbox.apache.org/repos/asf/doris.git
commit fc2ac5f6ae41f21291e201e032b5fe488d11c4f1
Author: eldenmoon <15...@163.com>
AuthorDate: Thu Jul 20 10:56:52 2023 +0800
rebase
---
regression-test/data/load_p0/stream_load/test_json_load.out | 6 ++----
regression-test/suites/load_p0/stream_load/test_json_load.groovy | 7 ++++---
2 files changed, 6 insertions(+), 7 deletions(-)
diff --git a/regression-test/data/load_p0/stream_load/test_json_load.out b/regression-test/data/load_p0/stream_load/test_json_load.out
index 07032bcf83..ad68ffa129 100644
--- a/regression-test/data/load_p0/stream_load/test_json_load.out
+++ b/regression-test/data/load_p0/stream_load/test_json_load.out
@@ -199,11 +199,9 @@
10 hefei 23456710
200 changsha 3456789
-<<<<<<< HEAD
-- !select1 --
John 30 New York {"email":"john@example.com","phone":"+1-123-456-7890"}
-=======
--- !select21 --
+
+-- !select22 --
11324 1321313082437 1678834024274 20230315 {"base_mac_value_null":24,"base_1_value_respiratoryrate":11,"base_3_value_heartrate":51,"base_3_status_onoroutofbed":3,"base_null_count_circulation":84,"base_1_status_onoroutofbed":3,"base_1_value_heartrate":51,"base_3_value_respiratoryrate":11,"base_3_value_bodyactivityenergy":43652,"base_2_value_respiratoryrate":11,"base_2_value_bodyactivityenergy":28831,"base_2_status_onoroutofbed":3,"base_1_value_bodyactivityenergy":56758,"base_2_value_heart [...]
->>>>>>> 7de147ac54 ([Fix](json reader) fix rapidjson `array->PushBack` may take ownership original document)
diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy b/regression-test/suites/load_p0/stream_load/test_json_load.groovy
index 4933a8d52f..c982196bb1 100644
--- a/regression-test/suites/load_p0/stream_load/test_json_load.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_json_load.groovy
@@ -590,6 +590,7 @@ suite("test_json_load", "p0") {
// case22: nested and it's member with jsonpath
try {
+ testTable = "test_json_load"
sql "DROP TABLE IF EXISTS ${testTable}"
sql """CREATE TABLE IF NOT EXISTS ${testTable}
(
@@ -605,10 +606,10 @@ suite("test_json_load", "p0") {
"replication_num" = "1"
);
"""
-
- load_json_data.call('with_jsonpath', '', 'true', 'json', """productid, deviceid, data, datatimestamp, dt=from_unixtime(substr(datatimestamp,1,10),'%Y%m%d')""",
+1
+ load_json_data.call("${testTable}", 'with_jsonpath', '', 'true', 'json', """productid, deviceid, data, datatimestamp, dt=from_unixtime(substr(datatimestamp,1,10),'%Y%m%d')""",
'["$.productid","$.deviceid","$.data","$.data.datatimestamp"]', '', '', '', 'with_jsonpath.json')
- qt_select21 "select * from ${testTable}"
+ qt_select22 "select * from ${testTable}"
} finally {
try_sql("DROP TABLE IF EXISTS ${testTable}")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/02: [Fix](json reader) fix rapidjson `array->PushBack` may take ownership original document
Posted by el...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch fix-json
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1f3f3b0f613a4310a82a48ea76d6f03cbb4bcc2a
Author: eldenmoon <15...@163.com>
AuthorDate: Wed Jul 19 17:10:52 2023 +0800
[Fix](json reader) fix rapidjson `array->PushBack` may take ownership original document
With bellow json path
`["$.data","$.data.datatimestamp"]`
After `array_obj->PushBack` the `data` field owner will be taken from array_obj, and lead to null values for json path `$.data.datatimestamp`
Rapidjson doc:
```
//! Append a GenericValue at the end of the array.
\note The ownership of \c value will be transferred to this array on success.
*/
GenericValue& PushBack(GenericValue& value, Allocator& allocator);
```
---
be/src/exprs/json_functions.cpp | 6 +++--
.../data/load_p0/stream_load/test_json_load.out | 5 +++++
.../data/load_p0/stream_load/with_jsonpath.json | 1 +
.../load_p0/stream_load/test_json_load.groovy | 26 ++++++++++++++++++++++
4 files changed, 36 insertions(+), 2 deletions(-)
diff --git a/be/src/exprs/json_functions.cpp b/be/src/exprs/json_functions.cpp
index 107217ed4f..1dce452e34 100644
--- a/be/src/exprs/json_functions.cpp
+++ b/be/src/exprs/json_functions.cpp
@@ -172,11 +172,13 @@ rapidjson::Value* JsonFunctions::get_json_array_from_parsed_json(
rapidjson::Value* root = match_value(parsed_paths, document, mem_allocator, true);
if (root == nullptr || root == document) { // not found
return nullptr;
- } else if (!root->IsArray()) {
+ } else if (!root->IsArray() && wrap_explicitly) {
rapidjson::Value* array_obj = nullptr;
array_obj = static_cast<rapidjson::Value*>(mem_allocator.Malloc(sizeof(rapidjson::Value)));
array_obj->SetArray();
- array_obj->PushBack(*root, mem_allocator);
+ rapidjson::Value copy;
+ copy.CopyFrom(*root, mem_allocator);
+ array_obj->PushBack(std::move(copy), mem_allocator);
// set `wrap_explicitly` to true, so that the caller knows that this Array is wrapped actively.
*wrap_explicitly = true;
return array_obj;
diff --git a/regression-test/data/load_p0/stream_load/test_json_load.out b/regression-test/data/load_p0/stream_load/test_json_load.out
index 6296f37099..07032bcf83 100644
--- a/regression-test/data/load_p0/stream_load/test_json_load.out
+++ b/regression-test/data/load_p0/stream_load/test_json_load.out
@@ -199,6 +199,11 @@
10 hefei 23456710
200 changsha 3456789
+<<<<<<< HEAD
-- !select1 --
John 30 New York {"email":"john@example.com","phone":"+1-123-456-7890"}
+=======
+-- !select21 --
+11324 1321313082437 1678834024274 20230315 {"base_mac_value_null":24,"base_1_value_respiratoryrate":11,"base_3_value_heartrate":51,"base_3_status_onoroutofbed":3,"base_null_count_circulation":84,"base_1_status_onoroutofbed":3,"base_1_value_heartrate":51,"base_3_value_respiratoryrate":11,"base_3_value_bodyactivityenergy":43652,"base_2_value_respiratoryrate":11,"base_2_value_bodyactivityenergy":28831,"base_2_status_onoroutofbed":3,"base_1_value_bodyactivityenergy":56758,"base_2_value_heart [...]
+>>>>>>> 7de147ac54 ([Fix](json reader) fix rapidjson `array->PushBack` may take ownership original document)
diff --git a/regression-test/data/load_p0/stream_load/with_jsonpath.json b/regression-test/data/load_p0/stream_load/with_jsonpath.json
new file mode 100644
index 0000000000..11d14310af
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/with_jsonpath.json
@@ -0,0 +1 @@
+{"data":{"base_mac_value_null":24,"base_1_value_respiratoryrate":11,"base_3_value_heartrate":51,"base_3_status_onoroutofbed":3,"base_null_count_circulation":84,"base_1_status_onoroutofbed":3,"base_1_value_heartrate":51,"base_3_value_respiratoryrate":11,"base_3_value_bodyactivityenergy":43652,"base_2_value_respiratoryrate":11,"base_2_value_bodyactivityenergy":28831,"base_2_status_onoroutofbed":3,"base_1_value_bodyactivityenergy":56758,"base_2_value_heartrate":51,"tsltype":"properties","da [...]
diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy b/regression-test/suites/load_p0/stream_load/test_json_load.groovy
index 7a69114d11..4933a8d52f 100644
--- a/regression-test/suites/load_p0/stream_load/test_json_load.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_json_load.groovy
@@ -588,6 +588,32 @@ suite("test_json_load", "p0") {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
+ // case22: nested and it's member with jsonpath
+ try {
+ sql "DROP TABLE IF EXISTS ${testTable}"
+ sql """CREATE TABLE IF NOT EXISTS ${testTable}
+ (
+ `productid` bigint NOT NULL COMMENT "productid",
+ `deviceid` bigint NOT NULL COMMENT "deviceid",
+ `datatimestamp` string NULL COMMENT "datatimestamp",
+ `dt` int NULL COMMENT "dt",
+ `data` string
+ )
+ DUPLICATE KEY(`productid`, `deviceid`)
+ DISTRIBUTED BY RANDOM BUCKETS auto
+ properties(
+ "replication_num" = "1"
+ );
+ """
+
+ load_json_data.call('with_jsonpath', '', 'true', 'json', """productid, deviceid, data, datatimestamp, dt=from_unixtime(substr(datatimestamp,1,10),'%Y%m%d')""",
+ '["$.productid","$.deviceid","$.data","$.data.datatimestamp"]', '', '', '', 'with_jsonpath.json')
+ qt_select21 "select * from ${testTable}"
+
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${testTable}")
+ }
+
// if 'enableHdfs' in regression-conf.groovy has been set to true,
// the test will run these case as below.
if (enableHdfs()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org