You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/11/22 15:54:14 UTC
(doris) branch branch-2.0 updated: [FIX](jsonb) fix jsonb in predict column #27325 (#27424)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 1212b561649 [FIX](jsonb) fix jsonb in predict column #27325 (#27424)
1212b561649 is described below
commit 1212b5616496b84d447c442099fc92d23f1e078c
Author: amory <wa...@selectdb.com>
AuthorDate: Wed Nov 22 23:54:05 2023 +0800
[FIX](jsonb) fix jsonb in predict column #27325 (#27424)
---
be/src/olap/schema.cpp | 1 +
.../data/json_p0/test_json_predict_is_null.out | 85 +++++++++++++++++++++
.../data/jsonb_p0/test_jsonb_predict_is_null.out | 85 +++++++++++++++++++++
.../json_p0/test_json_predict_is_null.groovy | 89 ++++++++++++++++++++++
.../jsonb_p0/test_jsonb_predict_is_null.groovy | 88 +++++++++++++++++++++
5 files changed, 348 insertions(+)
diff --git a/be/src/olap/schema.cpp b/be/src/olap/schema.cpp
index d6c737de03c..9677f94d5c8 100644
--- a/be/src/olap/schema.cpp
+++ b/be/src/olap/schema.cpp
@@ -180,6 +180,7 @@ vectorized::IColumn::MutablePtr Schema::get_predicate_column_ptr(const Field& fi
break;
case FieldType::OLAP_FIELD_TYPE_VARCHAR:
case FieldType::OLAP_FIELD_TYPE_STRING:
+ case FieldType::OLAP_FIELD_TYPE_JSONB:
if (config::enable_low_cardinality_optimize && reader_type == ReaderType::READER_QUERY) {
ptr = doris::vectorized::ColumnDictionary<doris::vectorized::Int32>::create(
field.type());
diff --git a/regression-test/data/json_p0/test_json_predict_is_null.out b/regression-test/data/json_p0/test_json_predict_is_null.out
new file mode 100644
index 00000000000..e472118b036
--- /dev/null
+++ b/regression-test/data/json_p0/test_json_predict_is_null.out
@@ -0,0 +1,85 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_extract --
+1 \N \N
+2 null \N
+3 true \N
+4 false \N
+5 100 \N
+6 10000 \N
+7 1000000000 \N
+8 1152921504606846976 \N
+9 6.18 \N
+10 "abcd" \N
+11 {} \N
+12 {"k1":"v31","k2":300} "v31"
+13 [] \N
+14 [123,456] \N
+15 ["abc","def"] \N
+16 [null,true,false,100,6.18,"abc"] \N
+17 [{"k1":"v41","k2":400},1,"a",3.14] \N
+18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31"
+26 \N \N
+27 {"k1":"v1","k2":200} "v1"
+28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
+29 12524337771678448270 \N
+30 -9223372036854775808 \N
+31 18446744073709551615 \N
+
+-- !select_pred --
+1 \N
+26 \N
+
+-- !select_delete --
+0
+
+-- !select_pred --
+2 null
+3 true
+4 false
+5 100
+6 10000
+7 1000000000
+8 1152921504606846976
+9 6.18
+10 "abcd"
+11 {}
+12 {"k1":"v31","k2":300}
+13 []
+14 [123,456]
+15 ["abc","def"]
+16 [null,true,false,100,6.18,"abc"]
+17 [{"k1":"v41","k2":400},1,"a",3.14]
+18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]}
+27 {"k1":"v1","k2":200}
+28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"}
+29 12524337771678448270
+30 -9223372036854775808
+31 18446744073709551615
+
+-- !select_drop --
+0
+
+-- !select --
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+27
+28
+29
+30
+31
+
diff --git a/regression-test/data/jsonb_p0/test_jsonb_predict_is_null.out b/regression-test/data/jsonb_p0/test_jsonb_predict_is_null.out
new file mode 100644
index 00000000000..e472118b036
--- /dev/null
+++ b/regression-test/data/jsonb_p0/test_jsonb_predict_is_null.out
@@ -0,0 +1,85 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !select_extract --
+1 \N \N
+2 null \N
+3 true \N
+4 false \N
+5 100 \N
+6 10000 \N
+7 1000000000 \N
+8 1152921504606846976 \N
+9 6.18 \N
+10 "abcd" \N
+11 {} \N
+12 {"k1":"v31","k2":300} "v31"
+13 [] \N
+14 [123,456] \N
+15 ["abc","def"] \N
+16 [null,true,false,100,6.18,"abc"] \N
+17 [{"k1":"v41","k2":400},1,"a",3.14] \N
+18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]} "v31"
+26 \N \N
+27 {"k1":"v1","k2":200} "v1"
+28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N
+29 12524337771678448270 \N
+30 -9223372036854775808 \N
+31 18446744073709551615 \N
+
+-- !select_pred --
+1 \N
+26 \N
+
+-- !select_delete --
+0
+
+-- !select_pred --
+2 null
+3 true
+4 false
+5 100
+6 10000
+7 1000000000
+8 1152921504606846976
+9 6.18
+10 "abcd"
+11 {}
+12 {"k1":"v31","k2":300}
+13 []
+14 [123,456]
+15 ["abc","def"]
+16 [null,true,false,100,6.18,"abc"]
+17 [{"k1":"v41","k2":400},1,"a",3.14]
+18 {"k1":"v31","k2":300,"a1":[{"k1":"v41","k2":400},1,"a",3.14]}
+27 {"k1":"v1","k2":200}
+28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"}
+29 12524337771678448270
+30 -9223372036854775808
+31 18446744073709551615
+
+-- !select_drop --
+0
+
+-- !select --
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+27
+28
+29
+30
+31
+
diff --git a/regression-test/suites/json_p0/test_json_predict_is_null.groovy b/regression-test/suites/json_p0/test_json_predict_is_null.groovy
new file mode 100644
index 00000000000..ed47a2934f8
--- /dev/null
+++ b/regression-test/suites/json_p0/test_json_predict_is_null.groovy
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_json_predict_is_null", "p0") {
+ sql """ set experimental_enable_nereids_planner = false """
+
+ sql """ set experimental_enable_nereids_planner = true """
+ sql """ set enable_fallback_to_original_planner = true """
+
+ sql "DROP TABLE IF EXISTS j_pred"
+
+ sql """
+ CREATE TABLE IF NOT EXISTS j_pred (
+ id INT,
+ j JSON
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 10
+ PROPERTIES("replication_num" = "1", "disable_auto_compaction" = "true");
+ """
+
+ // insert into valid json rows
+ sql """INSERT INTO j_pred VALUES(26, NULL)"""
+ sql """INSERT INTO j_pred VALUES(27, '{"k1":"v1", "k2": 200}')"""
+ sql """INSERT INTO j_pred VALUES(28, '{"a.b.c":{"k1.a1":"v31", "k2": 300},"a":"niu"}')"""
+ // int64 value
+ sql """INSERT INTO j_pred VALUES(29, '12524337771678448270')"""
+ // int64 min value
+ sql """INSERT INTO j_pred VALUES(30, '-9223372036854775808')"""
+ // int64 max value
+ sql """INSERT INTO j_pred VALUES(31, '18446744073709551615')"""
+
+ // load the jsonb data from csv file
+ streamLoad {
+ table "j_pred"
+
+ file "test_json.csv" // import csv file
+ set 'max_filter_ratio', '0.3'
+ time 10000 // limit inflight 10s
+ set 'strict_mode', 'true'
+
+ // if declared a check callback, the default check condition will ignore.
+ // So you must check all condition
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+
+ def (code, out, err) = curl("GET", json.ErrorURL)
+ log.info("error result: " + out)
+
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(25, json.NumberTotalRows)
+ assertEquals(18, json.NumberLoadedRows)
+ assertEquals(7, json.NumberFilteredRows)
+ assertTrue(json.LoadBytes > 0)
+ }
+ }
+
+ qt_select_extract "SELECT id, j, jsonb_extract(j, '\$.k1') FROM j_pred ORDER BY id"
+
+ qt_select_pred "select * from j_pred where j is null order by id"
+
+ qt_select_delete "delete from j_pred where j is null"
+
+ qt_select_pred "select * from j_pred order by id"
+
+ qt_select_drop "alter table j_pred DROP COLUMN j"
+
+ qt_select "select * from j_pred order by id"
+}
diff --git a/regression-test/suites/jsonb_p0/test_jsonb_predict_is_null.groovy b/regression-test/suites/jsonb_p0/test_jsonb_predict_is_null.groovy
new file mode 100644
index 00000000000..c97d0469908
--- /dev/null
+++ b/regression-test/suites/jsonb_p0/test_jsonb_predict_is_null.groovy
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_jsonb_predict_is_null", "p0") {
+ sql """ set experimental_enable_nereids_planner = false """
+
+ sql """ set experimental_enable_nereids_planner = true """
+ sql """ set enable_fallback_to_original_planner = true """
+
+ sql "DROP TABLE IF EXISTS jb_pred"
+
+ sql """
+ CREATE TABLE IF NOT EXISTS jb_pred (
+ id INT,
+ j JSONB
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 10
+ PROPERTIES("replication_num" = "1", "disable_auto_compaction" = "true");
+ """
+
+ // insert into valid json rows
+ sql """INSERT INTO jb_pred VALUES(26, NULL)"""
+ sql """INSERT INTO jb_pred VALUES(27, '{"k1":"v1", "k2": 200}')"""
+ sql """INSERT INTO jb_pred VALUES(28, '{"a.b.c":{"k1.a1":"v31", "k2": 300},"a":"niu"}')"""
+ // int64 value
+ sql """INSERT INTO jb_pred VALUES(29, '12524337771678448270')"""
+ // int64 min value
+ sql """INSERT INTO jb_pred VALUES(30, '-9223372036854775808')"""
+ // int64 max value
+ sql """INSERT INTO jb_pred VALUES(31, '18446744073709551615')"""
+
+ // load the jsonb data from csv file
+ streamLoad {
+ table "jb_pred"
+
+ file "test_jsonb.csv" // import csv file
+ set 'max_filter_ratio', '0.3'
+ time 10000 // limit inflight 10s
+ set 'strict_mode', 'true'
+
+ // if declared a check callback, the default check condition will ignore.
+ // So you must check all condition
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ def (code, out, err) = curl("GET", json.ErrorURL)
+ log.info("error result: " + out)
+
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(25, json.NumberTotalRows)
+ assertEquals(18, json.NumberLoadedRows)
+ assertEquals(7, json.NumberFilteredRows)
+ assertTrue(json.LoadBytes > 0)
+ }
+ }
+
+ qt_select_extract "SELECT id, j, jsonb_extract(j, '\$.k1') FROM jb_pred ORDER BY id"
+
+ qt_select_pred "select * from jb_pred where j is null order by id"
+
+ qt_select_delete "delete from jb_pred where j is null"
+
+ qt_select_pred "select * from jb_pred order by id"
+
+ qt_select_drop "alter table jb_pred DROP COLUMN j"
+
+ qt_select "select * from jb_pred order by id"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org