You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/02/27 05:58:31 UTC

(doris) branch branch-2.1 updated (4fd00e3ad73 -> f039ec8cfba)

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a change to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git


    from 4fd00e3ad73 [Fix](test)Fix workload group query queue regression test #31419
     new 735184b264c [test](auto partition)increase max_auto_partition_num configure value of auto partition (#31410)
     new 434396b4563 [docs](docs) Update docs' sidebar of Dev version (#31444)
     new f039ec8cfba [debug](Variant) sanitize variant type and column in `find_and_set_leave_value` (#31436)

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 be/src/vec/columns/column_object.cpp                | 19 ++++++++++++++-----
 docs/sidebars.json                                  |  1 +
 .../diff_data/stress_test_diff_date_list.groovy     |  4 +---
 .../stress_test_high_concurrency_load.groovy        |  4 +---
 .../same_data/stress_test_same_date_range.groovy    |  4 +---
 .../stress_test_two_stream_load.groovy              |  4 +---
 .../suites/variant_github_events_p0/load.groovy     | 21 ++++++++++-----------
 .../suites/variant_github_events_p0_new/load.groovy | 12 +++++++++++-
 8 files changed, 40 insertions(+), 29 deletions(-)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org


(doris) 02/03: [docs](docs) Update docs' sidebar of Dev version (#31444)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git

commit 434396b45631f91401d7d514419a39b4cd7ae46c
Author: KassieZ <13...@users.noreply.github.com>
AuthorDate: Tue Feb 27 12:07:24 2024 +0800

    [docs](docs) Update docs' sidebar of Dev version (#31444)
---
 docs/sidebars.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/sidebars.json b/docs/sidebars.json
index e21344bca35..8c749b25ece 100644
--- a/docs/sidebars.json
+++ b/docs/sidebars.json
@@ -187,6 +187,7 @@
                 "advanced/sql-mode",
                 "advanced/small-file-mgr",
                 "advanced/cold-hot-separation",
+                "advanced/is-being-synced",
                 "advanced/lateral-view",
                 "advanced/auto-increment"
             ]


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org


(doris) 03/03: [debug](Variant) sanitize variant type and column in `find_and_set_leave_value` (#31436)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git

commit f039ec8cfbadfcab43b734b2dd62dc2d66efe756
Author: lihangyu <15...@163.com>
AuthorDate: Tue Feb 27 13:03:58 2024 +0800

    [debug](Variant) sanitize variant type and column in `find_and_set_leave_value` (#31436)
---
 be/src/vec/columns/column_object.cpp                | 19 ++++++++++++++-----
 .../suites/variant_github_events_p0/load.groovy     | 21 ++++++++++-----------
 .../suites/variant_github_events_p0_new/load.groovy | 12 +++++++++++-
 3 files changed, 35 insertions(+), 17 deletions(-)

diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp
index 33ce8fad6aa..94ea8c8e56f 100644
--- a/be/src/vec/columns/column_object.cpp
+++ b/be/src/vec/columns/column_object.cpp
@@ -965,8 +965,15 @@ rapidjson::Value* find_leaf_node_by_path(rapidjson::Value& json, const PathInDat
 }
 
 void find_and_set_leave_value(const IColumn* column, const PathInData& path,
-                              const DataTypeSerDeSPtr& type, rapidjson::Value& root,
-                              rapidjson::Document::AllocatorType& allocator, int row) {
+                              const DataTypeSerDeSPtr& type_serde, const DataTypePtr& type,
+                              rapidjson::Value& root, rapidjson::Document::AllocatorType& allocator,
+                              int row) {
+    // sanitize type and column
+    if (column->get_name() != type->create_column()->get_name()) {
+        throw Exception(ErrorCode::INTERNAL_ERROR,
+                        "failed to set value for path {}, expected type {}, but got {} at row {}",
+                        path.get_path(), type->get_name(), column->get_name(), row);
+    }
     const auto* nullable = assert_cast<const ColumnNullable*>(column);
     if (nullable->is_null_at(row)) {
         return;
@@ -980,7 +987,7 @@ void find_and_set_leave_value(const IColumn* column, const PathInData& path,
         LOG(FATAL) << "could not find path " << path.get_path()
                    << ", root: " << std::string(buffer.GetString(), buffer.GetSize());
     }
-    type->write_one_cell_to_json(*column, *target, allocator, row);
+    type_serde->write_one_cell_to_json(*column, *target, allocator, row);
 }
 
 // compact null values
@@ -1088,7 +1095,8 @@ bool ColumnObject::serialize_one_row_to_json_format(int row, rapidjson::StringBu
 #endif
     for (const auto& subcolumn : subcolumns) {
         find_and_set_leave_value(subcolumn->data.get_finalized_column_ptr(), subcolumn->path,
-                                 subcolumn->data.get_least_common_type_serde(), root,
+                                 subcolumn->data.get_least_common_type_serde(),
+                                 subcolumn->data.get_least_common_type(), root,
                                  doc_structure->GetAllocator(), row);
     }
     compact_null_values(root, doc_structure->GetAllocator());
@@ -1151,7 +1159,8 @@ void ColumnObject::merge_sparse_to_root_column() {
                 continue;
             }
             find_and_set_leave_value(column, subcolumn->path,
-                                     subcolumn->data.get_least_common_type_serde(), root,
+                                     subcolumn->data.get_least_common_type_serde(),
+                                     subcolumn->data.get_least_common_type(), root,
                                      doc_structure->GetAllocator(), i);
         }
 
diff --git a/regression-test/suites/variant_github_events_p0/load.groovy b/regression-test/suites/variant_github_events_p0/load.groovy
index 1e8fe8ced1c..4bde4400f44 100644
--- a/regression-test/suites/variant_github_events_p0/load.groovy
+++ b/regression-test/suites/variant_github_events_p0/load.groovy
@@ -15,17 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("regression_test_variant_github_events_p0", "variant_type"){
+suite("regression_test_variant_github_events_p0", "nonConcurrent"){
+    def backendId_to_backendIP = [:]
+    def backendId_to_backendHttpPort = [:]
+    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
     def set_be_config = { key, value ->
-        String backend_id;
-        def backendId_to_backendIP = [:]
-        def backendId_to_backendHttpPort = [:]
-        getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
-
-        backend_id = backendId_to_backendIP.keySet()[0]
-        def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
-        logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
-    }
+        for (String backend_id: backendId_to_backendIP.keySet()) {
+            def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
+            logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+        }
+    } 
     sql "set enable_memtable_on_sink_node = true"
     def load_json_data = {table_name, file_name ->
         // load the json data
@@ -68,7 +67,7 @@ suite("regression_test_variant_github_events_p0", "variant_type"){
         DISTRIBUTED BY HASH(k) BUCKETS 4 
         properties("replication_num" = "1", "disable_auto_compaction" = "false");
     """
-    set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95")
+    set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1")
     // 2015
     load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-0.json'}""")
     load_json_data.call(table_name, """${getS3Url() + '/regression/gharchive.m/2015-01-01-1.json'}""")
diff --git a/regression-test/suites/variant_github_events_p0_new/load.groovy b/regression-test/suites/variant_github_events_p0_new/load.groovy
index 23471f1af1a..777befbd160 100644
--- a/regression-test/suites/variant_github_events_p0_new/load.groovy
+++ b/regression-test/suites/variant_github_events_p0_new/load.groovy
@@ -15,7 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
-suite("regression_test_variant_github_events_p0", "variant_type"){
+suite("regression_test_variant_github_events_p0", "nonConcurrent"){
+    def backendId_to_backendIP = [:]
+    def backendId_to_backendHttpPort = [:]
+    getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort);
+    def set_be_config = { key, value ->
+        for (String backend_id: backendId_to_backendIP.keySet()) {
+            def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value)
+            logger.info("update config: code=" + code + ", out=" + out + ", err=" + err)
+        }
+    } 
     def load_json_data = {table_name, file_name ->
         // load the json data
         streamLoad {
@@ -43,6 +52,7 @@ suite("regression_test_variant_github_events_p0", "variant_type"){
             }
         }
     }
+    set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "1")
 
     def table_name = "github_events"
     sql """DROP TABLE IF EXISTS ${table_name}"""


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org


(doris) 01/03: [test](auto partition)increase max_auto_partition_num configure value of auto partition (#31410)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git

commit 735184b264c730ed347ba34840f177de06756583
Author: zfr95 <87...@users.noreply.github.com>
AuthorDate: Tue Feb 27 11:19:54 2024 +0800

    [test](auto partition)increase max_auto_partition_num configure value of auto partition (#31410)
    
    * [test](auto partition)increase auto partition max_auto_partition_num configure value
    
    * fix bug
---
 .../auto_partition/diff_data/stress_test_diff_date_list.groovy        | 4 +---
 .../high_concur_load/stress_test_high_concurrency_load.groovy         | 4 +---
 .../auto_partition/same_data/stress_test_same_date_range.groovy       | 4 +---
 .../auto_partition/two_stream_load/stress_test_two_stream_load.groovy | 4 +---
 4 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/regression-test/suites/partition_p2/auto_partition/diff_data/stress_test_diff_date_list.groovy b/regression-test/suites/partition_p2/auto_partition/diff_data/stress_test_diff_date_list.groovy
index fa31ed66747..e01cd127a50 100644
--- a/regression-test/suites/partition_p2/auto_partition/diff_data/stress_test_diff_date_list.groovy
+++ b/regression-test/suites/partition_p2/auto_partition/diff_data/stress_test_diff_date_list.groovy
@@ -23,7 +23,7 @@ import java.io.File
 
 suite("stress_test_diff_date_list") {
 
-    sql """ADMIN SET FRONTEND CONFIG ('max_auto_partition_num' = '5000')"""
+    sql """ADMIN SET FRONTEND CONFIG ('max_auto_partition_num' = '10000000')"""
 
     // get doris-db from s3
     def dirPath = context.file.parent
@@ -221,6 +221,4 @@ suite("stress_test_diff_date_list") {
     // check data count
     assertTrue(count_rows == cur_rows * data_count + 1)
     assertTrue(origin_part.size() == 2)
-
-    sql """ADMIN SET FRONTEND CONFIG ('max_auto_partition_num' = '2000')"""
 }
diff --git a/regression-test/suites/partition_p2/auto_partition/high_concur_load/stress_test_high_concurrency_load.groovy b/regression-test/suites/partition_p2/auto_partition/high_concur_load/stress_test_high_concurrency_load.groovy
index 46c568b50c5..287b0dc00e3 100644
--- a/regression-test/suites/partition_p2/auto_partition/high_concur_load/stress_test_high_concurrency_load.groovy
+++ b/regression-test/suites/partition_p2/auto_partition/high_concur_load/stress_test_high_concurrency_load.groovy
@@ -23,7 +23,7 @@ import java.io.File
 
 suite("stress_test_high_concurrency_load") {
 
-    sql """ADMIN SET FRONTEND CONFIG ('max_auto_partition_num' = '5000')"""
+    sql """ADMIN SET FRONTEND CONFIG ('max_auto_partition_num' = '10000000')"""
 
     // get doris-db from s3
     def dirPath = context.file.parent
@@ -188,6 +188,4 @@ suite("stress_test_high_concurrency_load") {
             }
         }
     }
-
-    sql """ADMIN SET FRONTEND CONFIG ('max_auto_partition_num' = '2000')"""
 }
diff --git a/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy b/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy
index f25052a5e17..931856f8d25 100644
--- a/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy
+++ b/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy
@@ -23,7 +23,7 @@ import java.io.File
 
 suite("stress_test_same_date_range") {
 
-    sql """ADMIN SET FRONTEND CONFIG ('max_auto_partition_num' = '5000')"""
+    sql """ADMIN SET FRONTEND CONFIG ('max_auto_partition_num' = '10000000')"""
 
     // get doris-db from s3
     def dirPath = context.file.parent
@@ -196,6 +196,4 @@ suite("stress_test_same_date_range") {
     }
     def part_context_unique = part_context.clone().unique()
     assertTrue(part_context.size == part_context_unique.size)
-
-    sql """ADMIN SET FRONTEND CONFIG ('max_auto_partition_num' = '2000')"""
 }
diff --git a/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy b/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy
index ae795e3e854..f137e748204 100644
--- a/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy
+++ b/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy
@@ -23,7 +23,7 @@ import java.io.File
 
 suite("stress_test_two_stream_load") {
 
-    sql """ADMIN SET FRONTEND CONFIG ('max_auto_partition_num' = '100000')"""
+    sql """ADMIN SET FRONTEND CONFIG ('max_auto_partition_num' = '10000000')"""
 
     // get doris-db from s3
     def dirPath = context.file.parent
@@ -226,6 +226,4 @@ suite("stress_test_two_stream_load") {
     assertTrue(row_count_list[0][0] == partition_res_list.size)
 
     data_delete("list")
-
-    sql """ADMIN SET FRONTEND CONFIG ('max_auto_partition_num' = '2000')"""
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org