[incubator-doris] branch master updated: [Bug][stream-vec-load] Null data load do not skip the same place data (#9360)

[wa...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

wangbo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git


The following commit(s) were added to refs/heads/master by this push:
     new edc833ab76 [Bug][stream-vec-load] Null data load do not skip the same place data (#9360)
edc833ab76 is described below

commit edc833ab764b446ab1664d3a220311ddad4983b3
Author: HappenLee <ha...@hotmail.com>
AuthorDate: Fri May 6 16:07:45 2022 +0800

    [Bug][stream-vec-load] Null data load do not skip the same place data (#9360)
    
    Co-authored-by: lihaopeng <li...@baidu.com>
---
 be/src/olap/rowset/segment_v2/column_writer.cpp | 46 ++++++++++++++++++-------
 be/src/olap/rowset/segment_v2/column_writer.h   |  3 ++
 be/src/vec/sink/vtablet_sink.cpp                |  2 +-
 3 files changed, 37 insertions(+), 14 deletions(-)

diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp
index 7ebf917b45..5cb5140102 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -193,23 +193,43 @@ Status ColumnWriter::append_nullable(const uint8_t* is_null_bits, const void* da
     return Status::OK();
 }
 
+Status ColumnWriter::append_nullable(const uint8_t* null_map, const uint8_t** ptr,
+                                     size_t num_rows) {
+    size_t offset = 0;
+    auto next_run_step = [&]() {
+        size_t step = 1;
+        for (auto i = offset + 1; i < num_rows; ++i) {
+            if (null_map[offset] == null_map[i])
+                step++;
+            else
+                break;
+        }
+        return step;
+    };
+
+    do {
+        auto step = next_run_step();
+        if (null_map[offset]) {
+            RETURN_IF_ERROR(append_nulls(step));
+            *ptr += get_field()->size() * step;
+        } else {
+            // TODO:
+            //  1. `*ptr += get_field()->size() * step;` should do in this function, not append_data;
+            //  2. support array vectorized load and ptr offset add
+            RETURN_IF_ERROR(append_data(ptr, step));
+        }
+        offset += step;
+    } while (offset < num_rows);
+
+    return Status::OK();
+}
+
 Status ColumnWriter::append(const uint8_t* nullmap, const void* data, size_t num_rows) {
     assert(data && num_rows > 0);
+    const auto* ptr = (const uint8_t*)data;
     if (nullmap) {
-        size_t bitmap_size = BitmapSize(num_rows);
-        if (_null_bitmap.size() < bitmap_size) {
-            _null_bitmap.resize(bitmap_size);
-        }
-        uint8_t* bitmap_data = _null_bitmap.data();
-        memset(bitmap_data, 0, bitmap_size);
-        for (size_t i = 0; i < num_rows; ++i) {
-            if (nullmap[i]) {
-                BitmapSet(bitmap_data, i);
-            }
-        }
-        return append_nullable(bitmap_data, data, num_rows);
+        return append_nullable(nullmap, &ptr, num_rows);
     } else {
-        const uint8_t* ptr = (const uint8_t*)data;
         return append_data(&ptr, num_rows);
     }
 }
diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h
index 5e5e3d3019..0cacbf8547 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.h
+++ b/be/src/olap/rowset/segment_v2/column_writer.h
@@ -105,6 +105,9 @@ public:
 
     Status append_nullable(const uint8_t* nullmap, const void* data, size_t num_rows);
 
+    // use only in vectorized load
+    Status append_nullable(const uint8_t* null_map, const uint8_t** data, size_t num_rows);
+
     virtual Status append_nulls(size_t num_rows) = 0;
 
     virtual Status finish_current_page() = 0;
diff --git a/be/src/vec/sink/vtablet_sink.cpp b/be/src/vec/sink/vtablet_sink.cpp
index 78fd74a7d0..32e0e2148f 100644
--- a/be/src/vec/sink/vtablet_sink.cpp
+++ b/be/src/vec/sink/vtablet_sink.cpp
@@ -439,7 +439,7 @@ Status VOlapTableSink::send(RuntimeState* state, vectorized::Block* input_block)
     }
 
     // check intolerable failure
-    for (auto index_channel : _channels) {
+    for (const auto& index_channel : _channels) {
         RETURN_IF_ERROR(index_channel->check_intolerable_failure());
     }
     return Status::OK();


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org