You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2023/04/22 06:01:11 UTC
[doris] branch master updated: [Refact](type system)refact interconversion for jsonb with column (#18819)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1ffd34f6f1 [Refact](type system)refact interconversion for jsonb with column (#18819)
1ffd34f6f1 is described below
commit 1ffd34f6f1817184ee5c89c86622245cb9554b38
Author: amory <wa...@selectdb.com>
AuthorDate: Sat Apr 22 14:01:05 2023 +0800
[Refact](type system)refact interconversion for jsonb with column (#18819)
* refact jsonb to column
* update
* fix format
* fixed
* fix file head for compile
---
be/src/vec/columns/column_decimal.h | 2 +-
.../vec/data_types/serde/data_type_array_serde.cpp | 22 +-
.../vec/data_types/serde/data_type_array_serde.h | 5 +
.../data_types/serde/data_type_bitmap_serde.cpp | 32 ++-
.../vec/data_types/serde/data_type_bitmap_serde.h | 5 +
.../vec/data_types/serde/data_type_decimal_serde.h | 46 ++++
.../serde/data_type_fixedlengthobject_serde.h | 8 +
.../vec/data_types/serde/data_type_hll_serde.cpp | 20 ++
be/src/vec/data_types/serde/data_type_hll_serde.h | 5 +
.../vec/data_types/serde/data_type_map_serde.cpp | 18 +-
be/src/vec/data_types/serde/data_type_map_serde.h | 4 +
.../data_types/serde/data_type_nullable_serde.cpp | 22 ++
.../data_types/serde/data_type_nullable_serde.h | 5 +
.../vec/data_types/serde/data_type_number_serde.h | 60 +++++
.../vec/data_types/serde/data_type_object_serde.h | 8 +
.../serde/data_type_quantilestate_serde.h | 38 ++-
be/src/vec/data_types/serde/data_type_serde.h | 8 +-
.../data_types/serde/data_type_string_serde.cpp | 16 ++
.../vec/data_types/serde/data_type_string_serde.h | 4 +
.../data_types/serde/data_type_struct_serde.cpp | 19 +-
.../vec/data_types/serde/data_type_struct_serde.h | 4 +
be/src/vec/jsonb/serialize.cpp | 265 +--------------------
be/src/vec/jsonb/serialize.h | 1 +
be/test/vec/jsonb/serialize_test.cpp | 37 +++
24 files changed, 374 insertions(+), 280 deletions(-)
diff --git a/be/src/vec/columns/column_decimal.h b/be/src/vec/columns/column_decimal.h
index 0096789c12..26a9abde9a 100644
--- a/be/src/vec/columns/column_decimal.h
+++ b/be/src/vec/columns/column_decimal.h
@@ -247,7 +247,7 @@ public:
return false;
}
- void insert(const T value) { data.push_back(value); }
+ void insert_value(const T value) { data.push_back(value); }
Container& get_data() { return data; }
const Container& get_data() const { return data; }
const T& get_element(size_t n) const { return data[n]; }
diff --git a/be/src/vec/data_types/serde/data_type_array_serde.cpp b/be/src/vec/data_types/serde/data_type_array_serde.cpp
index 46029ac5d2..95161b164f 100644
--- a/be/src/vec/data_types/serde/data_type_array_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_array_serde.cpp
@@ -17,8 +17,26 @@
#include "data_type_array_serde.h"
-#include "vec/columns/column_array.cpp"
namespace doris {
-namespace vectorized {}
+namespace vectorized {
+
+void DataTypeArraySerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result,
+ Arena* mem_pool, int32_t col_id,
+ int row_num) const {
+ result.writeKey(col_id);
+ const char* begin = nullptr;
+ // maybe serialize_value_into_arena should move to here later.
+ StringRef value = column.serialize_value_into_arena(row_num, *mem_pool, begin);
+ result.writeStartBinary();
+ result.writeBinary(value.data, value.size);
+ result.writeEndBinary();
+}
+
+void DataTypeArraySerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const {
+ auto blob = static_cast<const JsonbBlobVal*>(arg);
+ column.deserialize_and_insert_from_arena(blob->getBlob());
+}
+
+} // namespace vectorized
} // namespace doris
\ No newline at end of file
diff --git a/be/src/vec/data_types/serde/data_type_array_serde.h b/be/src/vec/data_types/serde/data_type_array_serde.h
index 755b74391e..ef7f99da15 100644
--- a/be/src/vec/data_types/serde/data_type_array_serde.h
+++ b/be/src/vec/data_types/serde/data_type_array_serde.h
@@ -42,6 +42,11 @@ public:
LOG(FATAL) << "Not support read from pb to array";
}
+ void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena* mem_pool,
+ int32_t col_id, int row_num) const override;
+
+ void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override;
+
private:
DataTypeSerDeSPtr nested_serde;
};
diff --git a/be/src/vec/data_types/serde/data_type_bitmap_serde.cpp b/be/src/vec/data_types/serde/data_type_bitmap_serde.cpp
index e09b863240..3a6197fd29 100644
--- a/be/src/vec/data_types/serde/data_type_bitmap_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_bitmap_serde.cpp
@@ -17,13 +17,9 @@
#include "data_type_bitmap_serde.h"
-#include <gen_cpp/types.pb.h>
-
-#include <string>
-
-#include "util/bitmap_value.h"
#include "vec/columns/column_complex.h"
-#include "vec/common/assert_cast.h"
+#include "vec/common/arena.h"
+#include "vec/data_types/data_type.h"
namespace doris {
@@ -55,5 +51,29 @@ Status DataTypeBitMapSerDe::read_column_from_pb(IColumn& column, const PValues&
}
return Status::OK();
}
+
+void DataTypeBitMapSerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result,
+ Arena* mem_pool, int32_t col_id,
+ int row_num) const {
+ auto& data_column = assert_cast<const ColumnBitmap&>(column);
+ result.writeKey(col_id);
+ auto bitmap_value = const_cast<BitmapValue&>(data_column.get_element(row_num));
+ // serialize the content of string
+ auto size = bitmap_value.getSizeInBytes();
+ // serialize the content of string
+ auto ptr = mem_pool->alloc(size);
+ bitmap_value.write_to(const_cast<char*>(ptr));
+ result.writeStartBinary();
+ result.writeBinary(reinterpret_cast<const char*>(ptr), size);
+ result.writeEndBinary();
+}
+
+void DataTypeBitMapSerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const {
+ auto& col = reinterpret_cast<ColumnBitmap&>(column);
+ auto blob = static_cast<const JsonbBlobVal*>(arg);
+ BitmapValue bitmap_value(blob->getBlob());
+ col.insert_value(bitmap_value);
+}
+
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/data_types/serde/data_type_bitmap_serde.h b/be/src/vec/data_types/serde/data_type_bitmap_serde.h
index f59519b7f9..847f7d9742 100644
--- a/be/src/vec/data_types/serde/data_type_bitmap_serde.h
+++ b/be/src/vec/data_types/serde/data_type_bitmap_serde.h
@@ -31,6 +31,11 @@ public:
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
+
+ void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena* mem_pool,
+ int32_t col_id, int row_num) const override;
+
+ void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override;
};
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/data_types/serde/data_type_decimal_serde.h b/be/src/vec/data_types/serde/data_type_decimal_serde.h
index a269546eb8..1f37fbb8a7 100644
--- a/be/src/vec/data_types/serde/data_type_decimal_serde.h
+++ b/be/src/vec/data_types/serde/data_type_decimal_serde.h
@@ -41,6 +41,11 @@ public:
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
+
+ void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena* mem_pool,
+ int32_t col_id, int row_num) const override;
+
+ void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override;
};
template <typename T>
@@ -82,5 +87,46 @@ Status DataTypeDecimalSerDe<T>::read_column_from_pb(IColumn& column, const PValu
return Status::NotSupported("unknown ColumnType for reading from pb");
}
+
+template <typename T>
+void DataTypeDecimalSerDe<T>::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result,
+ Arena* mem_pool, int32_t col_id,
+ int row_num) const {
+ StringRef data_ref = column.get_data_at(row_num);
+ result.writeKey(col_id);
+ if constexpr (std::is_same_v<T, Decimal<Int128>>) {
+ Decimal128::NativeType val =
+ *reinterpret_cast<const Decimal128::NativeType*>(data_ref.data);
+ result.writeInt128(val);
+ } else if constexpr (std::is_same_v<T, Decimal<Int128I>>) {
+ Decimal64::NativeType val = *reinterpret_cast<const Decimal64::NativeType*>(data_ref.data);
+ result.writeInt128(val);
+ } else if constexpr (std::is_same_v<T, Decimal<Int32>>) {
+ Decimal32::NativeType val = *reinterpret_cast<const Decimal32::NativeType*>(data_ref.data);
+ result.writeInt32(val);
+ } else if constexpr (std::is_same_v<T, Decimal<Int64>>) {
+ Decimal64::NativeType val = *reinterpret_cast<const Decimal64::NativeType*>(data_ref.data);
+ result.writeInt64(val);
+ } else {
+ LOG(FATAL) << "unknown Column " << column.get_name() << " for writing to jsonb";
+ }
+}
+
+template <typename T>
+void DataTypeDecimalSerDe<T>::read_one_cell_from_jsonb(IColumn& column,
+ const JsonbValue* arg) const {
+ auto& col = reinterpret_cast<ColumnDecimal<T>&>(column);
+ if constexpr (std::is_same_v<T, Decimal<Int128>>) {
+ col.insert_value(static_cast<const JsonbInt128Val*>(arg)->val());
+ } else if constexpr (std::is_same_v<T, Decimal<Int128I>>) {
+ col.insert_value(static_cast<const JsonbInt128Val*>(arg)->val());
+ } else if constexpr (std::is_same_v<T, Decimal<Int32>>) {
+ col.insert_value(static_cast<const JsonbInt32Val*>(arg)->val());
+ } else if constexpr (std::is_same_v<T, Decimal<Int64>>) {
+ col.insert_value(static_cast<const JsonbInt64Val*>(arg)->val());
+ } else {
+ LOG(FATAL) << "unknown jsonb " << arg->typeName() << " for writing to column";
+ }
+}
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/data_types/serde/data_type_fixedlengthobject_serde.h b/be/src/vec/data_types/serde/data_type_fixedlengthobject_serde.h
index 563ef98523..6924f9d435 100644
--- a/be/src/vec/data_types/serde/data_type_fixedlengthobject_serde.h
+++ b/be/src/vec/data_types/serde/data_type_fixedlengthobject_serde.h
@@ -39,6 +39,14 @@ public:
Status read_column_from_pb(IColumn& column, const PValues& arg) const override {
LOG(FATAL) << "Not support read from pb to FixedLengthObject";
};
+ void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena* mem_pool,
+ int32_t col_id, int row_num) const override {
+ LOG(FATAL) << "Not support write FixedLengthObject column to jsonb";
+ }
+
+ void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override {
+ LOG(FATAL) << "Not support read from jsonb to FixedLengthObject";
+ }
};
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/data_types/serde/data_type_hll_serde.cpp b/be/src/vec/data_types/serde/data_type_hll_serde.cpp
index 96dbf23b25..4c27711d9f 100644
--- a/be/src/vec/data_types/serde/data_type_hll_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_hll_serde.cpp
@@ -26,6 +26,7 @@
#include "olap/hll.h"
#include "util/slice.h"
#include "vec/columns/column_complex.h"
+#include "vec/common/arena.h"
#include "vec/common/assert_cast.h"
namespace doris {
@@ -58,5 +59,24 @@ Status DataTypeHLLSerDe::read_column_from_pb(IColumn& column, const PValues& arg
return Status::OK();
}
+void DataTypeHLLSerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result,
+ Arena* mem_pool, int32_t col_id, int row_num) const {
+ result.writeKey(col_id);
+ auto& data_column = assert_cast<const ColumnHLL&>(column);
+ auto& hll_value = const_cast<HyperLogLog&>(data_column.get_element(row_num));
+ auto size = hll_value.max_serialized_size();
+ auto ptr = reinterpret_cast<char*>(mem_pool->alloc(size));
+ size_t actual_size = hll_value.serialize((uint8_t*)ptr);
+ result.writeStartBinary();
+ result.writeBinary(reinterpret_cast<const char*>(ptr), actual_size);
+ result.writeEndBinary();
+}
+void DataTypeHLLSerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const {
+ auto& col = reinterpret_cast<ColumnHLL&>(column);
+ auto blob = static_cast<const JsonbBlobVal*>(arg);
+ HyperLogLog hyper_log_log(Slice(blob->getBlob()));
+ col.insert_value(hyper_log_log);
+}
+
} // namespace vectorized
} // namespace doris
\ No newline at end of file
diff --git a/be/src/vec/data_types/serde/data_type_hll_serde.h b/be/src/vec/data_types/serde/data_type_hll_serde.h
index 538c35ed0a..d0633e627e 100644
--- a/be/src/vec/data_types/serde/data_type_hll_serde.h
+++ b/be/src/vec/data_types/serde/data_type_hll_serde.h
@@ -31,6 +31,11 @@ public:
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
+
+ void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena* mem_pool,
+ int32_t col_id, int row_num) const override;
+
+ void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override;
};
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/data_types/serde/data_type_map_serde.cpp b/be/src/vec/data_types/serde/data_type_map_serde.cpp
index 4a26c9d0d1..13c30fb2b5 100644
--- a/be/src/vec/data_types/serde/data_type_map_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_map_serde.cpp
@@ -18,5 +18,21 @@
#include "data_type_map_serde.h"
namespace doris {
-namespace vectorized {}
+namespace vectorized {
+void DataTypeMapSerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const {
+ auto blob = static_cast<const JsonbBlobVal*>(arg);
+ column.deserialize_and_insert_from_arena(blob->getBlob());
+}
+
+void DataTypeMapSerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result,
+ Arena* mem_pool, int32_t col_id, int row_num) const {
+ result.writeKey(col_id);
+ const char* begin = nullptr;
+ // maybe serialize_value_into_arena should move to here later.
+ StringRef value = column.serialize_value_into_arena(row_num, *mem_pool, begin);
+ result.writeStartBinary();
+ result.writeBinary(value.data, value.size);
+ result.writeEndBinary();
+}
+} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/data_types/serde/data_type_map_serde.h b/be/src/vec/data_types/serde/data_type_map_serde.h
index ac72465dc1..2769db1f2c 100644
--- a/be/src/vec/data_types/serde/data_type_map_serde.h
+++ b/be/src/vec/data_types/serde/data_type_map_serde.h
@@ -42,6 +42,10 @@ public:
Status read_column_from_pb(IColumn& column, const PValues& arg) const override {
LOG(FATAL) << "Not support read from pb to map";
}
+ void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena* mem_pool,
+ int32_t col_id, int row_num) const override;
+
+ void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override;
private:
DataTypeSerDeSPtr key_serde;
diff --git a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp
index aefb4f7575..27dee781fc 100644
--- a/be/src/vec/data_types/serde/data_type_nullable_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_nullable_serde.cpp
@@ -70,5 +70,27 @@ Status DataTypeNullableSerDe::read_column_from_pb(IColumn& column, const PValues
}
return Status::OK();
}
+void DataTypeNullableSerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result,
+ Arena* mem_pool, int32_t col_id,
+ int row_num) const {
+ auto& nullable_col = assert_cast<const ColumnNullable&>(column);
+ if (nullable_col.is_null_at(row_num)) {
+ // do not insert to jsonb
+ return;
+ }
+ result.writeKey(col_id);
+ nested_serde->write_one_cell_to_jsonb(nullable_col.get_nested_column(), result, mem_pool,
+ col_id, row_num);
+}
+void DataTypeNullableSerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const {
+ auto& col = reinterpret_cast<ColumnNullable&>(column);
+ if (!arg || arg->isNull()) {
+ col.insert_default();
+ return;
+ }
+ nested_serde->read_one_cell_from_jsonb(col.get_nested_column(), arg);
+ auto& null_map_data = col.get_null_map_data();
+ null_map_data.push_back(0);
+}
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/data_types/serde/data_type_nullable_serde.h b/be/src/vec/data_types/serde/data_type_nullable_serde.h
index 14aeabeb8a..4503b663d2 100644
--- a/be/src/vec/data_types/serde/data_type_nullable_serde.h
+++ b/be/src/vec/data_types/serde/data_type_nullable_serde.h
@@ -34,6 +34,11 @@ public:
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
+ void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena* mem_pool,
+ int32_t col_id, int row_num) const override;
+
+ void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override;
+
private:
DataTypeSerDeSPtr nested_serde;
};
diff --git a/be/src/vec/data_types/serde/data_type_number_serde.h b/be/src/vec/data_types/serde/data_type_number_serde.h
index bd76fd0de8..2dfec6925f 100644
--- a/be/src/vec/data_types/serde/data_type_number_serde.h
+++ b/be/src/vec/data_types/serde/data_type_number_serde.h
@@ -41,6 +41,11 @@ public:
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
+
+ void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena* mem_pool,
+ int32_t col_id, int row_num) const override;
+
+ void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override;
};
template <typename T>
@@ -167,5 +172,60 @@ Status DataTypeNumberSerDe<T>::write_column_to_pb(const IColumn& column, PValues
return Status::OK();
}
+template <typename T>
+void DataTypeNumberSerDe<T>::read_one_cell_from_jsonb(IColumn& column,
+ const JsonbValue* arg) const {
+ auto& col = reinterpret_cast<ColumnType&>(column);
+ if constexpr (std::is_same_v<T, Int8>) {
+ col.insert_value(static_cast<const JsonbInt8Val*>(arg)->val());
+ } else if constexpr (std::is_same_v<T, Int16> || std::is_same_v<T, UInt16>) {
+ col.insert_value(static_cast<const JsonbInt16Val*>(arg)->val());
+ } else if constexpr (std::is_same_v<T, Int32> || std::is_same_v<T, UInt32>) {
+ col.insert_value(static_cast<const JsonbInt32Val*>(arg)->val());
+ } else if constexpr (std::is_same_v<T, Int64> || std::is_same_v<T, UInt64>) {
+ col.insert_value(static_cast<const JsonbInt64Val*>(arg)->val());
+ } else if constexpr (std::is_same_v<T, Int128>) {
+ col.insert_value(static_cast<const JsonbInt128Val*>(arg)->val());
+ } else if constexpr (std::is_same_v<T, float>) {
+ col.insert_value(static_cast<const JsonbFloatVal*>(arg)->val());
+ } else if constexpr (std::is_same_v<T, double>) {
+ col.insert_value(static_cast<const JsonbDoubleVal*>(arg)->val());
+ } else {
+ LOG(FATAL) << "unknown jsonb type " << arg->typeName() << " for writing to column";
+ }
+}
+template <typename T>
+void DataTypeNumberSerDe<T>::write_one_cell_to_jsonb(const IColumn& column,
+ JsonbWriterT<JsonbOutStream>& result,
+ Arena* mem_pool, int32_t col_id,
+ int row_num) const {
+ result.writeKey(col_id);
+ StringRef data_ref = column.get_data_at(row_num);
+ if constexpr (std::is_same_v<T, Int8>) {
+ int8_t val = *reinterpret_cast<const int8_t*>(data_ref.data);
+ result.writeInt8(val);
+ } else if constexpr (std::is_same_v<T, Int16> || std::is_same_v<T, UInt16>) {
+ int16_t val = *reinterpret_cast<const int16_t*>(data_ref.data);
+ result.writeInt16(val);
+ } else if constexpr (std::is_same_v<T, Int32> || std::is_same_v<T, UInt32>) {
+ int32_t val = *reinterpret_cast<const int32_t*>(data_ref.data);
+ result.writeInt32(val);
+ } else if constexpr (std::is_same_v<T, Int64> || std::is_same_v<T, UInt64>) {
+ int64_t val = *reinterpret_cast<const int64_t*>(data_ref.data);
+ result.writeInt64(val);
+ } else if constexpr (std::is_same_v<T, Int128>) {
+ __int128_t val = *reinterpret_cast<const __int128_t*>(data_ref.data);
+ result.writeInt128(val);
+ } else if constexpr (std::is_same_v<T, float>) {
+ float val = *reinterpret_cast<const float*>(data_ref.data);
+ result.writeFloat(val);
+ } else if constexpr (std::is_same_v<T, double>) {
+ double val = *reinterpret_cast<const double*>(data_ref.data);
+ result.writeDouble(val);
+ } else {
+ LOG(FATAL) << "unknown column type " << column.get_name() << " for writing to jsonb";
+ }
+}
+
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/data_types/serde/data_type_object_serde.h b/be/src/vec/data_types/serde/data_type_object_serde.h
index 579176f405..54cb04ccf5 100644
--- a/be/src/vec/data_types/serde/data_type_object_serde.h
+++ b/be/src/vec/data_types/serde/data_type_object_serde.h
@@ -39,6 +39,14 @@ public:
Status read_column_from_pb(IColumn& column, const PValues& arg) const override {
LOG(FATAL) << "Not support read from pb to object";
}
+ void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena* mem_pool,
+ int32_t col_id, int row_num) const override {
+ LOG(FATAL) << "Not support write object column to json";
+ }
+
+ void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override {
+ LOG(FATAL) << "Not support write json object to column";
+ }
};
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/data_types/serde/data_type_quantilestate_serde.h b/be/src/vec/data_types/serde/data_type_quantilestate_serde.h
index 9dc41652e4..9f2bc8e7a7 100644
--- a/be/src/vec/data_types/serde/data_type_quantilestate_serde.h
+++ b/be/src/vec/data_types/serde/data_type_quantilestate_serde.h
@@ -17,13 +17,9 @@
#pragma once
-#include <gen_cpp/types.pb.h>
-#include <stddef.h>
-
-#include "common/status.h"
#include "data_type_serde.h"
-#include "vec/columns/column.h"
-#include "vec/common/string_ref.h"
+#include "vec/columns/column_complex.h"
+#include "vec/common/arena.h"
namespace doris {
@@ -35,6 +31,11 @@ public:
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
+
+ void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena* mem_pool,
+ int32_t col_id, int row_num) const override;
+
+ void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override;
};
template <typename T>
@@ -57,5 +58,30 @@ Status DataTypeQuantileStateSerDe<T>::read_column_from_pb(IColumn& column,
}
return Status::OK();
}
+
+template <typename T>
+void DataTypeQuantileStateSerDe<T>::write_one_cell_to_jsonb(const IColumn& column,
+ JsonbWriter& result, Arena* mem_pool,
+ int32_t col_id, int row_num) const {
+ auto& col = reinterpret_cast<const ColumnQuantileState<T>&>(column);
+ auto& val = const_cast<QuantileState<T>&>(col.get_element(row_num));
+ size_t actual_size = val.get_serialized_size();
+ auto ptr = mem_pool->alloc(actual_size);
+ result.writeKey(col_id);
+ result.writeStartBinary();
+ result.writeBinary(reinterpret_cast<const char*>(ptr), actual_size);
+ result.writeEndBinary();
+}
+
+template <typename T>
+void DataTypeQuantileStateSerDe<T>::read_one_cell_from_jsonb(IColumn& column,
+ const JsonbValue* arg) const {
+ auto& col = reinterpret_cast<ColumnQuantileState<T>&>(column);
+ auto blob = static_cast<const JsonbBlobVal*>(arg);
+ QuantileState<T> val;
+ val.deserialize(Slice(blob->getBlob()));
+ col.insert_value(val);
+}
+
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/data_types/serde/data_type_serde.h b/be/src/vec/data_types/serde/data_type_serde.h
index 0dc101f9a5..4d03a14dff 100644
--- a/be/src/vec/data_types/serde/data_type_serde.h
+++ b/be/src/vec/data_types/serde/data_type_serde.h
@@ -21,6 +21,8 @@
#include <vector>
#include "common/status.h"
+#include "util/jsonb_writer.h"
+#include "vec/columns/column.h"
namespace doris {
class PValues;
@@ -48,7 +50,11 @@ public:
virtual Status read_column_from_pb(IColumn& column, const PValues& arg) const = 0;
- // JSONB serializer and deserializer
+ // JSONB serializer and deserializer, should write col_id
+ virtual void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result,
+ Arena* mem_pool, int32_t col_id, int row_num) const = 0;
+
+ virtual void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const = 0;
// MySQL serializer and deserializer
diff --git a/be/src/vec/data_types/serde/data_type_string_serde.cpp b/be/src/vec/data_types/serde/data_type_string_serde.cpp
index 112fcc6bed..8c0970f32b 100644
--- a/be/src/vec/data_types/serde/data_type_string_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_string_serde.cpp
@@ -44,5 +44,21 @@ Status DataTypeStringSerDe::read_column_from_pb(IColumn& column, const PValues&
}
return Status::OK();
}
+
+void DataTypeStringSerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result,
+ Arena* mem_pool, int32_t col_id,
+ int row_num) const {
+ result.writeKey(col_id);
+ const auto& data_ref = column.get_data_at(row_num);
+ result.writeStartBinary();
+ result.writeBinary(reinterpret_cast<const char*>(data_ref.data), data_ref.size);
+ result.writeEndBinary();
+}
+void DataTypeStringSerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const {
+ assert(arg->isBinary());
+ auto& col = reinterpret_cast<ColumnString&>(column);
+ auto blob = static_cast<const JsonbBlobVal*>(arg);
+ col.insert_data(blob->getBlob(), blob->getBlobLen());
+}
} // namespace vectorized
} // namespace doris
\ No newline at end of file
diff --git a/be/src/vec/data_types/serde/data_type_string_serde.h b/be/src/vec/data_types/serde/data_type_string_serde.h
index 4ede2ef761..2715896d4b 100644
--- a/be/src/vec/data_types/serde/data_type_string_serde.h
+++ b/be/src/vec/data_types/serde/data_type_string_serde.h
@@ -31,6 +31,10 @@ public:
Status write_column_to_pb(const IColumn& column, PValues& result, int start,
int end) const override;
Status read_column_from_pb(IColumn& column, const PValues& arg) const override;
+ void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena* mem_pool,
+ int32_t col_id, int row_num) const override;
+
+ void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override;
};
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/data_types/serde/data_type_struct_serde.cpp b/be/src/vec/data_types/serde/data_type_struct_serde.cpp
index 730f5a6980..095a07226a 100644
--- a/be/src/vec/data_types/serde/data_type_struct_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_struct_serde.cpp
@@ -18,5 +18,22 @@
#include "data_type_struct_serde.h"
namespace doris {
-namespace vectorized {}
+namespace vectorized {
+void DataTypeStructSerDe::write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result,
+ Arena* mem_pool, int32_t col_id,
+ int row_num) const {
+ result.writeKey(col_id);
+ const char* begin = nullptr;
+ // maybe serialize_value_into_arena should move to here later.
+ StringRef value = column.serialize_value_into_arena(row_num, *mem_pool, begin);
+ result.writeStartBinary();
+ result.writeBinary(value.data, value.size);
+ result.writeEndBinary();
+}
+
+void DataTypeStructSerDe::read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const {
+ auto blob = static_cast<const JsonbBlobVal*>(arg);
+ column.deserialize_and_insert_from_arena(blob->getBlob());
+}
+} // namespace vectorized
} // namespace doris
\ No newline at end of file
diff --git a/be/src/vec/data_types/serde/data_type_struct_serde.h b/be/src/vec/data_types/serde/data_type_struct_serde.h
index e895513bd9..5b7854f43c 100644
--- a/be/src/vec/data_types/serde/data_type_struct_serde.h
+++ b/be/src/vec/data_types/serde/data_type_struct_serde.h
@@ -42,6 +42,10 @@ public:
Status read_column_from_pb(IColumn& column, const PValues& arg) const override {
LOG(FATAL) << "Not support read from pb to strut";
}
+ void write_one_cell_to_jsonb(const IColumn& column, JsonbWriter& result, Arena* mem_pool,
+ int32_t col_id, int row_num) const override;
+
+ void read_one_cell_from_jsonb(IColumn& column, const JsonbValue* arg) const override;
private:
DataTypeSerDeSPtrs elemSerDeSPtrs;
diff --git a/be/src/vec/jsonb/serialize.cpp b/be/src/vec/jsonb/serialize.cpp
index e1bf5f17b1..65df5ee99e 100644
--- a/be/src/vec/jsonb/serialize.cpp
+++ b/be/src/vec/jsonb/serialize.cpp
@@ -49,259 +49,6 @@
namespace doris::vectorized {
-static inline bool is_column_null_at(int row, const IColumn* column, const doris::FieldType& type,
- const StringRef& data_ref) {
- if (type != FieldType::OLAP_FIELD_TYPE_ARRAY) {
- return data_ref.data == nullptr;
- } else {
- Field array;
- column->get(row, array);
- return array.is_null();
- }
-}
-
-static bool is_jsonb_blob_type(FieldType type) {
- return type == FieldType::OLAP_FIELD_TYPE_VARCHAR || type == FieldType::OLAP_FIELD_TYPE_CHAR ||
- type == FieldType::OLAP_FIELD_TYPE_STRING || type == FieldType::OLAP_FIELD_TYPE_STRUCT ||
- type == FieldType::OLAP_FIELD_TYPE_ARRAY || type == FieldType::OLAP_FIELD_TYPE_MAP ||
- type == FieldType::OLAP_FIELD_TYPE_HLL || type == FieldType::OLAP_FIELD_TYPE_OBJECT ||
- type == FieldType::OLAP_FIELD_TYPE_JSONB;
-}
-
-// jsonb -> column value
-static void deserialize_column(PrimitiveType type, JsonbValue* slot_value, MutableColumnPtr& dst) {
- if (type == TYPE_ARRAY) {
- assert(slot_value->isBinary());
- auto blob = static_cast<JsonbBlobVal*>(slot_value);
- dst->deserialize_and_insert_from_arena(blob->getBlob());
- } else if (type == TYPE_OBJECT) {
- assert(slot_value->isBinary());
- auto blob = static_cast<JsonbBlobVal*>(slot_value);
- BitmapValue bitmap_value;
- bitmap_value.deserialize(blob->getBlob());
- dst->insert_data(reinterpret_cast<const char*>(&bitmap_value), sizeof(BitmapValue));
- } else if (type == TYPE_HLL) {
- assert(slot_value->isBinary());
- auto blob = static_cast<JsonbBlobVal*>(slot_value);
- HyperLogLog hyper_log_log;
- Slice data {blob->getBlob(), blob->getBlobLen()};
- hyper_log_log.deserialize(data);
- dst->insert_data(reinterpret_cast<const char*>(&hyper_log_log), sizeof(HyperLogLog));
- } else if (is_string_type(type)) {
- assert(slot_value->isBinary());
- auto blob = static_cast<JsonbBlobVal*>(slot_value);
- dst->insert_data(blob->getBlob(), blob->getBlobLen());
- } else {
- switch (type) {
- case TYPE_BOOLEAN: {
- assert(slot_value->isInt8());
- dst->insert(static_cast<JsonbInt8Val*>(slot_value)->val());
- break;
- }
- case TYPE_TINYINT: {
- assert(slot_value->isInt8());
- dst->insert(static_cast<JsonbInt8Val*>(slot_value)->val());
- break;
- }
- case TYPE_SMALLINT: {
- assert(slot_value->isInt16());
- dst->insert(static_cast<JsonbInt16Val*>(slot_value)->val());
- break;
- }
- case TYPE_INT: {
- assert(slot_value->isInt32());
- dst->insert(static_cast<JsonbInt32Val*>(slot_value)->val());
- break;
- }
- case TYPE_BIGINT: {
- assert(slot_value->isInt64());
- dst->insert(static_cast<JsonbInt64Val*>(slot_value)->val());
- break;
- }
- case TYPE_LARGEINT: {
- assert(slot_value->isInt128());
- dst->insert(static_cast<JsonbInt128Val*>(slot_value)->val());
- break;
- }
- case TYPE_FLOAT: {
- dst->insert(static_cast<JsonbFloatVal*>(slot_value)->val());
- break;
- }
- case TYPE_DOUBLE: {
- dst->insert(static_cast<JsonbDoubleVal*>(slot_value)->val());
- break;
- }
- case TYPE_DATE: {
- assert(slot_value->isInt32());
- int32_t val = static_cast<JsonbInt32Val*>(slot_value)->val();
- dst->insert_many_fix_len_data(reinterpret_cast<const char*>(&val), 1);
- break;
- }
- case TYPE_DATETIME: {
- assert(slot_value->isInt64());
- int64_t val = static_cast<JsonbInt64Val*>(slot_value)->val();
- dst->insert_many_fix_len_data(reinterpret_cast<const char*>(&val), 1);
- break;
- }
- case TYPE_DATEV2: {
- assert(slot_value->isInt32());
- dst->insert(static_cast<JsonbInt32Val*>(slot_value)->val());
- break;
- }
- case TYPE_DATETIMEV2: {
- assert(slot_value->isInt64());
- dst->insert(static_cast<JsonbInt64Val*>(slot_value)->val());
- break;
- }
- case TYPE_DECIMAL32: {
- assert(slot_value->isInt32());
- dst->insert(static_cast<JsonbInt32Val*>(slot_value)->val());
- break;
- }
- case TYPE_DECIMAL64: {
- assert(slot_value->isInt64());
- dst->insert(static_cast<JsonbInt64Val*>(slot_value)->val());
- break;
- }
- case TYPE_DECIMAL128I: {
- assert(slot_value->isInt128());
- dst->insert(static_cast<JsonbInt128Val*>(slot_value)->val());
- break;
- }
- default:
- LOG(FATAL) << "unknow type " << type;
- break;
- }
- }
-}
-
-// column value -> jsonb
-static void serialize_column(Arena* mem_pool, const TabletColumn& tablet_column,
- const IColumn* column, const StringRef& data_ref, int row,
- JsonbWriterT<JsonbOutStream>& jsonb_writer) {
- if (is_column_null_at(row, column, tablet_column.type(), data_ref)) {
- // Do nothing
- return;
- }
- jsonb_writer.writeKey(tablet_column.unique_id());
- if (tablet_column.is_array_type()) {
- const char* begin = nullptr;
- StringRef value = column->serialize_value_into_arena(row, *mem_pool, begin);
- jsonb_writer.writeStartBinary();
- jsonb_writer.writeBinary(value.data, value.size);
- jsonb_writer.writeEndBinary();
- } else if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_OBJECT) {
- auto bitmap_value = (BitmapValue*)(data_ref.data);
- auto size = bitmap_value->getSizeInBytes();
- // serialize the content of string
- auto ptr = mem_pool->alloc(size);
- bitmap_value->write_to(reinterpret_cast<char*>(ptr));
- jsonb_writer.writeStartBinary();
- jsonb_writer.writeBinary(reinterpret_cast<const char*>(ptr), size);
- jsonb_writer.writeEndBinary();
- } else if (tablet_column.type() == FieldType::OLAP_FIELD_TYPE_HLL) {
- auto hll_value = (HyperLogLog*)(data_ref.data);
- auto size = hll_value->max_serialized_size();
- auto ptr = reinterpret_cast<char*>(mem_pool->alloc(size));
- size_t actual_size = hll_value->serialize((uint8_t*)ptr);
- jsonb_writer.writeStartBinary();
- jsonb_writer.writeBinary(reinterpret_cast<const char*>(ptr), actual_size);
- jsonb_writer.writeEndBinary();
- } else if (is_jsonb_blob_type(tablet_column.type())) {
- jsonb_writer.writeStartBinary();
- jsonb_writer.writeBinary(reinterpret_cast<const char*>(data_ref.data), data_ref.size);
- jsonb_writer.writeEndBinary();
- } else {
- switch (tablet_column.type()) {
- case FieldType::OLAP_FIELD_TYPE_BOOL: {
- int8_t val = *reinterpret_cast<const int8_t*>(data_ref.data);
- jsonb_writer.writeInt8(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_TINYINT: {
- int8_t val = *reinterpret_cast<const int8_t*>(data_ref.data);
- jsonb_writer.writeInt8(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_SMALLINT: {
- int16_t val = *reinterpret_cast<const int16_t*>(data_ref.data);
- jsonb_writer.writeInt16(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_INT: {
- int32_t val = *reinterpret_cast<const int32_t*>(data_ref.data);
- jsonb_writer.writeInt32(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_BIGINT: {
- int64_t val = *reinterpret_cast<const int64_t*>(data_ref.data);
- jsonb_writer.writeInt64(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_LARGEINT: {
- __int128_t val = *reinterpret_cast<const __int128_t*>(data_ref.data);
- jsonb_writer.writeInt128(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_FLOAT: {
- float val = *reinterpret_cast<const float*>(data_ref.data);
- jsonb_writer.writeFloat(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DOUBLE: {
- double val = *reinterpret_cast<const double*>(data_ref.data);
- jsonb_writer.writeDouble(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DATE: {
- const auto* datetime_cur = reinterpret_cast<const VecDateTimeValue*>(data_ref.data);
- jsonb_writer.writeInt32(datetime_cur->to_olap_date());
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DATETIME: {
- const auto* datetime_cur = reinterpret_cast<const VecDateTimeValue*>(data_ref.data);
- jsonb_writer.writeInt64(datetime_cur->to_olap_datetime());
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DATEV2: {
- uint32_t val = *reinterpret_cast<const uint32_t*>(data_ref.data);
- jsonb_writer.writeInt32(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DATETIMEV2: {
- uint64_t val = *reinterpret_cast<const uint64_t*>(data_ref.data);
- jsonb_writer.writeInt64(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DECIMAL32: {
- Decimal32::NativeType val =
- *reinterpret_cast<const Decimal32::NativeType*>(data_ref.data);
- jsonb_writer.writeInt32(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DECIMAL64: {
- Decimal64::NativeType val =
- *reinterpret_cast<const Decimal64::NativeType*>(data_ref.data);
- jsonb_writer.writeInt64(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DECIMAL128I: {
- Decimal128I::NativeType val =
- *reinterpret_cast<const Decimal128I::NativeType*>(data_ref.data);
- jsonb_writer.writeInt128(val);
- break;
- }
- case FieldType::OLAP_FIELD_TYPE_DECIMAL:
- LOG(FATAL)
- << "FieldType::OLAP_FIELD_TYPE_DECIMAL not implemented use DecimalV3 instead";
- break;
- default:
- LOG(FATAL) << "unknow type " << int(tablet_column.type());
- break;
- }
- }
-}
-
void JsonbSerializeUtil::block_to_jsonb(const TabletSchema& schema, const Block& block,
ColumnString& dst, int num_cols) {
auto num_rows = block.rows();
@@ -317,9 +64,8 @@ void JsonbSerializeUtil::block_to_jsonb(const TabletSchema& schema, const Block&
// ignore dst row store column
continue;
}
- const auto& data_ref =
- !tablet_column.is_array_type() ? column->get_data_at(i) : StringRef();
- serialize_column(&pool, tablet_column, column.get(), data_ref, i, jsonb_writer);
+ block.get_data_type(j)->get_serde()->write_one_cell_to_jsonb(
+ *column, jsonb_writer, &pool, tablet_column.unique_id(), i);
}
jsonb_writer.writeEndObject();
dst.insert_data(jsonb_writer.getOutput()->getBuffer(), jsonb_writer.getOutput()->getSize());
@@ -344,12 +90,7 @@ void JsonbSerializeUtil::jsonb_to_block(const TupleDescriptor& desc, const char*
SlotDescriptor* slot = desc.slots()[j];
JsonbValue* slot_value = doc->find(slot->col_unique_id());
MutableColumnPtr dst_column = dst.get_by_position(j).column->assume_mutable();
- if (!slot_value || slot_value->isNull()) {
- // null or not exist
- dst_column->insert_default();
- continue;
- }
- deserialize_column(slot->type().type, slot_value, dst_column);
+ dst.get_data_type(j)->get_serde()->read_one_cell_from_jsonb(*dst_column, slot_value);
}
}
diff --git a/be/src/vec/jsonb/serialize.h b/be/src/vec/jsonb/serialize.h
index 0d9d13927d..7cbdc7498f 100644
--- a/be/src/vec/jsonb/serialize.h
+++ b/be/src/vec/jsonb/serialize.h
@@ -18,6 +18,7 @@
#pragma once
#include <stddef.h>
+#include "runtime/descriptors.h"
#include "vec/core/block.h"
namespace doris {
diff --git a/be/test/vec/jsonb/serialize_test.cpp b/be/test/vec/jsonb/serialize_test.cpp
index b39523248b..ccb91472f8 100644
--- a/be/test/vec/jsonb/serialize_test.cpp
+++ b/be/test/vec/jsonb/serialize_test.cpp
@@ -31,8 +31,10 @@
#include "vec/core/types.h"
#include "vec/data_types/data_type.h"
#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
#include "vec/data_types/data_type_date_time.h"
#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_hll.h"
#include "vec/data_types/data_type_nullable.h"
#include "vec/data_types/data_type_number.h"
#include "vec/data_types/data_type_string.h"
@@ -144,6 +146,8 @@ TEST(BlockSerializeTest, JsonbBlock) {
{"k1", FieldType::OLAP_FIELD_TYPE_INT, 1, TYPE_INT},
{"k2", FieldType::OLAP_FIELD_TYPE_STRING, 2, TYPE_STRING},
{"k3", FieldType::OLAP_FIELD_TYPE_DECIMAL128I, 3, TYPE_DECIMAL128I},
+ {"v1", FieldType::OLAP_FIELD_TYPE_OBJECT, 7, TYPE_OBJECT},
+ {"v2", FieldType::OLAP_FIELD_TYPE_HLL, 8, TYPE_HLL},
{"k4", FieldType::OLAP_FIELD_TYPE_STRING, 4, TYPE_STRING},
{"k5", FieldType::OLAP_FIELD_TYPE_DECIMAL128I, 5, TYPE_DECIMAL128I},
{"k6", FieldType::OLAP_FIELD_TYPE_INT, 6, TYPE_INT},
@@ -193,6 +197,39 @@ TEST(BlockSerializeTest, JsonbBlock) {
decimal_data_type, "test_decimal");
block.insert(type_and_name);
}
+ // bitmap
+ {
+ vectorized::DataTypePtr bitmap_data_type(std::make_shared<vectorized::DataTypeBitMap>());
+ auto bitmap_column = bitmap_data_type->create_column();
+ std::vector<BitmapValue>& container =
+ ((vectorized::ColumnComplexType<BitmapValue>*)bitmap_column.get())->get_data();
+ for (int i = 0; i < 1024; ++i) {
+ BitmapValue bv;
+ for (int j = 0; j <= i; ++j) {
+ bv.add(j);
+ }
+ container.push_back(bv);
+ }
+ vectorized::ColumnWithTypeAndName type_and_name(bitmap_column->get_ptr(), bitmap_data_type,
+ "test_bitmap");
+ block.insert(type_and_name);
+ }
+ // hll
+ {
+ vectorized::DataTypePtr hll_data_type(std::make_shared<vectorized::DataTypeHLL>());
+ auto hll_column = hll_data_type->create_column();
+ std::vector<HyperLogLog>& container =
+ ((vectorized::ColumnHLL*)hll_column.get())->get_data();
+ for (int i = 0; i < 1024; ++i) {
+ HyperLogLog hll;
+ hll.update(i);
+ container.push_back(hll);
+ }
+ vectorized::ColumnWithTypeAndName type_and_name(hll_column->get_ptr(), hll_data_type,
+ "test_hll");
+
+ block.insert(type_and_name);
+ }
// nullable string
{
vectorized::DataTypePtr string_data_type(std::make_shared<vectorized::DataTypeString>());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org