You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2022/12/29 14:02:37 UTC
[GitHub] [doris] github-actions[bot] commented on a diff in pull request #15491: Pq opt
github-actions[bot] commented on code in PR #15491:
URL: https://github.com/apache/doris/pull/15491#discussion_r1058970516
##########
be/src/olap/tablet_schema.cpp:
##########
@@ -920,11 +923,25 @@ bool operator==(const TabletSchema& a, const TabletSchema& b) {
if (a._is_in_memory != b._is_in_memory) return false;
if (a._delete_sign_idx != b._delete_sign_idx) return false;
if (a._disable_auto_compaction != b._disable_auto_compaction) return false;
+ if (a._store_row_column != b._store_row_column) return false;
Review Comment:
warning: statement should be inside braces [readability-braces-around-statements]
```suggestion
if (a._store_row_column != b._store_row_column) { return false;
}
```
##########
be/src/service/tablet_lookup_metric.h:
##########
@@ -0,0 +1,183 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+
+#include "butil/containers/doubly_buffered_data.h"
+#include "common/status.h"
+#include "gen_cpp/internal_service.pb.h"
+#include "gutil/int128.h"
+#include "olap/tablet.h"
+#include "util/runtime_profile.h"
+#include "vec/core/block.h"
+
+namespace doris {
+
+// For caching point lookup pre allocted blocks and exprs
+class Reusable {
+public:
+ ~Reusable();
+
+ bool is_expired(int64_t ttl_ms) {
Review Comment:
warning: method 'is_expired' can be made const [readability-make-member-function-const]
```suggestion
bool is_expired(int64_t ttl_ms) const {
```
##########
be/src/util/jsonb_document.h:
##########
@@ -168,28 +171,34 @@
const ObjectVal* operator->() const { return ((const ObjectVal*)payload_); }
public:
- bool operator==(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator==(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator!=(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator!=(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
Review Comment:
warning: function declared 'noreturn' should not return [clang-diagnostic-invalid-noreturn]
```cpp
}
^
```
##########
be/src/vec/sink/vmysql_result_writer.cpp:
##########
@@ -435,7 +421,17 @@ int VMysqlResultWriter::_add_one_cell(const ColumnPtr& column_ptr, size_t row_id
}
}
+<<<<<<< HEAD
Review Comment:
warning: version control conflict marker in file [clang-diagnostic-error]
```cpp
<<<<<<< HEAD
^
```
##########
be/test/vec/jsonb/serialize_test.cpp:
##########
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gtest/gtest.h>
+
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#define private public
+#include "olap/tablet_schema.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/jsonb/serialize.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+void fill_block_with_array_int(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnVector<int32_t>::create();
+ // init column array with [[1,2,3],[],[4],[5,6]]
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 3, 3, 4, 6};
+ std::vector<int32_t> vals = {1, 2, 3, 4, 5, 6};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data((const char*)(&v), 0);
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeInt32>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type,
+ "test_array_int");
+ block.insert(test_array_int);
+}
+
+void fill_block_with_array_string(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnString::create();
+ // init column array with [["abc","de"],["fg"],[], [""]];
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+ std::vector<std::string> vals = {"abc", "de", "fg", ""};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data(v.data(), v.size());
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeString>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,
+ "test_array_string");
+ block.insert(test_array_string);
+}
+
+TEST(BlockSerializeTest, Array) {
+ TabletSchema schema;
+ TabletColumn c1;
+ TabletColumn c2;
+ c1.set_name("k1");
+ c1.set_unique_id(1);
+ c1.set_type(OLAP_FIELD_TYPE_ARRAY);
+ c2.set_name("k2");
+ c2.set_unique_id(2);
Review Comment:
warning: no member named 'set_unique_id' in 'doris::TabletColumn' [clang-diagnostic-error]
```cpp
c2.set_unique_id(2);
^
```
##########
be/test/vec/jsonb/serialize_test.cpp:
##########
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gtest/gtest.h>
+
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#define private public
+#include "olap/tablet_schema.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/jsonb/serialize.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+void fill_block_with_array_int(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnVector<int32_t>::create();
+ // init column array with [[1,2,3],[],[4],[5,6]]
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 3, 3, 4, 6};
+ std::vector<int32_t> vals = {1, 2, 3, 4, 5, 6};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data((const char*)(&v), 0);
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeInt32>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type,
+ "test_array_int");
+ block.insert(test_array_int);
+}
+
+void fill_block_with_array_string(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnString::create();
+ // init column array with [["abc","de"],["fg"],[], [""]];
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+ std::vector<std::string> vals = {"abc", "de", "fg", ""};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data(v.data(), v.size());
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeString>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,
+ "test_array_string");
+ block.insert(test_array_string);
+}
+
+TEST(BlockSerializeTest, Array) {
+ TabletSchema schema;
+ TabletColumn c1;
+ TabletColumn c2;
+ c1.set_name("k1");
+ c1.set_unique_id(1);
+ c1.set_type(OLAP_FIELD_TYPE_ARRAY);
+ c2.set_name("k2");
+ c2.set_unique_id(2);
+ c2.set_type(OLAP_FIELD_TYPE_ARRAY);
+ schema.append_column(c1);
+ schema.append_column(c2);
+ // array int and array string
+ vectorized::Block block;
+ fill_block_with_array_int(block);
+ fill_block_with_array_string(block);
+ MutableColumnPtr col = ColumnString::create();
+ // serialize
+ JsonbSerializeUtil::block_to_jsonb(schema, block, static_cast<ColumnString&>(*col.get()),
+ block.columns());
+ // deserialize
+ TupleDescriptor read_desc(PTupleDescriptor(), true);
+ // slot1
+ PSlotDescriptor pslot1;
+ pslot1.set_col_name("k1");
+ TypeDescriptor type_desc(TYPE_ARRAY);
+ type_desc.children.push_back(TypeDescriptor(TYPE_INT));
+ type_desc.to_protobuf(pslot1.mutable_slot_type());
+ pslot1.set_col_unique_id(1);
+ SlotDescriptor* slot = new SlotDescriptor(pslot1);
+ read_desc.add_slot(slot);
+
+ // slot2
+ PSlotDescriptor pslot2;
+ pslot2.set_col_name("k2");
+ TypeDescriptor type_desc2(TYPE_ARRAY);
+ type_desc2.children.push_back(TypeDescriptor(TYPE_STRING));
+ type_desc2.to_protobuf(pslot2.mutable_slot_type());
+ pslot2.set_col_unique_id(2);
+ SlotDescriptor* slot2 = new SlotDescriptor(pslot2);
+ read_desc.add_slot(slot2);
Review Comment:
warning: 'add_slot' is a private member of 'doris::TupleDescriptor' [clang-diagnostic-error]
```cpp
read_desc.add_slot(slot2);
^
```
**be/src/runtime/descriptors.h:380:** declared private here
```cpp
void add_slot(SlotDescriptor* slot);
^
```
##########
be/test/vec/jsonb/serialize_test.cpp:
##########
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gtest/gtest.h>
+
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#define private public
+#include "olap/tablet_schema.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/jsonb/serialize.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+void fill_block_with_array_int(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnVector<int32_t>::create();
+ // init column array with [[1,2,3],[],[4],[5,6]]
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 3, 3, 4, 6};
+ std::vector<int32_t> vals = {1, 2, 3, 4, 5, 6};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data((const char*)(&v), 0);
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeInt32>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type,
+ "test_array_int");
+ block.insert(test_array_int);
+}
+
+void fill_block_with_array_string(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnString::create();
+ // init column array with [["abc","de"],["fg"],[], [""]];
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+ std::vector<std::string> vals = {"abc", "de", "fg", ""};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data(v.data(), v.size());
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeString>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,
+ "test_array_string");
+ block.insert(test_array_string);
+}
+
+TEST(BlockSerializeTest, Array) {
+ TabletSchema schema;
+ TabletColumn c1;
+ TabletColumn c2;
+ c1.set_name("k1");
+ c1.set_unique_id(1);
+ c1.set_type(OLAP_FIELD_TYPE_ARRAY);
+ c2.set_name("k2");
+ c2.set_unique_id(2);
+ c2.set_type(OLAP_FIELD_TYPE_ARRAY);
+ schema.append_column(c1);
+ schema.append_column(c2);
+ // array int and array string
+ vectorized::Block block;
+ fill_block_with_array_int(block);
+ fill_block_with_array_string(block);
+ MutableColumnPtr col = ColumnString::create();
+ // serialize
+ JsonbSerializeUtil::block_to_jsonb(schema, block, static_cast<ColumnString&>(*col.get()),
+ block.columns());
+ // deserialize
+ TupleDescriptor read_desc(PTupleDescriptor(), true);
+ // slot1
+ PSlotDescriptor pslot1;
Review Comment:
warning: variable has incomplete type 'doris::PSlotDescriptor' [clang-diagnostic-error]
```cpp
PSlotDescriptor pslot1;
^
```
**be/src/runtime/descriptors.h:52:** forward declaration of 'doris::PSlotDescriptor'
```cpp
class PSlotDescriptor;
^
```
##########
be/test/vec/jsonb/serialize_test.cpp:
##########
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gtest/gtest.h>
+
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#define private public
+#include "olap/tablet_schema.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/jsonb/serialize.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+void fill_block_with_array_int(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnVector<int32_t>::create();
+ // init column array with [[1,2,3],[],[4],[5,6]]
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 3, 3, 4, 6};
+ std::vector<int32_t> vals = {1, 2, 3, 4, 5, 6};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data((const char*)(&v), 0);
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeInt32>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type,
+ "test_array_int");
+ block.insert(test_array_int);
+}
+
+void fill_block_with_array_string(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnString::create();
+ // init column array with [["abc","de"],["fg"],[], [""]];
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+ std::vector<std::string> vals = {"abc", "de", "fg", ""};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data(v.data(), v.size());
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeString>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,
+ "test_array_string");
+ block.insert(test_array_string);
+}
+
+TEST(BlockSerializeTest, Array) {
+ TabletSchema schema;
+ TabletColumn c1;
+ TabletColumn c2;
+ c1.set_name("k1");
+ c1.set_unique_id(1);
+ c1.set_type(OLAP_FIELD_TYPE_ARRAY);
+ c2.set_name("k2");
+ c2.set_unique_id(2);
+ c2.set_type(OLAP_FIELD_TYPE_ARRAY);
+ schema.append_column(c1);
+ schema.append_column(c2);
+ // array int and array string
+ vectorized::Block block;
+ fill_block_with_array_int(block);
+ fill_block_with_array_string(block);
+ MutableColumnPtr col = ColumnString::create();
+ // serialize
+ JsonbSerializeUtil::block_to_jsonb(schema, block, static_cast<ColumnString&>(*col.get()),
+ block.columns());
+ // deserialize
+ TupleDescriptor read_desc(PTupleDescriptor(), true);
+ // slot1
+ PSlotDescriptor pslot1;
+ pslot1.set_col_name("k1");
+ TypeDescriptor type_desc(TYPE_ARRAY);
+ type_desc.children.push_back(TypeDescriptor(TYPE_INT));
+ type_desc.to_protobuf(pslot1.mutable_slot_type());
+ pslot1.set_col_unique_id(1);
+ SlotDescriptor* slot = new SlotDescriptor(pslot1);
+ read_desc.add_slot(slot);
+
+ // slot2
+ PSlotDescriptor pslot2;
+ pslot2.set_col_name("k2");
+ TypeDescriptor type_desc2(TYPE_ARRAY);
+ type_desc2.children.push_back(TypeDescriptor(TYPE_STRING));
+ type_desc2.to_protobuf(pslot2.mutable_slot_type());
+ pslot2.set_col_unique_id(2);
+ SlotDescriptor* slot2 = new SlotDescriptor(pslot2);
+ read_desc.add_slot(slot2);
+
+ Block new_block = block.clone_empty();
+ JsonbSerializeUtil::jsonb_to_block(read_desc, static_cast<ColumnString&>(*col.get()),
+ new_block);
+ std::cout << block.dump_data() << std::endl;
+ std::cout << new_block.dump_data() << std::endl;
+ EXPECT_EQ(block.dump_data(), new_block.dump_data());
+}
+
+TEST(BlockSerializeTest, JsonbBlock) {
+ vectorized::Block block;
+ TabletSchema schema;
+ std::vector<std::tuple<std::string, FieldType, int, PrimitiveType>> cols {
+ {"k1", OLAP_FIELD_TYPE_INT, 1, TYPE_INT},
+ {"k2", OLAP_FIELD_TYPE_STRING, 2, TYPE_STRING},
+ {"k3", OLAP_FIELD_TYPE_DECIMAL128, 3, TYPE_DECIMAL128},
+ {"k4", OLAP_FIELD_TYPE_STRING, 4, TYPE_STRING},
+ {"k5", OLAP_FIELD_TYPE_DECIMAL128, 5, TYPE_DECIMAL128},
Review Comment:
warning: use of undeclared identifier 'OLAP_FIELD_TYPE_DECIMAL128'; did you mean 'OLAP_FIELD_TYPE_DECIMAL128I'? [clang-diagnostic-error]
```suggestion
{"k5", OLAP_FIELD_TYPE_DECIMAL128I, 5, TYPE_DECIMAL128},
```
**be/src/olap/olap_common.h:153:** 'OLAP_FIELD_TYPE_DECIMAL128I' declared here
```cpp
OLAP_FIELD_TYPE_DECIMAL128I = 33,
^
```
##########
be/src/vec/sink/vmysql_result_writer.h:
##########
@@ -30,8 +30,11 @@ class TFetchDataResult;
namespace vectorized {
class VExprContext;
+template <bool is_binary_format = false>
class VMysqlResultWriter final : public VResultWriter {
public:
+ typedef std::vector<std::unique_ptr<TFetchDataResult>> ResultList;
Review Comment:
warning: use 'using' instead of 'typedef' [modernize-use-using]
```suggestion
using ResultList = std::vector<std::unique_ptr<TFetchDataResult>>;
```
##########
be/src/util/jsonb_document.h:
##########
@@ -168,28 +171,34 @@
const ObjectVal* operator->() const { return ((const ObjectVal*)payload_); }
public:
- bool operator==(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator==(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator!=(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator!=(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator<=(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator<=(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator>=(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator>=(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
Review Comment:
warning: function declared 'noreturn' should not return [clang-diagnostic-invalid-noreturn]
```cpp
}
^
```
##########
be/src/util/jsonb_document.h:
##########
@@ -168,28 +171,34 @@ class JsonbDocument {
const ObjectVal* operator->() const { return ((const ObjectVal*)payload_); }
public:
- bool operator==(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator==(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
Review Comment:
warning: function declared 'noreturn' should not return [clang-diagnostic-invalid-noreturn]
```cpp
}
^
```
##########
be/src/vec/sink/vmysql_result_writer.cpp:
##########
@@ -435,7 +421,17 @@
}
}
+<<<<<<< HEAD
Status VMysqlResultWriter::append_block(Block& input_block) {
Review Comment:
warning: 'VMysqlResultWriter' is not a class, namespace, or enumeration [clang-diagnostic-error]
```cpp
Status VMysqlResultWriter::append_block(Block& input_block) {
^
```
**be/src/vec/sink/vmysql_result_writer.h:33:** 'VMysqlResultWriter' declared here
```cpp
class VMysqlResultWriter final : public VResultWriter {
^
```
##########
be/src/util/jsonb_document.h:
##########
@@ -168,28 +171,34 @@
const ObjectVal* operator->() const { return ((const ObjectVal*)payload_); }
public:
- bool operator==(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator==(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator!=(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator!=(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator<=(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator<=(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
Review Comment:
warning: function declared 'noreturn' should not return [clang-diagnostic-invalid-noreturn]
```cpp
}
^
```
##########
be/test/vec/jsonb/serialize_test.cpp:
##########
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gtest/gtest.h>
+
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#define private public
+#include "olap/tablet_schema.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/jsonb/serialize.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+void fill_block_with_array_int(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnVector<int32_t>::create();
+ // init column array with [[1,2,3],[],[4],[5,6]]
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 3, 3, 4, 6};
+ std::vector<int32_t> vals = {1, 2, 3, 4, 5, 6};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data((const char*)(&v), 0);
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeInt32>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type,
+ "test_array_int");
+ block.insert(test_array_int);
+}
+
+void fill_block_with_array_string(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnString::create();
+ // init column array with [["abc","de"],["fg"],[], [""]];
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+ std::vector<std::string> vals = {"abc", "de", "fg", ""};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data(v.data(), v.size());
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeString>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,
+ "test_array_string");
+ block.insert(test_array_string);
+}
+
+TEST(BlockSerializeTest, Array) {
+ TabletSchema schema;
+ TabletColumn c1;
+ TabletColumn c2;
+ c1.set_name("k1");
+ c1.set_unique_id(1);
Review Comment:
warning: no member named 'set_unique_id' in 'doris::TabletColumn' [clang-diagnostic-error]
```cpp
c1.set_unique_id(1);
^
```
##########
be/src/util/jsonb_document.h:
##########
@@ -168,28 +171,34 @@
const ObjectVal* operator->() const { return ((const ObjectVal*)payload_); }
public:
- bool operator==(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator==(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator!=(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator!=(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator<=(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator<=(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator>=(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator>=(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator<(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator<(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
Review Comment:
warning: function declared 'noreturn' should not return [clang-diagnostic-invalid-noreturn]
```cpp
}
^
```
##########
be/test/vec/jsonb/serialize_test.cpp:
##########
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gtest/gtest.h>
+
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#define private public
+#include "olap/tablet_schema.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/jsonb/serialize.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+void fill_block_with_array_int(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnVector<int32_t>::create();
+ // init column array with [[1,2,3],[],[4],[5,6]]
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 3, 3, 4, 6};
+ std::vector<int32_t> vals = {1, 2, 3, 4, 5, 6};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data((const char*)(&v), 0);
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeInt32>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type,
+ "test_array_int");
+ block.insert(test_array_int);
+}
+
+void fill_block_with_array_string(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnString::create();
+ // init column array with [["abc","de"],["fg"],[], [""]];
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+ std::vector<std::string> vals = {"abc", "de", "fg", ""};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data(v.data(), v.size());
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeString>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,
+ "test_array_string");
+ block.insert(test_array_string);
+}
+
+TEST(BlockSerializeTest, Array) {
+ TabletSchema schema;
+ TabletColumn c1;
+ TabletColumn c2;
+ c1.set_name("k1");
+ c1.set_unique_id(1);
+ c1.set_type(OLAP_FIELD_TYPE_ARRAY);
Review Comment:
warning: no member named 'set_type' in 'doris::TabletColumn' [clang-diagnostic-error]
```cpp
c1.set_type(OLAP_FIELD_TYPE_ARRAY);
^
```
##########
be/src/util/jsonb_document.h:
##########
@@ -168,28 +171,34 @@
const ObjectVal* operator->() const { return ((const ObjectVal*)payload_); }
public:
- bool operator==(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator==(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator!=(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator!=(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator<=(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator<=(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator>=(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator>=(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator<(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator<(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
- bool operator>(const JsonbDocument& other) const {
- LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ [[noreturn]] bool operator>(const JsonbDocument& other) const {
+ // LOG(FATAL) << "comparing between JsonbDocument is not supported";
+ assert(false);
}
Review Comment:
warning: function declared 'noreturn' should not return [clang-diagnostic-invalid-noreturn]
```cpp
}
^
```
##########
be/test/vec/jsonb/serialize_test.cpp:
##########
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gtest/gtest.h>
+
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#define private public
+#include "olap/tablet_schema.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/jsonb/serialize.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+void fill_block_with_array_int(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnVector<int32_t>::create();
+ // init column array with [[1,2,3],[],[4],[5,6]]
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 3, 3, 4, 6};
+ std::vector<int32_t> vals = {1, 2, 3, 4, 5, 6};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data((const char*)(&v), 0);
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeInt32>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type,
+ "test_array_int");
+ block.insert(test_array_int);
+}
+
+void fill_block_with_array_string(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnString::create();
+ // init column array with [["abc","de"],["fg"],[], [""]];
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+ std::vector<std::string> vals = {"abc", "de", "fg", ""};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data(v.data(), v.size());
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeString>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,
+ "test_array_string");
+ block.insert(test_array_string);
+}
+
+TEST(BlockSerializeTest, Array) {
+ TabletSchema schema;
+ TabletColumn c1;
+ TabletColumn c2;
+ c1.set_name("k1");
+ c1.set_unique_id(1);
+ c1.set_type(OLAP_FIELD_TYPE_ARRAY);
+ c2.set_name("k2");
+ c2.set_unique_id(2);
+ c2.set_type(OLAP_FIELD_TYPE_ARRAY);
+ schema.append_column(c1);
+ schema.append_column(c2);
+ // array int and array string
+ vectorized::Block block;
+ fill_block_with_array_int(block);
+ fill_block_with_array_string(block);
+ MutableColumnPtr col = ColumnString::create();
+ // serialize
+ JsonbSerializeUtil::block_to_jsonb(schema, block, static_cast<ColumnString&>(*col.get()),
+ block.columns());
+ // deserialize
+ TupleDescriptor read_desc(PTupleDescriptor(), true);
Review Comment:
warning: invalid use of incomplete type 'doris::PTupleDescriptor' [clang-diagnostic-error]
```cpp
TupleDescriptor read_desc(PTupleDescriptor(), true);
^
```
**be/src/runtime/descriptors.h:51:** forward declaration of 'doris::PTupleDescriptor'
```cpp
class PTupleDescriptor;
^
```
##########
be/test/vec/jsonb/serialize_test.cpp:
##########
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gtest/gtest.h>
+
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#define private public
+#include "olap/tablet_schema.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/jsonb/serialize.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+void fill_block_with_array_int(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnVector<int32_t>::create();
+ // init column array with [[1,2,3],[],[4],[5,6]]
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 3, 3, 4, 6};
+ std::vector<int32_t> vals = {1, 2, 3, 4, 5, 6};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data((const char*)(&v), 0);
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeInt32>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type,
+ "test_array_int");
+ block.insert(test_array_int);
+}
+
+void fill_block_with_array_string(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnString::create();
+ // init column array with [["abc","de"],["fg"],[], [""]];
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+ std::vector<std::string> vals = {"abc", "de", "fg", ""};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data(v.data(), v.size());
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeString>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,
+ "test_array_string");
+ block.insert(test_array_string);
+}
+
+TEST(BlockSerializeTest, Array) {
+ TabletSchema schema;
+ TabletColumn c1;
+ TabletColumn c2;
+ c1.set_name("k1");
+ c1.set_unique_id(1);
+ c1.set_type(OLAP_FIELD_TYPE_ARRAY);
+ c2.set_name("k2");
+ c2.set_unique_id(2);
+ c2.set_type(OLAP_FIELD_TYPE_ARRAY);
+ schema.append_column(c1);
+ schema.append_column(c2);
+ // array int and array string
+ vectorized::Block block;
+ fill_block_with_array_int(block);
+ fill_block_with_array_string(block);
+ MutableColumnPtr col = ColumnString::create();
+ // serialize
+ JsonbSerializeUtil::block_to_jsonb(schema, block, static_cast<ColumnString&>(*col.get()),
+ block.columns());
+ // deserialize
+ TupleDescriptor read_desc(PTupleDescriptor(), true);
+ // slot1
+ PSlotDescriptor pslot1;
+ pslot1.set_col_name("k1");
+ TypeDescriptor type_desc(TYPE_ARRAY);
+ type_desc.children.push_back(TypeDescriptor(TYPE_INT));
+ type_desc.to_protobuf(pslot1.mutable_slot_type());
+ pslot1.set_col_unique_id(1);
+ SlotDescriptor* slot = new SlotDescriptor(pslot1);
+ read_desc.add_slot(slot);
Review Comment:
warning: 'add_slot' is a private member of 'doris::TupleDescriptor' [clang-diagnostic-error]
```cpp
read_desc.add_slot(slot);
^
```
**be/src/runtime/descriptors.h:380:** declared private here
```cpp
void add_slot(SlotDescriptor* slot);
^
```
##########
be/test/vec/jsonb/serialize_test.cpp:
##########
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gtest/gtest.h>
+
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#define private public
+#include "olap/tablet_schema.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/jsonb/serialize.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+void fill_block_with_array_int(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnVector<int32_t>::create();
+ // init column array with [[1,2,3],[],[4],[5,6]]
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 3, 3, 4, 6};
+ std::vector<int32_t> vals = {1, 2, 3, 4, 5, 6};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data((const char*)(&v), 0);
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeInt32>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type,
+ "test_array_int");
+ block.insert(test_array_int);
+}
+
+void fill_block_with_array_string(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnString::create();
+ // init column array with [["abc","de"],["fg"],[], [""]];
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+ std::vector<std::string> vals = {"abc", "de", "fg", ""};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data(v.data(), v.size());
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeString>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,
+ "test_array_string");
+ block.insert(test_array_string);
+}
+
+TEST(BlockSerializeTest, Array) {
+ TabletSchema schema;
+ TabletColumn c1;
+ TabletColumn c2;
+ c1.set_name("k1");
+ c1.set_unique_id(1);
+ c1.set_type(OLAP_FIELD_TYPE_ARRAY);
+ c2.set_name("k2");
+ c2.set_unique_id(2);
+ c2.set_type(OLAP_FIELD_TYPE_ARRAY);
+ schema.append_column(c1);
+ schema.append_column(c2);
+ // array int and array string
+ vectorized::Block block;
+ fill_block_with_array_int(block);
+ fill_block_with_array_string(block);
+ MutableColumnPtr col = ColumnString::create();
+ // serialize
+ JsonbSerializeUtil::block_to_jsonb(schema, block, static_cast<ColumnString&>(*col.get()),
+ block.columns());
+ // deserialize
+ TupleDescriptor read_desc(PTupleDescriptor(), true);
Review Comment:
warning: calling a private constructor of class 'doris::TupleDescriptor' [clang-diagnostic-error]
```cpp
TupleDescriptor read_desc(PTupleDescriptor(), true);
^
```
**be/src/runtime/descriptors.h:378:** declared private here
```cpp
TupleDescriptor(const PTupleDescriptor& tdesc, bool own_slot = false);
^
```
##########
be/test/vec/jsonb/serialize_test.cpp:
##########
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gtest/gtest.h>
+
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#define private public
+#include "olap/tablet_schema.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/jsonb/serialize.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+void fill_block_with_array_int(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnVector<int32_t>::create();
+ // init column array with [[1,2,3],[],[4],[5,6]]
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 3, 3, 4, 6};
+ std::vector<int32_t> vals = {1, 2, 3, 4, 5, 6};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data((const char*)(&v), 0);
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeInt32>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type,
+ "test_array_int");
+ block.insert(test_array_int);
+}
+
+void fill_block_with_array_string(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnString::create();
+ // init column array with [["abc","de"],["fg"],[], [""]];
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+ std::vector<std::string> vals = {"abc", "de", "fg", ""};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data(v.data(), v.size());
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeString>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,
+ "test_array_string");
+ block.insert(test_array_string);
+}
+
+TEST(BlockSerializeTest, Array) {
+ TabletSchema schema;
+ TabletColumn c1;
+ TabletColumn c2;
+ c1.set_name("k1");
+ c1.set_unique_id(1);
+ c1.set_type(OLAP_FIELD_TYPE_ARRAY);
+ c2.set_name("k2");
+ c2.set_unique_id(2);
+ c2.set_type(OLAP_FIELD_TYPE_ARRAY);
Review Comment:
warning: no member named 'set_type' in 'doris::TabletColumn' [clang-diagnostic-error]
```cpp
c2.set_type(OLAP_FIELD_TYPE_ARRAY);
^
```
##########
be/test/vec/jsonb/serialize_test.cpp:
##########
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gtest/gtest.h>
+
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#define private public
+#include "olap/tablet_schema.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/jsonb/serialize.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+void fill_block_with_array_int(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnVector<int32_t>::create();
+ // init column array with [[1,2,3],[],[4],[5,6]]
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 3, 3, 4, 6};
+ std::vector<int32_t> vals = {1, 2, 3, 4, 5, 6};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data((const char*)(&v), 0);
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeInt32>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type,
+ "test_array_int");
+ block.insert(test_array_int);
+}
+
+void fill_block_with_array_string(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnString::create();
+ // init column array with [["abc","de"],["fg"],[], [""]];
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+ std::vector<std::string> vals = {"abc", "de", "fg", ""};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data(v.data(), v.size());
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeString>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,
+ "test_array_string");
+ block.insert(test_array_string);
+}
+
+TEST(BlockSerializeTest, Array) {
+ TabletSchema schema;
+ TabletColumn c1;
+ TabletColumn c2;
+ c1.set_name("k1");
+ c1.set_unique_id(1);
+ c1.set_type(OLAP_FIELD_TYPE_ARRAY);
+ c2.set_name("k2");
+ c2.set_unique_id(2);
+ c2.set_type(OLAP_FIELD_TYPE_ARRAY);
+ schema.append_column(c1);
+ schema.append_column(c2);
+ // array int and array string
+ vectorized::Block block;
+ fill_block_with_array_int(block);
+ fill_block_with_array_string(block);
+ MutableColumnPtr col = ColumnString::create();
+ // serialize
+ JsonbSerializeUtil::block_to_jsonb(schema, block, static_cast<ColumnString&>(*col.get()),
+ block.columns());
+ // deserialize
+ TupleDescriptor read_desc(PTupleDescriptor(), true);
+ // slot1
+ PSlotDescriptor pslot1;
+ pslot1.set_col_name("k1");
+ TypeDescriptor type_desc(TYPE_ARRAY);
+ type_desc.children.push_back(TypeDescriptor(TYPE_INT));
+ type_desc.to_protobuf(pslot1.mutable_slot_type());
+ pslot1.set_col_unique_id(1);
+ SlotDescriptor* slot = new SlotDescriptor(pslot1);
+ read_desc.add_slot(slot);
+
+ // slot2
+ PSlotDescriptor pslot2;
+ pslot2.set_col_name("k2");
+ TypeDescriptor type_desc2(TYPE_ARRAY);
+ type_desc2.children.push_back(TypeDescriptor(TYPE_STRING));
+ type_desc2.to_protobuf(pslot2.mutable_slot_type());
+ pslot2.set_col_unique_id(2);
+ SlotDescriptor* slot2 = new SlotDescriptor(pslot2);
+ read_desc.add_slot(slot2);
+
+ Block new_block = block.clone_empty();
+ JsonbSerializeUtil::jsonb_to_block(read_desc, static_cast<ColumnString&>(*col.get()),
+ new_block);
+ std::cout << block.dump_data() << std::endl;
+ std::cout << new_block.dump_data() << std::endl;
+ EXPECT_EQ(block.dump_data(), new_block.dump_data());
+}
+
+TEST(BlockSerializeTest, JsonbBlock) {
+ vectorized::Block block;
+ TabletSchema schema;
+ std::vector<std::tuple<std::string, FieldType, int, PrimitiveType>> cols {
+ {"k1", OLAP_FIELD_TYPE_INT, 1, TYPE_INT},
+ {"k2", OLAP_FIELD_TYPE_STRING, 2, TYPE_STRING},
+ {"k3", OLAP_FIELD_TYPE_DECIMAL128, 3, TYPE_DECIMAL128},
Review Comment:
warning: use of undeclared identifier 'OLAP_FIELD_TYPE_DECIMAL128'; did you mean 'OLAP_FIELD_TYPE_DECIMAL128I'? [clang-diagnostic-error]
```suggestion
{"k3", OLAP_FIELD_TYPE_DECIMAL128I, 3, TYPE_DECIMAL128},
```
**be/src/olap/olap_common.h:153:** 'OLAP_FIELD_TYPE_DECIMAL128I' declared here
```cpp
OLAP_FIELD_TYPE_DECIMAL128I = 33,
^
```
##########
be/test/vec/jsonb/serialize_test.cpp:
##########
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gtest/gtest.h>
+
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#define private public
+#include "olap/tablet_schema.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/jsonb/serialize.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+void fill_block_with_array_int(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnVector<int32_t>::create();
+ // init column array with [[1,2,3],[],[4],[5,6]]
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 3, 3, 4, 6};
+ std::vector<int32_t> vals = {1, 2, 3, 4, 5, 6};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data((const char*)(&v), 0);
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeInt32>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type,
+ "test_array_int");
+ block.insert(test_array_int);
+}
+
+void fill_block_with_array_string(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnString::create();
+ // init column array with [["abc","de"],["fg"],[], [""]];
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+ std::vector<std::string> vals = {"abc", "de", "fg", ""};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data(v.data(), v.size());
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeString>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,
+ "test_array_string");
+ block.insert(test_array_string);
+}
+
+TEST(BlockSerializeTest, Array) {
+ TabletSchema schema;
+ TabletColumn c1;
+ TabletColumn c2;
+ c1.set_name("k1");
+ c1.set_unique_id(1);
+ c1.set_type(OLAP_FIELD_TYPE_ARRAY);
+ c2.set_name("k2");
+ c2.set_unique_id(2);
+ c2.set_type(OLAP_FIELD_TYPE_ARRAY);
+ schema.append_column(c1);
+ schema.append_column(c2);
+ // array int and array string
+ vectorized::Block block;
+ fill_block_with_array_int(block);
+ fill_block_with_array_string(block);
+ MutableColumnPtr col = ColumnString::create();
+ // serialize
+ JsonbSerializeUtil::block_to_jsonb(schema, block, static_cast<ColumnString&>(*col.get()),
+ block.columns());
+ // deserialize
+ TupleDescriptor read_desc(PTupleDescriptor(), true);
+ // slot1
+ PSlotDescriptor pslot1;
+ pslot1.set_col_name("k1");
+ TypeDescriptor type_desc(TYPE_ARRAY);
+ type_desc.children.push_back(TypeDescriptor(TYPE_INT));
+ type_desc.to_protobuf(pslot1.mutable_slot_type());
+ pslot1.set_col_unique_id(1);
+ SlotDescriptor* slot = new SlotDescriptor(pslot1);
+ read_desc.add_slot(slot);
+
+ // slot2
+ PSlotDescriptor pslot2;
Review Comment:
warning: variable has incomplete type 'doris::PSlotDescriptor' [clang-diagnostic-error]
```cpp
PSlotDescriptor pslot2;
^
```
**be/src/runtime/descriptors.h:52:** forward declaration of 'doris::PSlotDescriptor'
```cpp
class PSlotDescriptor;
^
```
##########
be/test/vec/jsonb/serialize_test.cpp:
##########
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#include <gtest/gtest.h>
+
+#include "vec/core/block.h"
+#include "vec/core/types.h"
+#define private public
+#include "olap/tablet_schema.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_decimal.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_string.h"
+#include "vec/columns/column_vector.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_bitmap.h"
+#include "vec/data_types/data_type_date.h"
+#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_nullable.h"
+#include "vec/data_types/data_type_number.h"
+#include "vec/data_types/data_type_string.h"
+#include "vec/jsonb/serialize.h"
+#include "vec/runtime/vdatetime_value.h"
+
+namespace doris::vectorized {
+
+void fill_block_with_array_int(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnVector<int32_t>::create();
+ // init column array with [[1,2,3],[],[4],[5,6]]
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 3, 3, 4, 6};
+ std::vector<int32_t> vals = {1, 2, 3, 4, 5, 6};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data((const char*)(&v), 0);
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeInt32>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_int(std::move(column_array_ptr), array_type,
+ "test_array_int");
+ block.insert(test_array_int);
+}
+
+void fill_block_with_array_string(vectorized::Block& block) {
+ auto off_column = vectorized::ColumnVector<vectorized::ColumnArray::Offset64>::create();
+ auto data_column = vectorized::ColumnString::create();
+ // init column array with [["abc","de"],["fg"],[], [""]];
+ std::vector<vectorized::ColumnArray::Offset64> offs = {0, 2, 3, 3, 4};
+ std::vector<std::string> vals = {"abc", "de", "fg", ""};
+ for (size_t i = 1; i < offs.size(); ++i) {
+ off_column->insert_data((const char*)(&offs[i]), 0);
+ }
+ for (auto& v : vals) {
+ data_column->insert_data(v.data(), v.size());
+ }
+
+ auto column_array_ptr =
+ vectorized::ColumnArray::create(std::move(data_column), std::move(off_column));
+ vectorized::DataTypePtr nested_type(std::make_shared<vectorized::DataTypeString>());
+ vectorized::DataTypePtr array_type(std::make_shared<vectorized::DataTypeArray>(nested_type));
+ vectorized::ColumnWithTypeAndName test_array_string(std::move(column_array_ptr), array_type,
+ "test_array_string");
+ block.insert(test_array_string);
+}
+
+TEST(BlockSerializeTest, Array) {
+ TabletSchema schema;
+ TabletColumn c1;
+ TabletColumn c2;
+ c1.set_name("k1");
+ c1.set_unique_id(1);
+ c1.set_type(OLAP_FIELD_TYPE_ARRAY);
+ c2.set_name("k2");
+ c2.set_unique_id(2);
+ c2.set_type(OLAP_FIELD_TYPE_ARRAY);
+ schema.append_column(c1);
+ schema.append_column(c2);
+ // array int and array string
+ vectorized::Block block;
+ fill_block_with_array_int(block);
+ fill_block_with_array_string(block);
+ MutableColumnPtr col = ColumnString::create();
+ // serialize
+ JsonbSerializeUtil::block_to_jsonb(schema, block, static_cast<ColumnString&>(*col.get()),
+ block.columns());
+ // deserialize
+ TupleDescriptor read_desc(PTupleDescriptor(), true);
+ // slot1
+ PSlotDescriptor pslot1;
+ pslot1.set_col_name("k1");
+ TypeDescriptor type_desc(TYPE_ARRAY);
+ type_desc.children.push_back(TypeDescriptor(TYPE_INT));
+ type_desc.to_protobuf(pslot1.mutable_slot_type());
+ pslot1.set_col_unique_id(1);
+ SlotDescriptor* slot = new SlotDescriptor(pslot1);
+ read_desc.add_slot(slot);
+
+ // slot2
+ PSlotDescriptor pslot2;
+ pslot2.set_col_name("k2");
+ TypeDescriptor type_desc2(TYPE_ARRAY);
+ type_desc2.children.push_back(TypeDescriptor(TYPE_STRING));
+ type_desc2.to_protobuf(pslot2.mutable_slot_type());
+ pslot2.set_col_unique_id(2);
+ SlotDescriptor* slot2 = new SlotDescriptor(pslot2);
+ read_desc.add_slot(slot2);
+
+ Block new_block = block.clone_empty();
+ JsonbSerializeUtil::jsonb_to_block(read_desc, static_cast<ColumnString&>(*col.get()),
+ new_block);
+ std::cout << block.dump_data() << std::endl;
+ std::cout << new_block.dump_data() << std::endl;
+ EXPECT_EQ(block.dump_data(), new_block.dump_data());
+}
+
+TEST(BlockSerializeTest, JsonbBlock) {
+ vectorized::Block block;
+ TabletSchema schema;
+ std::vector<std::tuple<std::string, FieldType, int, PrimitiveType>> cols {
+ {"k1", OLAP_FIELD_TYPE_INT, 1, TYPE_INT},
+ {"k2", OLAP_FIELD_TYPE_STRING, 2, TYPE_STRING},
+ {"k3", OLAP_FIELD_TYPE_DECIMAL128, 3, TYPE_DECIMAL128},
Review Comment:
warning: use of undeclared identifier 'TYPE_DECIMAL128'; did you mean 'TYPE_DECIMAL128I'? [clang-diagnostic-error]
```suggestion
{"k3", OLAP_FIELD_TYPE_DECIMAL128, 3, TYPE_DECIMAL128I},
```
**be/src/runtime/define_primitive_type.h:54:** 'TYPE_DECIMAL128I' declared here
```cpp
TYPE_DECIMAL128I, /* 30 */
^
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org