You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2023/01/16 07:51:36 UTC
[GitHub] [doris] amorynan opened a new pull request, #15966: feature-map
amorynan opened a new pull request, #15966:
URL: https://github.com/apache/doris/pull/15966
# Proposed changes
TO support doris complex type for map
Issue Number: close #xxx
## Problem summary
Describe your changes.
## Checklist(Required)
1. Does it affect the original behavior:
- [ ] Yes
- [ x ] No
- [ ] I don't know
2. Has unit tests been added:
- [ ] Yes
- [ ] No
- [ ] No Need
3. Has document been added or modified:
- [ ] Yes
- [ ] No
- [ x ] No Need
4. Does it need to update dependencies:
- [ ] Yes
- [ x ] No
5. Are there any changes that cannot be rolled back:
- [ ] Yes (If Yes, please explain WHY)
- [ ] No
## Further comments
If this is a relatively large or complex change, kick off the discussion at [dev@doris.apache.org](mailto:dev@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1090112475
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java:
##########
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+import org.apache.doris.thrift.TTypeDesc;
+import org.apache.doris.thrift.TTypeNode;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+// INSERT INTO table_map VALUES ({'key1':1, 'key2':10, 'k3':100}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30});
+// MapLiteral is one row-based literal
+public class MapLiteral extends LiteralExpr {
+
+ public MapLiteral() {
+ type = new MapType(Type.NULL, Type.NULL);
+ children = new ArrayList<>();
+ }
+
+ public MapLiteral(LiteralExpr... exprs) throws AnalysisException {
+ Type keyType = Type.NULL;
+ Type valueType = Type.NULL;
+ children = new ArrayList<>();
+ int idx = 0;
+ for (LiteralExpr expr : exprs) {
+ if (idx % 2 == 0) {
+ if (keyType == Type.NULL) {
+ keyType = expr.getType();
+ } else {
+ keyType = Type.getAssignmentCompatibleType(keyType, expr.getType(), false);
+ }
+ if (keyType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ } else {
+ if (valueType == Type.NULL) {
+ valueType = expr.getType();
+ } else {
+ valueType = Type.getAssignmentCompatibleType(valueType, expr.getType(), false);
+ }
+ if (valueType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ }
+ children.add(expr);
+ ++ idx;
+ }
+
+ type = new MapType(keyType, valueType);
+ }
+
+ protected MapLiteral(MapLiteral other) {
+ super(other);
+ }
+
+ @Override
+ public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
+ if (!targetType.isMapType()) {
+ return super.uncheckedCastTo(targetType);
+ }
+ MapLiteral literal = new MapLiteral(this);
+ Type keyType = ((MapType) targetType).getKeyType();
+ Type valueType = ((MapType) targetType).getValueType();
+
+ for (int i = 0; i < children.size(); ++ i) {
+ Expr child = children.get(i);
+ if ((i & 1) == 0) {
+ literal.children.set(i, child.uncheckedCastTo(keyType));
+ } else {
+ literal.children.set(i, child.uncheckedCastTo(valueType));
+ }
+ }
+ literal.setType(targetType);
+ return literal;
+ }
+
+ @Override
+ public void checkValueValid() throws AnalysisException {
+ for (Expr e : children) {
+ e.checkValueValid();
+ }
+ }
+
+ @Override
+ protected String toSqlImpl() {
+ List<String> list = new ArrayList<>(children.size());
+ for (int i = 0; i < children.size(); i += 2) {
+ list.add(children.get(i).toSqlImpl() + ":" + children.get(i + 1).toSqlImpl());
+ }
+ return "MAP{" + StringUtils.join(list, ", ") + "}";
+ }
+
+ @Override
+ protected void toThrift(TExprNode msg) {
+ msg.node_type = TExprNodeType.MAP_LITERAL;
+ TTypeDesc container = new TTypeDesc();
+ container.setTypes(new ArrayList<TTypeNode>());
+ type.toThrift(container);
+ msg.setType(container);
+ }
+
+ @Override
+ public Expr clone() {
+ return new MapLiteral(this);
+ }
+
+ @Override
+ public boolean isMinValue() {
+ return false;
+ }
+
+ @Override
+ public int compareLiteral(LiteralExpr expr) {
+ return 0;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ int size = in.readInt();
+ children = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ children.add(Expr.readIn(in));
+ }
+ }
+
+ public static MapLiteral read(DataInput in) throws IOException {
+ MapLiteral literal = new MapLiteral();
+ literal.readFields(in);
+ return literal;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ super.write(out);
+ out.writeInt(children.size());
+ for (Expr e : children) {
+ Expr.writeTo(e, out);
+ }
+ }
+
+ @Override
+ public String getStringValue() {
Review Comment:
done!
##########
fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlColType.java:
##########
@@ -52,7 +52,8 @@ public enum MysqlColType {
MYSQL_TYPE_BLOB(252, "BLOB"),
MYSQL_TYPE_VARSTRING(253, "VAR STRING"),
MYSQL_TYPE_STRING(254, "STRING"),
- MYSQL_TYPE_GEOMETRY(255, "GEOMETRY");
+ MYSQL_TYPE_GEOMETRY(255, "GEOMETRY"),
+ MYSQL_TYPE_MAP(256, "MAP");
Review Comment:
set 400 , done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089875811
##########
be/src/olap/types.h:
##########
@@ -430,6 +431,103 @@ class ArrayTypeInfo : public TypeInfo {
TypeInfoPtr _item_type_info;
const size_t _item_size;
};
+///====================== MapType Info ==========================///
+class MapTypeInfo : public TypeInfo {
+public:
+ explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info)
+ : _key_type_info(std::move(key_type_info)), _value_type_info(std::move(value_type_info)) {}
+ ~MapTypeInfo() override = default;
+
+ inline bool equal(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ return l_value->size() == r_value->size();
+ }
+
+ int cmp(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ uint32_t l_size = l_value->size();
+ uint32_t r_size = r_value->size();
+ if (l_size < r_size) {
+ return -1;
+ } else if (l_size > r_size) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ void shallow_copy(void* dest, const void* src) const override {
+ auto dest_value = reinterpret_cast<MapValue*>(dest);
+ auto src_value = reinterpret_cast<const MapValue*>(src);
+ dest_value->shallow_copy(src_value);
+ }
+
+ void deep_copy(void* dest, const void* src, MemPool* mem_pool) const override {
+ DCHECK(false);
+
+ }
+
+ void copy_object(void* dest, const void* src, MemPool* mem_pool) const override {
+ deep_copy(dest, src, mem_pool);
+ }
+
+ void direct_copy(void* dest, const void* src) const override {
+ CHECK(false);
+ }
+
+ void direct_copy(uint8_t** base, void* dest, const void* src) const {
+ CHECK(false);
+ }
+
+ void direct_copy_may_cut(void* dest, const void* src) const override { direct_copy(dest, src); }
+
+ Status convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool,
+ size_t variable_len = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ Status from_string(void* buf, const std::string& scan_key, const int precision = 0,
+ const int scale = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ std::string to_string(const void* src) const override {
+ return "{}";
Review Comment:
done!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] morningman commented on pull request #15966: feature-map
Posted by GitBox <gi...@apache.org>.
morningman commented on PR #15966:
URL: https://github.com/apache/doris/pull/15966#issuecomment-1384720667
Please change the title and rebase the code
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] xiaokang commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
xiaokang commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1072085398
##########
be/src/vec/data_types/data_type_map.cpp:
##########
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "data_type_map.h"
+
+#include "gen_cpp/data.pb.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/common/assert_cast.h"
+
+namespace doris::vectorized {
+
+DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_)
+{
+ key_type = keys_;
+ value_type = values_;
+
+ keys = std::make_shared<DataTypeArray>(key_type);
+ values = std::make_shared<DataTypeArray>(value_type);
+}
+
+std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ std::stringstream ss;
+ ss << "{";
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ss << ", ";
+ ss << "'" << keys->to_string(nested_keys, i);
+ ss << ':';
+ ss << "'" << values->to_string(nested_values, i);
+ }
+ ss << "}";
+ return ss.str();
+}
+
+void DataTypeMap::to_string(const class doris::vectorized::IColumn& column, size_t row_num,
Review Comment:
reuse to_string above
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] xiaokang commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
xiaokang commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1072087273
##########
be/src/vec/data_types/data_type_map.cpp:
##########
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "data_type_map.h"
+
+#include "gen_cpp/data.pb.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/common/assert_cast.h"
+
+namespace doris::vectorized {
+
+DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_)
+{
+ key_type = keys_;
+ value_type = values_;
+
+ keys = std::make_shared<DataTypeArray>(key_type);
+ values = std::make_shared<DataTypeArray>(value_type);
+}
+
+std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ std::stringstream ss;
+ ss << "{";
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ss << ", ";
+ ss << "'" << keys->to_string(nested_keys, i);
+ ss << ':';
+ ss << "'" << values->to_string(nested_values, i);
+ }
+ ss << "}";
+ return ss.str();
+}
+
+void DataTypeMap::to_string(const class doris::vectorized::IColumn& column, size_t row_num,
+ class doris::vectorized::BufferWritable& ostr) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ ostr.write("{", 1);
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ostr.write(", ", 2);
+ keys->to_string(nested_keys, i, ostr);
+ ostr.write(":", 1);
+ values->to_string(nested_values, i, ostr);
+ }
+ ostr.write("}", 1);
+}
+
+Status DataTypeMap::from_string(ReadBuffer& rb, IColumn* column) const {
+ DCHECK(!rb.eof());
+ // only support one level now
+ auto* map_column = assert_cast<ColumnMap*>(column);
+ // IColumn& nested_column = array_column->get_data();
+ if (*rb.position() != '{') {
+ return Status::InvalidArgument("map does not start with '{' character, found '{}'",
+ *rb.position());
+ }
+ keys->from_string(rb, &map_column->get_keys());
Review Comment:
should process key/value pair one by one, just like to_string()
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] carlvinhust2012 commented on a diff in pull request #15966: feature-map
Posted by GitBox <gi...@apache.org>.
carlvinhust2012 commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1071706504
##########
be/src/olap/field.h:
##########
@@ -450,6 +451,29 @@ uint32_t Field::hash_code(const CellType& cell, uint32_t seed) const {
return _type_info->hash_code(cell.cell_ptr(), seed);
}
+class MapField : public Field {
+public:
+ explicit MapField(const TabletColumn& column) : Field(column) {}
+
+ void consume(RowCursorCell* dst, const char* src, bool src_null, MemPool* mem_pool,
+ ObjectPool* agg_pool) const override {
+ dst->set_is_null(src_null);
+ if (src_null) {
+ return;
+ }
+ _type_info->deep_copy(dst->mutable_cell_ptr(), src, mem_pool);
+ }
+
+ // make variable_ptr memory allocate to cell_ptr as MapValue
Review Comment:
maybe need use the code formatter to format code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1083389580
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
Review Comment:
but I need make new array columns as map key and value column writer because I need wrap actually typed column with array column which actually may not init in init_column_meta(), what shoud I deal with the column_id/unique_id/encoding/compression?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1093010599
##########
be/src/runtime/types.cpp:
##########
@@ -223,6 +227,15 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
children.push_back(TypeDescriptor(types, idx));
break;
}
+ case TTypeNodeType::MAP: {
+ type = TYPE_MAP;
+ if (node.has_contains_null()) {
+ contains_null = node.contains_null();
Review Comment:
yes , map from fe has no contains_null yet, I will delete it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1084963919
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/MapType.java:
##########
@@ -98,4 +134,16 @@ public void toThrift(TTypeDesc container) {
keyType.toThrift(container);
valueType.toThrift(container);
}
+
+ @Override
+ public TColumnType toColumnTypeThrift() {
+ TColumnType thrift = new TColumnType();
+ thrift.type = PrimitiveType.MAP.toThrift();
Review Comment:
Yes, I think this is only map column declare . And I refer to ArrayType which not process its nested child column type too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1090117001
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
Review Comment:
Yes I refact it , pelease review it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089863981
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java:
##########
@@ -186,6 +188,11 @@ public void createChildrenColumn(Type type, Column column) {
Column c = new Column(COLUMN_ARRAY_CHILDREN, ((ArrayType) type).getItemType());
c.setIsAllowNull(((ArrayType) type).getContainsNull());
column.addChildrenColumn(c);
+ } else if (type.isMapType()) {
+ Column k = new Column(COLUMN_MAP_KEY, ((MapType) type).getKeyType());
+ Column v = new Column(COLUMN_MAP_VALUE, ((MapType) type).getValueType());
Review Comment:
column K, V in columnMap should not be null ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] github-actions[bot] commented on pull request #15966: [Feature](map) add map type to doris
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #15966:
URL: https://github.com/apache/doris/pull/15966#issuecomment-1414849763
PR approved by anyone and no changes requested.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] xy720 commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
xy720 commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1073120292
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java:
##########
@@ -318,6 +318,11 @@ public void analyze() throws AnalysisException {
type, Function.NullableMode.ALWAYS_NULLABLE,
Lists.newArrayList(Type.VARCHAR), false,
"doris::CastFunctions::cast_to_array_val", null, null, true);
+ } else if (type.isMapType()) {
+ fn = ScalarFunction.createBuiltin(getFnName(Type.MAP),
+ type, Function.NullableMode.ALWAYS_NULLABLE,
+ Lists.newArrayList(Type.VARCHAR), false,
+ "doris::CastFunctions::cast_to_map_val", null, null, true);
Review Comment:
Is this function exist?
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java:
##########
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+import org.apache.doris.thrift.TTypeDesc;
+import org.apache.doris.thrift.TTypeNode;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+// INSERT INTO table_map VALUES ({'key1':1, 'key2':10, 'k3':100}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30});
+// MapLiteral is one row-based literal
+public class MapLiteral extends LiteralExpr {
+
+ public MapLiteral() {
+ type = new MapType(Type.NULL, Type.NULL);
+ children = new ArrayList<>();
+ }
+
+ public MapLiteral(LiteralExpr... exprs) throws AnalysisException {
+ Type keyType = Type.NULL;
+ Type valueType = Type.NULL;
+ children = new ArrayList<>();
+ int idx = 0;
+ for (LiteralExpr expr : exprs) {
+ if (idx % 2 == 0) {
+ if (keyType == Type.NULL) {
+ keyType = expr.getType();
+ } else {
+ keyType = Type.getAssignmentCompatibleType(keyType, expr.getType(), false);
Review Comment:
```suggestion
keyType = Type.getAssignmentCompatibleType(keyType, expr.getType(), true);
```
to avoid loss of precision.
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/MapType.java:
##########
@@ -17,18 +17,25 @@
package org.apache.doris.catalog;
+import org.apache.doris.thrift.TColumnType;
import org.apache.doris.thrift.TTypeDesc;
import org.apache.doris.thrift.TTypeNode;
import org.apache.doris.thrift.TTypeNodeType;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.gson.annotations.SerializedName;
+
+import java.util.Objects;
Review Comment:
How is the nullable situation for key and value?
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java:
##########
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+import org.apache.doris.thrift.TTypeDesc;
+import org.apache.doris.thrift.TTypeNode;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+// INSERT INTO table_map VALUES ({'key1':1, 'key2':10, 'k3':100}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30});
+// MapLiteral is one row-based literal
+public class MapLiteral extends LiteralExpr {
+
+ public MapLiteral() {
+ type = new MapType(Type.NULL, Type.NULL);
+ children = new ArrayList<>();
+ }
+
+ public MapLiteral(LiteralExpr... exprs) throws AnalysisException {
+ Type keyType = Type.NULL;
+ Type valueType = Type.NULL;
+ children = new ArrayList<>();
+ int idx = 0;
+ for (LiteralExpr expr : exprs) {
+ if (idx % 2 == 0) {
+ if (keyType == Type.NULL) {
+ keyType = expr.getType();
+ } else {
+ keyType = Type.getAssignmentCompatibleType(keyType, expr.getType(), false);
+ }
+ if (keyType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ } else {
+ if (valueType == Type.NULL) {
+ valueType = expr.getType();
+ } else {
+ valueType = Type.getAssignmentCompatibleType(valueType, expr.getType(), false);
+ }
+ if (valueType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ }
+ children.add(expr);
+ ++ idx;
+ }
+
+ type = new MapType(keyType, valueType);
+ }
+
+ protected MapLiteral(MapLiteral other) {
+ super(other);
+ }
+
+ @Override
+ public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
+ if (!targetType.isMapType()) {
+ return super.uncheckedCastTo(targetType);
+ }
+ MapLiteral literal = new MapLiteral(this);
+ Type keyType = ((MapType) targetType).getKeyType();
+ Type valueType = ((MapType) targetType).getValueType();
+
+ for (int i = 0; i < children.size(); ++ i) {
+ Expr child = children.get(i);
+ if ((i & 1) == 0) {
+ literal.children.set(i, child.uncheckedCastTo(keyType));
+ } else {
+ literal.children.set(i, child.uncheckedCastTo(valueType));
+ }
+ }
+ literal.setType(targetType);
+ return literal;
+ }
+
+ @Override
+ public void checkValueValid() throws AnalysisException {
+ for (Expr e : children) {
+ e.checkValueValid();
+ }
+ }
+
+ @Override
+ protected String toSqlImpl() {
+ List<String> list = new ArrayList<>(children.size());
+ for (int i = 0; i < children.size(); i += 2) {
+ list.add(children.get(i).toSqlImpl() + ":" + children.get(i + 1).toSqlImpl());
+ }
+ return "MAP{" + StringUtils.join(list, ", ") + "}";
+ }
+
+ @Override
+ protected void toThrift(TExprNode msg) {
+ msg.node_type = TExprNodeType.MAP_LITERAL;
+ TTypeDesc container = new TTypeDesc();
+ container.setTypes(new ArrayList<TTypeNode>());
+ type.toThrift(container);
+ msg.setType(container);
+ }
+
+ @Override
+ public Expr clone() {
+ return new MapLiteral(this);
+ }
+
+ @Override
+ public boolean isMinValue() {
+ return false;
+ }
+
+ @Override
+ public int compareLiteral(LiteralExpr expr) {
+ return 0;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ int size = in.readInt();
+ children = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ children.add(Expr.readIn(in));
+ }
+ }
+
+ public static MapLiteral read(DataInput in) throws IOException {
+ MapLiteral literal = new MapLiteral();
+ literal.readFields(in);
+ return literal;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ super.write(out);
+ out.writeInt(children.size());
+ for (Expr e : children) {
+ Expr.writeTo(e, out);
+ }
+ }
+
+ @Override
+ public String getStringValue() {
+ List<String> list = new ArrayList<>(children.size());
+ children.forEach(v -> list.add(v.getStringValue()));
Review Comment:
check this logic
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java:
##########
@@ -396,37 +403,78 @@ public TColumn toThrift() {
return tColumn;
}
+
+ // here to make complex type column easy
+ private void setChildrenTColumn(Column children, TColumn tColumn) {
+ TColumn childrenTColumn = new TColumn();
+ childrenTColumn.setColumnName(children.name);
+
+ TColumnType childrenTColumnType = new TColumnType();
+ childrenTColumnType.setType(children.getDataType().toThrift());
+ childrenTColumnType.setLen(children.getStrLen());
+ childrenTColumnType.setPrecision(children.getPrecision());
+ childrenTColumnType.setScale(children.getScale());
+ childrenTColumnType.setIndexLen(children.getOlapColumnIndexSize());
+
+ childrenTColumn.setColumnType(childrenTColumnType);
+ childrenTColumn.setIsAllowNull(children.isAllowNull());
+ // TODO: If we don't set the aggregate type for children, the type will be
+ // considered as TAggregationType::SUM after deserializing in BE.
+ // For now, we make children inherit the aggregate type from their parent.
+ if (tColumn.getAggregationType() != null) {
+ childrenTColumn.setAggregationType(tColumn.getAggregationType());
+ }
+
+ tColumn.children_column.add(childrenTColumn);
+ toChildrenThrift(children, childrenTColumn);
+ }
+
+
Review Comment:
empty line
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java:
##########
@@ -186,6 +188,11 @@ public void createChildrenColumn(Type type, Column column) {
Column c = new Column(COLUMN_ARRAY_CHILDREN, ((ArrayType) type).getItemType());
c.setIsAllowNull(((ArrayType) type).getContainsNull());
column.addChildrenColumn(c);
+ } else if (type.isMapType()) {
+ Column k = new Column(COLUMN_MAP_KEY, ((MapType) type).getKeyType());
+ Column v = new Column(COLUMN_MAP_VALUE, ((MapType) type).getValueType());
Review Comment:
Here no need to setIsAllowNull for the column k and v?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1084840982
##########
be/src/olap/types.cpp:
##########
@@ -218,6 +233,11 @@ TypeInfoPtr get_type_info(const TabletColumn* col) {
TypeInfoPtr clone_type_info(const TypeInfo* type_info) {
if (is_scalar_type(type_info->type())) {
return create_static_type_info_ptr(type_info);
+ } else if (type_info->type() == OLAP_FIELD_TYPE_MAP) {
+ const auto map_type_info = dynamic_cast<const MapTypeInfo*>(type_info);
+ return create_dynamic_type_info_ptr(
+ new MapTypeInfo(clone_type_info(map_type_info->get_key_type_info()),
+ clone_type_info(map_type_info->get_value_type_info())));
} else {
Review Comment:
so no default type or else ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] xy720 commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
xy720 commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1072312487
##########
be/src/vec/core/field.h:
##########
@@ -85,6 +85,7 @@ using FieldVector = std::vector<Field>;
DEFINE_FIELD_VECTOR(Array);
DEFINE_FIELD_VECTOR(Tuple);
+DEFINE_FIELD_VECTOR(Map);
Review Comment:
Now is vector<Field>{key1, val1, key2, val2} with double size?
May be we should refactor this to use a map to store the keys and values
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1084955323
##########
fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java:
##########
@@ -185,6 +185,9 @@ public void finalize(Analyzer analyzer) throws UserException {
// corresponding output slot isn't being materialized)
materializedResultExprLists.clear();
Preconditions.checkState(resultExprLists.size() == children.size());
+ if (analyzer.getDescTbl().getTupleDesc(tupleId) == null) {
Review Comment:
I met a npe when I make unitest in
`List<SlotDescriptor> slots = analyzer.getDescTbl().getTupleDesc(tupleId).getSlots();`
because analyzer.getDescTbl().getTupleDesc(tupleId) return null...
so I add this, maybe it is not important , and it should not fix in this pr. I will delete this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1084974457
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java:
##########
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+import org.apache.doris.thrift.TTypeDesc;
+import org.apache.doris.thrift.TTypeNode;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+// INSERT INTO table_map VALUES ({'key1':1, 'key2':10, 'k3':100}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30});
+// MapLiteral is one row-based literal
+public class MapLiteral extends LiteralExpr {
+
+ public MapLiteral() {
+ type = new MapType(Type.NULL, Type.NULL);
+ children = new ArrayList<>();
+ }
+
+ public MapLiteral(LiteralExpr... exprs) throws AnalysisException {
+ Type keyType = Type.NULL;
+ Type valueType = Type.NULL;
+ children = new ArrayList<>();
+ int idx = 0;
+ for (LiteralExpr expr : exprs) {
+ if (idx % 2 == 0) {
+ if (keyType == Type.NULL) {
+ keyType = expr.getType();
+ } else {
+ keyType = Type.getAssignmentCompatibleType(keyType, expr.getType(), false);
+ }
+ if (keyType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ } else {
+ if (valueType == Type.NULL) {
+ valueType = expr.getType();
+ } else {
+ valueType = Type.getAssignmentCompatibleType(valueType, expr.getType(), false);
+ }
+ if (valueType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ }
+ children.add(expr);
+ ++ idx;
+ }
+
+ type = new MapType(keyType, valueType);
+ }
+
+ protected MapLiteral(MapLiteral other) {
+ super(other);
+ }
+
+ @Override
+ public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
+ if (!targetType.isMapType()) {
+ return super.uncheckedCastTo(targetType);
+ }
+ MapLiteral literal = new MapLiteral(this);
+ Type keyType = ((MapType) targetType).getKeyType();
+ Type valueType = ((MapType) targetType).getValueType();
+
+ for (int i = 0; i < children.size(); ++ i) {
+ Expr child = children.get(i);
+ if ((i & 1) == 0) {
+ literal.children.set(i, child.uncheckedCastTo(keyType));
+ } else {
+ literal.children.set(i, child.uncheckedCastTo(valueType));
+ }
+ }
+ literal.setType(targetType);
+ return literal;
+ }
+
+ @Override
+ public void checkValueValid() throws AnalysisException {
+ for (Expr e : children) {
+ e.checkValueValid();
+ }
+ }
+
+ @Override
+ protected String toSqlImpl() {
+ List<String> list = new ArrayList<>(children.size());
+ for (int i = 0; i < children.size(); i += 2) {
+ list.add(children.get(i).toSqlImpl() + ":" + children.get(i + 1).toSqlImpl());
+ }
+ return "MAP{" + StringUtils.join(list, ", ") + "}";
+ }
+
+ @Override
+ protected void toThrift(TExprNode msg) {
+ msg.node_type = TExprNodeType.MAP_LITERAL;
+ TTypeDesc container = new TTypeDesc();
+ container.setTypes(new ArrayList<TTypeNode>());
+ type.toThrift(container);
+ msg.setType(container);
+ }
+
+ @Override
+ public Expr clone() {
+ return new MapLiteral(this);
+ }
+
+ @Override
+ public boolean isMinValue() {
+ return false;
+ }
+
+ @Override
+ public int compareLiteral(LiteralExpr expr) {
+ return 0;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ int size = in.readInt();
+ children = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ children.add(Expr.readIn(in));
+ }
+ }
+
+ public static MapLiteral read(DataInput in) throws IOException {
+ MapLiteral literal = new MapLiteral();
+ literal.readFields(in);
+ return literal;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ super.write(out);
+ out.writeInt(children.size());
+ for (Expr e : children) {
+ Expr.writeTo(e, out);
+ }
+ }
+
+ @Override
+ public String getStringValue() {
+ List<String> list = new ArrayList<>(children.size());
+ children.forEach(v -> list.add(v.getStringValue()));
Review Comment:
Yep , I just call toSqlImp inner for it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1090112857
##########
be/src/vec/columns/column_map.cpp:
##########
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnMap.cpp
+// and modified by Doris
+
+#include "vec/columns/column_map.h"
+
+namespace doris::vectorized {
+
+/** A column of map values.
+ */
+std::string ColumnMap::get_name() const {
+ return "Map(" + keys->get_name() + ", " + values->get_name() + ")";
+}
+
+ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values)
+ : keys(std::move(keys)), values(std::move(values)) {
+ check_size();
+}
+
+ColumnArray::Offsets64& ColumnMap::get_offsets() const {
+ const ColumnArray & column_keys = assert_cast<const ColumnArray &> (get_keys());
+ // todo . did here check size ?
+ return const_cast<Offsets64&>(column_keys.get_offsets());
+}
+
+void ColumnMap::check_size() const {
+ const auto * key_array = typeid_cast<const ColumnArray *>(keys.get());
+ const auto * value_array = typeid_cast<const ColumnArray *>(values.get());
+ CHECK(key_array) << "ColumnMap keys can be created only from array";
+ CHECK(value_array) << "ColumnMap values can be created only from array";
+ CHECK_EQ(get_keys_ptr()->size(), get_values_ptr()->size());
+}
+
+// todo. here to resize every row map
+MutableColumnPtr ColumnMap::clone_resized(size_t to_size) const {
+ auto res = ColumnMap::create(keys->clone_resized(to_size), values->clone_resized(to_size));
+ return res;
+}
+
+// to support field functions
+Field ColumnMap::operator[](size_t n) const {
+ Map res(2);
+ keys->get(n, res[0]);
+ values->get(n, res[0]);
Review Comment:
done!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1090113298
##########
be/src/vec/functions/array/function_array_element.h:
##########
@@ -81,6 +94,79 @@ class FunctionArrayElement : public IFunction {
}
private:
+ //=========================== map element===========================//
+ ColumnPtr _get_mapped_idx(const ColumnArray& key_column,
+ const ColumnWithTypeAndName& argument) {
+ if (key_column.get_data().is_column_string()) {
+ return _mapped_key_string(key_column, argument);
Review Comment:
I make a _mapped_key function
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1090117069
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
+ key_opts.meta->set_column_id(4);
+ key_opts.meta->set_unique_id(4);
+ key_opts.meta->set_type(OLAP_FIELD_TYPE_ARRAY);
+ key_opts.meta->set_length(0);
+ key_opts.meta->set_encoding(BIT_SHUFFLE);
+ key_opts.meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
+ key_opts.need_zone_map = false;
+ // no need key array's null map
+ key_opts.meta->set_is_nullable(false);
+
+ ColumnMetaPB* child_meta = key_opts.meta->add_children_columns();
+ child_meta->set_column_id(5);
+ child_meta->set_unique_id(5);
+ child_meta->set_type(key_column.type());
+ child_meta->set_length(key_column.length());
+ child_meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
+ child_meta->set_encoding(DICT_ENCODING);
+ child_meta->set_is_nullable(key_column.is_nullable());
+ }
+
+ RETURN_IF_ERROR(
+ ColumnWriter::create(key_opts, &key_list_column, file_writer, &key_writer));
+
+
+ // create value writer
+ std::unique_ptr<ColumnWriter> value_writer;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] github-actions[bot] commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by github-actions.
github-actions[bot] commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089900145
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,6 +179,12 @@
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_null),
Review Comment:
warning: no member named 'contains_null' in 'doris::TypeDescriptor' [clang-diagnostic-error]
```cpp
create_data_type(col_desc.children[0], col_desc.contains_null),
^
```
##########
be/src/runtime/types.cpp:
##########
@@ -223,6 +227,15 @@
children.push_back(TypeDescriptor(types, idx));
break;
}
+ case TTypeNodeType::MAP: {
+ type = TYPE_MAP;
+ if (node.has_contains_null()) {
+ contains_null = node.contains_null();
Review Comment:
warning: no viable overloaded '=' [clang-diagnostic-error]
```cpp
contains_null = node.contains_null();
^
```
**/usr/include/c++/11/bits/stl_bvector.h:735:** candidate function not viable: no known conversion from 'bool' to 'const std::vector<bool>' for 1st argument
```cpp
operator=(const vector& __x)
^
```
**/usr/include/c++/11/bits/stl_bvector.h:766:** candidate function not viable: no known conversion from 'bool' to 'std::vector<bool>' for 1st argument
```cpp
operator=(vector&& __x) noexcept(_Bit_alloc_traits::_S_nothrow_move())
^
```
**/usr/include/c++/11/bits/stl_bvector.h:791:** candidate function not viable: no known conversion from 'bool' to 'initializer_list<bool>' for 1st argument
```cpp
operator=(initializer_list<bool> __l)
^
```
##########
be/src/vec/sink/vmysql_result_writer.cpp:
##########
@@ -715,6 +732,18 @@
}
break;
}
+ case TYPE_MAP: {
+ if (type_ptr->is_nullable()) {
+ auto& nested_type =
+ assert_cast<const DataTypeNullable&>(*type_ptr).get_nested_type(); //for map
+ status = _add_one_column<PrimitiveType::TYPE_MAP, true>(column_ptr, result,
+ nested_type);
+ } else {
+ status = _add_one_column<PrimitiveType::TYPE_MAP, false>(column_ptr, result,
Review Comment:
warning: no matching member function for call to '_add_one_column' [clang-diagnostic-error]
```cpp
status = _add_one_column<PrimitiveType::TYPE_MAP, false>(column_ptr, result,
^
```
**be/src/vec/sink/vmysql_result_writer.cpp:65:** candidate function template not viable: no known conversion from 'doris::vectorized::DataTypePtr' (aka 'shared_ptr<const doris::vectorized::IDataType>') to 'int' for 3rd argument
```cpp
Status VMysqlResultWriter::_add_one_column(const ColumnPtr& column_ptr,
^
```
##########
be/src/vec/sink/vmysql_result_writer.cpp:
##########
@@ -715,6 +732,18 @@
}
break;
}
+ case TYPE_MAP: {
+ if (type_ptr->is_nullable()) {
+ auto& nested_type =
+ assert_cast<const DataTypeNullable&>(*type_ptr).get_nested_type(); //for map
+ status = _add_one_column<PrimitiveType::TYPE_MAP, true>(column_ptr, result,
Review Comment:
warning: no matching member function for call to '_add_one_column' [clang-diagnostic-error]
```cpp
status = _add_one_column<PrimitiveType::TYPE_MAP, true>(column_ptr, result,
^
```
**be/src/vec/sink/vmysql_result_writer.cpp:65:** candidate function template not viable: no known conversion from 'const doris::vectorized::DataTypePtr' (aka 'const shared_ptr<const doris::vectorized::IDataType>') to 'int' for 3rd argument
```cpp
Status VMysqlResultWriter::_add_one_column(const ColumnPtr& column_ptr,
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,6 +179,12 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
+ case TYPE_MAP:
Review Comment:
warning: cannot jump from switch statement to this case label [clang-diagnostic-error]
```cpp
case TYPE_MAP:
^
```
**be/src/vec/data_types/data_type_factory.cpp:172:** jump bypasses variable initialization
```cpp
Strings names;
^
```
**be/src/vec/data_types/data_type_factory.cpp:171:** jump bypasses variable initialization
```cpp
DataTypes dataTypes;
^
```
**be/src/vec/data_types/data_type_factory.cpp:169:** jump bypasses variable initialization
```cpp
size_t child_size = col_desc.children.size();
^
```
##########
be/src/vec/sink/vmysql_result_writer.cpp:
##########
@@ -185,6 +187,21 @@ Status VMysqlResultWriter::_add_one_column(const ColumnPtr& column_ptr,
begin = false;
}
buf_ret = _buffer.push_string("]", 1);
+ _buffer.close_dynamic_mode();
+ result->result_batch.rows[i].append(_buffer.buf(), _buffer.length());
+ }
+ } else if constexpr (type == TYPE_MAP) {
+ auto& map_type = assert_cast<const DataTypeMap&>(*nested_type_ptr);
Review Comment:
warning: use of undeclared identifier 'nested_type_ptr' [clang-diagnostic-error]
```cpp
auto& map_type = assert_cast<const DataTypeMap&>(*nested_type_ptr);
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,6 +179,12 @@
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_null),
+ create_data_type(col_desc.children[1], col_desc.contains_null));
Review Comment:
warning: no member named 'contains_null' in 'doris::TypeDescriptor' [clang-diagnostic-error]
```cpp
create_data_type(col_desc.children[1], col_desc.contains_null));
^
```
##########
be/src/runtime/types.cpp:
##########
@@ -223,6 +227,15 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
children.push_back(TypeDescriptor(types, idx));
break;
}
+ case TTypeNodeType::MAP: {
+ type = TYPE_MAP;
+ if (node.has_contains_null()) {
+ contains_null = node.contains_null();
Review Comment:
warning: use of undeclared identifier 'contains_null'; did you mean 'contains_nulls'? [clang-diagnostic-error]
```suggestion
contains_nulls = node.contains_null();
```
**be/src/runtime/types.h:66:** 'contains_nulls' declared here
```cpp
std::vector<bool> contains_nulls;
^
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1092884152
##########
be/src/olap/field.h:
##########
@@ -49,7 +50,7 @@ class Field {
_index_size(column.index_length()),
_is_nullable(column.is_nullable()),
_unique_id(column.unique_id()) {
- if (column.type() == OLAP_FIELD_TYPE_ARRAY) {
+ if (column.type() == OLAP_FIELD_TYPE_ARRAY || column.type() == OLAP_FIELD_TYPE_MAP) {
_agg_info = get_aggregate_info(column.aggregation(), column.type(),
column.get_sub_column(0).type());
Review Comment:
I just go the default logic
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] github-actions[bot] commented on a diff in pull request #15966: feature-map
Posted by GitBox <gi...@apache.org>.
github-actions[bot] commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1070930875
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -695,5 +808,120 @@ Status ArrayColumnWriter::finish_current_page() {
return Status::NotSupported("array writer has no data, can not finish_current_page");
}
+/// ============================= MapColumnWriter =====================////
+MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
+ ScalarColumnWriter* null_writer,
+ std::unique_ptr<ColumnWriter> key_writer,
+ std::unique_ptr<ColumnWriter> value_writer)
+ : ColumnWriter(std::move(field), opts.meta->is_nullable()),
+ _key_writer(std::move(key_writer)),
+ _value_writer(std::move(value_writer)),
+ _opts(opts) {
+ if (is_nullable()) {
+ _null_writer.reset(null_writer);
+ }
+}
+
+Status MapColumnWriter::init() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->init());
+ }
+ RETURN_IF_ERROR(_key_writer->init());
+ RETURN_IF_ERROR(_value_writer->init());
+ return Status::OK();
+}
+
+uint64_t MapColumnWriter::estimate_buffer_size() {
+ size_t estimate = _key_writer->estimate_buffer_size() +
+ _value_writer->estimate_buffer_size();
+ if (is_nullable()) {
+ estimate += _null_writer->estimate_buffer_size();
+ }
+ return estimate;
+}
+
+Status MapColumnWriter::finish() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->finish());
+ }
+ RETURN_IF_ERROR(_key_writer->finish());
+ RETURN_IF_ERROR(_value_writer->finish());
+ return Status::OK();
+}
+
+// todo. make keys and values write
+Status MapColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
+ size_t remaining = num_rows;
+ const auto* col_cursor = reinterpret_cast<const MapValue*>(*ptr);
+ while (remaining > 0) {
+ size_t num_written = 1;
+ auto* key_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_key_data();
+ const uint8_t* key_ptr = (const uint8_t*)key_data_ptr;
+ RETURN_IF_ERROR(_key_writer->append_data(&key_ptr, 1));
+ auto* val_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_value_data();
+ const uint8_t* val_ptr = (const uint8_t*)val_data_ptr;
+ RETURN_IF_ERROR(_value_writer->append_data(&val_ptr, 1));
+ remaining -= num_written;
+ col_cursor += num_written;
+ *ptr += num_written * sizeof(MapValue);
+ }
+
+ if (is_nullable()) {
+ return write_null_column(num_rows, false);
Review Comment:
warning: use of undeclared identifier 'write_null_column' [clang-diagnostic-error]
```cpp
return write_null_column(num_rows, false);
^
```
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -695,5 +808,120 @@
return Status::NotSupported("array writer has no data, can not finish_current_page");
}
+/// ============================= MapColumnWriter =====================////
+MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
+ ScalarColumnWriter* null_writer,
+ std::unique_ptr<ColumnWriter> key_writer,
+ std::unique_ptr<ColumnWriter> value_writer)
+ : ColumnWriter(std::move(field), opts.meta->is_nullable()),
+ _key_writer(std::move(key_writer)),
+ _value_writer(std::move(value_writer)),
+ _opts(opts) {
+ if (is_nullable()) {
+ _null_writer.reset(null_writer);
+ }
+}
+
+Status MapColumnWriter::init() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->init());
+ }
+ RETURN_IF_ERROR(_key_writer->init());
+ RETURN_IF_ERROR(_value_writer->init());
+ return Status::OK();
+}
+
+uint64_t MapColumnWriter::estimate_buffer_size() {
+ size_t estimate = _key_writer->estimate_buffer_size() +
+ _value_writer->estimate_buffer_size();
+ if (is_nullable()) {
+ estimate += _null_writer->estimate_buffer_size();
+ }
+ return estimate;
+}
+
+Status MapColumnWriter::finish() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->finish());
+ }
+ RETURN_IF_ERROR(_key_writer->finish());
+ RETURN_IF_ERROR(_value_writer->finish());
+ return Status::OK();
+}
+
+// todo. make keys and values write
+Status MapColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
+ size_t remaining = num_rows;
+ const auto* col_cursor = reinterpret_cast<const MapValue*>(*ptr);
+ while (remaining > 0) {
+ size_t num_written = 1;
+ auto* key_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_key_data();
+ const uint8_t* key_ptr = (const uint8_t*)key_data_ptr;
+ RETURN_IF_ERROR(_key_writer->append_data(&key_ptr, 1));
+ auto* val_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_value_data();
+ const uint8_t* val_ptr = (const uint8_t*)val_data_ptr;
+ RETURN_IF_ERROR(_value_writer->append_data(&val_ptr, 1));
+ remaining -= num_written;
+ col_cursor += num_written;
+ *ptr += num_written * sizeof(MapValue);
+ }
+
+ if (is_nullable()) {
+ return write_null_column(num_rows, false);
+ }
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_data() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_data());
+ }
+ RETURN_IF_ERROR(_offset_writer->write_data());
Review Comment:
warning: use of undeclared identifier '_offset_writer' [clang-diagnostic-error]
```cpp
RETURN_IF_ERROR(_offset_writer->write_data());
^
```
##########
be/src/udf/udf.h:
##########
@@ -927,6 +952,7 @@ using doris_udf::DateTimeVal;
using doris_udf::HllVal;
using doris_udf::FunctionContext;
using doris_udf::CollectionVal;
+using doris_udf::MapVal;
Review Comment:
warning: using decl 'MapVal' is unused [misc-unused-using-decls]
```cpp
using doris_udf::MapVal;
^
```
**be/src/udf/udf.h:954:** remove the using
```cpp
using doris_udf::MapVal;
^
```
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -695,5 +808,120 @@
return Status::NotSupported("array writer has no data, can not finish_current_page");
}
+/// ============================= MapColumnWriter =====================////
+MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
+ ScalarColumnWriter* null_writer,
+ std::unique_ptr<ColumnWriter> key_writer,
+ std::unique_ptr<ColumnWriter> value_writer)
+ : ColumnWriter(std::move(field), opts.meta->is_nullable()),
+ _key_writer(std::move(key_writer)),
+ _value_writer(std::move(value_writer)),
+ _opts(opts) {
+ if (is_nullable()) {
+ _null_writer.reset(null_writer);
+ }
+}
+
+Status MapColumnWriter::init() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->init());
+ }
+ RETURN_IF_ERROR(_key_writer->init());
+ RETURN_IF_ERROR(_value_writer->init());
+ return Status::OK();
+}
+
+uint64_t MapColumnWriter::estimate_buffer_size() {
+ size_t estimate = _key_writer->estimate_buffer_size() +
+ _value_writer->estimate_buffer_size();
+ if (is_nullable()) {
+ estimate += _null_writer->estimate_buffer_size();
+ }
+ return estimate;
+}
+
+Status MapColumnWriter::finish() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->finish());
+ }
+ RETURN_IF_ERROR(_key_writer->finish());
+ RETURN_IF_ERROR(_value_writer->finish());
+ return Status::OK();
+}
+
+// todo. make keys and values write
+Status MapColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
+ size_t remaining = num_rows;
+ const auto* col_cursor = reinterpret_cast<const MapValue*>(*ptr);
+ while (remaining > 0) {
+ size_t num_written = 1;
+ auto* key_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_key_data();
+ const uint8_t* key_ptr = (const uint8_t*)key_data_ptr;
+ RETURN_IF_ERROR(_key_writer->append_data(&key_ptr, 1));
+ auto* val_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_value_data();
+ const uint8_t* val_ptr = (const uint8_t*)val_data_ptr;
+ RETURN_IF_ERROR(_value_writer->append_data(&val_ptr, 1));
+ remaining -= num_written;
+ col_cursor += num_written;
+ *ptr += num_written * sizeof(MapValue);
+ }
+
+ if (is_nullable()) {
+ return write_null_column(num_rows, false);
+ }
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_data() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_data());
+ }
+ RETURN_IF_ERROR(_offset_writer->write_data());
+ RETURN_IF_ERROR(_key_writer->write_data());
+ RETURN_IF_ERROR(_value_writer->write_data());
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_ordinal_index() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_ordinal_index());
+ }
+ RETURN_IF_ERROR(_key_writer->write_ordinal_index());
+ RETURN_IF_ERROR(_value_writer->write_ordinal_index());
+
+ return Status::OK();
+}
+
+Status MapColumnWriter::append_nulls(size_t num_rows) {
+ return write_null_column(num_rows, true);
+}
+
+Status MapColumnWriter::write_null_column(size_t num_rows, bool is_null) {
Review Comment:
warning: out-of-line definition of 'write_null_column' does not match any declaration in 'doris::segment_v2::MapColumnWriter' [clang-diagnostic-error]
```cpp
Status MapColumnWriter::write_null_column(size_t num_rows, bool is_null) {
^
```
##########
be/src/vec/data_types/data_type_map.cpp:
##########
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "data_type_map.h"
+
+#include "gen_cpp/data.pb.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/common/assert_cast.h"
+
+namespace doris::vectorized {
+
+DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_)
+{
+ key_type = keys_;
+ value_type = values_;
+
+ keys = std::make_shared<DataTypeArray>(key_type);
+ values = std::make_shared<DataTypeArray>(value_type);
+}
+
+std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ std::stringstream ss;
+ ss << "{";
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ss << ", ";
Review Comment:
warning: statement should be inside braces [readability-braces-around-statements]
```suggestion
if (i != offset) {
ss << ", ";
}
```
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -695,5 +808,120 @@
return Status::NotSupported("array writer has no data, can not finish_current_page");
}
+/// ============================= MapColumnWriter =====================////
+MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
+ ScalarColumnWriter* null_writer,
+ std::unique_ptr<ColumnWriter> key_writer,
+ std::unique_ptr<ColumnWriter> value_writer)
+ : ColumnWriter(std::move(field), opts.meta->is_nullable()),
+ _key_writer(std::move(key_writer)),
+ _value_writer(std::move(value_writer)),
+ _opts(opts) {
+ if (is_nullable()) {
+ _null_writer.reset(null_writer);
+ }
+}
+
+Status MapColumnWriter::init() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->init());
+ }
+ RETURN_IF_ERROR(_key_writer->init());
+ RETURN_IF_ERROR(_value_writer->init());
+ return Status::OK();
+}
+
+uint64_t MapColumnWriter::estimate_buffer_size() {
+ size_t estimate = _key_writer->estimate_buffer_size() +
+ _value_writer->estimate_buffer_size();
+ if (is_nullable()) {
+ estimate += _null_writer->estimate_buffer_size();
+ }
+ return estimate;
+}
+
+Status MapColumnWriter::finish() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->finish());
+ }
+ RETURN_IF_ERROR(_key_writer->finish());
+ RETURN_IF_ERROR(_value_writer->finish());
+ return Status::OK();
+}
+
+// todo. make keys and values write
+Status MapColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
+ size_t remaining = num_rows;
+ const auto* col_cursor = reinterpret_cast<const MapValue*>(*ptr);
+ while (remaining > 0) {
+ size_t num_written = 1;
+ auto* key_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_key_data();
+ const uint8_t* key_ptr = (const uint8_t*)key_data_ptr;
+ RETURN_IF_ERROR(_key_writer->append_data(&key_ptr, 1));
+ auto* val_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_value_data();
+ const uint8_t* val_ptr = (const uint8_t*)val_data_ptr;
+ RETURN_IF_ERROR(_value_writer->append_data(&val_ptr, 1));
+ remaining -= num_written;
+ col_cursor += num_written;
+ *ptr += num_written * sizeof(MapValue);
+ }
+
+ if (is_nullable()) {
+ return write_null_column(num_rows, false);
+ }
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_data() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_data());
+ }
+ RETURN_IF_ERROR(_offset_writer->write_data());
+ RETURN_IF_ERROR(_key_writer->write_data());
+ RETURN_IF_ERROR(_value_writer->write_data());
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_ordinal_index() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_ordinal_index());
+ }
+ RETURN_IF_ERROR(_key_writer->write_ordinal_index());
+ RETURN_IF_ERROR(_value_writer->write_ordinal_index());
+
+ return Status::OK();
+}
+
+Status MapColumnWriter::append_nulls(size_t num_rows) {
+ return write_null_column(num_rows, true);
Review Comment:
warning: use of undeclared identifier 'write_null_column' [clang-diagnostic-error]
```cpp
return write_null_column(num_rows, true);
^
```
##########
be/src/vec/data_types/data_type_map.cpp:
##########
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "data_type_map.h"
+
+#include "gen_cpp/data.pb.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/common/assert_cast.h"
+
+namespace doris::vectorized {
+
+DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_)
+{
+ key_type = keys_;
+ value_type = values_;
+
+ keys = std::make_shared<DataTypeArray>(key_type);
+ values = std::make_shared<DataTypeArray>(value_type);
+}
+
+std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ std::stringstream ss;
+ ss << "{";
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ss << ", ";
+ ss << "'" << keys->to_string(nested_keys, i);
+ ss << ':';
+ ss << "'" << values->to_string(nested_values, i);
+ }
+ ss << "}";
+ return ss.str();
+}
+
+void DataTypeMap::to_string(const class doris::vectorized::IColumn& column, size_t row_num,
+ class doris::vectorized::BufferWritable& ostr) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ ostr.write("{", 1);
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ostr.write(", ", 2);
Review Comment:
warning: statement should be inside braces [readability-braces-around-statements]
```suggestion
if (i != offset) {
ostr.write(", ", 2);
}
```
##########
be/src/vec/data_types/data_type_map.cpp:
##########
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "data_type_map.h"
+
+#include "gen_cpp/data.pb.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/common/assert_cast.h"
+
+namespace doris::vectorized {
+
+DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_)
+{
+ key_type = keys_;
+ value_type = values_;
+
+ keys = std::make_shared<DataTypeArray>(key_type);
+ values = std::make_shared<DataTypeArray>(value_type);
+}
+
+std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ std::stringstream ss;
+ ss << "{";
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ss << ", ";
+ ss << "'" << keys->to_string(nested_keys, i);
+ ss << ':';
+ ss << "'" << values->to_string(nested_values, i);
+ }
+ ss << "}";
+ return ss.str();
+}
+
+void DataTypeMap::to_string(const class doris::vectorized::IColumn& column, size_t row_num,
+ class doris::vectorized::BufferWritable& ostr) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ ostr.write("{", 1);
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ostr.write(", ", 2);
+ keys->to_string(nested_keys, i, ostr);
+ ostr.write(":", 1);
+ values->to_string(nested_values, i, ostr);
+ }
+ ostr.write("}", 1);
+}
+
+Status DataTypeMap::from_string(ReadBuffer& rb, IColumn* column) const {
+ DCHECK(!rb.eof());
+ // only support one level now
+ auto* map_column = assert_cast<ColumnMap*>(column);
+ // IColumn& nested_column = array_column->get_data();
+ if (*rb.position() != '{') {
+ return Status::InvalidArgument("map does not start with '{' character, found '{}'",
+ *rb.position());
+ }
+ keys->from_string(rb, &map_column->get_keys());
+ values->from_string(rb, &map_column->get_values());
+ if (*(rb.end() - 1) != '}') {
+ return Status::InvalidArgument("map does not end with '}' character, found '{}'",
+ *(rb.end() - 1));
+ }
+// keys->deserializeAsTextQuoted(extractElementColumn(column, 0), istr, settings);
+// assertChar(',', istr);
+// values->deserializeAsTextQuoted(extractElementColumn(column, 1), istr, settings);
+// assertChar('}', istr);
+ return Status::OK();
+}
+
+MutableColumnPtr DataTypeMap::create_column() const {
+ return ColumnMap::create(keys->create_column(), values->create_column());
+}
+
+void DataTypeMap::to_pb_column_meta(PColumnMeta* col_meta) const {
+ IDataType::to_pb_column_meta(col_meta);
+ auto key_children = col_meta->add_children();
+ auto value_children = col_meta->add_children();
+ keys->to_pb_column_meta(key_children);
+ values->to_pb_column_meta(value_children);
+}
+
+bool DataTypeMap::equals(const IDataType& rhs) const {
+ if (typeid(rhs) != typeid(*this))
+ return false;
Review Comment:
warning: statement should be inside braces [readability-braces-around-statements]
```suggestion
if (typeid(rhs) != typeid(*this)) {
return false;
}
```
##########
be/src/runtime/map_value.h:
##########
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <type_traits>
+#include "runtime/primitive_type.h"
+
+namespace doris_udf {
+class FunctionContext;
+struct AnyVal;
+} // namespace doris_udf
+
+namespace doris {
+
+using doris_udf::FunctionContext;
Review Comment:
warning: using decl 'FunctionContext' is unused [misc-unused-using-decls]
```cpp
using doris_udf::FunctionContext;
^
```
**be/src/runtime/map_value.h:29:** remove the using
```cpp
using doris_udf::FunctionContext;
^
```
##########
be/src/vec/exprs/vmap_literal.cpp:
##########
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exprs/vmap_literal.h"
+
+//insert into table_map values ({'name':'zhangsan', 'gender':'male'}), ({'name':'lisi', 'gender':'female'});
+namespace doris::vectorized {
+
+Status VMapLiteral::prepare(RuntimeState* state, const RowDescriptor& row_desc,
+ VExprContext* context) {
+ DCHECK_EQ(type().children.size(), 2) << "map children type not 2";
+
+ RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, row_desc, context));
+ // map-field should contain two vector field for keys and values
+ Field map = Map();
+ Field keys = Array();
+ Field values = Array();
+ // each child is slot with key1, value1, key2, value2...
+ for (int idx = 0; idx < _children.size(); ++idx ) {
+ Field item;
+ ColumnPtrWrapper* const_col_wrapper = nullptr;
+ RETURN_IF_ERROR(_children[idx]->get_const_col(context, &const_col_wrapper));
+ const_col_wrapper->column_ptr->get(0, item);
+
+ if ((idx & 1) == 0)
+ keys.get<Array>().push_back(item);
+ else
+ values.get<Array>().push_back(item);
Review Comment:
warning: statement should be inside braces [readability-braces-around-statements]
```suggestion
else {
values.get<Array>().push_back(item);
}
```
##########
be/src/runtime/map_value.h:
##########
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <type_traits>
+#include "runtime/primitive_type.h"
+
+namespace doris_udf {
+class FunctionContext;
+struct AnyVal;
+} // namespace doris_udf
+
+namespace doris {
+
+using doris_udf::FunctionContext;
+using doris_udf::AnyVal;
Review Comment:
warning: using decl 'AnyVal' is unused [misc-unused-using-decls]
```cpp
using doris_udf::AnyVal;
^
```
**be/src/runtime/map_value.h:30:** remove the using
```cpp
using doris_udf::AnyVal;
^
```
##########
be/src/vec/data_types/data_type_map.cpp:
##########
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "data_type_map.h"
+
+#include "gen_cpp/data.pb.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/common/assert_cast.h"
+
+namespace doris::vectorized {
+
+DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_)
+{
+ key_type = keys_;
+ value_type = values_;
+
+ keys = std::make_shared<DataTypeArray>(key_type);
+ values = std::make_shared<DataTypeArray>(value_type);
+}
+
+std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ std::stringstream ss;
+ ss << "{";
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ss << ", ";
+ ss << "'" << keys->to_string(nested_keys, i);
+ ss << ':';
+ ss << "'" << values->to_string(nested_values, i);
+ }
+ ss << "}";
+ return ss.str();
+}
+
+void DataTypeMap::to_string(const class doris::vectorized::IColumn& column, size_t row_num,
+ class doris::vectorized::BufferWritable& ostr) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ ostr.write("{", 1);
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ostr.write(", ", 2);
+ keys->to_string(nested_keys, i, ostr);
+ ostr.write(":", 1);
+ values->to_string(nested_values, i, ostr);
+ }
+ ostr.write("}", 1);
+}
+
+Status DataTypeMap::from_string(ReadBuffer& rb, IColumn* column) const {
+ DCHECK(!rb.eof());
+ // only support one level now
+ auto* map_column = assert_cast<ColumnMap*>(column);
+ // IColumn& nested_column = array_column->get_data();
+ if (*rb.position() != '{') {
+ return Status::InvalidArgument("map does not start with '{' character, found '{}'",
+ *rb.position());
+ }
+ keys->from_string(rb, &map_column->get_keys());
+ values->from_string(rb, &map_column->get_values());
+ if (*(rb.end() - 1) != '}') {
+ return Status::InvalidArgument("map does not end with '}' character, found '{}'",
+ *(rb.end() - 1));
+ }
+// keys->deserializeAsTextQuoted(extractElementColumn(column, 0), istr, settings);
+// assertChar(',', istr);
+// values->deserializeAsTextQuoted(extractElementColumn(column, 1), istr, settings);
+// assertChar('}', istr);
+ return Status::OK();
+}
+
+MutableColumnPtr DataTypeMap::create_column() const {
+ return ColumnMap::create(keys->create_column(), values->create_column());
+}
+
+void DataTypeMap::to_pb_column_meta(PColumnMeta* col_meta) const {
+ IDataType::to_pb_column_meta(col_meta);
+ auto key_children = col_meta->add_children();
+ auto value_children = col_meta->add_children();
+ keys->to_pb_column_meta(key_children);
+ values->to_pb_column_meta(value_children);
+}
+
+bool DataTypeMap::equals(const IDataType& rhs) const {
+ if (typeid(rhs) != typeid(*this))
+ return false;
+
+ const DataTypeMap & rhs_map = static_cast<const DataTypeMap &>(rhs);
+
+ if (!keys->equals(*rhs_map.keys))
+ return false;
+
+ if (!values->equals(*rhs_map.values))
+ return false;
Review Comment:
warning: statement should be inside braces [readability-braces-around-statements]
```suggestion
if (!values->equals(*rhs_map.values)) {
return false;
}
```
##########
be/src/vec/exprs/vmap_literal.cpp:
##########
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exprs/vmap_literal.h"
+
+//insert into table_map values ({'name':'zhangsan', 'gender':'male'}), ({'name':'lisi', 'gender':'female'});
+namespace doris::vectorized {
+
+Status VMapLiteral::prepare(RuntimeState* state, const RowDescriptor& row_desc,
+ VExprContext* context) {
+ DCHECK_EQ(type().children.size(), 2) << "map children type not 2";
+
+ RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, row_desc, context));
+ // map-field should contain two vector field for keys and values
+ Field map = Map();
+ Field keys = Array();
+ Field values = Array();
+ // each child is slot with key1, value1, key2, value2...
+ for (int idx = 0; idx < _children.size(); ++idx ) {
+ Field item;
+ ColumnPtrWrapper* const_col_wrapper = nullptr;
+ RETURN_IF_ERROR(_children[idx]->get_const_col(context, &const_col_wrapper));
+ const_col_wrapper->column_ptr->get(0, item);
+
+ if ((idx & 1) == 0)
Review Comment:
warning: statement should be inside braces [readability-braces-around-statements]
```suggestion
if ((idx & 1) == 0) {
```
be/src/vec/exprs/vmap_literal.cpp:40:
```diff
- else
+ } else
```
##########
be/src/vec/data_types/data_type_map.cpp:
##########
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "data_type_map.h"
+
+#include "gen_cpp/data.pb.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/common/assert_cast.h"
+
+namespace doris::vectorized {
+
+DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_)
+{
+ key_type = keys_;
+ value_type = values_;
+
+ keys = std::make_shared<DataTypeArray>(key_type);
+ values = std::make_shared<DataTypeArray>(value_type);
+}
+
+std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ std::stringstream ss;
+ ss << "{";
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ss << ", ";
+ ss << "'" << keys->to_string(nested_keys, i);
+ ss << ':';
+ ss << "'" << values->to_string(nested_values, i);
+ }
+ ss << "}";
+ return ss.str();
+}
+
+void DataTypeMap::to_string(const class doris::vectorized::IColumn& column, size_t row_num,
+ class doris::vectorized::BufferWritable& ostr) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ ostr.write("{", 1);
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ostr.write(", ", 2);
+ keys->to_string(nested_keys, i, ostr);
+ ostr.write(":", 1);
+ values->to_string(nested_values, i, ostr);
+ }
+ ostr.write("}", 1);
+}
+
+Status DataTypeMap::from_string(ReadBuffer& rb, IColumn* column) const {
+ DCHECK(!rb.eof());
+ // only support one level now
+ auto* map_column = assert_cast<ColumnMap*>(column);
+ // IColumn& nested_column = array_column->get_data();
+ if (*rb.position() != '{') {
+ return Status::InvalidArgument("map does not start with '{' character, found '{}'",
+ *rb.position());
+ }
+ keys->from_string(rb, &map_column->get_keys());
+ values->from_string(rb, &map_column->get_values());
+ if (*(rb.end() - 1) != '}') {
+ return Status::InvalidArgument("map does not end with '}' character, found '{}'",
+ *(rb.end() - 1));
+ }
+// keys->deserializeAsTextQuoted(extractElementColumn(column, 0), istr, settings);
+// assertChar(',', istr);
+// values->deserializeAsTextQuoted(extractElementColumn(column, 1), istr, settings);
+// assertChar('}', istr);
+ return Status::OK();
+}
+
+MutableColumnPtr DataTypeMap::create_column() const {
+ return ColumnMap::create(keys->create_column(), values->create_column());
+}
+
+void DataTypeMap::to_pb_column_meta(PColumnMeta* col_meta) const {
+ IDataType::to_pb_column_meta(col_meta);
+ auto key_children = col_meta->add_children();
+ auto value_children = col_meta->add_children();
+ keys->to_pb_column_meta(key_children);
+ values->to_pb_column_meta(value_children);
+}
+
+bool DataTypeMap::equals(const IDataType& rhs) const {
+ if (typeid(rhs) != typeid(*this))
+ return false;
+
+ const DataTypeMap & rhs_map = static_cast<const DataTypeMap &>(rhs);
+
+ if (!keys->equals(*rhs_map.keys))
+ return false;
Review Comment:
warning: statement should be inside braces [readability-braces-around-statements]
```suggestion
if (!keys->equals(*rhs_map.keys)) {
return false;
}
```
##########
be/src/vec/functions/array/function_array_element.h:
##########
@@ -81,6 +94,79 @@ class FunctionArrayElement : public IFunction {
}
private:
+ //=========================== map element===========================//
+ ColumnPtr _get_mapped_idx(const ColumnArray& key_column,
+ const ColumnWithTypeAndName& argument) {
+ if (key_column.get_data().is_column_string()) {
+ return _mapped_key_string(key_column, argument);
+ }
+ return nullptr;
+ }
+
+ ColumnPtr _get_mapped_value(const ColumnArray& val_column,
+ const IColumn& matched_indices,
+ const UInt8* src_null_map,
+ UInt8* dst_null_map) {
+ const UInt8* nested_null_map = nullptr;
+ ColumnPtr nested_column = nullptr;
+ if (is_column_nullable(val_column.get_data())) {
+ const auto& nested_null_column =
+ reinterpret_cast<const ColumnNullable&>(val_column.get_data());
+ nested_null_map = nested_null_column.get_null_map_column().get_data().data();
+ nested_column = nested_null_column.get_nested_column_ptr();
+ } else {
+ nested_column = val_column.get_data_ptr();
+ }
+ if (check_column<ColumnInt8>(nested_column)) {
+ return _execute_number<ColumnInt8>(val_column.get_offsets(), *nested_column,
+ src_null_map, matched_indices,
+ nested_null_map, dst_null_map);
+ } else if (check_column<ColumnInt32>(nested_column)) {
+ _execute_number<ColumnInt32>(val_column.get_offsets(), *nested_column,
+ src_null_map, matched_indices,
+ nested_null_map, dst_null_map);
+ }
+ return nullptr;
+ }
+
+ ColumnPtr _mapped_key_string(const ColumnArray& column,
+ const ColumnWithTypeAndName& argument) {
+ auto right_column = argument.column->convert_to_full_column_if_const();
+ const ColumnString& match_key = reinterpret_cast<const ColumnString&>(*right_column);
+ const ColumnArray::Offsets64& offsets = column.get_offsets();
+ ColumnPtr nested_ptr = nullptr;
+ if (is_column_nullable(column.get_data())) {
+ nested_ptr = reinterpret_cast<const ColumnNullable&>(column.get_data()).get_nested_column_ptr();
+ } else {
+ nested_ptr = column.get_data_ptr();
+ }
+ const ColumnString& nested_key = reinterpret_cast<const ColumnString&>(*nested_ptr);
+ size_t rows = offsets.size();
+ // prepare return data
+ auto matched_indices = ColumnVector<Int8>::create();
+ matched_indices->reserve(rows);
+
+ for (size_t i = 0; i < rows; i++)
+ {
+ bool matched = false;
+ size_t begin = offsets[i - 1];
+ size_t end = offsets[i];
+ for (size_t j = begin; j < end; j++) {
+ if (nested_key.get_data_at(j) == match_key.get_data_at(i)) {
+ matched_indices->insert_value(j-begin+1);
+ matched = true;
+ break;
+ }
+ }
+
+ if (!matched)
+ matched_indices->insert_value(end-begin+1); // make indices for null
Review Comment:
warning: statement should be inside braces [readability-braces-around-statements]
```suggestion
if (!matched) {
matched_indices->insert_value(end-begin+1); // make indices for null
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1085068246
##########
be/src/vec/functions/array/function_array_element.h:
##########
@@ -81,6 +94,79 @@ class FunctionArrayElement : public IFunction {
}
private:
+ //=========================== map element===========================//
+ ColumnPtr _get_mapped_idx(const ColumnArray& key_column,
+ const ColumnWithTypeAndName& argument) {
+ if (key_column.get_data().is_column_string()) {
+ return _mapped_key_string(key_column, argument);
+ }
+ return nullptr;
+ }
+
+ ColumnPtr _get_mapped_value(const ColumnArray& val_column,
+ const IColumn& matched_indices,
+ const UInt8* src_null_map,
+ UInt8* dst_null_map) {
+ const UInt8* nested_null_map = nullptr;
+ ColumnPtr nested_column = nullptr;
+ if (is_column_nullable(val_column.get_data())) {
+ const auto& nested_null_column =
+ reinterpret_cast<const ColumnNullable&>(val_column.get_data());
+ nested_null_map = nested_null_column.get_null_map_column().get_data().data();
+ nested_column = nested_null_column.get_nested_column_ptr();
+ } else {
+ nested_column = val_column.get_data_ptr();
+ }
+ if (check_column<ColumnInt8>(nested_column)) {
+ return _execute_number<ColumnInt8>(val_column.get_offsets(), *nested_column,
+ src_null_map, matched_indices,
+ nested_null_map, dst_null_map);
+ } else if (check_column<ColumnInt32>(nested_column)) {
+ _execute_number<ColumnInt32>(val_column.get_offsets(), *nested_column,
+ src_null_map, matched_indices,
+ nested_null_map, dst_null_map);
+ }
+ return nullptr;
+ }
+
+ ColumnPtr _mapped_key_string(const ColumnArray& column,
+ const ColumnWithTypeAndName& argument) {
+ auto right_column = argument.column->convert_to_full_column_if_const();
+ const ColumnString& match_key = reinterpret_cast<const ColumnString&>(*right_column);
+ const ColumnArray::Offsets64& offsets = column.get_offsets();
+ ColumnPtr nested_ptr = nullptr;
+ if (is_column_nullable(column.get_data())) {
+ nested_ptr = reinterpret_cast<const ColumnNullable&>(column.get_data()).get_nested_column_ptr();
+ } else {
+ nested_ptr = column.get_data_ptr();
+ }
+ const ColumnString& nested_key = reinterpret_cast<const ColumnString&>(*nested_ptr);
+ size_t rows = offsets.size();
+ // prepare return data
+ auto matched_indices = ColumnVector<Int8>::create();
+ matched_indices->reserve(rows);
+
+ for (size_t i = 0; i < rows; i++)
+ {
+ bool matched = false;
+ size_t begin = offsets[i - 1];
+ size_t end = offsets[i];
+ for (size_t j = begin; j < end; j++) {
+ if (nested_key.get_data_at(j) == match_key.get_data_at(i)) {
Review Comment:
done!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089841599
##########
be/src/olap/types.h:
##########
@@ -430,6 +431,103 @@ class ArrayTypeInfo : public TypeInfo {
TypeInfoPtr _item_type_info;
const size_t _item_size;
};
+///====================== MapType Info ==========================///
+class MapTypeInfo : public TypeInfo {
+public:
+ explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info)
+ : _key_type_info(std::move(key_type_info)), _value_type_info(std::move(value_type_info)) {}
Review Comment:
why ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1090117185
##########
be/src/vec/columns/column_map.h:
##########
@@ -0,0 +1,133 @@
+#pragma once
Review Comment:
done!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1093012479
##########
be/src/runtime/types.cpp:
##########
@@ -191,6 +197,14 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
children.push_back(TypeDescriptor(types, idx));
break;
}
+ case TTypeNodeType::MAP: {
+ type = TYPE_MAP;
Review Comment:
map now set has_contains_null == false from fe
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1093065559
##########
be/src/vec/columns/column_map.cpp:
##########
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnMap.cpp
+// and modified by Doris
+
+#include "vec/columns/column_map.h"
+
+namespace doris::vectorized {
+
+/** A column of map values.
+ */
+std::string ColumnMap::get_name() const {
+ return "Map(" + keys->get_name() + ", " + values->get_name() + ")";
+}
+
+ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values)
+ : keys(std::move(keys)), values(std::move(values)) {
+ check_size();
+}
+
+ColumnArray::Offsets64& ColumnMap::get_offsets() const {
+ const ColumnArray & column_keys = assert_cast<const ColumnArray &> (get_keys());
+ // todo . did here check size ?
+ return const_cast<Offsets64&>(column_keys.get_offsets());
+}
+
+void ColumnMap::check_size() const {
+ const auto * key_array = typeid_cast<const ColumnArray *>(keys.get());
+ const auto * value_array = typeid_cast<const ColumnArray *>(values.get());
+ CHECK(key_array) << "ColumnMap keys can be created only from array";
+ CHECK(value_array) << "ColumnMap values can be created only from array";
+ CHECK_EQ(get_keys_ptr()->size(), get_values_ptr()->size());
+}
+
+// todo. here to resize every row map
+MutableColumnPtr ColumnMap::clone_resized(size_t to_size) const {
+ auto res = ColumnMap::create(keys->clone_resized(to_size), values->clone_resized(to_size));
+ return res;
+}
+
+// to support field functions
+Field ColumnMap::operator[](size_t n) const {
+ Map res(2);
+ keys->get(n, res[0]);
+ values->get(n, res[0]);
+
+ return res;
+}
+
+// here to compare to below
+void ColumnMap::get(size_t n, Field & res) const {
+ Map map(2);
+ keys->get(n, map[0]);
+ values->get(n, map[1]);
+
+ res = map;
+}
+
+StringRef ColumnMap::get_data_at(size_t n) const {
+ LOG(FATAL) << "Method get_data_at is not supported for " << get_name();
+}
+
+void ColumnMap::insert_data(const char*, size_t) {
+ LOG(FATAL) << "Method insert_data is not supported for " << get_name();
+}
+
+void ColumnMap::insert(const Field& x) {
+ const auto& map = doris::vectorized::get<const Map&>(x);
+ // ({}, {}, {})
+ // ([], [])
+ CHECK_EQ(map.size(), 2);
+ keys->insert(map[0]);
+ values->insert(map[1]);
+}
+
+void ColumnMap::insert_default() {
Review Comment:
it should make array insert default to refresh offsets
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1093940463
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -695,5 +808,120 @@ Status ArrayColumnWriter::finish_current_page() {
return Status::NotSupported("array writer has no data, can not finish_current_page");
}
+/// ============================= MapColumnWriter =====================////
+MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
+ ScalarColumnWriter* null_writer,
+ std::unique_ptr<ColumnWriter> key_writer,
+ std::unique_ptr<ColumnWriter> value_writer)
+ : ColumnWriter(std::move(field), opts.meta->is_nullable()),
+ _key_writer(std::move(key_writer)),
+ _value_writer(std::move(value_writer)),
+ _opts(opts) {
+ if (is_nullable()) {
+ _null_writer.reset(null_writer);
+ }
+}
+
+Status MapColumnWriter::init() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->init());
+ }
+ RETURN_IF_ERROR(_key_writer->init());
+ RETURN_IF_ERROR(_value_writer->init());
+ return Status::OK();
+}
+
+uint64_t MapColumnWriter::estimate_buffer_size() {
+ size_t estimate = _key_writer->estimate_buffer_size() +
+ _value_writer->estimate_buffer_size();
+ if (is_nullable()) {
+ estimate += _null_writer->estimate_buffer_size();
+ }
+ return estimate;
+}
+
+Status MapColumnWriter::finish() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->finish());
+ }
+ RETURN_IF_ERROR(_key_writer->finish());
+ RETURN_IF_ERROR(_value_writer->finish());
+ return Status::OK();
+}
+
+// todo. make keys and values write
+Status MapColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
+ size_t remaining = num_rows;
+ const auto* col_cursor = reinterpret_cast<const MapValue*>(*ptr);
+ while (remaining > 0) {
+ size_t num_written = 1;
+ auto* key_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_key_data();
+ const uint8_t* key_ptr = (const uint8_t*)key_data_ptr;
+ RETURN_IF_ERROR(_key_writer->append_data(&key_ptr, 1));
+ auto* val_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_value_data();
+ const uint8_t* val_ptr = (const uint8_t*)val_data_ptr;
+ RETURN_IF_ERROR(_value_writer->append_data(&val_ptr, 1));
+ remaining -= num_written;
+ col_cursor += num_written;
+ *ptr += num_written * sizeof(MapValue);
+ }
+
+ if (is_nullable()) {
+ return write_null_column(num_rows, false);
+ }
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_data() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_data());
+ }
+ RETURN_IF_ERROR(_offset_writer->write_data());
+ RETURN_IF_ERROR(_key_writer->write_data());
+ RETURN_IF_ERROR(_value_writer->write_data());
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_ordinal_index() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_ordinal_index());
+ }
+ RETURN_IF_ERROR(_key_writer->write_ordinal_index());
+ RETURN_IF_ERROR(_value_writer->write_ordinal_index());
+
+ return Status::OK();
+}
+
+Status MapColumnWriter::append_nulls(size_t num_rows) {
+ return write_null_column(num_rows, true);
+}
+
+Status MapColumnWriter::write_null_column(size_t num_rows, bool is_null) {
+ uint8_t null_sign = is_null ? 1 : 0;
+ while (num_rows > 0) {
+ // TODO llj bulk write
+ const uint8_t* null_sign_ptr = &null_sign;
+ RETURN_IF_ERROR(_null_writer->append_data(&null_sign_ptr, 1));
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] github-actions[bot] commented on pull request #15966: [Feature](map) add map type to doris
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #15966:
URL: https://github.com/apache/doris/pull/15966#issuecomment-1414849661
PR approved by at least one committer and no changes requested.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089875841
##########
be/src/olap/types.h:
##########
@@ -430,6 +431,103 @@ class ArrayTypeInfo : public TypeInfo {
TypeInfoPtr _item_type_info;
const size_t _item_size;
};
+///====================== MapType Info ==========================///
+class MapTypeInfo : public TypeInfo {
+public:
+ explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info)
+ : _key_type_info(std::move(key_type_info)), _value_type_info(std::move(value_type_info)) {}
+ ~MapTypeInfo() override = default;
+
+ inline bool equal(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ return l_value->size() == r_value->size();
+ }
+
+ int cmp(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ uint32_t l_size = l_value->size();
+ uint32_t r_size = r_value->size();
+ if (l_size < r_size) {
+ return -1;
+ } else if (l_size > r_size) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ void shallow_copy(void* dest, const void* src) const override {
+ auto dest_value = reinterpret_cast<MapValue*>(dest);
+ auto src_value = reinterpret_cast<const MapValue*>(src);
+ dest_value->shallow_copy(src_value);
+ }
+
+ void deep_copy(void* dest, const void* src, MemPool* mem_pool) const override {
+ DCHECK(false);
+
+ }
+
+ void copy_object(void* dest, const void* src, MemPool* mem_pool) const override {
+ deep_copy(dest, src, mem_pool);
+ }
+
+ void direct_copy(void* dest, const void* src) const override {
+ CHECK(false);
+ }
+
+ void direct_copy(uint8_t** base, void* dest, const void* src) const {
+ CHECK(false);
+ }
+
+ void direct_copy_may_cut(void* dest, const void* src) const override { direct_copy(dest, src); }
+
+ Status convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool,
+ size_t variable_len = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ Status from_string(void* buf, const std::string& scan_key, const int precision = 0,
+ const int scale = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ std::string to_string(const void* src) const override {
+ return "{}";
+ }
+
+ void set_to_max(void* buf) const override {
+ DCHECK(false) << "set_to_max of list is not implemented.";
+ }
+
+ void set_to_min(void* buf) const override {
+ DCHECK(false) << "set_to_min of list is not implemented.";
+ }
+
+ uint32_t hash_code(const void* data, uint32_t seed) const override {
+ auto map_value = reinterpret_cast<const MapValue*>(data);
+ auto size = map_value->size();
+ uint32_t result = HashUtil::hash(&size, sizeof(size), seed);
+ result = seed * result + _key_type_info->hash_code(
+ map_value->key_data(), seed)
+ + _value_type_info->hash_code(
+ map_value->value_data(), seed);
+ return result;
+ }
+
+ // todo . is here only to need return 16 for two ptr?
+ const size_t size() const override { return 16; }
Review Comment:
ok
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089860257
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -94,8 +101,8 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo
break;
case TYPE_TIME:
case TYPE_TIMEV2:
- nested = std::make_shared<vectorized::DataTypeTime>();
- break;
+ nested = std::make_shared<vectorized::DataTypeTime>();
Review Comment:
code format ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1090114103
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
+ key_opts.meta->set_column_id(4);
+ key_opts.meta->set_unique_id(4);
+ key_opts.meta->set_type(OLAP_FIELD_TYPE_ARRAY);
+ key_opts.meta->set_length(0);
+ key_opts.meta->set_encoding(BIT_SHUFFLE);
+ key_opts.meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
+ key_opts.need_zone_map = false;
+ // no need key array's null map
+ key_opts.meta->set_is_nullable(false);
+
+ ColumnMetaPB* child_meta = key_opts.meta->add_children_columns();
+ child_meta->set_column_id(5);
+ child_meta->set_unique_id(5);
+ child_meta->set_type(key_column.type());
+ child_meta->set_length(key_column.length());
+ child_meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
+ child_meta->set_encoding(DICT_ENCODING);
Review Comment:
done!
##########
be/src/olap/types.h:
##########
@@ -430,6 +431,103 @@ class ArrayTypeInfo : public TypeInfo {
TypeInfoPtr _item_type_info;
const size_t _item_size;
};
+///====================== MapType Info ==========================///
+class MapTypeInfo : public TypeInfo {
+public:
+ explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info)
+ : _key_type_info(std::move(key_type_info)), _value_type_info(std::move(value_type_info)) {}
+ ~MapTypeInfo() override = default;
+
+ inline bool equal(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ return l_value->size() == r_value->size();
+ }
+
+ int cmp(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ uint32_t l_size = l_value->size();
+ uint32_t r_size = r_value->size();
+ if (l_size < r_size) {
+ return -1;
+ } else if (l_size > r_size) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ void shallow_copy(void* dest, const void* src) const override {
+ auto dest_value = reinterpret_cast<MapValue*>(dest);
+ auto src_value = reinterpret_cast<const MapValue*>(src);
+ dest_value->shallow_copy(src_value);
+ }
+
+ void deep_copy(void* dest, const void* src, MemPool* mem_pool) const override {
+ DCHECK(false);
+
+ }
+
+ void copy_object(void* dest, const void* src, MemPool* mem_pool) const override {
+ deep_copy(dest, src, mem_pool);
+ }
+
+ void direct_copy(void* dest, const void* src) const override {
+ CHECK(false);
+ }
+
+ void direct_copy(uint8_t** base, void* dest, const void* src) const {
+ CHECK(false);
+ }
+
+ void direct_copy_may_cut(void* dest, const void* src) const override { direct_copy(dest, src); }
+
+ Status convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool,
+ size_t variable_len = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ Status from_string(void* buf, const std::string& scan_key, const int precision = 0,
+ const int scale = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ std::string to_string(const void* src) const override {
+ return "{}";
+ }
+
+ void set_to_max(void* buf) const override {
+ DCHECK(false) << "set_to_max of list is not implemented.";
+ }
+
+ void set_to_min(void* buf) const override {
+ DCHECK(false) << "set_to_min of list is not implemented.";
+ }
+
+ uint32_t hash_code(const void* data, uint32_t seed) const override {
+ auto map_value = reinterpret_cast<const MapValue*>(data);
+ auto size = map_value->size();
+ uint32_t result = HashUtil::hash(&size, sizeof(size), seed);
+ result = seed * result + _key_type_info->hash_code(
+ map_value->key_data(), seed)
+ + _value_type_info->hash_code(
+ map_value->value_data(), seed);
+ return result;
+ }
+
+ // todo . is here only to need return 16 for two ptr?
+ const size_t size() const override { return 16; }
Review Comment:
done!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1090112631
##########
be/src/vec/data_types/data_type_map.cpp:
##########
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "data_type_map.h"
+
+#include "gen_cpp/data.pb.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/common/assert_cast.h"
+
+namespace doris::vectorized {
+
+DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_)
+{
+ key_type = keys_;
+ value_type = values_;
+
+ keys = std::make_shared<DataTypeArray>(key_type);
+ values = std::make_shared<DataTypeArray>(value_type);
+}
+
+std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ std::stringstream ss;
+ ss << "{";
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ss << ", ";
+ ss << "'" << keys->to_string(nested_keys, i);
Review Comment:
done!
##########
be/src/vec/data_types/data_type_map.cpp:
##########
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "data_type_map.h"
+
+#include "gen_cpp/data.pb.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/common/assert_cast.h"
+
+namespace doris::vectorized {
+
+DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_)
+{
+ key_type = keys_;
+ value_type = values_;
+
+ keys = std::make_shared<DataTypeArray>(key_type);
+ values = std::make_shared<DataTypeArray>(value_type);
+}
+
+std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ std::stringstream ss;
+ ss << "{";
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ss << ", ";
+ ss << "'" << keys->to_string(nested_keys, i);
+ ss << ':';
+ ss << "'" << values->to_string(nested_values, i);
+ }
+ ss << "}";
+ return ss.str();
+}
+
+void DataTypeMap::to_string(const class doris::vectorized::IColumn& column, size_t row_num,
Review Comment:
done!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] xy720 commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "xy720 (via GitHub)" <gi...@apache.org>.
xy720 commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1094130651
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -174,218 +174,220 @@ Status VExpr::create_expr(doris::ObjectPool* pool, const doris::TExprNode& texpr
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
Review Comment:
Why modify here?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1092997703
##########
be/src/vec/columns/column_map.h:
##########
@@ -0,0 +1,133 @@
+#pragma once
+
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1093017275
##########
be/src/vec/sink/vmysql_result_writer.cpp:
##########
@@ -715,6 +732,18 @@ Status VMysqlResultWriter::append_block(Block& input_block) {
}
break;
}
+ case TYPE_MAP: {
+ if (type_ptr->is_nullable()) {
+ auto& nested_type =
+ assert_cast<const DataTypeNullable&>(*type_ptr).get_nested_type(); //for map
+ status = _add_one_column<PrimitiveType::TYPE_MAP, true>(column_ptr, result,
Review Comment:
done !
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1093049679
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -695,5 +808,120 @@
return Status::NotSupported("array writer has no data, can not finish_current_page");
}
+/// ============================= MapColumnWriter =====================////
+MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
+ ScalarColumnWriter* null_writer,
+ std::unique_ptr<ColumnWriter> key_writer,
+ std::unique_ptr<ColumnWriter> value_writer)
+ : ColumnWriter(std::move(field), opts.meta->is_nullable()),
+ _key_writer(std::move(key_writer)),
+ _value_writer(std::move(value_writer)),
+ _opts(opts) {
+ if (is_nullable()) {
+ _null_writer.reset(null_writer);
+ }
+}
+
+Status MapColumnWriter::init() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->init());
+ }
+ RETURN_IF_ERROR(_key_writer->init());
+ RETURN_IF_ERROR(_value_writer->init());
+ return Status::OK();
+}
+
+uint64_t MapColumnWriter::estimate_buffer_size() {
+ size_t estimate = _key_writer->estimate_buffer_size() +
+ _value_writer->estimate_buffer_size();
+ if (is_nullable()) {
+ estimate += _null_writer->estimate_buffer_size();
+ }
+ return estimate;
+}
+
+Status MapColumnWriter::finish() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->finish());
+ }
+ RETURN_IF_ERROR(_key_writer->finish());
+ RETURN_IF_ERROR(_value_writer->finish());
+ return Status::OK();
+}
+
+// todo. make keys and values write
+Status MapColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
+ size_t remaining = num_rows;
+ const auto* col_cursor = reinterpret_cast<const MapValue*>(*ptr);
+ while (remaining > 0) {
+ size_t num_written = 1;
+ auto* key_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_key_data();
+ const uint8_t* key_ptr = (const uint8_t*)key_data_ptr;
+ RETURN_IF_ERROR(_key_writer->append_data(&key_ptr, 1));
+ auto* val_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_value_data();
+ const uint8_t* val_ptr = (const uint8_t*)val_data_ptr;
+ RETURN_IF_ERROR(_value_writer->append_data(&val_ptr, 1));
+ remaining -= num_written;
+ col_cursor += num_written;
+ *ptr += num_written * sizeof(MapValue);
+ }
+
+ if (is_nullable()) {
+ return write_null_column(num_rows, false);
+ }
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_data() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_data());
+ }
+ RETURN_IF_ERROR(_offset_writer->write_data());
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1082019883
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -449,13 +478,97 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this,null_iterator,
+ key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator: {}",
std::to_string(type));
}
}
}
+///====================== MapFileColumnIterator ============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::_peek_one_offset(ordinal_t* offset) {
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) {
+ return Status::OK();
+}
+
+
+Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(ord));
+ RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(ord));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+ const auto* column_map = vectorized::check_and_get_column<vectorized::ColumnMap>(
+ dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
+ : *dst);
+ auto column_key_ptr = column_map->get_keys().assume_mutable();
+ auto column_val_ptr = column_map->get_values().assume_mutable();
+ RETURN_IF_ERROR(_key_iterator->next_batch(n, column_key_ptr, has_null));
+ RETURN_IF_ERROR(_val_iterator->next_batch(n, column_val_ptr, has_null));
+
+ if (dst->is_nullable()) {
+ auto null_map_ptr =
+ static_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column_ptr();
+ size_t num_read = *n;
+ bool null_signs_has_null = false;
+ RETURN_IF_ERROR(_null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
+ DCHECK(num_read == *n);
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
+ vectorized::MutableColumnPtr& dst) {
+ for (size_t i = 0; i < count; ++i) {
+ RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
+ size_t num_read = 1;
+ RETURN_IF_ERROR(next_batch(&num_read, dst, nullptr));
Review Comment:
Here is inner array null map which will not have , so I do not think it will be wrong..
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1082016577
##########
be/src/olap/field.h:
##########
@@ -450,6 +451,29 @@ uint32_t Field::hash_code(const CellType& cell, uint32_t seed) const {
return _type_info->hash_code(cell.cell_ptr(), seed);
}
+class MapField : public Field {
+public:
+ explicit MapField(const TabletColumn& column) : Field(column) {}
+
+ void consume(RowCursorCell* dst, const char* src, bool src_null, MemPool* mem_pool,
+ ObjectPool* agg_pool) const override {
+ dst->set_is_null(src_null);
+ if (src_null) {
+ return;
+ }
+ _type_info->deep_copy(dst->mutable_cell_ptr(), src, mem_pool);
+ }
+
+ // make variable_ptr memory allocate to cell_ptr as MapValue
+ char* allocate_memory(char* cell_ptr, char* variable_ptr) const override {
+ return variable_ptr + _length;
+ }
+
+ size_t get_variable_len() const override {
+ return _length;
Review Comment:
_length just means variable for field , and here is row-based code ....
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1082016259
##########
be/src/olap/field.h:
##########
@@ -450,6 +451,29 @@ uint32_t Field::hash_code(const CellType& cell, uint32_t seed) const {
return _type_info->hash_code(cell.cell_ptr(), seed);
}
+class MapField : public Field {
+public:
+ explicit MapField(const TabletColumn& column) : Field(column) {}
+
+ void consume(RowCursorCell* dst, const char* src, bool src_null, MemPool* mem_pool,
+ ObjectPool* agg_pool) const override {
+ dst->set_is_null(src_null);
+ if (src_null) {
+ return;
+ }
+ _type_info->deep_copy(dst->mutable_cell_ptr(), src, mem_pool);
+ }
+
+ // make variable_ptr memory allocate to cell_ptr as MapValue
Review Comment:
I will try again
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1082024425
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
Review Comment:
is same as null writer ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1084803369
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
+ key_opts.meta->set_column_id(4);
+ key_opts.meta->set_unique_id(4);
+ key_opts.meta->set_type(OLAP_FIELD_TYPE_ARRAY);
+ key_opts.meta->set_length(0);
+ key_opts.meta->set_encoding(BIT_SHUFFLE);
+ key_opts.meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
+ key_opts.need_zone_map = false;
+ // no need key array's null map
+ key_opts.meta->set_is_nullable(false);
+
+ ColumnMetaPB* child_meta = key_opts.meta->add_children_columns();
Review Comment:
I can use opts.meta directly
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] github-actions[bot] commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1094172591
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,6 +179,12 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_nulls[0]),
+ create_data_type(col_desc.children[1], col_desc.contains_nulls[1]));
+ break;
}
case INVALID_TYPE:
Review Comment:
warning: 'case' statement not in switch statement [clang-diagnostic-error]
```cpp
case INVALID_TYPE:
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,6 +179,12 @@
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_nulls[0]),
+ create_data_type(col_desc.children[1], col_desc.contains_nulls[1]));
+ break;
}
case INVALID_TYPE:
default:
Review Comment:
warning: 'default' statement not in switch statement [clang-diagnostic-error]
```cpp
default:
^
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1073144858
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -449,13 +478,97 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this,null_iterator,
+ key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator: {}",
std::to_string(type));
}
}
}
+///====================== MapFileColumnIterator ============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::_peek_one_offset(ordinal_t* offset) {
+ return Status::OK();
Review Comment:
I will delete this method which is copied from array , but map does not use it ...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] xy720 merged pull request #15966: [Feature](map) add map type to doris
Posted by "xy720 (via GitHub)" <gi...@apache.org>.
xy720 merged PR #15966:
URL: https://github.com/apache/doris/pull/15966
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089854094
##########
be/src/vec/columns/column_map.cpp:
##########
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnMap.cpp
+// and modified by Doris
+
+#include "vec/columns/column_map.h"
+
+namespace doris::vectorized {
+
+/** A column of map values.
+ */
+std::string ColumnMap::get_name() const {
+ return "Map(" + keys->get_name() + ", " + values->get_name() + ")";
+}
+
+ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values)
+ : keys(std::move(keys)), values(std::move(values)) {
+ check_size();
+}
+
+ColumnArray::Offsets64& ColumnMap::get_offsets() const {
+ const ColumnArray & column_keys = assert_cast<const ColumnArray &> (get_keys());
+ // todo . did here check size ?
+ return const_cast<Offsets64&>(column_keys.get_offsets());
+}
+
+void ColumnMap::check_size() const {
+ const auto * key_array = typeid_cast<const ColumnArray *>(keys.get());
+ const auto * value_array = typeid_cast<const ColumnArray *>(values.get());
+ CHECK(key_array) << "ColumnMap keys can be created only from array";
+ CHECK(value_array) << "ColumnMap values can be created only from array";
+ CHECK_EQ(get_keys_ptr()->size(), get_values_ptr()->size());
+}
+
+// todo. here to resize every row map
+MutableColumnPtr ColumnMap::clone_resized(size_t to_size) const {
+ auto res = ColumnMap::create(keys->clone_resized(to_size), values->clone_resized(to_size));
+ return res;
+}
+
+// to support field functions
+Field ColumnMap::operator[](size_t n) const {
+ Map res(2);
+ keys->get(n, res[0]);
+ values->get(n, res[0]);
+
+ return res;
+}
+
+// here to compare to below
+void ColumnMap::get(size_t n, Field & res) const {
+ Map map(2);
+ keys->get(n, map[0]);
+ values->get(n, map[1]);
+
+ res = map;
+}
+
+StringRef ColumnMap::get_data_at(size_t n) const {
+ LOG(FATAL) << "Method get_data_at is not supported for " << get_name();
+}
+
+void ColumnMap::insert_data(const char*, size_t) {
+ LOG(FATAL) << "Method insert_data is not supported for " << get_name();
+}
+
+void ColumnMap::insert(const Field& x) {
+ const auto& map = doris::vectorized::get<const Map&>(x);
+ // ({}, {}, {})
+ // ([], [])
+ CHECK_EQ(map.size(), 2);
+ keys->insert(map[0]);
+ values->insert(map[1]);
+}
+
+void ColumnMap::insert_default() {
Review Comment:
so this function should do nothing?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] xy720 commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
xy720 commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1072095959
##########
be/src/olap/rowset/segment_v2/column_reader.h:
##########
@@ -393,6 +393,44 @@ class EmptyFileColumnIterator final : public ColumnIterator {
ordinal_t get_current_ordinal() const override { return 0; }
};
+// This iterator is used to read map value column
+class MapFileColumnIterator final : public ColumnIterator {
+public:
+ explicit MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator, ColumnIterator* val_iterator);
+
+ ~MapFileColumnIterator() override = default;
+
+ Status init(const ColumnIteratorOptions& opts) override;
+
+ Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) override;
+
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst, bool* has_null) override;
+
+ Status read_by_rowids(const rowid_t* rowids, const size_t count,
+ vectorized::MutableColumnPtr& dst) override;
+
+ Status seek_to_first() override {
+ RETURN_IF_ERROR(_key_iterator->seek_to_first());
+ RETURN_IF_ERROR(_val_iterator->seek_to_first());
+ return Status::OK();
+ }
+
+ Status seek_to_ordinal(ordinal_t ord) override;
+
+ ordinal_t get_current_ordinal() const override {
+ return _key_iterator->get_current_ordinal();
+ }
+
+private:
+ ColumnReader* _map_reader; // need ?
+ std::unique_ptr<ColumnIterator> _null_iterator;
+ std::unique_ptr<ColumnIterator> _key_iterator; // ArrayFileColumnIterator
+ std::unique_ptr<ColumnIterator> _val_iterator; // ArrayFileColumnIterator
+
+ Status _peek_one_offset(ordinal_t* offset);
Review Comment:
Do we need this method?
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -449,13 +478,97 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this,null_iterator,
+ key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator: {}",
std::to_string(type));
}
}
}
+///====================== MapFileColumnIterator ============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::_peek_one_offset(ordinal_t* offset) {
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) {
+ return Status::OK();
+}
+
+
+Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(ord));
+ RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(ord));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+ const auto* column_map = vectorized::check_and_get_column<vectorized::ColumnMap>(
+ dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
+ : *dst);
+ auto column_key_ptr = column_map->get_keys().assume_mutable();
+ auto column_val_ptr = column_map->get_values().assume_mutable();
+ RETURN_IF_ERROR(_key_iterator->next_batch(n, column_key_ptr, has_null));
Review Comment:
use a local value to pass into the method.
Such as
```
size_t num_read = *n;
RETURN_IF_ERROR(_key_iterator->next_batch(&num_read, column_key_ptr, has_null));
```
##########
be/src/olap/rowset/segment_v2/column_reader.h:
##########
@@ -393,6 +393,44 @@ class EmptyFileColumnIterator final : public ColumnIterator {
ordinal_t get_current_ordinal() const override { return 0; }
};
+// This iterator is used to read map value column
+class MapFileColumnIterator final : public ColumnIterator {
+public:
+ explicit MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator, ColumnIterator* val_iterator);
+
+ ~MapFileColumnIterator() override = default;
+
+ Status init(const ColumnIteratorOptions& opts) override;
+
+ Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) override;
+
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst, bool* has_null) override;
+
+ Status read_by_rowids(const rowid_t* rowids, const size_t count,
+ vectorized::MutableColumnPtr& dst) override;
+
+ Status seek_to_first() override {
+ RETURN_IF_ERROR(_key_iterator->seek_to_first());
+ RETURN_IF_ERROR(_val_iterator->seek_to_first());
Review Comment:
forget _null_iterator ?
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
Review Comment:
opts.meta and its children have been initialized outside.
See init_column_meta() function in segment_writer.cpp.
So here we should not change the column_id/unique_id/encoding/compression.
We just need to reset the type and nullable and length.
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
+ key_opts.meta->set_column_id(4);
+ key_opts.meta->set_unique_id(4);
+ key_opts.meta->set_type(OLAP_FIELD_TYPE_ARRAY);
+ key_opts.meta->set_length(0);
+ key_opts.meta->set_encoding(BIT_SHUFFLE);
+ key_opts.meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
+ key_opts.need_zone_map = false;
+ // no need key array's null map
+ key_opts.meta->set_is_nullable(false);
+
+ ColumnMetaPB* child_meta = key_opts.meta->add_children_columns();
+ child_meta->set_column_id(5);
+ child_meta->set_unique_id(5);
+ child_meta->set_type(key_column.type());
+ child_meta->set_length(key_column.length());
+ child_meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
+ child_meta->set_encoding(DICT_ENCODING);
Review Comment:
why the encoding is DICT_ENCODING?
May be DEFAULT_ENCODING ?
##########
be/src/olap/types.h:
##########
@@ -430,6 +431,103 @@ class ArrayTypeInfo : public TypeInfo {
TypeInfoPtr _item_type_info;
const size_t _item_size;
};
+///====================== MapType Info ==========================///
+class MapTypeInfo : public TypeInfo {
+public:
+ explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info)
+ : _key_type_info(std::move(key_type_info)), _value_type_info(std::move(value_type_info)) {}
+ ~MapTypeInfo() override = default;
+
+ inline bool equal(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ return l_value->size() == r_value->size();
+ }
+
+ int cmp(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ uint32_t l_size = l_value->size();
+ uint32_t r_size = r_value->size();
+ if (l_size < r_size) {
+ return -1;
+ } else if (l_size > r_size) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ void shallow_copy(void* dest, const void* src) const override {
+ auto dest_value = reinterpret_cast<MapValue*>(dest);
+ auto src_value = reinterpret_cast<const MapValue*>(src);
+ dest_value->shallow_copy(src_value);
+ }
+
+ void deep_copy(void* dest, const void* src, MemPool* mem_pool) const override {
+ DCHECK(false);
+
+ }
+
+ void copy_object(void* dest, const void* src, MemPool* mem_pool) const override {
+ deep_copy(dest, src, mem_pool);
+ }
+
+ void direct_copy(void* dest, const void* src) const override {
+ CHECK(false);
+ }
+
+ void direct_copy(uint8_t** base, void* dest, const void* src) const {
+ CHECK(false);
+ }
+
+ void direct_copy_may_cut(void* dest, const void* src) const override { direct_copy(dest, src); }
+
+ Status convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool,
+ size_t variable_len = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ Status from_string(void* buf, const std::string& scan_key, const int precision = 0,
+ const int scale = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ std::string to_string(const void* src) const override {
+ return "{}";
+ }
+
+ void set_to_max(void* buf) const override {
+ DCHECK(false) << "set_to_max of list is not implemented.";
+ }
+
+ void set_to_min(void* buf) const override {
+ DCHECK(false) << "set_to_min of list is not implemented.";
+ }
+
+ uint32_t hash_code(const void* data, uint32_t seed) const override {
+ auto map_value = reinterpret_cast<const MapValue*>(data);
+ auto size = map_value->size();
+ uint32_t result = HashUtil::hash(&size, sizeof(size), seed);
+ result = seed * result + _key_type_info->hash_code(
+ map_value->key_data(), seed)
+ + _value_type_info->hash_code(
+ map_value->value_data(), seed);
+ return result;
+ }
+
+ // todo . is here only to need return 16 for two ptr?
+ const size_t size() const override { return 16; }
Review Comment:
sizeof(MapValue)
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
+ key_opts.meta->set_column_id(4);
+ key_opts.meta->set_unique_id(4);
+ key_opts.meta->set_type(OLAP_FIELD_TYPE_ARRAY);
+ key_opts.meta->set_length(0);
+ key_opts.meta->set_encoding(BIT_SHUFFLE);
+ key_opts.meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
+ key_opts.need_zone_map = false;
+ // no need key array's null map
+ key_opts.meta->set_is_nullable(false);
+
+ ColumnMetaPB* child_meta = key_opts.meta->add_children_columns();
Review Comment:
Better set child_meta's values before set key_opts.meta's values,
because we can use the values in key_opts.meta to set child_meta's values.
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
+ key_opts.meta->set_column_id(4);
+ key_opts.meta->set_unique_id(4);
+ key_opts.meta->set_type(OLAP_FIELD_TYPE_ARRAY);
+ key_opts.meta->set_length(0);
+ key_opts.meta->set_encoding(BIT_SHUFFLE);
+ key_opts.meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
+ key_opts.need_zone_map = false;
+ // no need key array's null map
+ key_opts.meta->set_is_nullable(false);
+
+ ColumnMetaPB* child_meta = key_opts.meta->add_children_columns();
+ child_meta->set_column_id(5);
+ child_meta->set_unique_id(5);
+ child_meta->set_type(key_column.type());
+ child_meta->set_length(key_column.length());
+ child_meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
Review Comment:
the compression type should be same in one tablet_schema.
use key_opts.meta->compression().
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] xy720 commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
xy720 commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1072244337
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
+ key_opts.meta->set_column_id(4);
+ key_opts.meta->set_unique_id(4);
+ key_opts.meta->set_type(OLAP_FIELD_TYPE_ARRAY);
+ key_opts.meta->set_length(0);
+ key_opts.meta->set_encoding(BIT_SHUFFLE);
+ key_opts.meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
+ key_opts.need_zone_map = false;
+ // no need key array's null map
+ key_opts.meta->set_is_nullable(false);
+
+ ColumnMetaPB* child_meta = key_opts.meta->add_children_columns();
+ child_meta->set_column_id(5);
+ child_meta->set_unique_id(5);
+ child_meta->set_type(key_column.type());
+ child_meta->set_length(key_column.length());
+ child_meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
+ child_meta->set_encoding(DICT_ENCODING);
+ child_meta->set_is_nullable(key_column.is_nullable());
+ }
+
+ RETURN_IF_ERROR(
+ ColumnWriter::create(key_opts, &key_list_column, file_writer, &key_writer));
+
+
+ // create value writer
+ std::unique_ptr<ColumnWriter> value_writer;
Review Comment:
value_writer and key_writer look very same.
Why not use for loop to clean the code.
##########
be/src/vec/columns/column_map.h:
##########
@@ -0,0 +1,133 @@
+#pragma once
+
Review Comment:
Add a comment here if this file is copied from Clickhouse and modified by Doris.
##########
be/src/vec/core/field.h:
##########
@@ -85,6 +85,7 @@ using FieldVector = std::vector<Field>;
DEFINE_FIELD_VECTOR(Array);
DEFINE_FIELD_VECTOR(Tuple);
+DEFINE_FIELD_VECTOR(Map);
Review Comment:
Now is vector<Field>{key1, val1, key2, val2} with double size?
##########
be/src/vec/columns/column_map.h:
##########
@@ -0,0 +1,133 @@
+#pragma once
Review Comment:
Add licence.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] xy720 commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
xy720 commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1072312487
##########
be/src/vec/core/field.h:
##########
@@ -85,6 +85,7 @@ using FieldVector = std::vector<Field>;
DEFINE_FIELD_VECTOR(Array);
DEFINE_FIELD_VECTOR(Tuple);
+DEFINE_FIELD_VECTOR(Map);
Review Comment:
Now is vector<Field>{key1, val1, key2, val2} with double size?
May be we should refactor this to use a map to store the keys and values later.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089861089
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java:
##########
@@ -318,6 +318,11 @@ public void analyze() throws AnalysisException {
type, Function.NullableMode.ALWAYS_NULLABLE,
Lists.newArrayList(Type.VARCHAR), false,
"doris::CastFunctions::cast_to_array_val", null, null, true);
+ } else if (type.isMapType()) {
+ fn = ScalarFunction.createBuiltin(getFnName(Type.MAP),
+ type, Function.NullableMode.ALWAYS_NULLABLE,
+ Lists.newArrayList(Type.VARCHAR), false,
+ "doris::CastFunctions::cast_to_map_val", null, null, true);
Review Comment:
emm no, so its fake logic for passing map element function expr from fe to be
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089860918
##########
be/src/vec/functions/array/function_array_element.h:
##########
@@ -81,6 +94,79 @@ class FunctionArrayElement : public IFunction {
}
private:
+ //=========================== map element===========================//
+ ColumnPtr _get_mapped_idx(const ColumnArray& key_column,
+ const ColumnWithTypeAndName& argument) {
+ if (key_column.get_data().is_column_string()) {
+ return _mapped_key_string(key_column, argument);
+ }
+ return nullptr;
Review Comment:
using column.compare_at can cover other key types
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1093001951
##########
be/src/vec/olap/olap_data_convertor.cpp:
##########
@@ -129,6 +128,20 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co
return std::make_unique<OlapColumnDataConvertorArray>(
create_olap_column_data_convertor(sub_column));
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ const auto& key_column = column.get_sub_column(0);
+ const auto& value_column = column.get_sub_column(1);
+ return std::make_unique<OlapColumnDataConvertorMap>(
+ std::make_unique<OlapColumnDataConvertorArray>(
+ create_olap_column_data_convertor(key_column)),
+ std::make_unique<OlapColumnDataConvertorArray>(
+ create_olap_column_data_convertor(value_column)));
+ //const auto& key_column = column.get_sub_column(0);
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1093019844
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/MapType.java:
##########
@@ -17,18 +17,25 @@
package org.apache.doris.catalog;
+import org.apache.doris.thrift.TColumnType;
import org.apache.doris.thrift.TTypeDesc;
import org.apache.doris.thrift.TTypeNode;
import org.apache.doris.thrift.TTypeNodeType;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.gson.annotations.SerializedName;
+
+import java.util.Objects;
Review Comment:
now we set nullable true for key value column
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] github-actions[bot] commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by github-actions.
github-actions[bot] commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089889453
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -449,13 +476,91 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this, null_iterator, key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator: {}",
std::to_string(type));
}
}
}
+///====================== MapFileColumnIterator ============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) {
+ return Status::NotSupported("Not support next_batch");
+}
+
+Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(ord));
+ RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(ord));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+ const auto* column_map = vectorized::check_and_get_column<vectorized::ColumnMap>(
+ dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
+ : *dst);
+ size_t num_read = *n;
+ auto column_key_ptr = column_map->get_keys().assume_mutable();
+ auto column_val_ptr = column_map->get_values().assume_mutable();
+ RETURN_IF_ERROR(_key_iterator->next_batch(num_read, column_key_ptr, has_null));
Review Comment:
warning: no matching member function for call to 'next_batch' [clang-diagnostic-error]
```cpp
RETURN_IF_ERROR(_key_iterator->next_batch(num_read, column_key_ptr, has_null));
^
```
**be/src/common/status.h:487:** expanded from macro 'RETURN_IF_ERROR'
```cpp
Status _status_ = (stmt); \
^
```
**be/src/olap/rowset/segment_v2/column_reader.h:276:** candidate function not viable: no known conversion from 'size_t' (aka 'unsigned long') to 'size_t *' (aka 'unsigned long *') for 1st argument; take the address of the argument with &
```cpp
virtual Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst, bool* has_null) {
^
```
**be/src/olap/rowset/segment_v2/column_reader.h:274:** candidate function not viable: no known conversion from 'size_t' (aka 'unsigned long') to 'size_t *' (aka 'unsigned long *') for 1st argument; take the address of the argument with &
```cpp
virtual Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) = 0;
^
```
**be/src/olap/rowset/segment_v2/column_reader.h:261:** candidate function not viable: requires 2 arguments, but 3 were provided
```cpp
Status next_batch(size_t* n, ColumnBlockView* dst) {
^
```
**be/src/olap/rowset/segment_v2/column_reader.h:266:** candidate function not viable: requires 2 arguments, but 3 were provided
```cpp
Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst) {
^
```
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -449,13 +476,91 @@
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this, null_iterator, key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator: {}",
std::to_string(type));
}
}
}
+///====================== MapFileColumnIterator ============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) {
+ return Status::NotSupported("Not support next_batch");
+}
+
+Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(ord));
+ RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(ord));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+ const auto* column_map = vectorized::check_and_get_column<vectorized::ColumnMap>(
+ dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
+ : *dst);
+ size_t num_read = *n;
+ auto column_key_ptr = column_map->get_keys().assume_mutable();
+ auto column_val_ptr = column_map->get_values().assume_mutable();
+ RETURN_IF_ERROR(_key_iterator->next_batch(num_read, column_key_ptr, has_null));
+ RETURN_IF_ERROR(_val_iterator->next_batch(num_read, column_val_ptr, has_null));
Review Comment:
warning: no matching member function for call to 'next_batch' [clang-diagnostic-error]
```cpp
RETURN_IF_ERROR(_val_iterator->next_batch(num_read, column_val_ptr, has_null));
^
```
**be/src/common/status.h:487:** expanded from macro 'RETURN_IF_ERROR'
```cpp
Status _status_ = (stmt); \
^
```
**be/src/olap/rowset/segment_v2/column_reader.h:276:** candidate function not viable: no known conversion from 'size_t' (aka 'unsigned long') to 'size_t *' (aka 'unsigned long *') for 1st argument; take the address of the argument with &
```cpp
virtual Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst, bool* has_null) {
^
```
**be/src/olap/rowset/segment_v2/column_reader.h:274:** candidate function not viable: no known conversion from 'size_t' (aka 'unsigned long') to 'size_t *' (aka 'unsigned long *') for 1st argument; take the address of the argument with &
```cpp
virtual Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) = 0;
^
```
**be/src/olap/rowset/segment_v2/column_reader.h:261:** candidate function not viable: requires 2 arguments, but 3 were provided
```cpp
Status next_batch(size_t* n, ColumnBlockView* dst) {
^
```
**be/src/olap/rowset/segment_v2/column_reader.h:266:** candidate function not viable: requires 2 arguments, but 3 were provided
```cpp
Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst) {
^
```
##########
be/src/runtime/map_value.h:
##########
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <type_traits>
+
+#include "runtime/primitive_type.h"
+
+namespace doris_udf {
+class FunctionContext;
+struct AnyVal;
+} // namespace doris_udf
+
+namespace doris {
+
+using doris_udf::FunctionContext;
Review Comment:
warning: using decl 'FunctionContext' is unused [misc-unused-using-decls]
```cpp
using doris_udf::FunctionContext;
^
```
**be/src/runtime/map_value.h:30:** remove the using
```cpp
using doris_udf::FunctionContext;
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -45,6 +50,11 @@
if (col_desc.type() == OLAP_FIELD_TYPE_ARRAY) {
DCHECK(col_desc.get_subtype_count() == 1);
nested = std::make_shared<DataTypeArray>(create_data_type(col_desc.get_sub_column(0)));
+ } else if (col_desc.type() == OLAP_FIELD_TYPE_MAP) {
+ DCHECK(col_desc.get_subtype_count() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
Review Comment:
warning: no member named 'DataTypeMap' in namespace 'doris::vectorized' [clang-diagnostic-error]
```cpp
nested = std::make_shared<vectorized::DataTypeMap>(
^
```
##########
be/src/udf/udf.h:
##########
@@ -927,6 +951,7 @@ using doris_udf::DateTimeVal;
using doris_udf::HllVal;
using doris_udf::FunctionContext;
using doris_udf::CollectionVal;
+using doris_udf::MapVal;
Review Comment:
warning: using decl 'MapVal' is unused [misc-unused-using-decls]
```cpp
using doris_udf::MapVal;
^
```
**be/src/udf/udf.h:953:** remove the using
```cpp
using doris_udf::MapVal;
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -299,6 +315,13 @@
case PGenericType::FIXEDLENGTHOBJECT:
nested = std::make_shared<DataTypeFixedLengthObject>();
break;
+ case PGenericType::MAP:
+ DCHECK(pcolumn.children_size() == 2);
+ // here to check pcolumn is list?
+ nested = std::make_shared<vectorized::DataTypeMap>(
Review Comment:
warning: no member named 'DataTypeMap' in namespace 'doris::vectorized' [clang-diagnostic-error]
```cpp
nested = std::make_shared<vectorized::DataTypeMap>(
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -29,6 +29,11 @@ DataTypePtr DataTypeFactory::create_data_type(const doris::Field& col_desc) {
if (col_desc.type() == OLAP_FIELD_TYPE_ARRAY) {
DCHECK(col_desc.get_sub_field_count() == 1);
nested = std::make_shared<DataTypeArray>(create_data_type(*col_desc.get_sub_field(0)));
+ } else if (col_desc.type() == OLAP_FIELD_TYPE_MAP) {
+ DCHECK(col_desc.get_sub_field_count() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
Review Comment:
warning: no member named 'DataTypeMap' in namespace 'doris::vectorized' [clang-diagnostic-error]
```cpp
nested = std::make_shared<vectorized::DataTypeMap>(
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -131,6 +141,12 @@
nested = std::make_shared<vectorized::DataTypeArray>(
create_data_type(col_desc.children[0], col_desc.contains_null));
break;
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
Review Comment:
warning: no member named 'DataTypeMap' in namespace 'doris::vectorized' [clang-diagnostic-error]
```cpp
nested = std::make_shared<vectorized::DataTypeMap>(
^
```
##########
be/src/runtime/map_value.h:
##########
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <type_traits>
+
+#include "runtime/primitive_type.h"
+
+namespace doris_udf {
+class FunctionContext;
+struct AnyVal;
+} // namespace doris_udf
+
+namespace doris {
+
+using doris_udf::FunctionContext;
+using doris_udf::AnyVal;
Review Comment:
warning: using decl 'AnyVal' is unused [misc-unused-using-decls]
```cpp
using doris_udf::AnyVal;
^
```
**be/src/runtime/map_value.h:31:** remove the using
```cpp
using doris_udf::AnyVal;
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -368,6 +391,12 @@
nested = std::make_shared<vectorized::DataTypeArray>(
create_data_type(type->field(0)->type().get(), true));
break;
+ case ::arrow::Type::MAP:
+ DCHECK(type->num_fields() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
Review Comment:
warning: no member named 'DataTypeMap' in namespace 'doris::vectorized' [clang-diagnostic-error]
```cpp
nested = std::make_shared<vectorized::DataTypeMap>(
^
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1090113036
##########
be/src/vec/columns/column_map.cpp:
##########
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnMap.cpp
+// and modified by Doris
+
+#include "vec/columns/column_map.h"
+
+namespace doris::vectorized {
+
+/** A column of map values.
+ */
+std::string ColumnMap::get_name() const {
+ return "Map(" + keys->get_name() + ", " + values->get_name() + ")";
+}
+
+ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values)
+ : keys(std::move(keys)), values(std::move(values)) {
+ check_size();
+}
+
+ColumnArray::Offsets64& ColumnMap::get_offsets() const {
+ const ColumnArray & column_keys = assert_cast<const ColumnArray &> (get_keys());
+ // todo . did here check size ?
+ return const_cast<Offsets64&>(column_keys.get_offsets());
+}
+
+void ColumnMap::check_size() const {
+ const auto * key_array = typeid_cast<const ColumnArray *>(keys.get());
+ const auto * value_array = typeid_cast<const ColumnArray *>(values.get());
+ CHECK(key_array) << "ColumnMap keys can be created only from array";
+ CHECK(value_array) << "ColumnMap values can be created only from array";
+ CHECK_EQ(get_keys_ptr()->size(), get_values_ptr()->size());
+}
+
+// todo. here to resize every row map
+MutableColumnPtr ColumnMap::clone_resized(size_t to_size) const {
+ auto res = ColumnMap::create(keys->clone_resized(to_size), values->clone_resized(to_size));
+ return res;
+}
+
+// to support field functions
+Field ColumnMap::operator[](size_t n) const {
+ Map res(2);
+ keys->get(n, res[0]);
+ values->get(n, res[0]);
+
+ return res;
+}
+
+// here to compare to below
+void ColumnMap::get(size_t n, Field & res) const {
+ Map map(2);
+ keys->get(n, map[0]);
+ values->get(n, map[1]);
+
+ res = map;
+}
+
+StringRef ColumnMap::get_data_at(size_t n) const {
+ LOG(FATAL) << "Method get_data_at is not supported for " << get_name();
+}
+
+void ColumnMap::insert_data(const char*, size_t) {
+ LOG(FATAL) << "Method insert_data is not supported for " << get_name();
+}
+
+void ColumnMap::insert(const Field& x) {
+ const auto& map = doris::vectorized::get<const Map&>(x);
+ // ({}, {}, {})
Review Comment:
deleted !
##########
be/src/vec/columns/column_map.cpp:
##########
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnMap.cpp
+// and modified by Doris
+
+#include "vec/columns/column_map.h"
+
+namespace doris::vectorized {
+
+/** A column of map values.
+ */
+std::string ColumnMap::get_name() const {
+ return "Map(" + keys->get_name() + ", " + values->get_name() + ")";
+}
+
+ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values)
+ : keys(std::move(keys)), values(std::move(values)) {
+ check_size();
+}
+
+ColumnArray::Offsets64& ColumnMap::get_offsets() const {
+ const ColumnArray & column_keys = assert_cast<const ColumnArray &> (get_keys());
+ // todo . did here check size ?
+ return const_cast<Offsets64&>(column_keys.get_offsets());
+}
+
+void ColumnMap::check_size() const {
+ const auto * key_array = typeid_cast<const ColumnArray *>(keys.get());
+ const auto * value_array = typeid_cast<const ColumnArray *>(values.get());
+ CHECK(key_array) << "ColumnMap keys can be created only from array";
+ CHECK(value_array) << "ColumnMap values can be created only from array";
+ CHECK_EQ(get_keys_ptr()->size(), get_values_ptr()->size());
+}
+
+// todo. here to resize every row map
+MutableColumnPtr ColumnMap::clone_resized(size_t to_size) const {
+ auto res = ColumnMap::create(keys->clone_resized(to_size), values->clone_resized(to_size));
+ return res;
+}
+
+// to support field functions
+Field ColumnMap::operator[](size_t n) const {
+ Map res(2);
Review Comment:
done!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1082024425
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
Review Comment:
is same as null writer ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1085058385
##########
be/src/vec/functions/array/function_array_element.h:
##########
@@ -81,6 +94,79 @@ class FunctionArrayElement : public IFunction {
}
private:
+ //=========================== map element===========================//
+ ColumnPtr _get_mapped_idx(const ColumnArray& key_column,
+ const ColumnWithTypeAndName& argument) {
+ if (key_column.get_data().is_column_string()) {
+ return _mapped_key_string(key_column, argument);
+ }
+ return nullptr;
+ }
+
+ ColumnPtr _get_mapped_value(const ColumnArray& val_column,
Review Comment:
deleted !
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089863605
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java:
##########
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+import org.apache.doris.thrift.TTypeDesc;
+import org.apache.doris.thrift.TTypeNode;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+// INSERT INTO table_map VALUES ({'key1':1, 'key2':10, 'k3':100}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30});
+// MapLiteral is one row-based literal
+public class MapLiteral extends LiteralExpr {
+
+ public MapLiteral() {
+ type = new MapType(Type.NULL, Type.NULL);
+ children = new ArrayList<>();
+ }
+
+ public MapLiteral(LiteralExpr... exprs) throws AnalysisException {
+ Type keyType = Type.NULL;
+ Type valueType = Type.NULL;
+ children = new ArrayList<>();
+ int idx = 0;
+ for (LiteralExpr expr : exprs) {
+ if (idx % 2 == 0) {
+ if (keyType == Type.NULL) {
+ keyType = expr.getType();
+ } else {
+ keyType = Type.getAssignmentCompatibleType(keyType, expr.getType(), false);
+ }
+ if (keyType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ } else {
+ if (valueType == Type.NULL) {
+ valueType = expr.getType();
+ } else {
+ valueType = Type.getAssignmentCompatibleType(valueType, expr.getType(), false);
+ }
+ if (valueType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ }
+ children.add(expr);
+ ++ idx;
+ }
+
+ type = new MapType(keyType, valueType);
+ }
+
+ protected MapLiteral(MapLiteral other) {
+ super(other);
+ }
+
+ @Override
+ public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
+ if (!targetType.isMapType()) {
+ return super.uncheckedCastTo(targetType);
+ }
+ MapLiteral literal = new MapLiteral(this);
+ Type keyType = ((MapType) targetType).getKeyType();
+ Type valueType = ((MapType) targetType).getValueType();
+
+ for (int i = 0; i < children.size(); ++ i) {
+ Expr child = children.get(i);
+ if ((i & 1) == 0) {
+ literal.children.set(i, child.uncheckedCastTo(keyType));
+ } else {
+ literal.children.set(i, child.uncheckedCastTo(valueType));
+ }
+ }
+ literal.setType(targetType);
+ return literal;
+ }
+
+ @Override
+ public void checkValueValid() throws AnalysisException {
+ for (Expr e : children) {
+ e.checkValueValid();
+ }
+ }
+
+ @Override
+ protected String toSqlImpl() {
+ List<String> list = new ArrayList<>(children.size());
+ for (int i = 0; i < children.size(); i += 2) {
+ list.add(children.get(i).toSqlImpl() + ":" + children.get(i + 1).toSqlImpl());
+ }
+ return "MAP{" + StringUtils.join(list, ", ") + "}";
+ }
+
+ @Override
+ protected void toThrift(TExprNode msg) {
+ msg.node_type = TExprNodeType.MAP_LITERAL;
+ TTypeDesc container = new TTypeDesc();
+ container.setTypes(new ArrayList<TTypeNode>());
+ type.toThrift(container);
+ msg.setType(container);
Review Comment:
yes. when call this function outside, already confirm key/value type , see ( Expr.java : protected void treeToThriftHelper(TExpr container) )
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1084803369
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
+ key_opts.meta->set_column_id(4);
+ key_opts.meta->set_unique_id(4);
+ key_opts.meta->set_type(OLAP_FIELD_TYPE_ARRAY);
+ key_opts.meta->set_length(0);
+ key_opts.meta->set_encoding(BIT_SHUFFLE);
+ key_opts.meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
+ key_opts.need_zone_map = false;
+ // no need key array's null map
+ key_opts.meta->set_is_nullable(false);
+
+ ColumnMetaPB* child_meta = key_opts.meta->add_children_columns();
Review Comment:
maybe I can use opts.meta directly
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1086258307
##########
be/src/vec/olap/olap_data_convertor.cpp:
##########
@@ -712,4 +723,79 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap(
return Status::OK();
}
+Status OlapBlockDataConvertor::OlapColumnDataConvertorMap::convert_to_olap() {
+ const ColumnMap* column_map = nullptr;
+ const DataTypeMap* data_type_map = nullptr;
+ if (_nullmap) {
+ const auto* nullable_column =
+ assert_cast<const ColumnNullable*>(_typed_column.column.get());
+ column_map =
+ assert_cast<const ColumnMap*>(nullable_column->get_nested_column_ptr().get());
+ data_type_map = assert_cast<const DataTypeMap*>(
+ (assert_cast<const DataTypeNullable*>(_typed_column.type.get())->get_nested_type())
+ .get());
+ } else {
+ column_map = assert_cast<const ColumnMap*>(_typed_column.column.get());
+ data_type_map = assert_cast<const DataTypeMap*>(_typed_column.type.get());
+ }
+ assert(column_map);
+ assert(data_type_map);
+
+ return convert_to_olap(column_map, data_type_map);
+}
+
+Status OlapBlockDataConvertor::OlapColumnDataConvertorMap::convert_to_olap(
+ const ColumnMap* column_map,
+ const DataTypeMap* data_type_map) {
+
+ ColumnPtr key_data = column_map->get_keys_ptr();
+ ColumnPtr value_data = column_map->get_values_ptr();
+ if (column_map->get_keys().is_nullable()) {
+ const auto& key_nullable_column =
+ assert_cast<const ColumnNullable&>(column_map->get_keys());
+ key_data = key_nullable_column.get_nested_column_ptr();
+ }
+
+ if (column_map->get_values().is_nullable()) {
+ const auto& val_nullable_column =
+ assert_cast<const ColumnNullable&>(column_map->get_values());
+ value_data = val_nullable_column.get_nested_column_ptr();
+ }
+
+ const auto& offsets = column_map->get_offsets(); // use keys offsets
+
+ ColumnWithTypeAndName key_typed_column = {
+ key_data, remove_nullable(data_type_map->get_keys()),"map.key"};
+ _key_convertor->set_source_column(key_typed_column, _row_pos, _num_rows);
+ _key_convertor->convert_to_olap();
+
+ ColumnWithTypeAndName value_typed_column = {
+ value_data, remove_nullable(data_type_map->get_values()), "map.value"};
+ _value_convertor->set_source_column(value_typed_column, _row_pos, _num_rows);
+ _value_convertor->convert_to_olap();
+
+ MapValue* map_value = _values.data();
+ for (size_t i = 0; i < _num_rows; ++i, ++map_value) {
+ int64_t cur_pos = _row_pos + i;
+ int64_t prev_pos = cur_pos - 1;
+ if (_nullmap && _nullmap[cur_pos]) {
+ continue;
+ }
+ auto single_map_size = offsets[cur_pos] - offsets[prev_pos];
+ new (map_value) MapValue(single_map_size);
Review Comment:
? You mean one MapValue contains more than one rows ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1093018104
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java:
##########
@@ -396,37 +403,78 @@ public TColumn toThrift() {
return tColumn;
}
+
+ // here to make complex type column easy
+ private void setChildrenTColumn(Column children, TColumn tColumn) {
+ TColumn childrenTColumn = new TColumn();
+ childrenTColumn.setColumnName(children.name);
+
+ TColumnType childrenTColumnType = new TColumnType();
+ childrenTColumnType.setType(children.getDataType().toThrift());
+ childrenTColumnType.setLen(children.getStrLen());
+ childrenTColumnType.setPrecision(children.getPrecision());
+ childrenTColumnType.setScale(children.getScale());
+ childrenTColumnType.setIndexLen(children.getOlapColumnIndexSize());
+
+ childrenTColumn.setColumnType(childrenTColumnType);
+ childrenTColumn.setIsAllowNull(children.isAllowNull());
+ // TODO: If we don't set the aggregate type for children, the type will be
+ // considered as TAggregationType::SUM after deserializing in BE.
+ // For now, we make children inherit the aggregate type from their parent.
+ if (tColumn.getAggregationType() != null) {
+ childrenTColumn.setAggregationType(tColumn.getAggregationType());
+ }
+
+ tColumn.children_column.add(childrenTColumn);
+ toChildrenThrift(children, childrenTColumn);
+ }
+
+
private void toChildrenThrift(Column column, TColumn tColumn) {
if (column.type.isArrayType()) {
Column children = column.getChildren().get(0);
-
- TColumn childrenTColumn = new TColumn();
- childrenTColumn.setColumnName(children.name);
-
- TColumnType childrenTColumnType = new TColumnType();
- childrenTColumnType.setType(children.getDataType().toThrift());
- childrenTColumnType.setType(children.getDataType().toThrift());
- childrenTColumnType.setLen(children.getStrLen());
- childrenTColumnType.setPrecision(children.getPrecision());
- childrenTColumnType.setScale(children.getScale());
-
- childrenTColumnType.setIndexLen(children.getOlapColumnIndexSize());
- childrenTColumn.setColumnType(childrenTColumnType);
- childrenTColumn.setIsAllowNull(children.isAllowNull());
- // TODO: If we don't set the aggregate type for children, the type will be
- // considered as TAggregationType::SUM after deserializing in BE.
- // For now, we make children inherit the aggregate type from their parent.
- if (tColumn.getAggregationType() != null) {
- childrenTColumn.setAggregationType(tColumn.getAggregationType());
- }
-
tColumn.setChildrenColumn(new ArrayList<>());
- tColumn.children_column.add(childrenTColumn);
-
- toChildrenThrift(children, childrenTColumn);
+ setChildrenTColumn(children, tColumn);
+ } else if (column.type.isMapType()) {
+ Column k = column.getChildren().get(0);
+ Column v = column.getChildren().get(1);
+ tColumn.setChildrenColumn(new ArrayList<>());
+ setChildrenTColumn(k, tColumn);
+ setChildrenTColumn(v, tColumn);
}
}
+ //private void toChildrenThrift(Column column, TColumn tColumn) {
Review Comment:
done with struct code
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] github-actions[bot] commented on pull request #15966: [Feature](map) add map type to doris
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on PR #15966:
URL: https://github.com/apache/doris/pull/15966#issuecomment-1413438340
clang-tidy review says "All clean, LGTM! :+1:"
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] github-actions[bot] commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1093992392
##########
be/src/runtime/types.cpp:
##########
@@ -240,64 +251,67 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
default:
DCHECK(false) << node.type();
}
-}
-
-std::string TypeDescriptor::debug_string() const {
- std::stringstream ss;
- switch (type) {
- case TYPE_CHAR:
- ss << "CHAR(" << len << ")";
- return ss.str();
- case TYPE_DECIMALV2:
- ss << "DECIMALV2(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL32:
- ss << "DECIMAL32(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL64:
- ss << "DECIMAL64(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL128I:
- ss << "DECIMAL128(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_ARRAY: {
- ss << "ARRAY<" << children[0].debug_string() << ">";
- return ss.str();
}
- case TYPE_STRUCT: {
- ss << "STRUCT<";
- for (size_t i = 0; i < children.size(); i++) {
- ss << field_names[i];
- ss << ":";
- ss << children[i].debug_string();
- if (i != children.size() - 1) {
- ss << ",";
+
+ std::string TypeDescriptor::debug_string() const {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
std::string TypeDescriptor::debug_string() const {
^
```
##########
be/src/runtime/types.cpp:
##########
@@ -240,64 +251,67 @@
default:
DCHECK(false) << node.type();
}
-}
-
-std::string TypeDescriptor::debug_string() const {
- std::stringstream ss;
- switch (type) {
- case TYPE_CHAR:
- ss << "CHAR(" << len << ")";
- return ss.str();
- case TYPE_DECIMALV2:
- ss << "DECIMALV2(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL32:
- ss << "DECIMAL32(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL64:
- ss << "DECIMAL64(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL128I:
- ss << "DECIMAL128(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_ARRAY: {
- ss << "ARRAY<" << children[0].debug_string() << ">";
- return ss.str();
}
- case TYPE_STRUCT: {
- ss << "STRUCT<";
- for (size_t i = 0; i < children.size(); i++) {
- ss << field_names[i];
- ss << ":";
- ss << children[i].debug_string();
- if (i != children.size() - 1) {
- ss << ",";
+
+ std::string TypeDescriptor::debug_string() const {
+ std::stringstream ss;
+ switch (type) {
+ case TYPE_CHAR:
+ ss << "CHAR(" << len << ")";
+ return ss.str();
+ case TYPE_DECIMALV2:
+ ss << "DECIMALV2(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL32:
+ ss << "DECIMAL32(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL64:
+ ss << "DECIMAL64(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL128I:
+ ss << "DECIMAL128(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_ARRAY: {
+ ss << "ARRAY<" << children[0].debug_string() << ">";
+ return ss.str();
+ }
+ case TYPE_MAP:
+ ss << "MAP<" << children[0].debug_string() << ", " << children[1].debug_string() << ">";
+ return ss.str();
+ case TYPE_STRUCT: {
+ ss << "STRUCT<";
+ for (size_t i = 0; i < children.size(); i++) {
+ ss << field_names[i];
+ ss << ":";
+ ss << children[i].debug_string();
+ if (i != children.size() - 1) {
+ ss << ",";
+ }
}
+ ss << ">";
+ return ss.str();
+ }
+ default:
+ return type_to_string(type);
}
- ss << ">";
- return ss.str();
- }
- default:
- return type_to_string(type);
}
-}
-std::ostream& operator<<(std::ostream& os, const TypeDescriptor& type) {
- os << type.debug_string();
- return os;
-}
+ std::ostream& operator<<(std::ostream& os, const TypeDescriptor& type) {
+ os << type.debug_string();
+ return os;
+ }
-TTypeDesc create_type_desc(PrimitiveType type, int precision, int scale) {
- TTypeDesc type_desc;
- std::vector<TTypeNode> node_type;
- node_type.emplace_back();
- TScalarType scalarType;
- scalarType.__set_type(to_thrift(type));
- scalarType.__set_len(-1);
- scalarType.__set_precision(precision);
- scalarType.__set_scale(scale);
- node_type.back().__set_scalar_type(scalarType);
- type_desc.__set_types(node_type);
- return type_desc;
-}
+ TTypeDesc create_type_desc(PrimitiveType type, int precision, int scale) {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
TTypeDesc create_type_desc(PrimitiveType type, int precision, int scale) {
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,17 +179,23 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
- }
- case INVALID_TYPE:
- default:
- DCHECK(false) << "invalid PrimitiveType:" << (int)col_desc.type;
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_nulls[0]),
+ create_data_type(col_desc.children[1], col_desc.contains_nulls[1]));
break;
}
+case INVALID_TYPE:
Review Comment:
warning: 'case' statement not in switch statement [clang-diagnostic-error]
```cpp
case INVALID_TYPE:
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,17 +179,23 @@
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
- }
- case INVALID_TYPE:
- default:
- DCHECK(false) << "invalid PrimitiveType:" << (int)col_desc.type;
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_nulls[0]),
+ create_data_type(col_desc.children[1], col_desc.contains_nulls[1]));
break;
}
+case INVALID_TYPE:
+default:
+ DCHECK(false) << "invalid PrimitiveType:" << (int)col_desc.type;
+ break;
+}
- if (nested && is_nullable) {
- return std::make_shared<vectorized::DataTypeNullable>(nested);
- }
- return nested;
+if (nested && is_nullable) {
Review Comment:
warning: expected unqualified-id [clang-diagnostic-error]
```cpp
if (nested && is_nullable) {
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,17 +179,23 @@
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
- }
- case INVALID_TYPE:
- default:
- DCHECK(false) << "invalid PrimitiveType:" << (int)col_desc.type;
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_nulls[0]),
+ create_data_type(col_desc.children[1], col_desc.contains_nulls[1]));
break;
}
+case INVALID_TYPE:
+default:
+ DCHECK(false) << "invalid PrimitiveType:" << (int)col_desc.type;
+ break;
+}
- if (nested && is_nullable) {
- return std::make_shared<vectorized::DataTypeNullable>(nested);
- }
- return nested;
+if (nested && is_nullable) {
+ return std::make_shared<vectorized::DataTypeNullable>(nested);
+}
+return nested;
}
DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type, int precision,
Review Comment:
warning: use of undeclared identifier 'DataTypeFactory'; did you mean 'doris::vectorized::DataTypeFactory'? [clang-diagnostic-error]
```suggestion
DataTypePtr doris::vectorized::DataTypeFactory::_create_primitive_data_type(const FieldType& type, int precision,
```
**be/src/vec/data_types/data_type_factory.hpp:48:** 'doris::vectorized::DataTypeFactory' declared here
```cpp
class DataTypeFactory {
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
+ }
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
}
- return Status::OK();
-}
-Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
+ ctxs->clear();
+ for (int i = 0; i < texprs.size(); ++i) {
+ VExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
+ ctxs->push_back(ctx);
+ }
+ return Status::OK();
}
- return Status::OK();
-}
-void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (auto ctx : ctxs) {
- ctx->close(state);
+ Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ const RowDescriptor& row_desc) {
+ for (auto ctx : ctxs) {
+ RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ }
+ return Status::OK();
}
-}
-Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->open(state));
+ void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@ Status VExpr::create_expr(doris::ObjectPool* pool, const doris::TExprNode& texpr
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
int* node_idx, VExpr** root_expr, VExprContext** ctx) {
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
+ }
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
}
- return Status::OK();
-}
-Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
+ ctxs->clear();
+ for (int i = 0; i < texprs.size(); ++i) {
+ VExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
+ ctxs->push_back(ctx);
+ }
+ return Status::OK();
}
- return Status::OK();
-}
-void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (auto ctx : ctxs) {
- ctx->close(state);
+ Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ const RowDescriptor& row_desc) {
+ for (auto ctx : ctxs) {
+ RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ }
+ return Status::OK();
}
-}
-Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->open(state));
+ void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (auto ctx : ctxs) {
+ ctx->close(state);
+ }
}
- return Status::OK();
-}
-Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- std::vector<VExprContext*>* new_ctxs) {
- DCHECK(new_ctxs != nullptr);
- if (!new_ctxs->empty()) {
- // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
- DCHECK_EQ(new_ctxs->size(), ctxs.size());
- for (int i = 0; i < new_ctxs->size(); ++i) {
- DCHECK((*new_ctxs)[i]->_is_clone);
+ Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->open(state));
}
return Status::OK();
}
- new_ctxs->resize(ctxs.size());
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
- }
- return Status::OK();
-}
-std::string VExpr::debug_string() const {
- // TODO: implement partial debug string for member vars
- std::stringstream out;
- out << " type=" << _type.debug_string();
- out << " codegen="
- << "false";
- if (!_children.empty()) {
- out << " children=" << debug_string(_children);
+ Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ std::vector<VExprContext*>* new_ctxs) {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
std::vector<VExprContext*>* new_ctxs) {
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
+ }
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
}
- return Status::OK();
-}
-Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
+ ctxs->clear();
+ for (int i = 0; i < texprs.size(); ++i) {
+ VExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
+ ctxs->push_back(ctx);
+ }
+ return Status::OK();
}
- return Status::OK();
-}
-void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (auto ctx : ctxs) {
- ctx->close(state);
+ Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ const RowDescriptor& row_desc) {
+ for (auto ctx : ctxs) {
+ RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ }
+ return Status::OK();
}
-}
-Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->open(state));
+ void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (auto ctx : ctxs) {
+ ctx->close(state);
+ }
}
- return Status::OK();
-}
-Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- std::vector<VExprContext*>* new_ctxs) {
- DCHECK(new_ctxs != nullptr);
- if (!new_ctxs->empty()) {
- // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
- DCHECK_EQ(new_ctxs->size(), ctxs.size());
- for (int i = 0; i < new_ctxs->size(); ++i) {
- DCHECK((*new_ctxs)[i]->_is_clone);
+ Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->open(state));
}
return Status::OK();
}
- new_ctxs->resize(ctxs.size());
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
- }
- return Status::OK();
-}
-std::string VExpr::debug_string() const {
- // TODO: implement partial debug string for member vars
- std::stringstream out;
- out << " type=" << _type.debug_string();
- out << " codegen="
- << "false";
- if (!_children.empty()) {
- out << " children=" << debug_string(_children);
+ Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ std::vector<VExprContext*>* new_ctxs) {
+ DCHECK(new_ctxs != nullptr);
+ if (!new_ctxs->empty()) {
+ // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
+ DCHECK_EQ(new_ctxs->size(), ctxs.size());
+ for (int i = 0; i < new_ctxs->size(); ++i) {
+ DCHECK((*new_ctxs)[i]->_is_clone);
+ }
+ return Status::OK();
+ }
+ new_ctxs->resize(ctxs.size());
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
+ }
+ return Status::OK();
}
+ std::string VExpr::debug_string() const {
+ // TODO: implement partial debug string for member vars
+ std::stringstream out;
+ out << " type=" << _type.debug_string();
+ out << " codegen="
+ << "false";
- return out.str();
-}
-
-std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
- std::stringstream out;
- out << "[";
+ if (!_children.empty()) {
+ out << " children=" << debug_string(_children);
+ }
- for (int i = 0; i < exprs.size(); ++i) {
- out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
+ return out.str();
}
- out << "]";
- return out.str();
-}
+ std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
+ std::stringstream out;
+ out << "[";
-std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
- std::vector<VExpr*> exprs;
- for (int i = 0; i < ctxs.size(); ++i) {
- exprs.push_back(ctxs[i]->root());
+ for (int i = 0; i < exprs.size(); ++i) {
+ out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
+ }
+
+ out << "]";
+ return out.str();
}
- return debug_string(exprs);
-}
-bool VExpr::is_constant() const {
- for (int i = 0; i < _children.size(); ++i) {
- if (!_children[i]->is_constant()) {
- return false;
+ std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
+ }
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
}
- return Status::OK();
-}
-Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
+ ctxs->clear();
+ for (int i = 0; i < texprs.size(); ++i) {
+ VExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
+ ctxs->push_back(ctx);
+ }
+ return Status::OK();
}
- return Status::OK();
-}
-void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (auto ctx : ctxs) {
- ctx->close(state);
+ Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ const RowDescriptor& row_desc) {
+ for (auto ctx : ctxs) {
+ RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ }
+ return Status::OK();
}
-}
-Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->open(state));
+ void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (auto ctx : ctxs) {
+ ctx->close(state);
+ }
}
- return Status::OK();
-}
-Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- std::vector<VExprContext*>* new_ctxs) {
- DCHECK(new_ctxs != nullptr);
- if (!new_ctxs->empty()) {
- // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
- DCHECK_EQ(new_ctxs->size(), ctxs.size());
- for (int i = 0; i < new_ctxs->size(); ++i) {
- DCHECK((*new_ctxs)[i]->_is_clone);
+ Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->open(state));
}
return Status::OK();
}
- new_ctxs->resize(ctxs.size());
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
- }
- return Status::OK();
-}
-std::string VExpr::debug_string() const {
- // TODO: implement partial debug string for member vars
- std::stringstream out;
- out << " type=" << _type.debug_string();
- out << " codegen="
- << "false";
- if (!_children.empty()) {
- out << " children=" << debug_string(_children);
+ Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ std::vector<VExprContext*>* new_ctxs) {
+ DCHECK(new_ctxs != nullptr);
+ if (!new_ctxs->empty()) {
+ // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
+ DCHECK_EQ(new_ctxs->size(), ctxs.size());
+ for (int i = 0; i < new_ctxs->size(); ++i) {
+ DCHECK((*new_ctxs)[i]->_is_clone);
+ }
+ return Status::OK();
+ }
+ new_ctxs->resize(ctxs.size());
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
+ }
+ return Status::OK();
}
+ std::string VExpr::debug_string() const {
+ // TODO: implement partial debug string for member vars
+ std::stringstream out;
+ out << " type=" << _type.debug_string();
+ out << " codegen="
+ << "false";
- return out.str();
-}
-
-std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
- std::stringstream out;
- out << "[";
+ if (!_children.empty()) {
+ out << " children=" << debug_string(_children);
+ }
- for (int i = 0; i < exprs.size(); ++i) {
- out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
+ return out.str();
}
- out << "]";
- return out.str();
-}
+ std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
+ std::stringstream out;
+ out << "[";
-std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
- std::vector<VExpr*> exprs;
- for (int i = 0; i < ctxs.size(); ++i) {
- exprs.push_back(ctxs[i]->root());
+ for (int i = 0; i < exprs.size(); ++i) {
+ out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
+ }
+
+ out << "]";
+ return out.str();
}
- return debug_string(exprs);
-}
-bool VExpr::is_constant() const {
- for (int i = 0; i < _children.size(); ++i) {
- if (!_children[i]->is_constant()) {
- return false;
+ std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
+ std::vector<VExpr*> exprs;
+ for (int i = 0; i < ctxs.size(); ++i) {
+ exprs.push_back(ctxs[i]->root());
}
+ return debug_string(exprs);
}
- return true;
-}
+ bool VExpr::is_constant() const {
+ for (int i = 0; i < _children.size(); ++i) {
+ if (!_children[i]->is_constant()) {
+ return false;
+ }
+ }
-Status VExpr::get_const_col(VExprContext* context, ColumnPtrWrapper** output) {
- *output = nullptr;
- if (!is_constant()) {
- return Status::OK();
+ return true;
}
- if (_constant_col != nullptr) {
+ Status VExpr::get_const_col(VExprContext * context, ColumnPtrWrapper * *output) {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
Status VExpr::get_const_col(VExprContext * context, ColumnPtrWrapper * *output) {
^
```
##########
be/src/runtime/types.cpp:
##########
@@ -240,64 +251,67 @@
default:
DCHECK(false) << node.type();
}
-}
-
-std::string TypeDescriptor::debug_string() const {
- std::stringstream ss;
- switch (type) {
- case TYPE_CHAR:
- ss << "CHAR(" << len << ")";
- return ss.str();
- case TYPE_DECIMALV2:
- ss << "DECIMALV2(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL32:
- ss << "DECIMAL32(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL64:
- ss << "DECIMAL64(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL128I:
- ss << "DECIMAL128(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_ARRAY: {
- ss << "ARRAY<" << children[0].debug_string() << ">";
- return ss.str();
}
- case TYPE_STRUCT: {
- ss << "STRUCT<";
- for (size_t i = 0; i < children.size(); i++) {
- ss << field_names[i];
- ss << ":";
- ss << children[i].debug_string();
- if (i != children.size() - 1) {
- ss << ",";
+
+ std::string TypeDescriptor::debug_string() const {
+ std::stringstream ss;
+ switch (type) {
+ case TYPE_CHAR:
+ ss << "CHAR(" << len << ")";
+ return ss.str();
+ case TYPE_DECIMALV2:
+ ss << "DECIMALV2(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL32:
+ ss << "DECIMAL32(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL64:
+ ss << "DECIMAL64(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL128I:
+ ss << "DECIMAL128(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_ARRAY: {
+ ss << "ARRAY<" << children[0].debug_string() << ">";
+ return ss.str();
+ }
+ case TYPE_MAP:
+ ss << "MAP<" << children[0].debug_string() << ", " << children[1].debug_string() << ">";
+ return ss.str();
+ case TYPE_STRUCT: {
+ ss << "STRUCT<";
+ for (size_t i = 0; i < children.size(); i++) {
+ ss << field_names[i];
+ ss << ":";
+ ss << children[i].debug_string();
+ if (i != children.size() - 1) {
+ ss << ",";
+ }
}
+ ss << ">";
+ return ss.str();
+ }
+ default:
+ return type_to_string(type);
}
- ss << ">";
- return ss.str();
- }
- default:
- return type_to_string(type);
}
-}
-std::ostream& operator<<(std::ostream& os, const TypeDescriptor& type) {
- os << type.debug_string();
- return os;
-}
+ std::ostream& operator<<(std::ostream& os, const TypeDescriptor& type) {
+ os << type.debug_string();
+ return os;
+ }
-TTypeDesc create_type_desc(PrimitiveType type, int precision, int scale) {
- TTypeDesc type_desc;
- std::vector<TTypeNode> node_type;
- node_type.emplace_back();
- TScalarType scalarType;
- scalarType.__set_type(to_thrift(type));
- scalarType.__set_len(-1);
- scalarType.__set_precision(precision);
- scalarType.__set_scale(scale);
- node_type.back().__set_scalar_type(scalarType);
- type_desc.__set_types(node_type);
- return type_desc;
-}
+ TTypeDesc create_type_desc(PrimitiveType type, int precision, int scale) {
+ TTypeDesc type_desc;
+ std::vector<TTypeNode> node_type;
+ node_type.emplace_back();
+ TScalarType scalarType;
+ scalarType.__set_type(to_thrift(type));
+ scalarType.__set_len(-1);
+ scalarType.__set_precision(precision);
+ scalarType.__set_scale(scale);
+ node_type.back().__set_scalar_type(scalarType);
+ type_desc.__set_types(node_type);
+ return type_desc;
+ }
} // namespace doris
Review Comment:
warning: expected '}' [clang-diagnostic-error]
```cpp
} // namespace doris
^
```
**be/src/runtime/types.cpp:24:** to match this '{'
```cpp
namespace doris {
^
```
##########
be/src/runtime/types.cpp:
##########
@@ -240,64 +251,67 @@
default:
DCHECK(false) << node.type();
}
-}
-
-std::string TypeDescriptor::debug_string() const {
- std::stringstream ss;
- switch (type) {
- case TYPE_CHAR:
- ss << "CHAR(" << len << ")";
- return ss.str();
- case TYPE_DECIMALV2:
- ss << "DECIMALV2(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL32:
- ss << "DECIMAL32(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL64:
- ss << "DECIMAL64(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL128I:
- ss << "DECIMAL128(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_ARRAY: {
- ss << "ARRAY<" << children[0].debug_string() << ">";
- return ss.str();
}
- case TYPE_STRUCT: {
- ss << "STRUCT<";
- for (size_t i = 0; i < children.size(); i++) {
- ss << field_names[i];
- ss << ":";
- ss << children[i].debug_string();
- if (i != children.size() - 1) {
- ss << ",";
+
+ std::string TypeDescriptor::debug_string() const {
+ std::stringstream ss;
+ switch (type) {
+ case TYPE_CHAR:
+ ss << "CHAR(" << len << ")";
+ return ss.str();
+ case TYPE_DECIMALV2:
+ ss << "DECIMALV2(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL32:
+ ss << "DECIMAL32(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL64:
+ ss << "DECIMAL64(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL128I:
+ ss << "DECIMAL128(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_ARRAY: {
+ ss << "ARRAY<" << children[0].debug_string() << ">";
+ return ss.str();
+ }
+ case TYPE_MAP:
+ ss << "MAP<" << children[0].debug_string() << ", " << children[1].debug_string() << ">";
+ return ss.str();
+ case TYPE_STRUCT: {
+ ss << "STRUCT<";
+ for (size_t i = 0; i < children.size(); i++) {
+ ss << field_names[i];
+ ss << ":";
+ ss << children[i].debug_string();
+ if (i != children.size() - 1) {
+ ss << ",";
+ }
}
+ ss << ">";
+ return ss.str();
+ }
+ default:
+ return type_to_string(type);
}
- ss << ">";
- return ss.str();
- }
- default:
- return type_to_string(type);
}
-}
-std::ostream& operator<<(std::ostream& os, const TypeDescriptor& type) {
- os << type.debug_string();
- return os;
-}
+ std::ostream& operator<<(std::ostream& os, const TypeDescriptor& type) {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
std::ostream& operator<<(std::ostream& os, const TypeDescriptor& type) {
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,17 +179,23 @@
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
- }
- case INVALID_TYPE:
- default:
- DCHECK(false) << "invalid PrimitiveType:" << (int)col_desc.type;
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_nulls[0]),
+ create_data_type(col_desc.children[1], col_desc.contains_nulls[1]));
break;
}
+case INVALID_TYPE:
+default:
Review Comment:
warning: 'default' statement not in switch statement [clang-diagnostic-error]
```cpp
default:
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,17 +179,23 @@
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
- }
- case INVALID_TYPE:
- default:
- DCHECK(false) << "invalid PrimitiveType:" << (int)col_desc.type;
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_nulls[0]),
+ create_data_type(col_desc.children[1], col_desc.contains_nulls[1]));
break;
}
+case INVALID_TYPE:
+default:
+ DCHECK(false) << "invalid PrimitiveType:" << (int)col_desc.type;
+ break;
Review Comment:
warning: 'break' statement not in loop or switch statement [clang-diagnostic-error]
```cpp
break;
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,17 +179,23 @@
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
- }
- case INVALID_TYPE:
- default:
- DCHECK(false) << "invalid PrimitiveType:" << (int)col_desc.type;
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_nulls[0]),
+ create_data_type(col_desc.children[1], col_desc.contains_nulls[1]));
break;
}
+case INVALID_TYPE:
+default:
+ DCHECK(false) << "invalid PrimitiveType:" << (int)col_desc.type;
+ break;
+}
- if (nested && is_nullable) {
- return std::make_shared<vectorized::DataTypeNullable>(nested);
- }
- return nested;
+if (nested && is_nullable) {
+ return std::make_shared<vectorized::DataTypeNullable>(nested);
+}
+return nested;
}
DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type, int precision,
Review Comment:
warning: unknown type name 'DataTypePtr'; did you mean 'doris::vectorized::DataTypePtr'? [clang-diagnostic-error]
```suggestion
doris::vectorized::DataTypePtr DataTypeFactory::_create_primitive_data_type(const FieldType& type, int precision,
```
**be/src/vec/aggregate_functions/aggregate_function.h:44:** 'doris::vectorized::DataTypePtr' declared here
```cpp
using DataTypePtr = std::shared_ptr<const IDataType>;
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,17 +179,23 @@
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
- }
- case INVALID_TYPE:
- default:
- DCHECK(false) << "invalid PrimitiveType:" << (int)col_desc.type;
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_nulls[0]),
+ create_data_type(col_desc.children[1], col_desc.contains_nulls[1]));
break;
}
+case INVALID_TYPE:
+default:
+ DCHECK(false) << "invalid PrimitiveType:" << (int)col_desc.type;
+ break;
+}
- if (nested && is_nullable) {
- return std::make_shared<vectorized::DataTypeNullable>(nested);
- }
- return nested;
+if (nested && is_nullable) {
+ return std::make_shared<vectorized::DataTypeNullable>(nested);
+}
+return nested;
Review Comment:
warning: expected unqualified-id [clang-diagnostic-error]
```cpp
return nested;
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
+ }
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
}
- return Status::OK();
-}
-Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
+ ctxs->clear();
+ for (int i = 0; i < texprs.size(); ++i) {
+ VExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
+ ctxs->push_back(ctx);
+ }
+ return Status::OK();
}
- return Status::OK();
-}
-void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (auto ctx : ctxs) {
- ctx->close(state);
+ Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ const RowDescriptor& row_desc) {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
const RowDescriptor& row_desc) {
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
+ }
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
}
- return Status::OK();
-}
-Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
std::vector<VExprContext*>* ctxs) {
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
VExprContext** ctx) {
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
+ }
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
}
- return Status::OK();
-}
-Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
+ ctxs->clear();
+ for (int i = 0; i < texprs.size(); ++i) {
+ VExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
+ ctxs->push_back(ctx);
+ }
+ return Status::OK();
}
- return Status::OK();
-}
-void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (auto ctx : ctxs) {
- ctx->close(state);
+ Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ const RowDescriptor& row_desc) {
+ for (auto ctx : ctxs) {
+ RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ }
+ return Status::OK();
}
-}
-Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->open(state));
+ void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (auto ctx : ctxs) {
+ ctx->close(state);
+ }
}
- return Status::OK();
-}
-Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- std::vector<VExprContext*>* new_ctxs) {
- DCHECK(new_ctxs != nullptr);
- if (!new_ctxs->empty()) {
- // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
- DCHECK_EQ(new_ctxs->size(), ctxs.size());
- for (int i = 0; i < new_ctxs->size(); ++i) {
- DCHECK((*new_ctxs)[i]->_is_clone);
+ Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
+ }
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
}
- return Status::OK();
-}
-Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
+ ctxs->clear();
+ for (int i = 0; i < texprs.size(); ++i) {
+ VExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
+ ctxs->push_back(ctx);
+ }
+ return Status::OK();
}
- return Status::OK();
-}
-void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (auto ctx : ctxs) {
- ctx->close(state);
+ Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ const RowDescriptor& row_desc) {
+ for (auto ctx : ctxs) {
+ RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ }
+ return Status::OK();
}
-}
-Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->open(state));
+ void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (auto ctx : ctxs) {
+ ctx->close(state);
+ }
}
- return Status::OK();
-}
-Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- std::vector<VExprContext*>* new_ctxs) {
- DCHECK(new_ctxs != nullptr);
- if (!new_ctxs->empty()) {
- // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
- DCHECK_EQ(new_ctxs->size(), ctxs.size());
- for (int i = 0; i < new_ctxs->size(); ++i) {
- DCHECK((*new_ctxs)[i]->_is_clone);
+ Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->open(state));
}
return Status::OK();
}
- new_ctxs->resize(ctxs.size());
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
- }
- return Status::OK();
-}
-std::string VExpr::debug_string() const {
- // TODO: implement partial debug string for member vars
- std::stringstream out;
- out << " type=" << _type.debug_string();
- out << " codegen="
- << "false";
- if (!_children.empty()) {
- out << " children=" << debug_string(_children);
+ Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ std::vector<VExprContext*>* new_ctxs) {
+ DCHECK(new_ctxs != nullptr);
+ if (!new_ctxs->empty()) {
+ // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
+ DCHECK_EQ(new_ctxs->size(), ctxs.size());
+ for (int i = 0; i < new_ctxs->size(); ++i) {
+ DCHECK((*new_ctxs)[i]->_is_clone);
+ }
+ return Status::OK();
+ }
+ new_ctxs->resize(ctxs.size());
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
+ }
+ return Status::OK();
}
+ std::string VExpr::debug_string() const {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
std::string VExpr::debug_string() const {
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
+ }
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
}
- return Status::OK();
-}
-Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
+ ctxs->clear();
+ for (int i = 0; i < texprs.size(); ++i) {
+ VExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
+ ctxs->push_back(ctx);
+ }
+ return Status::OK();
}
- return Status::OK();
-}
-void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (auto ctx : ctxs) {
- ctx->close(state);
+ Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ const RowDescriptor& row_desc) {
+ for (auto ctx : ctxs) {
+ RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ }
+ return Status::OK();
}
-}
-Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->open(state));
+ void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (auto ctx : ctxs) {
+ ctx->close(state);
+ }
}
- return Status::OK();
-}
-Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- std::vector<VExprContext*>* new_ctxs) {
- DCHECK(new_ctxs != nullptr);
- if (!new_ctxs->empty()) {
- // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
- DCHECK_EQ(new_ctxs->size(), ctxs.size());
- for (int i = 0; i < new_ctxs->size(); ++i) {
- DCHECK((*new_ctxs)[i]->_is_clone);
+ Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->open(state));
}
return Status::OK();
}
- new_ctxs->resize(ctxs.size());
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
- }
- return Status::OK();
-}
-std::string VExpr::debug_string() const {
- // TODO: implement partial debug string for member vars
- std::stringstream out;
- out << " type=" << _type.debug_string();
- out << " codegen="
- << "false";
- if (!_children.empty()) {
- out << " children=" << debug_string(_children);
+ Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ std::vector<VExprContext*>* new_ctxs) {
+ DCHECK(new_ctxs != nullptr);
+ if (!new_ctxs->empty()) {
+ // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
+ DCHECK_EQ(new_ctxs->size(), ctxs.size());
+ for (int i = 0; i < new_ctxs->size(); ++i) {
+ DCHECK((*new_ctxs)[i]->_is_clone);
+ }
+ return Status::OK();
+ }
+ new_ctxs->resize(ctxs.size());
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
+ }
+ return Status::OK();
}
+ std::string VExpr::debug_string() const {
+ // TODO: implement partial debug string for member vars
+ std::stringstream out;
+ out << " type=" << _type.debug_string();
+ out << " codegen="
+ << "false";
- return out.str();
-}
-
-std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
- std::stringstream out;
- out << "[";
+ if (!_children.empty()) {
+ out << " children=" << debug_string(_children);
+ }
- for (int i = 0; i < exprs.size(); ++i) {
- out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
+ return out.str();
}
- out << "]";
- return out.str();
-}
+ std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
+ }
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
}
- return Status::OK();
-}
-Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
+ ctxs->clear();
+ for (int i = 0; i < texprs.size(); ++i) {
+ VExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
+ ctxs->push_back(ctx);
+ }
+ return Status::OK();
}
- return Status::OK();
-}
-void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (auto ctx : ctxs) {
- ctx->close(state);
+ Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ const RowDescriptor& row_desc) {
+ for (auto ctx : ctxs) {
+ RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ }
+ return Status::OK();
}
-}
-Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->open(state));
+ void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (auto ctx : ctxs) {
+ ctx->close(state);
+ }
}
- return Status::OK();
-}
-Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- std::vector<VExprContext*>* new_ctxs) {
- DCHECK(new_ctxs != nullptr);
- if (!new_ctxs->empty()) {
- // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
- DCHECK_EQ(new_ctxs->size(), ctxs.size());
- for (int i = 0; i < new_ctxs->size(); ++i) {
- DCHECK((*new_ctxs)[i]->_is_clone);
+ Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->open(state));
}
return Status::OK();
}
- new_ctxs->resize(ctxs.size());
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
- }
- return Status::OK();
-}
-std::string VExpr::debug_string() const {
- // TODO: implement partial debug string for member vars
- std::stringstream out;
- out << " type=" << _type.debug_string();
- out << " codegen="
- << "false";
- if (!_children.empty()) {
- out << " children=" << debug_string(_children);
+ Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ std::vector<VExprContext*>* new_ctxs) {
+ DCHECK(new_ctxs != nullptr);
+ if (!new_ctxs->empty()) {
+ // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
+ DCHECK_EQ(new_ctxs->size(), ctxs.size());
+ for (int i = 0; i < new_ctxs->size(); ++i) {
+ DCHECK((*new_ctxs)[i]->_is_clone);
+ }
+ return Status::OK();
+ }
+ new_ctxs->resize(ctxs.size());
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
+ }
+ return Status::OK();
}
+ std::string VExpr::debug_string() const {
+ // TODO: implement partial debug string for member vars
+ std::stringstream out;
+ out << " type=" << _type.debug_string();
+ out << " codegen="
+ << "false";
- return out.str();
-}
-
-std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
- std::stringstream out;
- out << "[";
+ if (!_children.empty()) {
+ out << " children=" << debug_string(_children);
+ }
- for (int i = 0; i < exprs.size(); ++i) {
- out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
+ return out.str();
}
- out << "]";
- return out.str();
-}
+ std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
+ std::stringstream out;
+ out << "[";
-std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
- std::vector<VExpr*> exprs;
- for (int i = 0; i < ctxs.size(); ++i) {
- exprs.push_back(ctxs[i]->root());
+ for (int i = 0; i < exprs.size(); ++i) {
+ out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
+ }
+
+ out << "]";
+ return out.str();
}
- return debug_string(exprs);
-}
-bool VExpr::is_constant() const {
- for (int i = 0; i < _children.size(); ++i) {
- if (!_children[i]->is_constant()) {
- return false;
+ std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
+ std::vector<VExpr*> exprs;
+ for (int i = 0; i < ctxs.size(); ++i) {
+ exprs.push_back(ctxs[i]->root());
}
+ return debug_string(exprs);
}
- return true;
-}
+ bool VExpr::is_constant() const {
+ for (int i = 0; i < _children.size(); ++i) {
+ if (!_children[i]->is_constant()) {
+ return false;
+ }
+ }
-Status VExpr::get_const_col(VExprContext* context, ColumnPtrWrapper** output) {
- *output = nullptr;
- if (!is_constant()) {
- return Status::OK();
+ return true;
}
- if (_constant_col != nullptr) {
+ Status VExpr::get_const_col(VExprContext * context, ColumnPtrWrapper * *output) {
+ *output = nullptr;
+ if (!is_constant()) {
+ return Status::OK();
+ }
+
+ if (_constant_col != nullptr) {
+ *output = _constant_col.get();
+ return Status::OK();
+ }
+
+ int result = -1;
+ Block block;
+ // If block is empty, some functions will produce no result. So we insert a column with
+ // single value here.
+ block.insert({ColumnUInt8::create(1), std::make_shared<DataTypeUInt8>(), ""});
+ RETURN_IF_ERROR(execute(context, &block, &result));
+ DCHECK(result != -1);
+ const auto& column = block.get_by_position(result).column;
+ _constant_col = std::make_shared<ColumnPtrWrapper>(column);
*output = _constant_col.get();
return Status::OK();
}
- int result = -1;
- Block block;
- // If block is empty, some functions will produce no result. So we insert a column with
- // single value here.
- block.insert({ColumnUInt8::create(1), std::make_shared<DataTypeUInt8>(), ""});
- RETURN_IF_ERROR(execute(context, &block, &result));
- DCHECK(result != -1);
- const auto& column = block.get_by_position(result).column;
- _constant_col = std::make_shared<ColumnPtrWrapper>(column);
- *output = _constant_col.get();
- return Status::OK();
-}
+ void VExpr::register_function_context(doris::RuntimeState * state, VExprContext * context) {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
void VExpr::register_function_context(doris::RuntimeState * state, VExprContext * context) {
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
+ }
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
}
- return Status::OK();
-}
-Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
+ ctxs->clear();
+ for (int i = 0; i < texprs.size(); ++i) {
+ VExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
+ ctxs->push_back(ctx);
+ }
+ return Status::OK();
}
- return Status::OK();
-}
-void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (auto ctx : ctxs) {
- ctx->close(state);
+ Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ const RowDescriptor& row_desc) {
+ for (auto ctx : ctxs) {
+ RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ }
+ return Status::OK();
}
-}
-Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->open(state));
+ void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (auto ctx : ctxs) {
+ ctx->close(state);
+ }
}
- return Status::OK();
-}
-Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- std::vector<VExprContext*>* new_ctxs) {
- DCHECK(new_ctxs != nullptr);
- if (!new_ctxs->empty()) {
- // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
- DCHECK_EQ(new_ctxs->size(), ctxs.size());
- for (int i = 0; i < new_ctxs->size(); ++i) {
- DCHECK((*new_ctxs)[i]->_is_clone);
+ Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->open(state));
}
return Status::OK();
}
- new_ctxs->resize(ctxs.size());
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
- }
- return Status::OK();
-}
-std::string VExpr::debug_string() const {
- // TODO: implement partial debug string for member vars
- std::stringstream out;
- out << " type=" << _type.debug_string();
- out << " codegen="
- << "false";
- if (!_children.empty()) {
- out << " children=" << debug_string(_children);
+ Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ std::vector<VExprContext*>* new_ctxs) {
+ DCHECK(new_ctxs != nullptr);
+ if (!new_ctxs->empty()) {
+ // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
+ DCHECK_EQ(new_ctxs->size(), ctxs.size());
+ for (int i = 0; i < new_ctxs->size(); ++i) {
+ DCHECK((*new_ctxs)[i]->_is_clone);
+ }
+ return Status::OK();
+ }
+ new_ctxs->resize(ctxs.size());
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
+ }
+ return Status::OK();
}
+ std::string VExpr::debug_string() const {
+ // TODO: implement partial debug string for member vars
+ std::stringstream out;
+ out << " type=" << _type.debug_string();
+ out << " codegen="
+ << "false";
- return out.str();
-}
-
-std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
- std::stringstream out;
- out << "[";
+ if (!_children.empty()) {
+ out << " children=" << debug_string(_children);
+ }
- for (int i = 0; i < exprs.size(); ++i) {
- out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
+ return out.str();
}
- out << "]";
- return out.str();
-}
+ std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
+ std::stringstream out;
+ out << "[";
-std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
- std::vector<VExpr*> exprs;
- for (int i = 0; i < ctxs.size(); ++i) {
- exprs.push_back(ctxs[i]->root());
+ for (int i = 0; i < exprs.size(); ++i) {
+ out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
+ }
+
+ out << "]";
+ return out.str();
}
- return debug_string(exprs);
-}
-bool VExpr::is_constant() const {
- for (int i = 0; i < _children.size(); ++i) {
- if (!_children[i]->is_constant()) {
- return false;
+ std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
+ std::vector<VExpr*> exprs;
+ for (int i = 0; i < ctxs.size(); ++i) {
+ exprs.push_back(ctxs[i]->root());
}
+ return debug_string(exprs);
}
- return true;
-}
+ bool VExpr::is_constant() const {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
bool VExpr::is_constant() const {
^
```
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -171,218 +174,220 @@
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
+ }
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
}
- return Status::OK();
-}
-Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
+ ctxs->clear();
+ for (int i = 0; i < texprs.size(); ++i) {
+ VExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
+ ctxs->push_back(ctx);
+ }
+ return Status::OK();
}
- return Status::OK();
-}
-void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (auto ctx : ctxs) {
- ctx->close(state);
+ Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ const RowDescriptor& row_desc) {
+ for (auto ctx : ctxs) {
+ RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ }
+ return Status::OK();
}
-}
-Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->open(state));
+ void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (auto ctx : ctxs) {
+ ctx->close(state);
+ }
}
- return Status::OK();
-}
-Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- std::vector<VExprContext*>* new_ctxs) {
- DCHECK(new_ctxs != nullptr);
- if (!new_ctxs->empty()) {
- // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
- DCHECK_EQ(new_ctxs->size(), ctxs.size());
- for (int i = 0; i < new_ctxs->size(); ++i) {
- DCHECK((*new_ctxs)[i]->_is_clone);
+ Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->open(state));
}
return Status::OK();
}
- new_ctxs->resize(ctxs.size());
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
- }
- return Status::OK();
-}
-std::string VExpr::debug_string() const {
- // TODO: implement partial debug string for member vars
- std::stringstream out;
- out << " type=" << _type.debug_string();
- out << " codegen="
- << "false";
- if (!_children.empty()) {
- out << " children=" << debug_string(_children);
+ Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ std::vector<VExprContext*>* new_ctxs) {
+ DCHECK(new_ctxs != nullptr);
+ if (!new_ctxs->empty()) {
+ // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
+ DCHECK_EQ(new_ctxs->size(), ctxs.size());
+ for (int i = 0; i < new_ctxs->size(); ++i) {
+ DCHECK((*new_ctxs)[i]->_is_clone);
+ }
+ return Status::OK();
+ }
+ new_ctxs->resize(ctxs.size());
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
+ }
+ return Status::OK();
}
+ std::string VExpr::debug_string() const {
+ // TODO: implement partial debug string for member vars
+ std::stringstream out;
+ out << " type=" << _type.debug_string();
+ out << " codegen="
+ << "false";
- return out.str();
-}
-
-std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
- std::stringstream out;
- out << "[";
+ if (!_children.empty()) {
+ out << " children=" << debug_string(_children);
+ }
- for (int i = 0; i < exprs.size(); ++i) {
- out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
+ return out.str();
}
- out << "]";
- return out.str();
-}
+ std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
+ std::stringstream out;
+ out << "[";
-std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
- std::vector<VExpr*> exprs;
- for (int i = 0; i < ctxs.size(); ++i) {
- exprs.push_back(ctxs[i]->root());
+ for (int i = 0; i < exprs.size(); ++i) {
+ out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
+ }
+
+ out << "]";
+ return out.str();
}
- return debug_string(exprs);
-}
-bool VExpr::is_constant() const {
- for (int i = 0; i < _children.size(); ++i) {
- if (!_children[i]->is_constant()) {
- return false;
+ std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
+ std::vector<VExpr*> exprs;
+ for (int i = 0; i < ctxs.size(); ++i) {
+ exprs.push_back(ctxs[i]->root());
}
+ return debug_string(exprs);
}
- return true;
-}
+ bool VExpr::is_constant() const {
+ for (int i = 0; i < _children.size(); ++i) {
+ if (!_children[i]->is_constant()) {
+ return false;
+ }
+ }
-Status VExpr::get_const_col(VExprContext* context, ColumnPtrWrapper** output) {
- *output = nullptr;
- if (!is_constant()) {
- return Status::OK();
+ return true;
}
- if (_constant_col != nullptr) {
+ Status VExpr::get_const_col(VExprContext * context, ColumnPtrWrapper * *output) {
+ *output = nullptr;
+ if (!is_constant()) {
+ return Status::OK();
+ }
+
+ if (_constant_col != nullptr) {
+ *output = _constant_col.get();
+ return Status::OK();
+ }
+
+ int result = -1;
+ Block block;
+ // If block is empty, some functions will produce no result. So we insert a column with
+ // single value here.
+ block.insert({ColumnUInt8::create(1), std::make_shared<DataTypeUInt8>(), ""});
+ RETURN_IF_ERROR(execute(context, &block, &result));
+ DCHECK(result != -1);
+ const auto& column = block.get_by_position(result).column;
+ _constant_col = std::make_shared<ColumnPtrWrapper>(column);
*output = _constant_col.get();
return Status::OK();
}
- int result = -1;
- Block block;
- // If block is empty, some functions will produce no result. So we insert a column with
- // single value here.
- block.insert({ColumnUInt8::create(1), std::make_shared<DataTypeUInt8>(), ""});
- RETURN_IF_ERROR(execute(context, &block, &result));
- DCHECK(result != -1);
- const auto& column = block.get_by_position(result).column;
- _constant_col = std::make_shared<ColumnPtrWrapper>(column);
- *output = _constant_col.get();
- return Status::OK();
-}
+ void VExpr::register_function_context(doris::RuntimeState * state, VExprContext * context) {
+ FunctionContext::TypeDesc return_type = AnyValUtil::column_type_to_type_desc(_type);
+ std::vector<FunctionContext::TypeDesc> arg_types;
+ for (int i = 0; i < _children.size(); ++i) {
+ arg_types.push_back(AnyValUtil::column_type_to_type_desc(_children[i]->type()));
+ }
-void VExpr::register_function_context(doris::RuntimeState* state, VExprContext* context) {
- FunctionContext::TypeDesc return_type = AnyValUtil::column_type_to_type_desc(_type);
- std::vector<FunctionContext::TypeDesc> arg_types;
- for (int i = 0; i < _children.size(); ++i) {
- arg_types.push_back(AnyValUtil::column_type_to_type_desc(_children[i]->type()));
+ _fn_context_index = context->register_func(state, return_type, arg_types, 0);
}
- _fn_context_index = context->register_func(state, return_type, arg_types, 0);
-}
-
-Status VExpr::init_function_context(VExprContext* context,
- FunctionContext::FunctionStateScope scope,
- const FunctionBasePtr& function) const {
- FunctionContext* fn_ctx = context->fn_context(_fn_context_index);
- if (scope == FunctionContext::FRAGMENT_LOCAL) {
- std::vector<ColumnPtrWrapper*> constant_cols;
- for (auto c : _children) {
- ColumnPtrWrapper* const_col_wrapper = nullptr;
- RETURN_IF_ERROR(c->get_const_col(context, &const_col_wrapper));
- constant_cols.push_back(const_col_wrapper);
+ Status VExpr::init_function_context(VExprContext * context,
+ FunctionContext::FunctionStateScope scope,
+ const FunctionBasePtr& function) const {
Review Comment:
warning: function definition is not allowed here [clang-diagnostic-error]
```cpp
const FunctionBasePtr& function) const {
^
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1093944836
##########
be/src/vec/data_types/data_type_map.cpp:
##########
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "data_type_map.h"
+
+#include "gen_cpp/data.pb.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/common/assert_cast.h"
+
+namespace doris::vectorized {
+
+DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_)
+{
+ key_type = keys_;
+ value_type = values_;
+
+ keys = std::make_shared<DataTypeArray>(key_type);
+ values = std::make_shared<DataTypeArray>(value_type);
+}
+
+std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ std::stringstream ss;
+ ss << "{";
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ss << ", ";
+ ss << "'" << keys->to_string(nested_keys, i);
+ ss << ':';
+ ss << "'" << values->to_string(nested_values, i);
+ }
+ ss << "}";
+ return ss.str();
+}
+
+void DataTypeMap::to_string(const class doris::vectorized::IColumn& column, size_t row_num,
+ class doris::vectorized::BufferWritable& ostr) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ ostr.write("{", 1);
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ostr.write(", ", 2);
+ keys->to_string(nested_keys, i, ostr);
+ ostr.write(":", 1);
+ values->to_string(nested_values, i, ostr);
+ }
+ ostr.write("}", 1);
+}
+
+Status DataTypeMap::from_string(ReadBuffer& rb, IColumn* column) const {
+ DCHECK(!rb.eof());
+ // only support one level now
+ auto* map_column = assert_cast<ColumnMap*>(column);
+ // IColumn& nested_column = array_column->get_data();
+ if (*rb.position() != '{') {
+ return Status::InvalidArgument("map does not start with '{' character, found '{}'",
+ *rb.position());
+ }
+ keys->from_string(rb, &map_column->get_keys());
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] github-actions[bot] commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "github-actions[bot] (via GitHub)" <gi...@apache.org>.
github-actions[bot] commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1094185768
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,7 +179,12 @@
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
- }
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_nulls[0]),
+ create_data_type(col_desc.children[1], col_desc.contains_nulls[1]));
+ break;
case INVALID_TYPE:
default:
Review Comment:
warning: cannot jump from switch statement to this case label [clang-diagnostic-error]
```cpp
default:
^
```
**be/src/vec/data_types/data_type_factory.cpp:172:** jump bypasses variable initialization
```cpp
Strings names;
^
```
**be/src/vec/data_types/data_type_factory.cpp:171:** jump bypasses variable initialization
```cpp
DataTypes dataTypes;
^
```
**be/src/vec/data_types/data_type_factory.cpp:169:** jump bypasses variable initialization
```cpp
size_t child_size = col_desc.children.size();
^
```
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -169,7 +179,12 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
- }
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_nulls[0]),
+ create_data_type(col_desc.children[1], col_desc.contains_nulls[1]));
+ break;
case INVALID_TYPE:
Review Comment:
warning: cannot jump from switch statement to this case label [clang-diagnostic-error]
```cpp
case INVALID_TYPE:
^
```
**be/src/vec/data_types/data_type_factory.cpp:172:** jump bypasses variable initialization
```cpp
Strings names;
^
```
**be/src/vec/data_types/data_type_factory.cpp:171:** jump bypasses variable initialization
```cpp
DataTypes dataTypes;
^
```
**be/src/vec/data_types/data_type_factory.cpp:169:** jump bypasses variable initialization
```cpp
size_t child_size = col_desc.children.size();
^
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] xiaokang commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
xiaokang commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1071929373
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java:
##########
@@ -396,37 +403,78 @@ public TColumn toThrift() {
return tColumn;
}
+
+ // here to make complex type column easy
+ private void setChildrenTColumn(Column children, TColumn tColumn) {
+ TColumn childrenTColumn = new TColumn();
+ childrenTColumn.setColumnName(children.name);
+
+ TColumnType childrenTColumnType = new TColumnType();
+ childrenTColumnType.setType(children.getDataType().toThrift());
+ childrenTColumnType.setLen(children.getStrLen());
+ childrenTColumnType.setPrecision(children.getPrecision());
+ childrenTColumnType.setScale(children.getScale());
+ childrenTColumnType.setIndexLen(children.getOlapColumnIndexSize());
+
+ childrenTColumn.setColumnType(childrenTColumnType);
+ childrenTColumn.setIsAllowNull(children.isAllowNull());
+ // TODO: If we don't set the aggregate type for children, the type will be
+ // considered as TAggregationType::SUM after deserializing in BE.
+ // For now, we make children inherit the aggregate type from their parent.
+ if (tColumn.getAggregationType() != null) {
+ childrenTColumn.setAggregationType(tColumn.getAggregationType());
+ }
+
+ tColumn.children_column.add(childrenTColumn);
+ toChildrenThrift(children, childrenTColumn);
+ }
+
+
private void toChildrenThrift(Column column, TColumn tColumn) {
if (column.type.isArrayType()) {
Column children = column.getChildren().get(0);
-
- TColumn childrenTColumn = new TColumn();
- childrenTColumn.setColumnName(children.name);
-
- TColumnType childrenTColumnType = new TColumnType();
- childrenTColumnType.setType(children.getDataType().toThrift());
- childrenTColumnType.setType(children.getDataType().toThrift());
- childrenTColumnType.setLen(children.getStrLen());
- childrenTColumnType.setPrecision(children.getPrecision());
- childrenTColumnType.setScale(children.getScale());
-
- childrenTColumnType.setIndexLen(children.getOlapColumnIndexSize());
- childrenTColumn.setColumnType(childrenTColumnType);
- childrenTColumn.setIsAllowNull(children.isAllowNull());
- // TODO: If we don't set the aggregate type for children, the type will be
- // considered as TAggregationType::SUM after deserializing in BE.
- // For now, we make children inherit the aggregate type from their parent.
- if (tColumn.getAggregationType() != null) {
- childrenTColumn.setAggregationType(tColumn.getAggregationType());
- }
-
tColumn.setChildrenColumn(new ArrayList<>());
- tColumn.children_column.add(childrenTColumn);
-
- toChildrenThrift(children, childrenTColumn);
+ setChildrenTColumn(children, tColumn);
+ } else if (column.type.isMapType()) {
+ Column k = column.getChildren().get(0);
+ Column v = column.getChildren().get(1);
+ tColumn.setChildrenColumn(new ArrayList<>());
+ setChildrenTColumn(k, tColumn);
+ setChildrenTColumn(v, tColumn);
}
}
+ //private void toChildrenThrift(Column column, TColumn tColumn) {
Review Comment:
delete useless code
##########
fe/fe-core/src/main/java/org/apache/doris/catalog/MapType.java:
##########
@@ -98,4 +134,16 @@ public void toThrift(TTypeDesc container) {
keyType.toThrift(container);
valueType.toThrift(container);
}
+
+ @Override
+ public TColumnType toColumnTypeThrift() {
+ TColumnType thrift = new TColumnType();
+ thrift.type = PrimitiveType.MAP.toThrift();
Review Comment:
no need to process keyType and valType?
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java:
##########
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+import org.apache.doris.thrift.TTypeDesc;
+import org.apache.doris.thrift.TTypeNode;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+// INSERT INTO table_map VALUES ({'key1':1, 'key2':10, 'k3':100}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30});
+// MapLiteral is one row-based literal
+public class MapLiteral extends LiteralExpr {
+
+ public MapLiteral() {
+ type = new MapType(Type.NULL, Type.NULL);
+ children = new ArrayList<>();
+ }
+
+ public MapLiteral(LiteralExpr... exprs) throws AnalysisException {
+ Type keyType = Type.NULL;
+ Type valueType = Type.NULL;
+ children = new ArrayList<>();
+ int idx = 0;
+ for (LiteralExpr expr : exprs) {
+ if (idx % 2 == 0) {
+ if (keyType == Type.NULL) {
+ keyType = expr.getType();
+ } else {
+ keyType = Type.getAssignmentCompatibleType(keyType, expr.getType(), false);
+ }
+ if (keyType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ } else {
+ if (valueType == Type.NULL) {
+ valueType = expr.getType();
+ } else {
+ valueType = Type.getAssignmentCompatibleType(valueType, expr.getType(), false);
+ }
+ if (valueType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ }
+ children.add(expr);
+ ++ idx;
+ }
+
+ type = new MapType(keyType, valueType);
+ }
+
+ protected MapLiteral(MapLiteral other) {
+ super(other);
+ }
+
+ @Override
+ public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
+ if (!targetType.isMapType()) {
+ return super.uncheckedCastTo(targetType);
+ }
+ MapLiteral literal = new MapLiteral(this);
+ Type keyType = ((MapType) targetType).getKeyType();
+ Type valueType = ((MapType) targetType).getValueType();
+
+ for (int i = 0; i < children.size(); ++ i) {
+ Expr child = children.get(i);
+ if ((i & 1) == 0) {
Review Comment:
use a consistent way to check key/value index
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java:
##########
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+import org.apache.doris.thrift.TTypeDesc;
+import org.apache.doris.thrift.TTypeNode;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+// INSERT INTO table_map VALUES ({'key1':1, 'key2':10, 'k3':100}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30});
+// MapLiteral is one row-based literal
+public class MapLiteral extends LiteralExpr {
+
+ public MapLiteral() {
+ type = new MapType(Type.NULL, Type.NULL);
+ children = new ArrayList<>();
+ }
+
+ public MapLiteral(LiteralExpr... exprs) throws AnalysisException {
+ Type keyType = Type.NULL;
+ Type valueType = Type.NULL;
+ children = new ArrayList<>();
+ int idx = 0;
+ for (LiteralExpr expr : exprs) {
+ if (idx % 2 == 0) {
+ if (keyType == Type.NULL) {
+ keyType = expr.getType();
+ } else {
+ keyType = Type.getAssignmentCompatibleType(keyType, expr.getType(), false);
+ }
+ if (keyType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ } else {
+ if (valueType == Type.NULL) {
+ valueType = expr.getType();
+ } else {
+ valueType = Type.getAssignmentCompatibleType(valueType, expr.getType(), false);
+ }
+ if (valueType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ }
+ children.add(expr);
+ ++ idx;
+ }
+
+ type = new MapType(keyType, valueType);
+ }
+
+ protected MapLiteral(MapLiteral other) {
+ super(other);
+ }
+
+ @Override
+ public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
+ if (!targetType.isMapType()) {
+ return super.uncheckedCastTo(targetType);
+ }
+ MapLiteral literal = new MapLiteral(this);
+ Type keyType = ((MapType) targetType).getKeyType();
+ Type valueType = ((MapType) targetType).getValueType();
+
+ for (int i = 0; i < children.size(); ++ i) {
+ Expr child = children.get(i);
+ if ((i & 1) == 0) {
+ literal.children.set(i, child.uncheckedCastTo(keyType));
+ } else {
+ literal.children.set(i, child.uncheckedCastTo(valueType));
+ }
+ }
+ literal.setType(targetType);
+ return literal;
+ }
+
+ @Override
+ public void checkValueValid() throws AnalysisException {
+ for (Expr e : children) {
+ e.checkValueValid();
+ }
+ }
+
+ @Override
+ protected String toSqlImpl() {
+ List<String> list = new ArrayList<>(children.size());
+ for (int i = 0; i < children.size(); i += 2) {
+ list.add(children.get(i).toSqlImpl() + ":" + children.get(i + 1).toSqlImpl());
+ }
+ return "MAP{" + StringUtils.join(list, ", ") + "}";
+ }
+
+ @Override
+ protected void toThrift(TExprNode msg) {
+ msg.node_type = TExprNodeType.MAP_LITERAL;
+ TTypeDesc container = new TTypeDesc();
+ container.setTypes(new ArrayList<TTypeNode>());
+ type.toThrift(container);
+ msg.setType(container);
Review Comment:
no key/value data?
##########
fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlColType.java:
##########
@@ -52,7 +52,8 @@ public enum MysqlColType {
MYSQL_TYPE_BLOB(252, "BLOB"),
MYSQL_TYPE_VARSTRING(253, "VAR STRING"),
MYSQL_TYPE_STRING(254, "STRING"),
- MYSQL_TYPE_GEOMETRY(255, "GEOMETRY");
+ MYSQL_TYPE_GEOMETRY(255, "GEOMETRY"),
+ MYSQL_TYPE_MAP(256, "MAP");
Review Comment:
we can use a larger value, eg. 400, instead of continous value to avoid conflict with new data type in mysql
##########
fe/fe-core/src/main/java/org/apache/doris/planner/SetOperationNode.java:
##########
@@ -185,6 +185,9 @@ public void finalize(Analyzer analyzer) throws UserException {
// corresponding output slot isn't being materialized)
materializedResultExprLists.clear();
Preconditions.checkState(resultExprLists.size() == children.size());
+ if (analyzer.getDescTbl().getTupleDesc(tupleId) == null) {
Review Comment:
what's the effect of this?
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java:
##########
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+import org.apache.doris.thrift.TTypeDesc;
+import org.apache.doris.thrift.TTypeNode;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+// INSERT INTO table_map VALUES ({'key1':1, 'key2':10, 'k3':100}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30});
+// MapLiteral is one row-based literal
+public class MapLiteral extends LiteralExpr {
+
+ public MapLiteral() {
+ type = new MapType(Type.NULL, Type.NULL);
+ children = new ArrayList<>();
+ }
+
+ public MapLiteral(LiteralExpr... exprs) throws AnalysisException {
+ Type keyType = Type.NULL;
+ Type valueType = Type.NULL;
+ children = new ArrayList<>();
+ int idx = 0;
+ for (LiteralExpr expr : exprs) {
+ if (idx % 2 == 0) {
+ if (keyType == Type.NULL) {
+ keyType = expr.getType();
+ } else {
+ keyType = Type.getAssignmentCompatibleType(keyType, expr.getType(), false);
+ }
+ if (keyType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ } else {
+ if (valueType == Type.NULL) {
+ valueType = expr.getType();
+ } else {
+ valueType = Type.getAssignmentCompatibleType(valueType, expr.getType(), false);
+ }
+ if (valueType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ }
+ children.add(expr);
+ ++ idx;
+ }
+
+ type = new MapType(keyType, valueType);
+ }
+
+ protected MapLiteral(MapLiteral other) {
+ super(other);
+ }
+
+ @Override
+ public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
+ if (!targetType.isMapType()) {
+ return super.uncheckedCastTo(targetType);
+ }
+ MapLiteral literal = new MapLiteral(this);
+ Type keyType = ((MapType) targetType).getKeyType();
+ Type valueType = ((MapType) targetType).getValueType();
+
+ for (int i = 0; i < children.size(); ++ i) {
+ Expr child = children.get(i);
+ if ((i & 1) == 0) {
+ literal.children.set(i, child.uncheckedCastTo(keyType));
+ } else {
+ literal.children.set(i, child.uncheckedCastTo(valueType));
+ }
+ }
+ literal.setType(targetType);
+ return literal;
+ }
+
+ @Override
+ public void checkValueValid() throws AnalysisException {
+ for (Expr e : children) {
+ e.checkValueValid();
+ }
+ }
+
+ @Override
+ protected String toSqlImpl() {
+ List<String> list = new ArrayList<>(children.size());
+ for (int i = 0; i < children.size(); i += 2) {
+ list.add(children.get(i).toSqlImpl() + ":" + children.get(i + 1).toSqlImpl());
+ }
+ return "MAP{" + StringUtils.join(list, ", ") + "}";
+ }
+
+ @Override
+ protected void toThrift(TExprNode msg) {
+ msg.node_type = TExprNodeType.MAP_LITERAL;
+ TTypeDesc container = new TTypeDesc();
+ container.setTypes(new ArrayList<TTypeNode>());
+ type.toThrift(container);
+ msg.setType(container);
+ }
+
+ @Override
+ public Expr clone() {
+ return new MapLiteral(this);
+ }
+
+ @Override
+ public boolean isMinValue() {
+ return false;
+ }
+
+ @Override
+ public int compareLiteral(LiteralExpr expr) {
+ return 0;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ int size = in.readInt();
+ children = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ children.add(Expr.readIn(in));
+ }
+ }
+
+ public static MapLiteral read(DataInput in) throws IOException {
+ MapLiteral literal = new MapLiteral();
+ literal.readFields(in);
+ return literal;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ super.write(out);
+ out.writeInt(children.size());
+ for (Expr e : children) {
+ Expr.writeTo(e, out);
+ }
+ }
+
+ @Override
+ public String getStringValue() {
Review Comment:
should be the same as toSqlImpl()
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] xiaokang commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
xiaokang commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1072084853
##########
be/src/vec/data_types/data_type_map.cpp:
##########
@@ -0,0 +1,157 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "data_type_map.h"
+
+#include "gen_cpp/data.pb.h"
+#include "vec/data_types/data_type_factory.hpp"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/common/assert_cast.h"
+
+namespace doris::vectorized {
+
+DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_)
+{
+ key_type = keys_;
+ value_type = values_;
+
+ keys = std::make_shared<DataTypeArray>(key_type);
+ values = std::make_shared<DataTypeArray>(value_type);
+}
+
+std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const {
+ const ColumnMap & map_column = assert_cast<const ColumnMap &>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const IColumn & nested_keys = map_column.get_keys();
+ const IColumn & nested_values = map_column.get_values();
+
+ std::stringstream ss;
+ ss << "{";
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ ss << ", ";
+ ss << "'" << keys->to_string(nested_keys, i);
Review Comment:
'' is only needed for key of type string
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] xiaokang commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
xiaokang commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1072094803
##########
be/src/vec/data_types/data_type_factory.cpp:
##########
@@ -94,8 +101,8 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo
break;
case TYPE_TIME:
case TYPE_TIMEV2:
- nested = std::make_shared<vectorized::DataTypeTime>();
- break;
+ nested = std::make_shared<vectorized::DataTypeTime>();
Review Comment:
mistake
##########
be/src/vec/columns/column_map.cpp:
##########
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnMap.cpp
+// and modified by Doris
+
+#include "vec/columns/column_map.h"
+
+namespace doris::vectorized {
+
+/** A column of map values.
+ */
+std::string ColumnMap::get_name() const {
+ return "Map(" + keys->get_name() + ", " + values->get_name() + ")";
+}
+
+ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values)
+ : keys(std::move(keys)), values(std::move(values)) {
+ check_size();
+}
+
+ColumnArray::Offsets64& ColumnMap::get_offsets() const {
+ const ColumnArray & column_keys = assert_cast<const ColumnArray &> (get_keys());
+ // todo . did here check size ?
+ return const_cast<Offsets64&>(column_keys.get_offsets());
+}
+
+void ColumnMap::check_size() const {
+ const auto * key_array = typeid_cast<const ColumnArray *>(keys.get());
+ const auto * value_array = typeid_cast<const ColumnArray *>(values.get());
+ CHECK(key_array) << "ColumnMap keys can be created only from array";
+ CHECK(value_array) << "ColumnMap values can be created only from array";
+ CHECK_EQ(get_keys_ptr()->size(), get_values_ptr()->size());
+}
+
+// todo. here to resize every row map
+MutableColumnPtr ColumnMap::clone_resized(size_t to_size) const {
+ auto res = ColumnMap::create(keys->clone_resized(to_size), values->clone_resized(to_size));
+ return res;
+}
+
+// to support field functions
+Field ColumnMap::operator[](size_t n) const {
+ Map res(2);
+ keys->get(n, res[0]);
+ values->get(n, res[0]);
+
+ return res;
+}
+
+// here to compare to below
+void ColumnMap::get(size_t n, Field & res) const {
+ Map map(2);
+ keys->get(n, map[0]);
+ values->get(n, map[1]);
+
+ res = map;
+}
+
+StringRef ColumnMap::get_data_at(size_t n) const {
+ LOG(FATAL) << "Method get_data_at is not supported for " << get_name();
+}
+
+void ColumnMap::insert_data(const char*, size_t) {
+ LOG(FATAL) << "Method insert_data is not supported for " << get_name();
+}
+
+void ColumnMap::insert(const Field& x) {
+ const auto& map = doris::vectorized::get<const Map&>(x);
+ // ({}, {}, {})
Review Comment:
confusing comment
##########
be/src/vec/data_types/data_type_factory.hpp:
##########
@@ -33,10 +33,12 @@
#include "vec/data_types/data_type_bitmap.h"
#include "vec/data_types/data_type_date.h"
#include "vec/data_types/data_type_date_time.h"
+#include "vec/data_types/data_type_date_time.h"
#include "vec/data_types/data_type_decimal.h"
#include "vec/data_types/data_type_fixed_length_object.h"
#include "vec/data_types/data_type_hll.h"
#include "vec/data_types/data_type_jsonb.h"
+#include "vec/data_types/data_type_map.h"
Review Comment:
not used. should process map like array.
##########
be/src/vec/columns/column_map.cpp:
##########
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnMap.cpp
+// and modified by Doris
+
+#include "vec/columns/column_map.h"
+
+namespace doris::vectorized {
+
+/** A column of map values.
+ */
+std::string ColumnMap::get_name() const {
+ return "Map(" + keys->get_name() + ", " + values->get_name() + ")";
+}
+
+ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values)
+ : keys(std::move(keys)), values(std::move(values)) {
+ check_size();
+}
+
+ColumnArray::Offsets64& ColumnMap::get_offsets() const {
+ const ColumnArray & column_keys = assert_cast<const ColumnArray &> (get_keys());
+ // todo . did here check size ?
+ return const_cast<Offsets64&>(column_keys.get_offsets());
+}
+
+void ColumnMap::check_size() const {
+ const auto * key_array = typeid_cast<const ColumnArray *>(keys.get());
+ const auto * value_array = typeid_cast<const ColumnArray *>(values.get());
+ CHECK(key_array) << "ColumnMap keys can be created only from array";
+ CHECK(value_array) << "ColumnMap values can be created only from array";
+ CHECK_EQ(get_keys_ptr()->size(), get_values_ptr()->size());
+}
+
+// todo. here to resize every row map
+MutableColumnPtr ColumnMap::clone_resized(size_t to_size) const {
+ auto res = ColumnMap::create(keys->clone_resized(to_size), values->clone_resized(to_size));
+ return res;
+}
+
+// to support field functions
+Field ColumnMap::operator[](size_t n) const {
+ Map res(2);
+ keys->get(n, res[0]);
+ values->get(n, res[0]);
Review Comment:
res[1]
##########
be/src/vec/columns/column_map.cpp:
##########
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnMap.cpp
+// and modified by Doris
+
+#include "vec/columns/column_map.h"
+
+namespace doris::vectorized {
+
+/** A column of map values.
+ */
+std::string ColumnMap::get_name() const {
+ return "Map(" + keys->get_name() + ", " + values->get_name() + ")";
+}
+
+ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values)
+ : keys(std::move(keys)), values(std::move(values)) {
+ check_size();
+}
+
+ColumnArray::Offsets64& ColumnMap::get_offsets() const {
+ const ColumnArray & column_keys = assert_cast<const ColumnArray &> (get_keys());
+ // todo . did here check size ?
+ return const_cast<Offsets64&>(column_keys.get_offsets());
+}
+
+void ColumnMap::check_size() const {
+ const auto * key_array = typeid_cast<const ColumnArray *>(keys.get());
+ const auto * value_array = typeid_cast<const ColumnArray *>(values.get());
+ CHECK(key_array) << "ColumnMap keys can be created only from array";
+ CHECK(value_array) << "ColumnMap values can be created only from array";
+ CHECK_EQ(get_keys_ptr()->size(), get_values_ptr()->size());
+}
+
+// todo. here to resize every row map
+MutableColumnPtr ColumnMap::clone_resized(size_t to_size) const {
+ auto res = ColumnMap::create(keys->clone_resized(to_size), values->clone_resized(to_size));
+ return res;
+}
+
+// to support field functions
+Field ColumnMap::operator[](size_t n) const {
+ Map res(2);
+ keys->get(n, res[0]);
+ values->get(n, res[0]);
+
+ return res;
+}
+
+// here to compare to below
+void ColumnMap::get(size_t n, Field & res) const {
+ Map map(2);
+ keys->get(n, map[0]);
+ values->get(n, map[1]);
+
+ res = map;
+}
+
+StringRef ColumnMap::get_data_at(size_t n) const {
+ LOG(FATAL) << "Method get_data_at is not supported for " << get_name();
+}
+
+void ColumnMap::insert_data(const char*, size_t) {
+ LOG(FATAL) << "Method insert_data is not supported for " << get_name();
+}
+
+void ColumnMap::insert(const Field& x) {
+ const auto& map = doris::vectorized::get<const Map&>(x);
+ // ({}, {}, {})
+ // ([], [])
+ CHECK_EQ(map.size(), 2);
+ keys->insert(map[0]);
+ values->insert(map[1]);
+}
+
+void ColumnMap::insert_default() {
Review Comment:
map default should be empty, so do not need to call keys/values->insert_default()
##########
be/src/vec/columns/column_map.cpp:
##########
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnMap.cpp
+// and modified by Doris
+
+#include "vec/columns/column_map.h"
+
+namespace doris::vectorized {
+
+/** A column of map values.
+ */
+std::string ColumnMap::get_name() const {
+ return "Map(" + keys->get_name() + ", " + values->get_name() + ")";
+}
+
+ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values)
+ : keys(std::move(keys)), values(std::move(values)) {
+ check_size();
+}
+
+ColumnArray::Offsets64& ColumnMap::get_offsets() const {
+ const ColumnArray & column_keys = assert_cast<const ColumnArray &> (get_keys());
+ // todo . did here check size ?
+ return const_cast<Offsets64&>(column_keys.get_offsets());
+}
+
+void ColumnMap::check_size() const {
+ const auto * key_array = typeid_cast<const ColumnArray *>(keys.get());
+ const auto * value_array = typeid_cast<const ColumnArray *>(values.get());
+ CHECK(key_array) << "ColumnMap keys can be created only from array";
+ CHECK(value_array) << "ColumnMap values can be created only from array";
+ CHECK_EQ(get_keys_ptr()->size(), get_values_ptr()->size());
+}
+
+// todo. here to resize every row map
+MutableColumnPtr ColumnMap::clone_resized(size_t to_size) const {
+ auto res = ColumnMap::create(keys->clone_resized(to_size), values->clone_resized(to_size));
+ return res;
+}
+
+// to support field functions
+Field ColumnMap::operator[](size_t n) const {
+ Map res(2);
Review Comment:
add comment for type Map
##########
be/src/runtime/types.cpp:
##########
@@ -131,7 +132,7 @@ void TypeDescriptor::to_thrift(TTypeDesc* thrift_type) const {
}
void TypeDescriptor::to_protobuf(PTypeDesc* ptype) const {
- DCHECK(!is_complex_type() || type == TYPE_ARRAY)
+ DCHECK(!is_complex_type() || type == TYPE_ARRAY || type == TYPE_MAP)
Review Comment:
it's strange and may be a lagacy check
##########
be/src/vec/columns/column_map.cpp:
##########
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnMap.cpp
+// and modified by Doris
+
+#include "vec/columns/column_map.h"
+
+namespace doris::vectorized {
+
+/** A column of map values.
+ */
+std::string ColumnMap::get_name() const {
+ return "Map(" + keys->get_name() + ", " + values->get_name() + ")";
+}
+
+ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values)
+ : keys(std::move(keys)), values(std::move(values)) {
+ check_size();
+}
+
+ColumnArray::Offsets64& ColumnMap::get_offsets() const {
+ const ColumnArray & column_keys = assert_cast<const ColumnArray &> (get_keys());
+ // todo . did here check size ?
+ return const_cast<Offsets64&>(column_keys.get_offsets());
+}
+
+void ColumnMap::check_size() const {
+ const auto * key_array = typeid_cast<const ColumnArray *>(keys.get());
+ const auto * value_array = typeid_cast<const ColumnArray *>(values.get());
+ CHECK(key_array) << "ColumnMap keys can be created only from array";
+ CHECK(value_array) << "ColumnMap values can be created only from array";
+ CHECK_EQ(get_keys_ptr()->size(), get_values_ptr()->size());
+}
+
+// todo. here to resize every row map
+MutableColumnPtr ColumnMap::clone_resized(size_t to_size) const {
+ auto res = ColumnMap::create(keys->clone_resized(to_size), values->clone_resized(to_size));
+ return res;
+}
+
+// to support field functions
+Field ColumnMap::operator[](size_t n) const {
+ Map res(2);
+ keys->get(n, res[0]);
+ values->get(n, res[0]);
+
+ return res;
+}
+
+// here to compare to below
+void ColumnMap::get(size_t n, Field & res) const {
+ Map map(2);
+ keys->get(n, map[0]);
+ values->get(n, map[1]);
+
+ res = map;
+}
+
+StringRef ColumnMap::get_data_at(size_t n) const {
+ LOG(FATAL) << "Method get_data_at is not supported for " << get_name();
+}
+
+void ColumnMap::insert_data(const char*, size_t) {
+ LOG(FATAL) << "Method insert_data is not supported for " << get_name();
+}
+
+void ColumnMap::insert(const Field& x) {
+ const auto& map = doris::vectorized::get<const Map&>(x);
+ // ({}, {}, {})
+ // ([], [])
+ CHECK_EQ(map.size(), 2);
+ keys->insert(map[0]);
+ values->insert(map[1]);
+}
+
+void ColumnMap::insert_default() {
+ keys->insert_default();
+ values->insert_default();
+}
+
+void ColumnMap::pop_back(size_t n) {
+ keys->pop_back(n);
+ values->pop_back(n);
+}
+
+StringRef ColumnMap::serialize_value_into_arena(size_t n, Arena & arena, char const*& begin)
+ const {
+ StringRef res(begin, 0);
+ auto keys_ref = keys->serialize_value_into_arena(n, arena, begin);
+ res.data = keys_ref.data - res.size;
+ res.size += keys_ref.size;
+ auto value_ref = values->serialize_value_into_arena(n, arena, begin);
+ res.data = value_ref.data - res.size;
+ res.size += value_ref.size;
+
+ return res;
+}
+
+void ColumnMap::insert_from(const IColumn& src_, size_t n) {
+ const ColumnMap& src = assert_cast<const ColumnMap&>(src_);
+
+ if ((!get_keys().is_nullable() && src.get_keys().is_nullable())
+ || (!get_values().is_nullable() && src.get_values().is_nullable())) {
+ DCHECK(false);
+ } else if ((get_keys().is_nullable() && !src.get_keys().is_nullable())
+ || (get_values().is_nullable() && !src.get_values().is_nullable())) {
+ DCHECK(false);
+ } else {
+ keys->insert_from(*assert_cast<const ColumnMap&>(src_).keys, n);
+ values->insert_from(*assert_cast<const ColumnMap&>(src_).values, n);
+ }
+}
+
+void ColumnMap::insert_indices_from(const IColumn& src, const int* indices_begin,
+ const int* indices_end) {
+ for (auto x = indices_begin; x != indices_end; ++x) {
+ if (*x == -1) {
+ ColumnMap::insert_default();
+ } else {
+ ColumnMap::insert_from(src, *x);
+ }
+ }
+}
+
+const char* ColumnMap::deserialize_and_insert_from_arena(const char* pos) {
+ pos = keys->deserialize_and_insert_from_arena(pos);
+ pos = values->deserialize_and_insert_from_arena(pos);
+
+ return pos;
+}
+
+void ColumnMap::update_hash_with_value(size_t n, SipHash & hash) const {
+ keys->update_hash_with_value(n, hash);
+ values->update_hash_with_value(n, hash);
+}
+
+void ColumnMap::insert_range_from(const IColumn& src, size_t start, size_t length) {
+ keys->insert_range_from(*assert_cast<const ColumnMap&>(src).keys, start, length);
+ values->insert_range_from(*assert_cast<const ColumnMap&>(src).values, start, length);
+}
+
+ColumnPtr ColumnMap::filter(const Filter& filt, ssize_t result_size_hint) const {
+ return ColumnMap::create(keys->filter(filt, result_size_hint), values->filter(filt, result_size_hint));
+}
+
+ColumnPtr ColumnMap::permute(const Permutation& perm, size_t limit) const {
+ return ColumnMap::create(keys->permute(perm, limit), values->permute(perm, limit));
+}
+
+ColumnPtr ColumnMap::replicate(const Offsets& offsets) const {
+ return ColumnMap::create(keys->replicate(offsets), values->replicate(offsets));
+}
+
+//MutableColumns ColumnMap::scatter(ColumnIndex num_columns, const Selector& selector) const {
+//
+// MutableColumns keys_scatter = keys->scatter(num_columns, selector);
+// MutableColumns values_scatter = values->scatter(num_columns, selector);
+//
+// MutableColumns res(num_columns);
+//
+// for (size_t scattered_idx = 0; scattered_idx < num_columns; ++scattered_idx)
+// {
+// MutableColumns new_columns(2);
+// for (size_t map_element_idx = 0; map_element_idx < 2; ++map_element_idx)
+// new_columns[map_element_idx] = std::move(scattered_map_elements[map_element_idx][scattered_idx]);
+// res[scattered_idx] = ColumnMap::create(std::move(new_columns));
+// }
+//
+//
+// return res;
+//}
+
+
+void ColumnMap::reserve(size_t n) {
+ get_keys().reserve(n);
+ get_values().reserve(n);
+}
+
+size_t ColumnMap::byte_size() const {
+ return get_keys().byte_size() + get_values().byte_size();
+}
+
+size_t ColumnMap::allocated_bytes() const {
+ return get_keys().allocated_bytes() + get_values().allocated_bytes();
+}
+
+void ColumnMap::protect() {
+ get_keys().protect();
+ get_values().protect();
+}
+
+void ColumnMap::get_extremes(Field & min, Field & max) const {
+ Map min_map(2);
+ Map max_map(2);
+
+ keys->get_extremes(min_map[0], max_map[0]);
+ values->get_extremes(min_map[1], max_map[1]);
Review Comment:
should get index of min/max key, and get min/max value by the index of min/max key
##########
be/src/olap/types.h:
##########
@@ -430,6 +431,103 @@ class ArrayTypeInfo : public TypeInfo {
TypeInfoPtr _item_type_info;
const size_t _item_size;
};
+///====================== MapType Info ==========================///
+class MapTypeInfo : public TypeInfo {
+public:
+ explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info)
+ : _key_type_info(std::move(key_type_info)), _value_type_info(std::move(value_type_info)) {}
+ ~MapTypeInfo() override = default;
+
+ inline bool equal(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ return l_value->size() == r_value->size();
Review Comment:
should also check key and value type
##########
be/src/olap/types.h:
##########
@@ -430,6 +431,103 @@ class ArrayTypeInfo : public TypeInfo {
TypeInfoPtr _item_type_info;
const size_t _item_size;
};
+///====================== MapType Info ==========================///
+class MapTypeInfo : public TypeInfo {
+public:
+ explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info)
+ : _key_type_info(std::move(key_type_info)), _value_type_info(std::move(value_type_info)) {}
Review Comment:
std::move may be not approprate here
##########
be/src/olap/field.h:
##########
@@ -49,7 +50,7 @@ class Field {
_index_size(column.index_length()),
_is_nullable(column.is_nullable()),
_unique_id(column.unique_id()) {
- if (column.type() == OLAP_FIELD_TYPE_ARRAY) {
+ if (column.type() == OLAP_FIELD_TYPE_ARRAY || column.type() == OLAP_FIELD_TYPE_MAP) {
_agg_info = get_aggregate_info(column.aggregation(), column.type(),
column.get_sub_column(0).type());
Review Comment:
is it right to just use key type for map?
##########
be/src/olap/types.cpp:
##########
@@ -218,6 +233,11 @@ TypeInfoPtr get_type_info(const TabletColumn* col) {
TypeInfoPtr clone_type_info(const TypeInfo* type_info) {
if (is_scalar_type(type_info->type())) {
return create_static_type_info_ptr(type_info);
+ } else if (type_info->type() == OLAP_FIELD_TYPE_MAP) {
+ const auto map_type_info = dynamic_cast<const MapTypeInfo*>(type_info);
+ return create_dynamic_type_info_ptr(
+ new MapTypeInfo(clone_type_info(map_type_info->get_key_type_info()),
+ clone_type_info(map_type_info->get_value_type_info())));
} else {
Review Comment:
add check for array type
##########
be/src/olap/field.h:
##########
@@ -450,6 +451,29 @@ uint32_t Field::hash_code(const CellType& cell, uint32_t seed) const {
return _type_info->hash_code(cell.cell_ptr(), seed);
}
+class MapField : public Field {
+public:
+ explicit MapField(const TabletColumn& column) : Field(column) {}
+
+ void consume(RowCursorCell* dst, const char* src, bool src_null, MemPool* mem_pool,
+ ObjectPool* agg_pool) const override {
+ dst->set_is_null(src_null);
+ if (src_null) {
+ return;
+ }
+ _type_info->deep_copy(dst->mutable_cell_ptr(), src, mem_pool);
+ }
+
+ // make variable_ptr memory allocate to cell_ptr as MapValue
+ char* allocate_memory(char* cell_ptr, char* variable_ptr) const override {
+ return variable_ptr + _length;
+ }
+
+ size_t get_variable_len() const override {
+ return _length;
Review Comment:
how is _length caculated?
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -695,5 +808,120 @@ Status ArrayColumnWriter::finish_current_page() {
return Status::NotSupported("array writer has no data, can not finish_current_page");
}
+/// ============================= MapColumnWriter =====================////
+MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
+ ScalarColumnWriter* null_writer,
+ std::unique_ptr<ColumnWriter> key_writer,
+ std::unique_ptr<ColumnWriter> value_writer)
+ : ColumnWriter(std::move(field), opts.meta->is_nullable()),
+ _key_writer(std::move(key_writer)),
+ _value_writer(std::move(value_writer)),
+ _opts(opts) {
+ if (is_nullable()) {
+ _null_writer.reset(null_writer);
+ }
+}
+
+Status MapColumnWriter::init() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->init());
+ }
+ RETURN_IF_ERROR(_key_writer->init());
+ RETURN_IF_ERROR(_value_writer->init());
+ return Status::OK();
+}
+
+uint64_t MapColumnWriter::estimate_buffer_size() {
+ size_t estimate = _key_writer->estimate_buffer_size() +
+ _value_writer->estimate_buffer_size();
+ if (is_nullable()) {
+ estimate += _null_writer->estimate_buffer_size();
+ }
+ return estimate;
+}
+
+Status MapColumnWriter::finish() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->finish());
+ }
+ RETURN_IF_ERROR(_key_writer->finish());
+ RETURN_IF_ERROR(_value_writer->finish());
+ return Status::OK();
+}
+
+// todo. make keys and values write
+Status MapColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
+ size_t remaining = num_rows;
+ const auto* col_cursor = reinterpret_cast<const MapValue*>(*ptr);
+ while (remaining > 0) {
+ size_t num_written = 1;
+ auto* key_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_key_data();
Review Comment:
just use key_data()
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -695,5 +808,120 @@ Status ArrayColumnWriter::finish_current_page() {
return Status::NotSupported("array writer has no data, can not finish_current_page");
}
+/// ============================= MapColumnWriter =====================////
+MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
+ ScalarColumnWriter* null_writer,
+ std::unique_ptr<ColumnWriter> key_writer,
+ std::unique_ptr<ColumnWriter> value_writer)
+ : ColumnWriter(std::move(field), opts.meta->is_nullable()),
+ _key_writer(std::move(key_writer)),
+ _value_writer(std::move(value_writer)),
+ _opts(opts) {
+ if (is_nullable()) {
+ _null_writer.reset(null_writer);
+ }
+}
+
+Status MapColumnWriter::init() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->init());
+ }
+ RETURN_IF_ERROR(_key_writer->init());
+ RETURN_IF_ERROR(_value_writer->init());
+ return Status::OK();
+}
+
+uint64_t MapColumnWriter::estimate_buffer_size() {
+ size_t estimate = _key_writer->estimate_buffer_size() +
+ _value_writer->estimate_buffer_size();
+ if (is_nullable()) {
+ estimate += _null_writer->estimate_buffer_size();
+ }
+ return estimate;
+}
+
+Status MapColumnWriter::finish() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->finish());
+ }
+ RETURN_IF_ERROR(_key_writer->finish());
+ RETURN_IF_ERROR(_value_writer->finish());
+ return Status::OK();
+}
+
+// todo. make keys and values write
+Status MapColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
+ size_t remaining = num_rows;
+ const auto* col_cursor = reinterpret_cast<const MapValue*>(*ptr);
+ while (remaining > 0) {
+ size_t num_written = 1;
+ auto* key_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_key_data();
+ const uint8_t* key_ptr = (const uint8_t*)key_data_ptr;
+ RETURN_IF_ERROR(_key_writer->append_data(&key_ptr, 1));
+ auto* val_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_value_data();
+ const uint8_t* val_ptr = (const uint8_t*)val_data_ptr;
+ RETURN_IF_ERROR(_value_writer->append_data(&val_ptr, 1));
+ remaining -= num_written;
+ col_cursor += num_written;
+ *ptr += num_written * sizeof(MapValue);
+ }
+
+ if (is_nullable()) {
+ return write_null_column(num_rows, false);
+ }
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_data() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_data());
+ }
+ RETURN_IF_ERROR(_offset_writer->write_data());
+ RETURN_IF_ERROR(_key_writer->write_data());
+ RETURN_IF_ERROR(_value_writer->write_data());
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_ordinal_index() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_ordinal_index());
+ }
+ RETURN_IF_ERROR(_key_writer->write_ordinal_index());
+ RETURN_IF_ERROR(_value_writer->write_ordinal_index());
+
+ return Status::OK();
+}
+
+Status MapColumnWriter::append_nulls(size_t num_rows) {
+ return write_null_column(num_rows, true);
+}
+
+Status MapColumnWriter::write_null_column(size_t num_rows, bool is_null) {
+ uint8_t null_sign = is_null ? 1 : 0;
+ while (num_rows > 0) {
+ // TODO llj bulk write
+ const uint8_t* null_sign_ptr = &null_sign;
+ RETURN_IF_ERROR(_null_writer->append_data(&null_sign_ptr, 1));
Review Comment:
bulk write
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -449,13 +478,97 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this,null_iterator,
+ key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator: {}",
std::to_string(type));
}
}
}
+///====================== MapFileColumnIterator ============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::_peek_one_offset(ordinal_t* offset) {
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) {
+ return Status::OK();
Review Comment:
return error
##########
be/src/vec/functions/array/function_array_element.h:
##########
@@ -44,12 +46,18 @@ class FunctionArrayElement : public IFunction {
size_t get_number_of_arguments() const override { return 2; }
DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
- DCHECK(is_array(arguments[0]))
- << "first argument for function: " << name << " should be DataTypeArray";
- DCHECK(is_integer(arguments[1]))
- << "second argument for function: " << name << " should be Integer";
- return make_nullable(
- check_and_get_data_type<DataTypeArray>(arguments[0].get())->get_nested_type());
+ DCHECK(is_array(arguments[0]) || is_map(arguments[0]))
+ << "first argument for function: " << name << " should be DataTypeArray or DataTypeMap";
+ if (is_array(arguments[0])) {
+ DCHECK(is_integer(arguments[1])) << "second argument for function: " << name << " should be Integer for array element";
+ return make_nullable(
+ check_and_get_data_type<DataTypeArray>(arguments[0].get())->get_nested_type());
+ } else {
Review Comment:
check map type
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -449,13 +478,97 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this,null_iterator,
+ key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator: {}",
std::to_string(type));
}
}
}
+///====================== MapFileColumnIterator ============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::_peek_one_offset(ordinal_t* offset) {
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) {
+ return Status::OK();
+}
+
+
+Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(ord));
+ RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(ord));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+ const auto* column_map = vectorized::check_and_get_column<vectorized::ColumnMap>(
+ dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
+ : *dst);
+ auto column_key_ptr = column_map->get_keys().assume_mutable();
+ auto column_val_ptr = column_map->get_values().assume_mutable();
+ RETURN_IF_ERROR(_key_iterator->next_batch(n, column_key_ptr, has_null));
+ RETURN_IF_ERROR(_val_iterator->next_batch(n, column_val_ptr, has_null));
+
+ if (dst->is_nullable()) {
+ auto null_map_ptr =
+ static_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column_ptr();
+ size_t num_read = *n;
+ bool null_signs_has_null = false;
+ RETURN_IF_ERROR(_null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
+ DCHECK(num_read == *n);
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
+ vectorized::MutableColumnPtr& dst) {
+ for (size_t i = 0; i < count; ++i) {
+ RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
+ size_t num_read = 1;
+ RETURN_IF_ERROR(next_batch(&num_read, dst, nullptr));
Review Comment:
is nullptr always right?
##########
be/src/runtime/map_value.h:
##########
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <type_traits>
+#include "runtime/primitive_type.h"
+
+namespace doris_udf {
+class FunctionContext;
+struct AnyVal;
+} // namespace doris_udf
+
+namespace doris {
+
+using doris_udf::FunctionContext;
+using doris_udf::AnyVal;
+
+/**
+ * MapValue is for map type in memory
+ */
+class MapValue {
+public:
+ MapValue() = default;
+
+ explicit MapValue(int32_t length)
+ : _key_data(nullptr), _value_data(nullptr), _length(length){}
Review Comment:
it's not valid if _key/value_data is null and _length > 0
##########
be/src/olap/rowset/segment_v2/column_reader.h:
##########
@@ -393,6 +393,44 @@ class EmptyFileColumnIterator final : public ColumnIterator {
ordinal_t get_current_ordinal() const override { return 0; }
};
+// This iterator is used to read map value column
+class MapFileColumnIterator final : public ColumnIterator {
+public:
+ explicit MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator, ColumnIterator* val_iterator);
+
+ ~MapFileColumnIterator() override = default;
+
+ Status init(const ColumnIteratorOptions& opts) override;
+
+ Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) override;
+
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst, bool* has_null) override;
+
+ Status read_by_rowids(const rowid_t* rowids, const size_t count,
+ vectorized::MutableColumnPtr& dst) override;
+
+ Status seek_to_first() override {
+ RETURN_IF_ERROR(_key_iterator->seek_to_first());
+ RETURN_IF_ERROR(_val_iterator->seek_to_first());
+ return Status::OK();
+ }
+
+ Status seek_to_ordinal(ordinal_t ord) override;
+
+ ordinal_t get_current_ordinal() const override {
+ return _key_iterator->get_current_ordinal();
+ }
+
+private:
+ ColumnReader* _map_reader; // need ?
Review Comment:
it is used for nullable check
##########
be/src/runtime/map_value.h:
##########
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <type_traits>
+#include "runtime/primitive_type.h"
+
+namespace doris_udf {
+class FunctionContext;
+struct AnyVal;
+} // namespace doris_udf
+
+namespace doris {
+
+using doris_udf::FunctionContext;
+using doris_udf::AnyVal;
+
+/**
+ * MapValue is for map type in memory
+ */
+class MapValue {
+public:
+ MapValue() = default;
+
+ explicit MapValue(int32_t length)
+ : _key_data(nullptr), _value_data(nullptr), _length(length){}
+
+ MapValue(void* k_data, void* v_data, int32_t length)
+ : _key_data(k_data), _value_data(v_data), _length(length) {}
+
+ void to_map_val(MapVal* val) const;
+
+ int32_t size() const { return _length; }
+
+ int32_t length() const { return _length; }
+
+ void shallow_copy(const MapValue* other);
Review Comment:
is deep_copy needed?
##########
be/src/olap/types.h:
##########
@@ -430,6 +431,103 @@ class ArrayTypeInfo : public TypeInfo {
TypeInfoPtr _item_type_info;
const size_t _item_size;
};
+///====================== MapType Info ==========================///
+class MapTypeInfo : public TypeInfo {
+public:
+ explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info)
+ : _key_type_info(std::move(key_type_info)), _value_type_info(std::move(value_type_info)) {}
+ ~MapTypeInfo() override = default;
+
+ inline bool equal(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ return l_value->size() == r_value->size();
+ }
+
+ int cmp(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ uint32_t l_size = l_value->size();
+ uint32_t r_size = r_value->size();
+ if (l_size < r_size) {
+ return -1;
+ } else if (l_size > r_size) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ void shallow_copy(void* dest, const void* src) const override {
+ auto dest_value = reinterpret_cast<MapValue*>(dest);
+ auto src_value = reinterpret_cast<const MapValue*>(src);
+ dest_value->shallow_copy(src_value);
+ }
+
+ void deep_copy(void* dest, const void* src, MemPool* mem_pool) const override {
+ DCHECK(false);
+
+ }
+
+ void copy_object(void* dest, const void* src, MemPool* mem_pool) const override {
+ deep_copy(dest, src, mem_pool);
+ }
+
+ void direct_copy(void* dest, const void* src) const override {
+ CHECK(false);
+ }
+
+ void direct_copy(uint8_t** base, void* dest, const void* src) const {
+ CHECK(false);
+ }
+
+ void direct_copy_may_cut(void* dest, const void* src) const override { direct_copy(dest, src); }
+
+ Status convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool,
+ size_t variable_len = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ Status from_string(void* buf, const std::string& scan_key, const int precision = 0,
+ const int scale = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ std::string to_string(const void* src) const override {
+ return "{}";
+ }
+
+ void set_to_max(void* buf) const override {
+ DCHECK(false) << "set_to_max of list is not implemented.";
+ }
+
+ void set_to_min(void* buf) const override {
+ DCHECK(false) << "set_to_min of list is not implemented.";
+ }
+
+ uint32_t hash_code(const void* data, uint32_t seed) const override {
+ auto map_value = reinterpret_cast<const MapValue*>(data);
+ auto size = map_value->size();
+ uint32_t result = HashUtil::hash(&size, sizeof(size), seed);
+ result = seed * result + _key_type_info->hash_code(
+ map_value->key_data(), seed)
+ + _value_type_info->hash_code(
+ map_value->value_data(), seed);
+ return result;
+ }
+
+ // todo . is here only to need return 16 for two ptr?
+ const size_t size() const override { return 16; }
Review Comment:
?
##########
be/src/vec/sink/vmysql_result_writer.cpp:
##########
@@ -186,6 +188,58 @@ Status VMysqlResultWriter::_add_one_column(const ColumnPtr& column_ptr,
_buffer.close_dynamic_mode();
result->result_batch.rows[i].append(_buffer.buf(), _buffer.length());
}
+ } else if constexpr (type == TYPE_MAP) {
+ auto& column_map = assert_cast<const ColumnMap&>(*column);
+ auto& offsets = column_map.get_offsets();
+ auto& column_key_array = assert_cast<const ColumnArray&>(column_map.get_keys());
+ auto& column_val_array = assert_cast<const ColumnArray&>(column_map.get_values());
+ auto& map_type = assert_cast<const DataTypeMap&>(*nested_type_ptr);
+ auto& key_nested_type_ptr = map_type.get_key_type();
+ auto& val_nested_type_ptr = map_type.get_value_type();
+ for (ssize_t i = 0; i < row_size; ++i) {
+ if (0 != buf_ret) {
+ return Status::InternalError("pack mysql buffer failed.");
+ }
+ _buffer.reset();
+
+ _buffer.open_dynamic_mode();
+ buf_ret = _buffer.push_string("{", 1);
Review Comment:
can DataTypeMap::to_string() be reused here to reduce redundant code ?
##########
be/src/olap/types.h:
##########
@@ -430,6 +431,103 @@ class ArrayTypeInfo : public TypeInfo {
TypeInfoPtr _item_type_info;
const size_t _item_size;
};
+///====================== MapType Info ==========================///
+class MapTypeInfo : public TypeInfo {
+public:
+ explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info)
+ : _key_type_info(std::move(key_type_info)), _value_type_info(std::move(value_type_info)) {}
+ ~MapTypeInfo() override = default;
+
+ inline bool equal(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ return l_value->size() == r_value->size();
+ }
+
+ int cmp(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ uint32_t l_size = l_value->size();
+ uint32_t r_size = r_value->size();
+ if (l_size < r_size) {
+ return -1;
+ } else if (l_size > r_size) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ void shallow_copy(void* dest, const void* src) const override {
+ auto dest_value = reinterpret_cast<MapValue*>(dest);
+ auto src_value = reinterpret_cast<const MapValue*>(src);
+ dest_value->shallow_copy(src_value);
+ }
+
+ void deep_copy(void* dest, const void* src, MemPool* mem_pool) const override {
+ DCHECK(false);
+
+ }
+
+ void copy_object(void* dest, const void* src, MemPool* mem_pool) const override {
+ deep_copy(dest, src, mem_pool);
+ }
+
+ void direct_copy(void* dest, const void* src) const override {
+ CHECK(false);
+ }
+
+ void direct_copy(uint8_t** base, void* dest, const void* src) const {
+ CHECK(false);
+ }
+
+ void direct_copy_may_cut(void* dest, const void* src) const override { direct_copy(dest, src); }
+
+ Status convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool,
+ size_t variable_len = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ Status from_string(void* buf, const std::string& scan_key, const int precision = 0,
+ const int scale = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ std::string to_string(const void* src) const override {
+ return "{}";
Review Comment:
not valid
##########
be/src/runtime/types.cpp:
##########
@@ -191,6 +197,14 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
children.push_back(TypeDescriptor(types, idx));
break;
}
+ case TTypeNodeType::MAP: {
+ type = TYPE_MAP;
Review Comment:
has_contains_null is missed
##########
be/src/vec/functions/array/function_array_element.h:
##########
@@ -81,6 +94,79 @@ class FunctionArrayElement : public IFunction {
}
private:
+ //=========================== map element===========================//
+ ColumnPtr _get_mapped_idx(const ColumnArray& key_column,
+ const ColumnWithTypeAndName& argument) {
+ if (key_column.get_data().is_column_string()) {
+ return _mapped_key_string(key_column, argument);
Review Comment:
key compare can be unified for all type using column.compare_at
##########
be/src/olap/types.h:
##########
@@ -430,6 +431,103 @@ class ArrayTypeInfo : public TypeInfo {
TypeInfoPtr _item_type_info;
const size_t _item_size;
};
+///====================== MapType Info ==========================///
+class MapTypeInfo : public TypeInfo {
+public:
+ explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info)
+ : _key_type_info(std::move(key_type_info)), _value_type_info(std::move(value_type_info)) {}
+ ~MapTypeInfo() override = default;
+
+ inline bool equal(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ return l_value->size() == r_value->size();
+ }
+
+ int cmp(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ uint32_t l_size = l_value->size();
+ uint32_t r_size = r_value->size();
+ if (l_size < r_size) {
+ return -1;
+ } else if (l_size > r_size) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ void shallow_copy(void* dest, const void* src) const override {
+ auto dest_value = reinterpret_cast<MapValue*>(dest);
+ auto src_value = reinterpret_cast<const MapValue*>(src);
+ dest_value->shallow_copy(src_value);
+ }
+
+ void deep_copy(void* dest, const void* src, MemPool* mem_pool) const override {
+ DCHECK(false);
Review Comment:
add impl
##########
be/src/olap/field.h:
##########
@@ -792,6 +824,15 @@ class FieldFactory {
local->add_sub_field(std::move(item_field));
return local;
}
+ case OLAP_FIELD_TYPE_MAP: {
Review Comment:
we should consider if array/map is valid in value column of agg table
##########
be/src/vec/functions/array/function_array_element.h:
##########
@@ -81,6 +94,79 @@ class FunctionArrayElement : public IFunction {
}
private:
+ //=========================== map element===========================//
+ ColumnPtr _get_mapped_idx(const ColumnArray& key_column,
+ const ColumnWithTypeAndName& argument) {
+ if (key_column.get_data().is_column_string()) {
+ return _mapped_key_string(key_column, argument);
+ }
+ return nullptr;
+ }
+
+ ColumnPtr _get_mapped_value(const ColumnArray& val_column,
Review Comment:
useless method
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
+ key_opts.meta->set_column_id(4);
+ key_opts.meta->set_unique_id(4);
+ key_opts.meta->set_type(OLAP_FIELD_TYPE_ARRAY);
+ key_opts.meta->set_length(0);
+ key_opts.meta->set_encoding(BIT_SHUFFLE);
Review Comment:
hardcoded encoding
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
Review Comment:
this code need to be refactor and cleaned
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -449,13 +478,97 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this,null_iterator,
+ key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator: {}",
std::to_string(type));
}
}
}
+///====================== MapFileColumnIterator ============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::_peek_one_offset(ordinal_t* offset) {
+ return Status::OK();
Review Comment:
return error
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
+ key_opts.meta->set_column_id(4);
Review Comment:
magic number 4
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -173,6 +173,119 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ // todo . here key and value is array only?
+ const TabletColumn& key_column = column->get_sub_column(0); // field_type is true key and value
+ const TabletColumn& value_column = column->get_sub_column(1);
+
+ // create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key writer
+ std::unique_ptr<ColumnWriter> key_writer;
+ ColumnWriterOptions key_opts;
+ TabletColumn key_list_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+ {
+ key_list_column.add_sub_column(const_cast<TabletColumn&>(key_column));
+// key_list_column.add_sub_column(key_column);
+ key_list_column.set_name("map.key");
+ key_list_column.set_index_length(-1);
+
+
+ key_opts.meta = opts.meta->mutable_children_columns(0);
+ key_opts.meta->set_column_id(4);
+ key_opts.meta->set_unique_id(4);
+ key_opts.meta->set_type(OLAP_FIELD_TYPE_ARRAY);
+ key_opts.meta->set_length(0);
+ key_opts.meta->set_encoding(BIT_SHUFFLE);
+ key_opts.meta->set_compression(segment_v2::CompressionTypePB::LZ4F);
Review Comment:
hardcoded compression
##########
be/src/vec/functions/array/function_array_element.h:
##########
@@ -81,6 +94,79 @@ class FunctionArrayElement : public IFunction {
}
private:
+ //=========================== map element===========================//
+ ColumnPtr _get_mapped_idx(const ColumnArray& key_column,
+ const ColumnWithTypeAndName& argument) {
+ if (key_column.get_data().is_column_string()) {
+ return _mapped_key_string(key_column, argument);
+ }
+ return nullptr;
+ }
+
+ ColumnPtr _get_mapped_value(const ColumnArray& val_column,
+ const IColumn& matched_indices,
+ const UInt8* src_null_map,
+ UInt8* dst_null_map) {
+ const UInt8* nested_null_map = nullptr;
+ ColumnPtr nested_column = nullptr;
+ if (is_column_nullable(val_column.get_data())) {
+ const auto& nested_null_column =
+ reinterpret_cast<const ColumnNullable&>(val_column.get_data());
+ nested_null_map = nested_null_column.get_null_map_column().get_data().data();
+ nested_column = nested_null_column.get_nested_column_ptr();
+ } else {
+ nested_column = val_column.get_data_ptr();
+ }
+ if (check_column<ColumnInt8>(nested_column)) {
+ return _execute_number<ColumnInt8>(val_column.get_offsets(), *nested_column,
+ src_null_map, matched_indices,
+ nested_null_map, dst_null_map);
+ } else if (check_column<ColumnInt32>(nested_column)) {
+ _execute_number<ColumnInt32>(val_column.get_offsets(), *nested_column,
+ src_null_map, matched_indices,
+ nested_null_map, dst_null_map);
+ }
+ return nullptr;
+ }
+
+ ColumnPtr _mapped_key_string(const ColumnArray& column,
+ const ColumnWithTypeAndName& argument) {
+ auto right_column = argument.column->convert_to_full_column_if_const();
+ const ColumnString& match_key = reinterpret_cast<const ColumnString&>(*right_column);
+ const ColumnArray::Offsets64& offsets = column.get_offsets();
+ ColumnPtr nested_ptr = nullptr;
+ if (is_column_nullable(column.get_data())) {
+ nested_ptr = reinterpret_cast<const ColumnNullable&>(column.get_data()).get_nested_column_ptr();
+ } else {
+ nested_ptr = column.get_data_ptr();
+ }
+ const ColumnString& nested_key = reinterpret_cast<const ColumnString&>(*nested_ptr);
+ size_t rows = offsets.size();
+ // prepare return data
+ auto matched_indices = ColumnVector<Int8>::create();
+ matched_indices->reserve(rows);
+
+ for (size_t i = 0; i < rows; i++)
+ {
+ bool matched = false;
+ size_t begin = offsets[i - 1];
+ size_t end = offsets[i];
+ for (size_t j = begin; j < end; j++) {
+ if (nested_key.get_data_at(j) == match_key.get_data_at(i)) {
Review Comment:
use general compare method column.compare_at()
##########
be/src/vec/olap/olap_data_convertor.cpp:
##########
@@ -712,4 +723,79 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap(
return Status::OK();
}
+Status OlapBlockDataConvertor::OlapColumnDataConvertorMap::convert_to_olap() {
+ const ColumnMap* column_map = nullptr;
+ const DataTypeMap* data_type_map = nullptr;
+ if (_nullmap) {
+ const auto* nullable_column =
+ assert_cast<const ColumnNullable*>(_typed_column.column.get());
+ column_map =
+ assert_cast<const ColumnMap*>(nullable_column->get_nested_column_ptr().get());
+ data_type_map = assert_cast<const DataTypeMap*>(
+ (assert_cast<const DataTypeNullable*>(_typed_column.type.get())->get_nested_type())
+ .get());
+ } else {
+ column_map = assert_cast<const ColumnMap*>(_typed_column.column.get());
+ data_type_map = assert_cast<const DataTypeMap*>(_typed_column.type.get());
+ }
+ assert(column_map);
+ assert(data_type_map);
+
+ return convert_to_olap(column_map, data_type_map);
+}
+
+Status OlapBlockDataConvertor::OlapColumnDataConvertorMap::convert_to_olap(
+ const ColumnMap* column_map,
+ const DataTypeMap* data_type_map) {
+
+ ColumnPtr key_data = column_map->get_keys_ptr();
+ ColumnPtr value_data = column_map->get_values_ptr();
+ if (column_map->get_keys().is_nullable()) {
+ const auto& key_nullable_column =
+ assert_cast<const ColumnNullable&>(column_map->get_keys());
+ key_data = key_nullable_column.get_nested_column_ptr();
+ }
+
+ if (column_map->get_values().is_nullable()) {
+ const auto& val_nullable_column =
+ assert_cast<const ColumnNullable&>(column_map->get_values());
+ value_data = val_nullable_column.get_nested_column_ptr();
+ }
+
+ const auto& offsets = column_map->get_offsets(); // use keys offsets
+
+ ColumnWithTypeAndName key_typed_column = {
+ key_data, remove_nullable(data_type_map->get_keys()),"map.key"};
+ _key_convertor->set_source_column(key_typed_column, _row_pos, _num_rows);
+ _key_convertor->convert_to_olap();
+
+ ColumnWithTypeAndName value_typed_column = {
+ value_data, remove_nullable(data_type_map->get_values()), "map.value"};
+ _value_convertor->set_source_column(value_typed_column, _row_pos, _num_rows);
+ _value_convertor->convert_to_olap();
+
+ MapValue* map_value = _values.data();
+ for (size_t i = 0; i < _num_rows; ++i, ++map_value) {
+ int64_t cur_pos = _row_pos + i;
+ int64_t prev_pos = cur_pos - 1;
+ if (_nullmap && _nullmap[cur_pos]) {
+ continue;
+ }
+ auto single_map_size = offsets[cur_pos] - offsets[prev_pos];
+ new (map_value) MapValue(single_map_size);
Review Comment:
it's more efficient to allocate _num_rows MapValue once.
##########
be/src/vec/functions/array/function_array_element.h:
##########
@@ -81,6 +94,79 @@ class FunctionArrayElement : public IFunction {
}
private:
+ //=========================== map element===========================//
+ ColumnPtr _get_mapped_idx(const ColumnArray& key_column,
+ const ColumnWithTypeAndName& argument) {
+ if (key_column.get_data().is_column_string()) {
+ return _mapped_key_string(key_column, argument);
+ }
+ return nullptr;
Review Comment:
add TODO to support other key types
##########
be/src/olap/rowset/segment_v2/column_writer.cpp:
##########
@@ -695,5 +808,120 @@
return Status::NotSupported("array writer has no data, can not finish_current_page");
}
+/// ============================= MapColumnWriter =====================////
+MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
+ ScalarColumnWriter* null_writer,
+ std::unique_ptr<ColumnWriter> key_writer,
+ std::unique_ptr<ColumnWriter> value_writer)
+ : ColumnWriter(std::move(field), opts.meta->is_nullable()),
+ _key_writer(std::move(key_writer)),
+ _value_writer(std::move(value_writer)),
+ _opts(opts) {
+ if (is_nullable()) {
+ _null_writer.reset(null_writer);
+ }
+}
+
+Status MapColumnWriter::init() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->init());
+ }
+ RETURN_IF_ERROR(_key_writer->init());
+ RETURN_IF_ERROR(_value_writer->init());
+ return Status::OK();
+}
+
+uint64_t MapColumnWriter::estimate_buffer_size() {
+ size_t estimate = _key_writer->estimate_buffer_size() +
+ _value_writer->estimate_buffer_size();
+ if (is_nullable()) {
+ estimate += _null_writer->estimate_buffer_size();
+ }
+ return estimate;
+}
+
+Status MapColumnWriter::finish() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->finish());
+ }
+ RETURN_IF_ERROR(_key_writer->finish());
+ RETURN_IF_ERROR(_value_writer->finish());
+ return Status::OK();
+}
+
+// todo. make keys and values write
+Status MapColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
+ size_t remaining = num_rows;
+ const auto* col_cursor = reinterpret_cast<const MapValue*>(*ptr);
+ while (remaining > 0) {
+ size_t num_written = 1;
+ auto* key_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_key_data();
+ const uint8_t* key_ptr = (const uint8_t*)key_data_ptr;
+ RETURN_IF_ERROR(_key_writer->append_data(&key_ptr, 1));
+ auto* val_data_ptr = const_cast<MapValue*>(col_cursor)->mutable_value_data();
+ const uint8_t* val_ptr = (const uint8_t*)val_data_ptr;
+ RETURN_IF_ERROR(_value_writer->append_data(&val_ptr, 1));
+ remaining -= num_written;
+ col_cursor += num_written;
+ *ptr += num_written * sizeof(MapValue);
+ }
+
+ if (is_nullable()) {
+ return write_null_column(num_rows, false);
+ }
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_data() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_data());
+ }
+ RETURN_IF_ERROR(_offset_writer->write_data());
Review Comment:
no _offset_writer any more
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by GitBox <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1080686467
##########
be/src/olap/rowset/segment_v2/column_reader.h:
##########
@@ -393,6 +393,44 @@ class EmptyFileColumnIterator final : public ColumnIterator {
ordinal_t get_current_ordinal() const override { return 0; }
};
+// This iterator is used to read map value column
+class MapFileColumnIterator final : public ColumnIterator {
+public:
+ explicit MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator, ColumnIterator* val_iterator);
+
+ ~MapFileColumnIterator() override = default;
+
+ Status init(const ColumnIteratorOptions& opts) override;
+
+ Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) override;
+
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst, bool* has_null) override;
+
+ Status read_by_rowids(const rowid_t* rowids, const size_t count,
+ vectorized::MutableColumnPtr& dst) override;
+
+ Status seek_to_first() override {
+ RETURN_IF_ERROR(_key_iterator->seek_to_first());
+ RETURN_IF_ERROR(_val_iterator->seek_to_first());
+ return Status::OK();
+ }
+
+ Status seek_to_ordinal(ordinal_t ord) override;
+
+ ordinal_t get_current_ordinal() const override {
+ return _key_iterator->get_current_ordinal();
+ }
+
+private:
+ ColumnReader* _map_reader; // need ?
+ std::unique_ptr<ColumnIterator> _null_iterator;
+ std::unique_ptr<ColumnIterator> _key_iterator; // ArrayFileColumnIterator
+ std::unique_ptr<ColumnIterator> _val_iterator; // ArrayFileColumnIterator
+
+ Status _peek_one_offset(ordinal_t* offset);
Review Comment:
So I delete it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1086264303
##########
be/src/vec/functions/array/function_array_element.h:
##########
@@ -44,12 +46,18 @@ class FunctionArrayElement : public IFunction {
size_t get_number_of_arguments() const override { return 2; }
DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
- DCHECK(is_array(arguments[0]))
- << "first argument for function: " << name << " should be DataTypeArray";
- DCHECK(is_integer(arguments[1]))
- << "second argument for function: " << name << " should be Integer";
- return make_nullable(
- check_and_get_data_type<DataTypeArray>(arguments[0].get())->get_nested_type());
+ DCHECK(is_array(arguments[0]) || is_map(arguments[0]))
+ << "first argument for function: " << name << " should be DataTypeArray or DataTypeMap";
+ if (is_array(arguments[0])) {
+ DCHECK(is_integer(arguments[1])) << "second argument for function: " << name << " should be Integer for array element";
+ return make_nullable(
+ check_and_get_data_type<DataTypeArray>(arguments[0].get())->get_nested_type());
+ } else {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089859869
##########
be/src/vec/core/field.h:
##########
@@ -85,6 +85,7 @@ using FieldVector = std::vector<Field>;
DEFINE_FIELD_VECTOR(Array);
DEFINE_FIELD_VECTOR(Tuple);
+DEFINE_FIELD_VECTOR(Map);
Review Comment:
Yes， it decides from fe which pass value to be
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089863363
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java:
##########
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+import org.apache.doris.thrift.TTypeDesc;
+import org.apache.doris.thrift.TTypeNode;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+// INSERT INTO table_map VALUES ({'key1':1, 'key2':10, 'k3':100}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30});
+// MapLiteral is one row-based literal
+public class MapLiteral extends LiteralExpr {
+
+ public MapLiteral() {
+ type = new MapType(Type.NULL, Type.NULL);
+ children = new ArrayList<>();
+ }
+
+ public MapLiteral(LiteralExpr... exprs) throws AnalysisException {
+ Type keyType = Type.NULL;
+ Type valueType = Type.NULL;
+ children = new ArrayList<>();
+ int idx = 0;
+ for (LiteralExpr expr : exprs) {
+ if (idx % 2 == 0) {
+ if (keyType == Type.NULL) {
+ keyType = expr.getType();
+ } else {
+ keyType = Type.getAssignmentCompatibleType(keyType, expr.getType(), false);
+ }
+ if (keyType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ } else {
+ if (valueType == Type.NULL) {
+ valueType = expr.getType();
+ } else {
+ valueType = Type.getAssignmentCompatibleType(valueType, expr.getType(), false);
+ }
+ if (valueType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ }
+ children.add(expr);
+ ++ idx;
+ }
+
+ type = new MapType(keyType, valueType);
+ }
+
+ protected MapLiteral(MapLiteral other) {
+ super(other);
+ }
+
+ @Override
+ public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
+ if (!targetType.isMapType()) {
+ return super.uncheckedCastTo(targetType);
+ }
+ MapLiteral literal = new MapLiteral(this);
+ Type keyType = ((MapType) targetType).getKeyType();
+ Type valueType = ((MapType) targetType).getValueType();
+
+ for (int i = 0; i < children.size(); ++ i) {
+ Expr child = children.get(i);
+ if ((i & 1) == 0) {
Review Comment:
done!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089861009
##########
be/src/vec/sink/vmysql_result_writer.cpp:
##########
@@ -186,6 +188,58 @@ Status VMysqlResultWriter::_add_one_column(const ColumnPtr& column_ptr,
_buffer.close_dynamic_mode();
result->result_batch.rows[i].append(_buffer.buf(), _buffer.length());
}
+ } else if constexpr (type == TYPE_MAP) {
+ auto& column_map = assert_cast<const ColumnMap&>(*column);
+ auto& offsets = column_map.get_offsets();
+ auto& column_key_array = assert_cast<const ColumnArray&>(column_map.get_keys());
+ auto& column_val_array = assert_cast<const ColumnArray&>(column_map.get_values());
+ auto& map_type = assert_cast<const DataTypeMap&>(*nested_type_ptr);
+ auto& key_nested_type_ptr = map_type.get_key_type();
+ auto& val_nested_type_ptr = map_type.get_value_type();
+ for (ssize_t i = 0; i < row_size; ++i) {
+ if (0 != buf_ret) {
+ return Status::InternalError("pack mysql buffer failed.");
+ }
+ _buffer.reset();
+
+ _buffer.open_dynamic_mode();
+ buf_ret = _buffer.push_string("{", 1);
Review Comment:
Yes . I will
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1089857885
##########
be/src/vec/columns/column_map.cpp:
##########
@@ -0,0 +1,218 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnMap.cpp
+// and modified by Doris
+
+#include "vec/columns/column_map.h"
+
+namespace doris::vectorized {
+
+/** A column of map values.
+ */
+std::string ColumnMap::get_name() const {
+ return "Map(" + keys->get_name() + ", " + values->get_name() + ")";
+}
+
+ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values)
+ : keys(std::move(keys)), values(std::move(values)) {
+ check_size();
+}
+
+ColumnArray::Offsets64& ColumnMap::get_offsets() const {
+ const ColumnArray & column_keys = assert_cast<const ColumnArray &> (get_keys());
+ // todo . did here check size ?
+ return const_cast<Offsets64&>(column_keys.get_offsets());
+}
+
+void ColumnMap::check_size() const {
+ const auto * key_array = typeid_cast<const ColumnArray *>(keys.get());
+ const auto * value_array = typeid_cast<const ColumnArray *>(values.get());
+ CHECK(key_array) << "ColumnMap keys can be created only from array";
+ CHECK(value_array) << "ColumnMap values can be created only from array";
+ CHECK_EQ(get_keys_ptr()->size(), get_values_ptr()->size());
+}
+
+// todo. here to resize every row map
+MutableColumnPtr ColumnMap::clone_resized(size_t to_size) const {
+ auto res = ColumnMap::create(keys->clone_resized(to_size), values->clone_resized(to_size));
+ return res;
+}
+
+// to support field functions
+Field ColumnMap::operator[](size_t n) const {
+ Map res(2);
+ keys->get(n, res[0]);
+ values->get(n, res[0]);
+
+ return res;
+}
+
+// here to compare to below
+void ColumnMap::get(size_t n, Field & res) const {
+ Map map(2);
+ keys->get(n, map[0]);
+ values->get(n, map[1]);
+
+ res = map;
+}
+
+StringRef ColumnMap::get_data_at(size_t n) const {
+ LOG(FATAL) << "Method get_data_at is not supported for " << get_name();
+}
+
+void ColumnMap::insert_data(const char*, size_t) {
+ LOG(FATAL) << "Method insert_data is not supported for " << get_name();
+}
+
+void ColumnMap::insert(const Field& x) {
+ const auto& map = doris::vectorized::get<const Map&>(x);
+ // ({}, {}, {})
+ // ([], [])
+ CHECK_EQ(map.size(), 2);
+ keys->insert(map[0]);
+ values->insert(map[1]);
+}
+
+void ColumnMap::insert_default() {
+ keys->insert_default();
+ values->insert_default();
+}
+
+void ColumnMap::pop_back(size_t n) {
+ keys->pop_back(n);
+ values->pop_back(n);
+}
+
+StringRef ColumnMap::serialize_value_into_arena(size_t n, Arena & arena, char const*& begin)
+ const {
+ StringRef res(begin, 0);
+ auto keys_ref = keys->serialize_value_into_arena(n, arena, begin);
+ res.data = keys_ref.data - res.size;
+ res.size += keys_ref.size;
+ auto value_ref = values->serialize_value_into_arena(n, arena, begin);
+ res.data = value_ref.data - res.size;
+ res.size += value_ref.size;
+
+ return res;
+}
+
+void ColumnMap::insert_from(const IColumn& src_, size_t n) {
+ const ColumnMap& src = assert_cast<const ColumnMap&>(src_);
+
+ if ((!get_keys().is_nullable() && src.get_keys().is_nullable())
+ || (!get_values().is_nullable() && src.get_values().is_nullable())) {
+ DCHECK(false);
+ } else if ((get_keys().is_nullable() && !src.get_keys().is_nullable())
+ || (get_values().is_nullable() && !src.get_values().is_nullable())) {
+ DCHECK(false);
+ } else {
+ keys->insert_from(*assert_cast<const ColumnMap&>(src_).keys, n);
+ values->insert_from(*assert_cast<const ColumnMap&>(src_).values, n);
+ }
+}
+
+void ColumnMap::insert_indices_from(const IColumn& src, const int* indices_begin,
+ const int* indices_end) {
+ for (auto x = indices_begin; x != indices_end; ++x) {
+ if (*x == -1) {
+ ColumnMap::insert_default();
+ } else {
+ ColumnMap::insert_from(src, *x);
+ }
+ }
+}
+
+const char* ColumnMap::deserialize_and_insert_from_arena(const char* pos) {
+ pos = keys->deserialize_and_insert_from_arena(pos);
+ pos = values->deserialize_and_insert_from_arena(pos);
+
+ return pos;
+}
+
+void ColumnMap::update_hash_with_value(size_t n, SipHash & hash) const {
+ keys->update_hash_with_value(n, hash);
+ values->update_hash_with_value(n, hash);
+}
+
+void ColumnMap::insert_range_from(const IColumn& src, size_t start, size_t length) {
+ keys->insert_range_from(*assert_cast<const ColumnMap&>(src).keys, start, length);
+ values->insert_range_from(*assert_cast<const ColumnMap&>(src).values, start, length);
+}
+
+ColumnPtr ColumnMap::filter(const Filter& filt, ssize_t result_size_hint) const {
+ return ColumnMap::create(keys->filter(filt, result_size_hint), values->filter(filt, result_size_hint));
+}
+
+ColumnPtr ColumnMap::permute(const Permutation& perm, size_t limit) const {
+ return ColumnMap::create(keys->permute(perm, limit), values->permute(perm, limit));
+}
+
+ColumnPtr ColumnMap::replicate(const Offsets& offsets) const {
+ return ColumnMap::create(keys->replicate(offsets), values->replicate(offsets));
+}
+
+//MutableColumns ColumnMap::scatter(ColumnIndex num_columns, const Selector& selector) const {
+//
+// MutableColumns keys_scatter = keys->scatter(num_columns, selector);
+// MutableColumns values_scatter = values->scatter(num_columns, selector);
+//
+// MutableColumns res(num_columns);
+//
+// for (size_t scattered_idx = 0; scattered_idx < num_columns; ++scattered_idx)
+// {
+// MutableColumns new_columns(2);
+// for (size_t map_element_idx = 0; map_element_idx < 2; ++map_element_idx)
+// new_columns[map_element_idx] = std::move(scattered_map_elements[map_element_idx][scattered_idx]);
+// res[scattered_idx] = ColumnMap::create(std::move(new_columns));
+// }
+//
+//
+// return res;
+//}
+
+
+void ColumnMap::reserve(size_t n) {
+ get_keys().reserve(n);
+ get_values().reserve(n);
+}
+
+size_t ColumnMap::byte_size() const {
+ return get_keys().byte_size() + get_values().byte_size();
+}
+
+size_t ColumnMap::allocated_bytes() const {
+ return get_keys().allocated_bytes() + get_values().allocated_bytes();
+}
+
+void ColumnMap::protect() {
+ get_keys().protect();
+ get_values().protect();
+}
+
+void ColumnMap::get_extremes(Field & min, Field & max) const {
+ Map min_map(2);
+ Map max_map(2);
+
+ keys->get_extremes(min_map[0], max_map[0]);
+ values->get_extremes(min_map[1], max_map[1]);
Review Comment:
Or Maybe no need to impl get_extremes function like array, because We should make some functions for map to calculate values for map
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1090113853
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -449,13 +478,97 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this,null_iterator,
+ key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator: {}",
std::to_string(type));
}
}
}
+///====================== MapFileColumnIterator ============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::_peek_one_offset(ordinal_t* offset) {
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) {
+ return Status::OK();
+}
+
+
+Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(ord));
+ RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(ord));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+ const auto* column_map = vectorized::check_and_get_column<vectorized::ColumnMap>(
+ dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
+ : *dst);
+ auto column_key_ptr = column_map->get_keys().assume_mutable();
+ auto column_val_ptr = column_map->get_values().assume_mutable();
+ RETURN_IF_ERROR(_key_iterator->next_batch(n, column_key_ptr, has_null));
Review Comment:
done!
##########
be/src/olap/rowset/segment_v2/column_reader.h:
##########
@@ -393,6 +393,44 @@ class EmptyFileColumnIterator final : public ColumnIterator {
ordinal_t get_current_ordinal() const override { return 0; }
};
+// This iterator is used to read map value column
+class MapFileColumnIterator final : public ColumnIterator {
+public:
+ explicit MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator, ColumnIterator* val_iterator);
+
+ ~MapFileColumnIterator() override = default;
+
+ Status init(const ColumnIteratorOptions& opts) override;
+
+ Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) override;
+
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst, bool* has_null) override;
+
+ Status read_by_rowids(const rowid_t* rowids, const size_t count,
+ vectorized::MutableColumnPtr& dst) override;
+
+ Status seek_to_first() override {
+ RETURN_IF_ERROR(_key_iterator->seek_to_first());
+ RETURN_IF_ERROR(_val_iterator->seek_to_first());
Review Comment:
add it!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] xy720 commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "xy720 (via GitHub)" <gi...@apache.org>.
xy720 commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1092726577
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -485,13 +512,91 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this, null_iterator, key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator: {}",
std::to_string(type));
}
}
}
+///====================== MapFileColumnIterator ============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) {
+ return Status::NotSupported("Not support next_batch");
+}
+
+Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(ord));
+ RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(ord));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+ const auto* column_map = vectorized::check_and_get_column<vectorized::ColumnMap>(
+ dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
+ : *dst);
+ size_t num_read = *n;
+ auto column_key_ptr = column_map->get_keys().assume_mutable();
+ auto column_val_ptr = column_map->get_values().assume_mutable();
+ RETURN_IF_ERROR(_key_iterator->next_batch(num_read, column_key_ptr, has_null));
Review Comment:
```suggestion
RETURN_IF_ERROR(_key_iterator->next_batch(&num_read, column_key_ptr, has_null));
```
##########
be/src/olap/types.h:
##########
@@ -431,6 +432,93 @@ class ArrayTypeInfo : public TypeInfo {
TypeInfoPtr _item_type_info;
const size_t _item_size;
};
+///====================== MapType Info ==========================///
+class MapTypeInfo : public TypeInfo {
+public:
+ explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info)
+ : _key_type_info(std::move(key_type_info)),
+ _value_type_info(std::move(value_type_info)) {}
+ ~MapTypeInfo() override = default;
+
+ inline bool equal(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ return l_value->size() == r_value->size();
+ }
+
+ int cmp(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ uint32_t l_size = l_value->size();
+ uint32_t r_size = r_value->size();
+ if (l_size < r_size) {
+ return -1;
+ } else if (l_size > r_size) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ void shallow_copy(void* dest, const void* src) const override {
+ auto dest_value = reinterpret_cast<MapValue*>(dest);
+ auto src_value = reinterpret_cast<const MapValue*>(src);
+ dest_value->shallow_copy(src_value);
+ }
+
+ void deep_copy(void* dest, const void* src, MemPool* mem_pool) const override { DCHECK(false); }
+
+ void copy_object(void* dest, const void* src, MemPool* mem_pool) const override {
+ deep_copy(dest, src, mem_pool);
+ }
+
+ void direct_copy(void* dest, const void* src) const override { CHECK(false); }
+
+ void direct_copy(uint8_t** base, void* dest, const void* src) const { CHECK(false); }
+
+ void direct_copy_may_cut(void* dest, const void* src) const override { direct_copy(dest, src); }
+
+ Status convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool,
+ size_t variable_len = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ Status from_string(void* buf, const std::string& scan_key, const int precision = 0,
+ const int scale = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ std::string to_string(const void* src) const override { return "{}"; }
+
+ void set_to_max(void* buf) const override {
+ DCHECK(false) << "set_to_max of list is not implemented.";
+ }
+
+ void set_to_min(void* buf) const override {
+ DCHECK(false) << "set_to_min of list is not implemented.";
+ }
+
+ uint32_t hash_code(const void* data, uint32_t seed) const override {
+ auto map_value = reinterpret_cast<const MapValue*>(data);
+ auto size = map_value->size();
+ uint32_t result = HashUtil::hash(&size, sizeof(size), seed);
+ result = seed * result + _key_type_info->hash_code(map_value->key_data(), seed) +
+ _value_type_info->hash_code(map_value->value_data(), seed);
+ return result;
+ }
+
+ // todo . is here only to need return 16 for two ptr?
+ const size_t size() const override { return 16; }
Review Comment:
should use sizeof(MapValue)
16 bytes(2 pointers) + 4 bytes(_length) = 20 bytes => align to 8 bytes => total 24 bytes
##########
be/src/runtime/types.cpp:
##########
@@ -223,6 +227,15 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
children.push_back(TypeDescriptor(types, idx));
break;
}
+ case TTypeNodeType::MAP: {
+ type = TYPE_MAP;
+ if (node.has_contains_null()) {
+ contains_null = node.contains_null();
Review Comment:
contains_null has been modified by struct, please check this logic.
##########
be/src/vec/olap/olap_data_convertor.cpp:
##########
@@ -129,6 +128,20 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co
return std::make_unique<OlapColumnDataConvertorArray>(
create_olap_column_data_convertor(sub_column));
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ const auto& key_column = column.get_sub_column(0);
+ const auto& value_column = column.get_sub_column(1);
+ return std::make_unique<OlapColumnDataConvertorMap>(
+ std::make_unique<OlapColumnDataConvertorArray>(
+ create_olap_column_data_convertor(key_column)),
+ std::make_unique<OlapColumnDataConvertorArray>(
+ create_olap_column_data_convertor(value_column)));
+ //const auto& key_column = column.get_sub_column(0);
Review Comment:
remove unused code
##########
be/src/olap/rowset/segment_v2/column_reader.cpp:
##########
@@ -485,13 +512,91 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this, null_iterator, key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator: {}",
std::to_string(type));
}
}
}
+///====================== MapFileColumnIterator ============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) {
+ return Status::NotSupported("Not support next_batch");
+}
+
+Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(ord));
+ RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(ord));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+ const auto* column_map = vectorized::check_and_get_column<vectorized::ColumnMap>(
+ dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
+ : *dst);
+ size_t num_read = *n;
+ auto column_key_ptr = column_map->get_keys().assume_mutable();
+ auto column_val_ptr = column_map->get_values().assume_mutable();
+ RETURN_IF_ERROR(_key_iterator->next_batch(num_read, column_key_ptr, has_null));
+ RETURN_IF_ERROR(_val_iterator->next_batch(num_read, column_val_ptr, has_null));
Review Comment:
```suggestion
RETURN_IF_ERROR(_val_iterator->next_batch(&num_read, column_val_ptr, has_null));
```
##########
fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java:
##########
@@ -318,6 +318,11 @@ public void analyze() throws AnalysisException {
type, Function.NullableMode.ALWAYS_NULLABLE,
Lists.newArrayList(Type.VARCHAR), false,
"doris::CastFunctions::cast_to_array_val", null, null, true);
+ } else if (type.isMapType()) {
+ fn = ScalarFunction.createBuiltin(getFnName(Type.MAP),
+ type, Function.NullableMode.ALWAYS_NULLABLE,
+ Lists.newArrayList(Type.VARCHAR), false,
+ "doris::CastFunctions::cast_to_map_val", null, null, true);
Review Comment:
I see. No need to change here.
##########
be/src/vec/sink/vmysql_result_writer.cpp:
##########
@@ -715,6 +732,18 @@ Status VMysqlResultWriter::append_block(Block& input_block) {
}
break;
}
+ case TYPE_MAP: {
+ if (type_ptr->is_nullable()) {
+ auto& nested_type =
+ assert_cast<const DataTypeNullable&>(*type_ptr).get_nested_type(); //for map
+ status = _add_one_column<PrimitiveType::TYPE_MAP, true>(column_ptr, result,
Review Comment:
_add_one_column has been modified by struct, please check this logic.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1093944636
##########
be/src/runtime/map_value.h:
##########
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <type_traits>
+#include "runtime/primitive_type.h"
+
+namespace doris_udf {
+class FunctionContext;
+struct AnyVal;
+} // namespace doris_udf
+
+namespace doris {
+
+using doris_udf::FunctionContext;
+using doris_udf::AnyVal;
+
+/**
+ * MapValue is for map type in memory
+ */
+class MapValue {
+public:
+ MapValue() = default;
+
+ explicit MapValue(int32_t length)
+ : _key_data(nullptr), _value_data(nullptr), _length(length){}
+
+ MapValue(void* k_data, void* v_data, int32_t length)
+ : _key_data(k_data), _value_data(v_data), _length(length) {}
+
+ void to_map_val(MapVal* val) const;
+
+ int32_t size() const { return _length; }
+
+ int32_t length() const { return _length; }
+
+ void shallow_copy(const MapValue* other);
Review Comment:
needed not yet
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[GitHub] [doris] amorynan commented on a diff in pull request #15966: [Feature](map)support complex struct for doris
Posted by "amorynan (via GitHub)" <gi...@apache.org>.
amorynan commented on code in PR #15966:
URL: https://github.com/apache/doris/pull/15966#discussion_r1094149710
##########
be/src/vec/exprs/vexpr.cpp:
##########
@@ -174,218 +174,220 @@ Status VExpr::create_expr(doris::ObjectPool* pool, const doris::TExprNode& texpr
default:
return Status::InternalError("Unknown expr node type: {}", texpr_node.node_type);
}
- return Status::OK();
-}
+ return Status::OK();
+ }
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
Review Comment:
when I use clang-format.sh , it make these change...
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org