You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by xu...@apache.org on 2023/02/06 07:03:47 UTC
[doris] branch struct-map-type-rebase created (now c02a706c9a)
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a change to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
at c02a706c9a rebase master and fix compile error
This branch includes the following new commits:
new 6628baae93 test
new 5e9a6e40ba save
new 48a0b4e80a [test] (test) test modify branch (#14375)
new 80f6a7b0ab [feature](struct-type) add the class ColumnStruct and class DataTypeStruct implement (#14545)
new 3d68b650eb [WIP](struct-type) support struct-type in vectorize engine (#15665)
new 5137ca6da0 [fix](struct-type) struct create fix and add more struct codes (#15879)
new 3818973f96 [feature](struct-type) adjust the vexpr for struct type (#15669)
new 9fa4b05c5d save (#15906)
new 273bc1b703 save (#15908)
new 747cc04cc3 [fix](struct-type) support to remove double or single quote (#15977)
new 11c8eca9f0 [fix](struct-type) support struct type insert (#15941)
new 7a573c9404 [hotfix](struct-type) Fix type descriptor make array load core (#16020)
new e6c298e9c3 [fix](struct-type) select of struct return wrong result (#16032)
new eb70160b96 [fix](struct-type) fix insert null and explain for struct type (#16054)
new ebfd14ca2c [Feature](map) add map type to doris (#15966)
new 454ff4c69a [fix](map-type) fix compile error (#16413)
new eccddd910a [fix](complex_type) fix clone_type_info function return default scalar type (#16400)
new c02a706c9a rebase master and fix compile error
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/18: [test] (test) test modify branch (#14375)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 48a0b4e80ad964a36de94e023a9a2cf1eadcaf3a
Author: xy720 <22...@users.noreply.github.com>
AuthorDate: Thu Nov 17 19:27:40 2022 +0800
[test] (test) test modify branch (#14375)
---
README.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/README.md b/README.md
index 709f323252..a89d2fabbd 100644
--- a/README.md
+++ b/README.md
@@ -231,3 +231,4 @@ some Doris features to be complied with Apache 2.0 License. For details, refer t
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 17/18: [fix](complex_type) fix clone_type_info function return default scalar type (#16400)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit eccddd910aff380d165128188addef817c79e3d8
Author: amory <wa...@selectdb.com>
AuthorDate: Mon Feb 6 10:50:46 2023 +0800
[fix](complex_type) fix clone_type_info function return default scalar type (#16400)
---
be/src/olap/types.cpp | 11 +-
be/src/runtime/types.cpp | 114 ++++++++--------
be/src/vec/exprs/vexpr.cpp | 333 ++++++++++++++++++++++-----------------------
3 files changed, 229 insertions(+), 229 deletions(-)
diff --git a/be/src/olap/types.cpp b/be/src/olap/types.cpp
index bee790e066..0ccf4b98b1 100644
--- a/be/src/olap/types.cpp
+++ b/be/src/olap/types.cpp
@@ -262,14 +262,13 @@ TypeInfoPtr get_type_info(const TabletColumn* col) {
}
TypeInfoPtr clone_type_info(const TypeInfo* type_info) {
- if (is_scalar_type(type_info->type())) {
- return create_static_type_info_ptr(type_info);
- } else if (type_info->type() == OLAP_FIELD_TYPE_MAP) {
+ auto type = type_info->type();
+ if (UNLIKELY(type == OLAP_FIELD_TYPE_MAP)) {
const auto map_type_info = dynamic_cast<const MapTypeInfo*>(type_info);
return create_dynamic_type_info_ptr(
new MapTypeInfo(clone_type_info(map_type_info->get_key_type_info()),
clone_type_info(map_type_info->get_value_type_info())));
- } else if (type_info->type() == OLAP_FIELD_TYPE_STRUCT) {
+ } else if (UNLIKELY(type == OLAP_FIELD_TYPE_STRUCT)) {
const auto struct_type_info = dynamic_cast<const StructTypeInfo*>(type_info);
std::vector<TypeInfoPtr> clone_type_infos;
const std::vector<TypeInfoPtr>* sub_type_infos = struct_type_info->type_infos();
@@ -278,10 +277,12 @@ TypeInfoPtr clone_type_info(const TypeInfo* type_info) {
clone_type_infos.push_back(clone_type_info((*sub_type_infos)[i].get()));
}
return create_dynamic_type_info_ptr(new StructTypeInfo(clone_type_infos));
- } else if (type_info->type() == OLAP_FIELD_TYPE_ARRAY) {
+ } else if (UNLIKELY(type == OLAP_FIELD_TYPE_ARRAY)) {
const auto array_type_info = dynamic_cast<const ArrayTypeInfo*>(type_info);
return create_dynamic_type_info_ptr(
new ArrayTypeInfo(clone_type_info(array_type_info->item_type_info())));
+ } else {
+ return create_static_type_info_ptr(type_info);
}
}
diff --git a/be/src/runtime/types.cpp b/be/src/runtime/types.cpp
index 2945bfd33f..002fe1ed07 100644
--- a/be/src/runtime/types.cpp
+++ b/be/src/runtime/types.cpp
@@ -252,67 +252,67 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
default:
DCHECK(false) << node.type();
}
- }
+}
- std::string TypeDescriptor::debug_string() const {
- std::stringstream ss;
- switch (type) {
- case TYPE_CHAR:
- ss << "CHAR(" << len << ")";
- return ss.str();
- case TYPE_DECIMALV2:
- ss << "DECIMALV2(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL32:
- ss << "DECIMAL32(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL64:
- ss << "DECIMAL64(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL128I:
- ss << "DECIMAL128(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_ARRAY: {
- ss << "ARRAY<" << children[0].debug_string() << ">";
- return ss.str();
- }
- case TYPE_MAP:
- ss << "MAP<" << children[0].debug_string() << ", " << children[1].debug_string() << ">";
- return ss.str();
- case TYPE_STRUCT: {
- ss << "STRUCT<";
- for (size_t i = 0; i < children.size(); i++) {
- ss << field_names[i];
- ss << ":";
- ss << children[i].debug_string();
- if (i != children.size() - 1) {
- ss << ",";
- }
+std::string TypeDescriptor::debug_string() const {
+ std::stringstream ss;
+ switch (type) {
+ case TYPE_CHAR:
+ ss << "CHAR(" << len << ")";
+ return ss.str();
+ case TYPE_DECIMALV2:
+ ss << "DECIMALV2(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL32:
+ ss << "DECIMAL32(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL64:
+ ss << "DECIMAL64(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL128I:
+ ss << "DECIMAL128(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_ARRAY: {
+ ss << "ARRAY<" << children[0].debug_string() << ">";
+ return ss.str();
+ }
+ case TYPE_MAP:
+ ss << "MAP<" << children[0].debug_string() << ", " << children[1].debug_string() << ">";
+ return ss.str();
+ case TYPE_STRUCT: {
+ ss << "STRUCT<";
+ for (size_t i = 0; i < children.size(); i++) {
+ ss << field_names[i];
+ ss << ":";
+ ss << children[i].debug_string();
+ if (i != children.size() - 1) {
+ ss << ",";
}
- ss << ">";
- return ss.str();
- }
- default:
- return type_to_string(type);
}
+ ss << ">";
+ return ss.str();
}
-
- std::ostream& operator<<(std::ostream& os, const TypeDescriptor& type) {
- os << type.debug_string();
- return os;
+ default:
+ return type_to_string(type);
}
+}
- TTypeDesc create_type_desc(PrimitiveType type, int precision, int scale) {
- TTypeDesc type_desc;
- std::vector<TTypeNode> node_type;
- node_type.emplace_back();
- TScalarType scalarType;
- scalarType.__set_type(to_thrift(type));
- scalarType.__set_len(-1);
- scalarType.__set_precision(precision);
- scalarType.__set_scale(scale);
- node_type.back().__set_scalar_type(scalarType);
- type_desc.__set_types(node_type);
- return type_desc;
- }
+std::ostream& operator<<(std::ostream& os, const TypeDescriptor& type) {
+ os << type.debug_string();
+ return os;
+}
+
+TTypeDesc create_type_desc(PrimitiveType type, int precision, int scale) {
+ TTypeDesc type_desc;
+ std::vector<TTypeNode> node_type;
+ node_type.emplace_back();
+ TScalarType scalarType;
+ scalarType.__set_type(to_thrift(type));
+ scalarType.__set_len(-1);
+ scalarType.__set_precision(precision);
+ scalarType.__set_scale(scale);
+ node_type.back().__set_scalar_type(scalarType);
+ type_desc.__set_types(node_type);
+ return type_desc;
+}
} // namespace doris
diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp
index 573a2b705c..9029802dd0 100644
--- a/be/src/vec/exprs/vexpr.cpp
+++ b/be/src/vec/exprs/vexpr.cpp
@@ -180,90 +180,90 @@ Status VExpr::create_expr(doris::ObjectPool* pool, const doris::TExprNode& texpr
return Status::OK();
}
- Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
+Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- }
- return Status::OK();
}
+ return Status::OK();
+}
- Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
- return Status::OK();
- }
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were "
- "used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
+Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
}
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
+}
- Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
- }
- return Status::OK();
+Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
+ ctxs->clear();
+ for (int i = 0; i < texprs.size(); ++i) {
+ VExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
+ ctxs->push_back(ctx);
}
+ return Status::OK();
+}
- Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
- }
- return Status::OK();
+Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ const RowDescriptor& row_desc) {
+ for (auto ctx : ctxs) {
+ RETURN_IF_ERROR(ctx->prepare(state, row_desc));
}
+ return Status::OK();
+}
- void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (auto ctx : ctxs) {
- ctx->close(state);
- }
+void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (auto ctx : ctxs) {
+ ctx->close(state);
}
+}
- Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->open(state));
- }
- return Status::OK();
+Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->open(state));
}
+ return Status::OK();
+}
FunctionContext::TypeDesc VExpr::column_type_to_type_desc(const TypeDescriptor& type) {
FunctionContext::TypeDesc out;
@@ -367,132 +367,131 @@ FunctionContext::TypeDesc VExpr::column_type_to_type_desc(const TypeDescriptor&
return out;
}
- Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- std::vector<VExprContext*>* new_ctxs) {
- DCHECK(new_ctxs != nullptr);
- if (!new_ctxs->empty()) {
- // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
- DCHECK_EQ(new_ctxs->size(), ctxs.size());
- for (int i = 0; i < new_ctxs->size(); ++i) {
- DCHECK((*new_ctxs)[i]->_is_clone);
- }
- return Status::OK();
- }
- new_ctxs->resize(ctxs.size());
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
+Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ std::vector<VExprContext*>* new_ctxs) {
+ DCHECK(new_ctxs != nullptr);
+ if (!new_ctxs->empty()) {
+ // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
+ DCHECK_EQ(new_ctxs->size(), ctxs.size());
+ for (int i = 0; i < new_ctxs->size(); ++i) {
+ DCHECK((*new_ctxs)[i]->_is_clone);
}
return Status::OK();
}
- std::string VExpr::debug_string() const {
- // TODO: implement partial debug string for member vars
- std::stringstream out;
- out << " type=" << _type.debug_string();
- out << " codegen="
- << "false";
-
- if (!_children.empty()) {
- out << " children=" << debug_string(_children);
- }
+ new_ctxs->resize(ctxs.size());
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
+ }
+ return Status::OK();
+}
+std::string VExpr::debug_string() const {
+ // TODO: implement partial debug string for member vars
+ std::stringstream out;
+ out << " type=" << _type.debug_string();
+ out << " codegen="
+ << "false";
- return out.str();
+ if (!_children.empty()) {
+ out << " children=" << debug_string(_children);
}
- std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
- std::stringstream out;
- out << "[";
+ return out.str();
+}
- for (int i = 0; i < exprs.size(); ++i) {
- out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
- }
+std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
+ std::stringstream out;
+ out << "[";
- out << "]";
- return out.str();
+ for (int i = 0; i < exprs.size(); ++i) {
+ out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
}
- std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
- std::vector<VExpr*> exprs;
- for (int i = 0; i < ctxs.size(); ++i) {
- exprs.push_back(ctxs[i]->root());
- }
- return debug_string(exprs);
+ out << "]";
+ return out.str();
+}
+
+std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
+ std::vector<VExpr*> exprs;
+ for (int i = 0; i < ctxs.size(); ++i) {
+ exprs.push_back(ctxs[i]->root());
}
+ return debug_string(exprs);
+}
- bool VExpr::is_constant() const {
- for (int i = 0; i < _children.size(); ++i) {
- if (!_children[i]->is_constant()) {
- return false;
- }
+bool VExpr::is_constant() const {
+ for (int i = 0; i < _children.size(); ++i) {
+ if (!_children[i]->is_constant()) {
+ return false;
}
-
- return true;
}
- Status VExpr::get_const_col(VExprContext * context, ColumnPtrWrapper * *output) {
- *output = nullptr;
- if (!is_constant()) {
- return Status::OK();
- }
+ return true;
+}
- if (_constant_col != nullptr) {
- *output = _constant_col.get();
- return Status::OK();
- }
+Status VExpr::get_const_col(VExprContext* context, ColumnPtrWrapper** output) {
+ *output = nullptr;
+ if (!is_constant()) {
+ return Status::OK();
+ }
- int result = -1;
- Block block;
- // If block is empty, some functions will produce no result. So we insert a column with
- // single value here.
- block.insert({ColumnUInt8::create(1), std::make_shared<DataTypeUInt8>(), ""});
- RETURN_IF_ERROR(execute(context, &block, &result));
- DCHECK(result != -1);
- const auto& column = block.get_by_position(result).column;
- _constant_col = std::make_shared<ColumnPtrWrapper>(column);
+ if (_constant_col != nullptr) {
*output = _constant_col.get();
return Status::OK();
}
- void VExpr::register_function_context(doris::RuntimeState * state, VExprContext * context) {
- FunctionContext::TypeDesc return_type = VExpr::column_type_to_type_desc(_type);
- std::vector<FunctionContext::TypeDesc> arg_types;
- for (int i = 0; i < _children.size(); ++i) {
- arg_types.push_back(VExpr::column_type_to_type_desc(_children[i]->type()));
- }
+ int result = -1;
+ Block block;
+ // If block is empty, some functions will produce no result. So we insert a column with
+ // single value here.
+ block.insert({ColumnUInt8::create(1), std::make_shared<DataTypeUInt8>(), ""});
+ RETURN_IF_ERROR(execute(context, &block, &result));
+ DCHECK(result != -1);
+ const auto& column = block.get_by_position(result).column;
+ _constant_col = std::make_shared<ColumnPtrWrapper>(column);
+ *output = _constant_col.get();
+ return Status::OK();
+}
- _fn_context_index = context->register_func(state, return_type, arg_types, 0);
+void VExpr::register_function_context(doris::RuntimeState* state, VExprContext* context) {
+ FunctionContext::TypeDesc return_type = VExpr::column_type_to_type_desc(_type);
+ std::vector<FunctionContext::TypeDesc> arg_types;
+ for (int i = 0; i < _children.size(); ++i) {
+ arg_types.push_back(VExpr::column_type_to_type_desc(_children[i]->type()));
}
- Status VExpr::init_function_context(VExprContext * context,
- FunctionContext::FunctionStateScope scope,
- const FunctionBasePtr& function) const {
- FunctionContext* fn_ctx = context->fn_context(_fn_context_index);
- if (scope == FunctionContext::FRAGMENT_LOCAL) {
- std::vector<ColumnPtrWrapper*> constant_cols;
- for (auto c : _children) {
- ColumnPtrWrapper* const_col_wrapper = nullptr;
- RETURN_IF_ERROR(c->get_const_col(context, &const_col_wrapper));
- constant_cols.push_back(const_col_wrapper);
- }
- fn_ctx->impl()->set_constant_cols(constant_cols);
- }
+ _fn_context_index = context->register_func(state, return_type, arg_types, 0);
+}
- if (scope == FunctionContext::FRAGMENT_LOCAL) {
- RETURN_IF_ERROR(function->prepare(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
+Status VExpr::init_function_context(VExprContext* context,
+ FunctionContext::FunctionStateScope scope,
+ const FunctionBasePtr& function) const {
+ FunctionContext* fn_ctx = context->fn_context(_fn_context_index);
+ if (scope == FunctionContext::FRAGMENT_LOCAL) {
+ std::vector<ColumnPtrWrapper*> constant_cols;
+ for (auto c : _children) {
+ ColumnPtrWrapper* const_col_wrapper = nullptr;
+ RETURN_IF_ERROR(c->get_const_col(context, &const_col_wrapper));
+ constant_cols.push_back(const_col_wrapper);
}
- RETURN_IF_ERROR(function->prepare(fn_ctx, FunctionContext::THREAD_LOCAL));
- return Status::OK();
+ fn_ctx->impl()->set_constant_cols(constant_cols);
}
- void VExpr::close_function_context(VExprContext * context,
- FunctionContext::FunctionStateScope scope,
- const FunctionBasePtr& function) const {
- if (_fn_context_index != -1 && !context->_stale) {
- FunctionContext* fn_ctx = context->fn_context(_fn_context_index);
- function->close(fn_ctx, FunctionContext::THREAD_LOCAL);
- if (scope == FunctionContext::FRAGMENT_LOCAL) {
- function->close(fn_ctx, FunctionContext::FRAGMENT_LOCAL);
- }
+ if (scope == FunctionContext::FRAGMENT_LOCAL) {
+ RETURN_IF_ERROR(function->prepare(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
+ }
+ RETURN_IF_ERROR(function->prepare(fn_ctx, FunctionContext::THREAD_LOCAL));
+ return Status::OK();
+}
+
+void VExpr::close_function_context(VExprContext* context, FunctionContext::FunctionStateScope scope,
+ const FunctionBasePtr& function) const {
+ if (_fn_context_index != -1 && !context->_stale) {
+ FunctionContext* fn_ctx = context->fn_context(_fn_context_index);
+ function->close(fn_ctx, FunctionContext::THREAD_LOCAL);
+ if (scope == FunctionContext::FRAGMENT_LOCAL) {
+ function->close(fn_ctx, FunctionContext::FRAGMENT_LOCAL);
}
}
+}
} // namespace doris::vectorized
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 07/18: [feature](struct-type) adjust the vexpr for struct type (#15669)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3818973f96a19de961e1bcf2f34882cc3d33bbc8
Author: carlvinhust2012 <hu...@126.com>
AuthorDate: Fri Jan 13 17:14:11 2023 +0800
[feature](struct-type) adjust the vexpr for struct type (#15669)
---
be/src/vec/columns/column_struct.cpp | 7 --
be/src/vec/columns/column_struct.h | 1 -
be/src/vec/data_types/data_type_factory.hpp | 8 ++
be/src/vec/data_types/data_type_struct.cpp | 99 +++++++++++++++++++---
be/src/vec/data_types/data_type_struct.h | 3 +
be/src/vec/functions/function_cast.h | 13 ++-
.../main/java/org/apache/doris/catalog/Type.java | 14 ++-
.../java/org/apache/doris/analysis/CastExpr.java | 8 ++
.../java/org/apache/doris/analysis/TypeDef.java | 2 +-
.../main/java/org/apache/doris/catalog/Column.java | 1 +
10 files changed, 134 insertions(+), 22 deletions(-)
diff --git a/be/src/vec/columns/column_struct.cpp b/be/src/vec/columns/column_struct.cpp
index d51bebb536..afef00c314 100644
--- a/be/src/vec/columns/column_struct.cpp
+++ b/be/src/vec/columns/column_struct.cpp
@@ -22,13 +22,6 @@
namespace doris::vectorized {
-namespace ErrorCodes {
-extern const int ILLEGAL_COLUMN;
-extern const int NOT_IMPLEMENTED;
-extern const int CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE;
-extern const int LOGICAL_ERROR;
-} // namespace ErrorCodes
-
std::string ColumnStruct::get_name() const {
std::stringstream res;
res << "Struct(";
diff --git a/be/src/vec/columns/column_struct.h b/be/src/vec/columns/column_struct.h
index a66d91f3e0..393b2c275f 100644
--- a/be/src/vec/columns/column_struct.h
+++ b/be/src/vec/columns/column_struct.h
@@ -103,7 +103,6 @@ public:
bool can_be_inside_nullable() const override { return true; }
MutableColumnPtr clone_empty() const override;
MutableColumnPtr clone_resized(size_t size) const override;
-
size_t size() const override { return columns.at(0)->size(); }
Field operator[](size_t n) const override;
diff --git a/be/src/vec/data_types/data_type_factory.hpp b/be/src/vec/data_types/data_type_factory.hpp
index d46dd3435d..9bc5e20d5c 100644
--- a/be/src/vec/data_types/data_type_factory.hpp
+++ b/be/src/vec/data_types/data_type_factory.hpp
@@ -112,6 +112,14 @@ public:
return entity.second;
}
}
+ if (type_ptr->get_type_id() == TypeIndex::Struct) {
+ DataTypeFactory::instance().register_data_type(type_ptr->get_name(), type_ptr);
+ for (const auto& entity : _invert_data_type_map) {
+ if (entity.first->equals(*type_ptr)) {
+ return entity.second;
+ }
+ }
+ }
return _empty_string;
}
diff --git a/be/src/vec/data_types/data_type_struct.cpp b/be/src/vec/data_types/data_type_struct.cpp
index 4364f5c34f..87b5c4f110 100644
--- a/be/src/vec/data_types/data_type_struct.cpp
+++ b/be/src/vec/data_types/data_type_struct.cpp
@@ -22,17 +22,6 @@
namespace doris::vectorized {
-namespace ErrorCodes {
-extern const int BAD_ARGUMENTS;
-extern const int DUPLICATE_COLUMN;
-extern const int EMPTY_DATA_PASSED;
-extern const int NOT_FOUND_COLUMN_IN_BLOCK;
-extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-extern const int SIZES_OF_COLUMNS_IN_TUPLE_DOESNT_MATCH;
-extern const int ILLEGAL_INDEX;
-extern const int LOGICAL_ERROR;
-} // namespace ErrorCodes
-
DataTypeStruct::DataTypeStruct(const DataTypes& elems_)
: elems(elems_), have_explicit_names(false) {
/// Automatically assigned names in form of '1', '2', ...
@@ -91,6 +80,94 @@ std::string DataTypeStruct::do_get_name() const {
return s.str();
}
+Status DataTypeStruct::from_string(ReadBuffer& rb, IColumn* column) const {
+ DCHECK(!rb.eof());
+ auto* struct_column = assert_cast<ColumnStruct*>(column);
+
+ if (*rb.position() != '{') {
+ return Status::InvalidArgument("Struct does not start with '{' character, found '{}'",
+ *rb.position());
+ }
+ if (rb.count() < 2 || *(rb.end() - 1) != '}') {
+ return Status::InvalidArgument("Struct does not end with '}' character, found '{}'",
+ *(rb.end() - 1));
+ }
+
+ // here need handle the empty struct '{}'
+ if (rb.count() == 2) {
+ return Status::OK();
+ }
+
+ ++rb.position();
+ std::vector<ReadBuffer> field_rbs;
+ field_rbs.reserve(elems.size());
+
+ // here get the value "jack" and 20 from {"name":"jack","age":20}
+ while (!rb.eof()) {
+ size_t field_len = 0;
+ auto start = rb.position();
+ while (!rb.eof() && *start != ',' && *start != '}') {
+ field_len++;
+ start++;
+ }
+ if (field_len >= rb.count()) {
+ return Status::InvalidArgument("Invalid Length");
+ }
+ ReadBuffer field_rb(rb.position(), field_len);
+ size_t len = 0;
+ auto start_rb = field_rb.position();
+ while (!field_rb.eof() && *start_rb != ':') {
+ len++;
+ start_rb++;
+ }
+ ReadBuffer field(field_rb.position() + len + 1, field_rb.count() - len - 1);
+
+ if (field.count() < 2 || *field.position() != '"' || *field.end() != '"') {
+ field_rbs.push_back(field);
+ } else {
+ ReadBuffer field_has_quote(field.position() + 1, field.count() - 2);
+ field_rbs.push_back(field_has_quote);
+ }
+
+ rb.position() += field_len + 1;
+ }
+
+ for (size_t idx = 0; idx < elems.size(); idx++) {
+ elems[idx]->from_string(field_rbs[idx], &struct_column->get_column(idx));
+ }
+
+ return Status::OK();
+}
+
+std::string DataTypeStruct::to_string(const IColumn& column, size_t row_num) const {
+ auto ptr = column.convert_to_full_column_if_const();
+ auto& struct_column = assert_cast<const ColumnStruct&>(*ptr.get());
+
+ std::stringstream ss;
+ ss << "<";
+ for (size_t idx = 0; idx < elems.size(); idx++) {
+ if (idx != 0) {
+ ss << ", ";
+ }
+ ss << elems[idx]->to_string(struct_column.get_column(idx), row_num);
+ }
+ ss << ">";
+ return ss.str();
+}
+
+void DataTypeStruct::to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const {
+ auto ptr = column.convert_to_full_column_if_const();
+ auto& struct_column = assert_cast<const ColumnStruct&>(*ptr.get());
+ ostr.write("<", 1);
+ for (size_t idx = 0; idx < elems.size(); idx++) {
+ if (idx != 0) {
+ ostr.write(", ", 2);
+ }
+ elems[idx]->to_string(struct_column.get_column(idx), row_num, ostr);
+ }
+ ostr.write(">", 1);
+}
+
static inline IColumn& extract_element_column(IColumn& column, size_t idx) {
return assert_cast<ColumnStruct&>(column).get_column(idx);
}
diff --git a/be/src/vec/data_types/data_type_struct.h b/be/src/vec/data_types/data_type_struct.h
index 88255de38c..47cfaab1a3 100644
--- a/be/src/vec/data_types/data_type_struct.h
+++ b/be/src/vec/data_types/data_type_struct.h
@@ -96,6 +96,9 @@ public:
const char* deserialize(const char* buf, IColumn* column, int be_exec_version) const override;
void to_pb_column_meta(PColumnMeta* col_meta) const override;
+ Status from_string(ReadBuffer& rb, IColumn* column) const override;
+ std::string to_string(const IColumn& column, size_t row_num) const override;
+ void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
// bool is_parametric() const { return true; }
// SerializationPtr do_get_default_serialization() const override;
// SerializationPtr get_serialization(const SerializationInfo& info) const override;
diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h
index 9c2878db5b..f68e14578d 100644
--- a/be/src/vec/functions/function_cast.h
+++ b/be/src/vec/functions/function_cast.h
@@ -1540,6 +1540,16 @@ private:
return &ConvertImplGenericToJsonb::execute;
}
}
+ // check struct value type and get to_type value
+ // TODO: need handle another type to cast struct
+ WrapperType create_struct_wrapper(const DataTypePtr& from_type,
+ const DataTypeStruct& to_type) const {
+ switch (from_type->get_type_id()) {
+ case TypeIndex::String:
+ default:
+ return &ConvertImplGenericFromString<ColumnString>::execute;
+ }
+ }
WrapperType prepare_unpack_dictionaries(FunctionContext* context, const DataTypePtr& from_type,
const DataTypePtr& to_type) const {
@@ -1714,6 +1724,8 @@ private:
case TypeIndex::Array:
return create_array_wrapper(context, from_type,
static_cast<const DataTypeArray&>(*to_type));
+ case TypeIndex::Struct:
+ return create_struct_wrapper(from_type, static_cast<const DataTypeStruct&>(*to_type));
default:
break;
}
@@ -1759,7 +1771,6 @@ protected:
// TODO(xy): support return struct type for factory
auto type = DataTypeFactory::instance().get(type_col->get_value<String>());
DCHECK(type != nullptr);
-
bool need_to_be_nullable = false;
// 1. from_type is nullable
need_to_be_nullable |= arguments[0].type->is_nullable();
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index 3084a3eb93..551b85e4b6 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -183,8 +183,20 @@ public abstract class Type {
arraySubTypes.add(STRING);
structSubTypes = Lists.newArrayList();
- structSubTypes.add(INT);
+ structSubTypes.addAll(numericTypes);
+ structSubTypes.add(BOOLEAN);
+ structSubTypes.add(VARCHAR);
structSubTypes.add(STRING);
+ structSubTypes.add(CHAR);
+ structSubTypes.add(DATE);
+ structSubTypes.add(DATETIME);
+ structSubTypes.add(DATEV2);
+ structSubTypes.add(DATETIMEV2);
+ structSubTypes.add(TIME);
+ structSubTypes.add(TIMEV2);
+ structSubTypes.add(DECIMAL32);
+ structSubTypes.add(DECIMAL64);
+ structSubTypes.add(DECIMAL128);
}
public static final Set<Class> DATE_SUPPORTED_JAVA_TYPE = Sets.newHashSet(LocalDate.class, java.util.Date.class,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
index d7d5c07c38..87dc3d680b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
@@ -319,9 +319,17 @@ public class CastExpr extends Expr {
type, Function.NullableMode.ALWAYS_NULLABLE,
Lists.newArrayList(Type.VARCHAR), false,
"doris::CastFunctions::cast_to_array_val", null, null, true);
+ } else if (type.isStructType()) {
+ fn = ScalarFunction.createBuiltin(getFnName(Type.STRUCT),
+ type, Function.NullableMode.ALWAYS_NULLABLE,
+ Lists.newArrayList(Type.VARCHAR), false,
+ "doris::CastFunctions::cast_to_struct_val", null, null, true);
}
if (fn == null) {
+ if (type.isStructType() && childType.isStringType()) {
+ return;
+ }
if (childType.isNull() && Type.canCastTo(childType, type)) {
return;
} else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
index 5690c115c1..c89c659207 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
@@ -129,7 +129,7 @@ public class TypeDef implements ParseNode {
// check whether the sub-type is supported
if (!parent.supportSubType(child)) {
throw new AnalysisException(
- parent.getPrimitiveType() + "unsupported sub-type: " + child.toSql());
+ parent.getPrimitiveType() + " unsupported sub-type: " + child.toSql());
}
if (child.getPrimitiveType().isStringType() && !child.isLengthSet()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 18cd580788..3551908d7a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -61,6 +61,7 @@ public class Column implements Writable, GsonPostProcessable {
public static final String ROWID_COL = "__DORIS_ROWID_COL__";
public static final String ROW_STORE_COL = "__DORIS_ROW_STORE_COL__";
private static final String COLUMN_ARRAY_CHILDREN = "item";
+ private static final String COLUMN_STRUCT_CHILDREN = "field";
public static final int COLUMN_UNIQUE_ID_INIT_VALUE = -1;
public static final Column UNSUPPORTED_COLUMN = new Column("unknown",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 08/18: save (#15906)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9fa4b05c5d83df7a0ceffe13926753db2e0f7411
Author: xy720 <22...@users.noreply.github.com>
AuthorDate: Fri Jan 13 17:29:42 2023 +0800
save (#15906)
---
be/src/vec/data_types/data_type_factory.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp
index 553f117a48..c4fbfbd61b 100644
--- a/be/src/vec/data_types/data_type_factory.cpp
+++ b/be/src/vec/data_types/data_type_factory.cpp
@@ -38,7 +38,7 @@ DataTypePtr DataTypeFactory::create_data_type(const doris::Field& col_desc) {
names.reserve(field_size);
for (size_t i = 0; i < field_size; i++) {
dataTypes.push_back(create_data_type(*col_desc.get_sub_field(i)));
- names.push_back(col_desc.name());
+ names.push_back(col_desc.get_sub_field(i).name());
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
} else {
@@ -66,7 +66,7 @@ DataTypePtr DataTypeFactory::create_data_type(const TabletColumn& col_desc, bool
names.reserve(col_size);
for (size_t i = 0; i < col_size; i++) {
dataTypes.push_back(create_data_type(col_desc.get_sub_column(i)));
- names.push_back(col_desc.name());
+ names.push_back(col_desc.get_sub_field(i).name());
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
} else {
@@ -347,7 +347,7 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) {
names.reserve(col_size);
for (size_t i = 0; i < col_size; i++) {
dataTypes.push_back(create_data_type(pcolumn.children(i)));
- names.push_back(pcolumn.name());
+ names.push_back(pcolumn.children(i).name());
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 18/18: rebase master and fix compile error
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c02a706c9aad619ed7112f5f5670d1cc057295a6
Author: xy720 <xu...@baidu.com>
AuthorDate: Mon Feb 6 14:12:13 2023 +0800
rebase master and fix compile error
---
be/src/olap/field.h | 106 +-----------------------
be/src/olap/rowset/segment_v2/column_reader.cpp | 8 --
be/src/olap/rowset/segment_v2/column_reader.h | 4 -
be/src/olap/types.h | 96 +--------------------
be/src/vec/sink/vmysql_result_writer.cpp | 57 ++++++-------
5 files changed, 29 insertions(+), 242 deletions(-)
diff --git a/be/src/olap/field.h b/be/src/olap/field.h
index dd3c455b90..198b371bf8 100644
--- a/be/src/olap/field.h
+++ b/be/src/olap/field.h
@@ -260,113 +260,10 @@ private:
int32_t _unique_id;
};
-<<<<<<< HEAD
-=======
-template <typename LhsCellType, typename RhsCellType>
-int Field::index_cmp(const LhsCellType& lhs, const RhsCellType& rhs) const {
- bool l_null = lhs.is_null();
- bool r_null = rhs.is_null();
- if (l_null != r_null) {
- return l_null ? -1 : 1;
- } else if (l_null) {
- return 0;
- }
-
- int32_t res = 0;
- if (type() == OLAP_FIELD_TYPE_VARCHAR || type() == OLAP_FIELD_TYPE_STRING) {
- const Slice* l_slice = reinterpret_cast<const Slice*>(lhs.cell_ptr());
- const Slice* r_slice = reinterpret_cast<const Slice*>(rhs.cell_ptr());
- uint32_t max_bytes =
- type() == OLAP_FIELD_TYPE_VARCHAR ? OLAP_VARCHAR_MAX_BYTES : OLAP_STRING_MAX_BYTES;
- if (r_slice->size + max_bytes > _index_size || l_slice->size + max_bytes > _index_size) {
- // if the actual length of the field is longer than the short key, only the prefix is compared,
- // make sure all blocks with the same short key are scanned
- // Otherwise, the short key and field can be compared directly
- int compare_size = _index_size - max_bytes;
- // l_slice size and r_slice size may be less than compare_size
- // so calculate the min of the three size as new compare_size
- compare_size = std::min(std::min(compare_size, (int)l_slice->size), (int)r_slice->size);
-
- // This function is used to compare prefix index.
- // Only the fixed length of prefix index should be compared.
- // If r_slice->size > l_slice->size, ignore the extra parts directly.
- res = strncmp(l_slice->data, r_slice->data, compare_size);
- if (res == 0 && compare_size != (_index_size - max_bytes)) {
- if (l_slice->size < r_slice->size) {
- res = -1;
- } else if (l_slice->size > r_slice->size) {
- res = 1;
- } else {
- res = 0;
- }
- }
- } else {
- res = l_slice->compare(*r_slice);
- }
- } else {
- res = _type_info->cmp(lhs.cell_ptr(), rhs.cell_ptr());
- }
-
- return res;
-}
-
-template <typename DstCellType, typename SrcCellType>
-void Field::to_index(DstCellType* dst, const SrcCellType& src) const {
- bool is_null = src.is_null();
- dst->set_is_null(is_null);
- if (is_null) {
- return;
- }
-
- if (type() == OLAP_FIELD_TYPE_VARCHAR) {
- // clear before copy
- memset(dst->mutable_cell_ptr(), 0, _index_size);
- const Slice* slice = reinterpret_cast<const Slice*>(src.cell_ptr());
- size_t copy_size = slice->size < _index_size - OLAP_VARCHAR_MAX_BYTES
- ? slice->size
- : _index_size - OLAP_VARCHAR_MAX_BYTES;
- *reinterpret_cast<VarcharLengthType*>(dst->mutable_cell_ptr()) = copy_size;
- memory_copy((char*)dst->mutable_cell_ptr() + OLAP_VARCHAR_MAX_BYTES, slice->data,
- copy_size);
- } else if (type() == OLAP_FIELD_TYPE_STRING) {
- // clear before copy
- memset(dst->mutable_cell_ptr(), 0, _index_size);
- const Slice* slice = reinterpret_cast<const Slice*>(src.cell_ptr());
- size_t copy_size = slice->size < _index_size - OLAP_STRING_MAX_BYTES
- ? slice->size
- : _index_size - OLAP_STRING_MAX_BYTES;
- *reinterpret_cast<StringLengthType*>(dst->mutable_cell_ptr()) = copy_size;
- memory_copy((char*)dst->mutable_cell_ptr() + OLAP_STRING_MAX_BYTES, slice->data, copy_size);
- } else if (type() == OLAP_FIELD_TYPE_CHAR) {
- // clear before copy
- memset(dst->mutable_cell_ptr(), 0, _index_size);
- const Slice* slice = reinterpret_cast<const Slice*>(src.cell_ptr());
- memory_copy(dst->mutable_cell_ptr(), slice->data, _index_size);
- } else {
- memory_copy(dst->mutable_cell_ptr(), src.cell_ptr(), size());
- }
-}
-
-template <typename CellType>
-uint32_t Field::hash_code(const CellType& cell, uint32_t seed) const {
- bool is_null = cell.is_null();
- if (is_null) {
- return HashUtil::hash(&is_null, sizeof(is_null), seed);
- }
- return _type_info->hash_code(cell.cell_ptr(), seed);
-}
-
class MapField : public Field {
public:
explicit MapField(const TabletColumn& column) : Field(column) {}
- void consume(RowCursorCell* dst, const char* src, bool src_null, MemPool* mem_pool,
- ObjectPool* agg_pool) const override {
- dst->set_is_null(src_null);
- if (src_null) {
- return;
- }
- _type_info->deep_copy(dst->mutable_cell_ptr(), src, mem_pool);
- }
+
// make variable_ptr memory allocate to cell_ptr as MapValue
char* allocate_memory(char* cell_ptr, char* variable_ptr) const override {
return variable_ptr + _length;
@@ -397,7 +294,6 @@ public:
}
};
->>>>>>> [WIP](struct-type) support struct-type in vectorize engine (#15665)
class ArrayField : public Field {
public:
explicit ArrayField(const TabletColumn& column) : Field(column) {}
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 3bc4895da1..95e83ee049 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -561,10 +561,6 @@ Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
return Status::OK();
}
-Status MapFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) {
- return Status::NotSupported("Not support next_batch");
-}
-
Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(ord));
RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(ord));
@@ -631,10 +627,6 @@ Status StructFileColumnIterator::init(const ColumnIteratorOptions& opts) {
return Status::OK();
}
-Status StructFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) {
- return Status::NotSupported("not supported");
-}
-
Status StructFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
bool* has_null) {
const auto* column_struct = vectorized::check_and_get_column<vectorized::ColumnStruct>(
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 5a0d4e8bdd..b72f22bb4a 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -386,8 +386,6 @@ public:
Status init(const ColumnIteratorOptions& opts) override;
- Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) override;
-
Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst, bool* has_null) override;
Status read_by_rowids(const rowid_t* rowids, const size_t count,
@@ -420,8 +418,6 @@ public:
Status init(const ColumnIteratorOptions& opts) override;
- Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) override;
-
Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst, bool* has_null) override;
Status read_by_rowids(const rowid_t* rowids, const size_t count,
diff --git a/be/src/olap/types.h b/be/src/olap/types.h
index 73a0e4c080..cb71ef2228 100644
--- a/be/src/olap/types.h
+++ b/be/src/olap/types.h
@@ -332,12 +332,6 @@ public:
_value_type_info(std::move(value_type_info)) {}
~MapTypeInfo() override = default;
- inline bool equal(const void* left, const void* right) const override {
- auto l_value = reinterpret_cast<const MapValue*>(left);
- auto r_value = reinterpret_cast<const MapValue*>(right);
- return l_value->size() == r_value->size();
- }
-
int cmp(const void* left, const void* right) const override {
auto l_value = reinterpret_cast<const MapValue*>(left);
auto r_value = reinterpret_cast<const MapValue*>(right);
@@ -352,29 +346,14 @@ public:
}
}
- void shallow_copy(void* dest, const void* src) const override {
- auto dest_value = reinterpret_cast<MapValue*>(dest);
- auto src_value = reinterpret_cast<const MapValue*>(src);
- dest_value->shallow_copy(src_value);
- }
-
void deep_copy(void* dest, const void* src, MemPool* mem_pool) const override { DCHECK(false); }
- void copy_object(void* dest, const void* src, MemPool* mem_pool) const override {
- deep_copy(dest, src, mem_pool);
- }
-
void direct_copy(void* dest, const void* src) const override { CHECK(false); }
void direct_copy(uint8_t** base, void* dest, const void* src) const { CHECK(false); }
void direct_copy_may_cut(void* dest, const void* src) const override { direct_copy(dest, src); }
- Status convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool,
- size_t variable_len = 0) const override {
- return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
- }
-
Status from_string(void* buf, const std::string& scan_key, const int precision = 0,
const int scale = 0) const override {
return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
@@ -390,17 +369,8 @@ public:
DCHECK(false) << "set_to_min of list is not implemented.";
}
- uint32_t hash_code(const void* data, uint32_t seed) const override {
- auto map_value = reinterpret_cast<const MapValue*>(data);
- auto size = map_value->size();
- uint32_t result = HashUtil::hash(&size, sizeof(size), seed);
- result = seed * result + _key_type_info->hash_code(map_value->key_data(), seed) +
- _value_type_info->hash_code(map_value->value_data(), seed);
- return result;
- }
-
// todo . is here only to need return 16 for two ptr?
- const size_t size() const override { return sizeof(MapValue); }
+ size_t size() const override { return sizeof(MapValue); }
FieldType type() const override { return OLAP_FIELD_TYPE_MAP; }
@@ -421,39 +391,6 @@ public:
}
~StructTypeInfo() override = default;
- bool equal(const void* left, const void* right) const override {
- auto l_value = reinterpret_cast<const StructValue*>(left);
- auto r_value = reinterpret_cast<const StructValue*>(right);
- if (l_value->size() != r_value->size()) {
- return false;
- }
- uint32_t size = l_value->size();
-
- if (!l_value->has_null() && !r_value->has_null()) {
- for (size_t i = 0; i < size; ++i) {
- if (!_type_infos[i]->equal(l_value->child_value(i), r_value->child_value(i))) {
- return false;
- }
- }
- } else {
- for (size_t i = 0; i < size; ++i) {
- if (l_value->is_null_at(i)) {
- if (r_value->is_null_at(i)) { // both are null
- continue;
- } else { // left is null & right is not null
- return false;
- }
- } else if (r_value->is_null_at(i)) { // left is not null & right is null
- return false;
- }
- if (!_type_infos[i]->equal(l_value->child_value(i), r_value->child_value(i))) {
- return false;
- }
- }
- }
- return true;
- }
-
int cmp(const void* left, const void* right) const override {
auto l_value = reinterpret_cast<const StructValue*>(left);
auto r_value = reinterpret_cast<const StructValue*>(right);
@@ -498,12 +435,6 @@ public:
}
}
- void shallow_copy(void* dest, const void* src) const override {
- auto dest_value = reinterpret_cast<StructValue*>(dest);
- auto src_value = reinterpret_cast<const StructValue*>(src);
- dest_value->shallow_copy(src_value);
- }
-
void deep_copy(void* dest, const void* src, MemPool* mem_pool) const override {
auto dest_value = reinterpret_cast<StructValue*>(dest);
auto src_value = reinterpret_cast<const StructValue*>(src);
@@ -542,10 +473,6 @@ public:
}
}
- void copy_object(void* dest, const void* src, MemPool* mem_pool) const override {
- deep_copy(dest, src, mem_pool);
- }
-
void direct_copy(void* dest, const void* src) const override {
auto dest_value = static_cast<StructValue*>(dest);
auto base = reinterpret_cast<uint8_t*>(dest_value->mutable_values());
@@ -594,11 +521,6 @@ public:
void direct_copy_may_cut(void* dest, const void* src) const override { direct_copy(dest, src); }
- Status convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool,
- size_t variable_len = 0) const override {
- return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
- }
-
Status from_string(void* buf, const std::string& scan_key, const int precision = 0,
const int scale = 0) const override {
return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
@@ -627,21 +549,7 @@ public:
DCHECK(false) << "set_to_min of list is not implemented.";
}
- uint32_t hash_code(const void* data, uint32_t seed) const override {
- auto struct_value = reinterpret_cast<const StructValue*>(data);
- auto size = struct_value->size();
- uint32_t result = HashUtil::hash(&size, sizeof(size), seed);
- for (size_t i = 0; i < size; ++i) {
- if (struct_value->is_null_at(i)) {
- result = seed * result;
- } else {
- result = seed * result + _type_infos[i]->hash_code(struct_value->values()[i], seed);
- }
- }
- return result;
- }
-
- const size_t size() const override { return sizeof(StructValue); }
+ size_t size() const override { return sizeof(StructValue); }
FieldType type() const override { return OLAP_FIELD_TYPE_STRUCT; }
diff --git a/be/src/vec/sink/vmysql_result_writer.cpp b/be/src/vec/sink/vmysql_result_writer.cpp
index 6ec225f961..849db2c58d 100644
--- a/be/src/vec/sink/vmysql_result_writer.cpp
+++ b/be/src/vec/sink/vmysql_result_writer.cpp
@@ -175,11 +175,11 @@ Status VMysqlResultWriter<is_binary_format>::_add_one_column(
buf_ret = rows_buffer[i].push_string("NULL", strlen("NULL"));
} else {
if (WhichDataType(remove_nullable(sub_types[0])).is_string()) {
- buf_ret = _buffer.push_string("'", 1);
- buf_ret = _add_one_cell(data, j, sub_types[0], _buffer);
- buf_ret = _buffer.push_string("'", 1);
+ buf_ret = rows_buffer[i].push_string("'", 1);
+ buf_ret = _add_one_cell(data, j, sub_types[0], rows_buffer[i]);
+ buf_ret = rows_buffer[i].push_string("'", 1);
} else {
- buf_ret = _add_one_cell(data, j, sub_types[0], _buffer);
+ buf_ret = _add_one_cell(data, j, sub_types[0], rows_buffer[i]);
}
}
begin = false;
@@ -194,14 +194,12 @@ Status VMysqlResultWriter<is_binary_format>::_add_one_column(
if (0 != buf_ret) {
return Status::InternalError("pack mysql buffer failed.");
}
- _buffer.reset();
- _buffer.open_dynamic_mode();
+ rows_buffer[i].open_dynamic_mode();
std::string cell_str = map_type.to_string(*column, i);
- buf_ret = _buffer.push_string(cell_str.c_str(), strlen(cell_str.c_str()));
+ buf_ret = rows_buffer[i].push_string(cell_str.c_str(), strlen(cell_str.c_str()));
- _buffer.close_dynamic_mode();
- result->result_batch.rows[i].append(_buffer.buf(), _buffer.length());
+ rows_buffer[i].close_dynamic_mode();
}
} else if constexpr (type == TYPE_STRUCT) {
DCHECK_GE(sub_types.size(), 1);
@@ -210,40 +208,37 @@ Status VMysqlResultWriter<is_binary_format>::_add_one_column(
if (0 != buf_ret) {
return Status::InternalError("pack mysql buffer failed.");
}
- _buffer.reset();
if constexpr (is_nullable) {
if (column_ptr->is_null_at(i)) {
- buf_ret = _buffer.push_null();
- result->result_batch.rows[i].append(_buffer.buf(), _buffer.length());
+ buf_ret = rows_buffer[i].push_null();
continue;
}
}
- _buffer.open_dynamic_mode();
- buf_ret = _buffer.push_string("{", 1);
+ rows_buffer[i].open_dynamic_mode();
+ buf_ret = rows_buffer[i].push_string("{", 1);
bool begin = true;
for (size_t j = 0; j < sub_types.size(); ++j) {
if (!begin) {
- buf_ret = _buffer.push_string(", ", 2);
+ buf_ret = rows_buffer[i].push_string(", ", 2);
}
const auto& data = column_struct.get_column_ptr(j);
if (data->is_null_at(i)) {
- buf_ret = _buffer.push_string("NULL", strlen("NULL"));
+ buf_ret = rows_buffer[i].push_string("NULL", strlen("NULL"));
} else {
if (WhichDataType(remove_nullable(sub_types[j])).is_string()) {
- buf_ret = _buffer.push_string("'", 1);
- buf_ret = _add_one_cell(data, i, sub_types[j], _buffer);
- buf_ret = _buffer.push_string("'", 1);
+ buf_ret = rows_buffer[i].push_string("'", 1);
+ buf_ret = _add_one_cell(data, i, sub_types[j], rows_buffer[i]);
+ buf_ret = rows_buffer[i].push_string("'", 1);
} else {
- buf_ret = _add_one_cell(data, i, sub_types[j], _buffer);
+ buf_ret = _add_one_cell(data, i, sub_types[j], rows_buffer[i]);
}
}
begin = false;
}
- buf_ret = _buffer.push_string("}", 1);
- _buffer.close_dynamic_mode();
- result->result_batch.rows[i].append(_buffer.buf(), _buffer.length());
+ buf_ret = rows_buffer[i].push_string("}", 1);
+ rows_buffer[i].close_dynamic_mode();
}
} else if constexpr (type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 ||
type == TYPE_DECIMAL128I) {
@@ -758,12 +753,12 @@ Status VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
auto& nested_type =
assert_cast<const DataTypeNullable&>(*type_ptr).get_nested_type();
auto& sub_types = assert_cast<const DataTypeStruct&>(*nested_type).get_elements();
- status = _add_one_column<PrimitiveType::TYPE_STRUCT, true>(column_ptr, result,
- scale, sub_types);
+ status = _add_one_column<PrimitiveType::TYPE_STRUCT, true>(
+ column_ptr, result, rows_buffer, scale, sub_types);
} else {
auto& sub_types = assert_cast<const DataTypeStruct&>(*type_ptr).get_elements();
- status = _add_one_column<PrimitiveType::TYPE_STRUCT, false>(column_ptr, result,
- scale, sub_types);
+ status = _add_one_column<PrimitiveType::TYPE_STRUCT, false>(
+ column_ptr, result, rows_buffer, scale, sub_types);
}
break;
}
@@ -771,11 +766,11 @@ Status VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
if (type_ptr->is_nullable()) {
auto& nested_type =
assert_cast<const DataTypeNullable&>(*type_ptr).get_nested_type(); //for map
- status = _add_one_column<PrimitiveType::TYPE_MAP, true>(column_ptr, result, scale,
- {nested_type});
+ status = _add_one_column<PrimitiveType::TYPE_MAP, true>(
+ column_ptr, result, rows_buffer, scale, {nested_type});
} else {
- status = _add_one_column<PrimitiveType::TYPE_MAP, false>(column_ptr, result, scale,
- {type_ptr});
+ status = _add_one_column<PrimitiveType::TYPE_MAP, false>(
+ column_ptr, result, rows_buffer, scale, {type_ptr});
}
break;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 11/18: [fix](struct-type) support struct type insert (#15941)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 11c8eca9f0ae4db23dabff76e1589156238a8ea5
Author: camby <10...@qq.com>
AuthorDate: Mon Jan 16 21:10:54 2023 +0800
[fix](struct-type) support struct type insert (#15941)
Co-authored-by: cambyzju <zh...@baidu.com>
---
be/src/olap/rowset/segment_v2/column_writer.cpp | 28 ++++++++++++-------------
be/src/olap/rowset/segment_v2/column_writer.h | 1 -
be/src/vec/olap/olap_data_convertor.cpp | 8 +++----
be/src/vec/sink/vmysql_result_writer.cpp | 6 +++---
4 files changed, 21 insertions(+), 22 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp
index 5b37d48fa6..d94a0200f0 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -647,21 +647,21 @@ Status StructColumnWriter::write_inverted_index() {
return Status::OK();
}
-Status StructColumnWriter::append_nullable(const uint8_t* null_map, const uint8_t** ptr,
- size_t num_rows) {
- RETURN_IF_ERROR(append_data(ptr, num_rows));
- RETURN_IF_ERROR(_null_writer->append_data(&null_map, num_rows));
- return Status::OK();
-}
-
Status StructColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
- auto data_cursor = reinterpret_cast<const void**>(ptr);
- auto null_map_cursor = data_cursor + _num_sub_column_writers;
- for (auto& column_writer : _sub_column_writers) {
- RETURN_IF_ERROR(column_writer->append(reinterpret_cast<const uint8_t*>(*null_map_cursor),
- *data_cursor, num_rows));
- data_cursor++;
- null_map_cursor++;
+ auto results = reinterpret_cast<const uint64_t*>(*ptr);
+ for (size_t i = 0; i < _num_sub_column_writers; ++i) {
+ auto nullmap = *(results + _num_sub_column_writers + i);
+ auto data = *(results + i);
+ RETURN_IF_ERROR(_sub_column_writers[i]->append(reinterpret_cast<const uint8_t*>(nullmap),
+ reinterpret_cast<const void*>(data),
+ num_rows));
+ }
+ if (is_nullable()) {
+ uint8_t null_sign = 0;
+ const uint8_t* null_sign_ptr = &null_sign;
+ for (size_t i = 0; i < num_rows; ++i) {
+ RETURN_IF_ERROR(_null_writer->append_data(&null_sign_ptr, 1));
+ }
}
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h
index 007a69a6d9..c1329f2404 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.h
+++ b/be/src/olap/rowset/segment_v2/column_writer.h
@@ -274,7 +274,6 @@ public:
Status init() override;
- Status append_nullable(const uint8_t* null_map, const uint8_t** data, size_t num_rows);
Status append_data(const uint8_t** ptr, size_t num_rows) override;
uint64_t estimate_buffer_size() override;
diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp
index cd683e5283..da0910ca2c 100644
--- a/be/src/vec/olap/olap_data_convertor.cpp
+++ b/be/src/vec/olap/olap_data_convertor.cpp
@@ -660,7 +660,7 @@ void OlapBlockDataConvertor::OlapColumnDataConvertorStruct::set_source_column(
}
const void* OlapBlockDataConvertor::OlapColumnDataConvertorStruct::get_data() const {
- return _results[0];
+ return _results.data();
}
const void* OlapBlockDataConvertor::OlapColumnDataConvertorStruct::get_data_at(
@@ -688,10 +688,10 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorStruct::convert_to_olap()
assert(column_struct);
assert(data_type_struct);
- size_t data_size = column_struct->tuple_size();
+ size_t fields_num = column_struct->tuple_size();
size_t data_cursor = 0;
- size_t null_map_cursor = data_cursor + data_size;
- for (size_t i = 0; i < data_size; i++) {
+ size_t null_map_cursor = data_cursor + fields_num;
+ for (size_t i = 0; i < fields_num; i++) {
ColumnPtr sub_column = column_struct->get_column_ptr(i);
DataTypePtr sub_type = data_type_struct->get_element(i);
ColumnWithTypeAndName sub_typed_column = {sub_column, sub_type, ""};
diff --git a/be/src/vec/sink/vmysql_result_writer.cpp b/be/src/vec/sink/vmysql_result_writer.cpp
index 4d30e17227..78fa7d4ccb 100644
--- a/be/src/vec/sink/vmysql_result_writer.cpp
+++ b/be/src/vec/sink/vmysql_result_writer.cpp
@@ -211,15 +211,15 @@ Status VMysqlResultWriter<is_binary_format>::_add_one_column(
buf_ret = _buffer.push_string(", ", 2);
}
const auto& data = column_struct.get_column_ptr(j);
- if (data->is_null_at(j)) {
+ if (data->is_null_at(i)) {
buf_ret = _buffer.push_string("NULL", strlen("NULL"));
} else {
if (WhichDataType(remove_nullable(sub_types[j])).is_string()) {
buf_ret = _buffer.push_string("'", 1);
- buf_ret = _add_one_cell(data, j, sub_types[j], _buffer);
+ buf_ret = _add_one_cell(data, i, sub_types[j], _buffer);
buf_ret = _buffer.push_string("'", 1);
} else {
- buf_ret = _add_one_cell(data, j, sub_types[j], _buffer);
+ buf_ret = _add_one_cell(data, i, sub_types[j], _buffer);
}
}
begin = false;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 14/18: [fix](struct-type) fix insert null and explain for struct type (#16054)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit eb70160b962a275a33e8bf733ebb364b2815f3ff
Author: camby <10...@qq.com>
AuthorDate: Wed Jan 18 16:11:07 2023 +0800
[fix](struct-type) fix insert null and explain for struct type (#16054)
1、support insert null for struct type;
2、fix explain const exprs for strut type got wrong result;
Co-authored-by: cambyzju <zh...@baidu.com>
---
be/src/olap/rowset/segment_v2/column_writer.cpp | 25 ++++++++++++++++------
be/src/olap/rowset/segment_v2/column_writer.h | 3 ++-
be/src/vec/data_types/data_type_struct.cpp | 2 +-
.../org/apache/doris/analysis/StructLiteral.java | 2 +-
4 files changed, 23 insertions(+), 9 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp
index d94a0200f0..7949ed02ce 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -647,6 +647,13 @@ Status StructColumnWriter::write_inverted_index() {
return Status::OK();
}
+Status StructColumnWriter::append_nullable(const uint8_t* null_map, const uint8_t** ptr,
+ size_t num_rows) {
+ RETURN_IF_ERROR(append_data(ptr, num_rows));
+ RETURN_IF_ERROR(_null_writer->append_data(&null_map, num_rows));
+ return Status::OK();
+}
+
Status StructColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
auto results = reinterpret_cast<const uint64_t*>(*ptr);
for (size_t i = 0; i < _num_sub_column_writers; ++i) {
@@ -657,11 +664,9 @@ Status StructColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
num_rows));
}
if (is_nullable()) {
- uint8_t null_sign = 0;
- const uint8_t* null_sign_ptr = &null_sign;
- for (size_t i = 0; i < num_rows; ++i) {
- RETURN_IF_ERROR(_null_writer->append_data(&null_sign_ptr, 1));
- }
+ std::vector<vectorized::UInt8> null_signs(num_rows, 0);
+ const uint8_t* null_sign_ptr = null_signs.data();
+ RETURN_IF_ERROR(_null_writer->append_data(&null_sign_ptr, num_rows));
}
return Status::OK();
}
@@ -706,7 +711,15 @@ Status StructColumnWriter::write_ordinal_index() {
}
Status StructColumnWriter::append_nulls(size_t num_rows) {
- return Status::NotSupported("struct writer not support append nulls");
+ for (auto& column_writer : _sub_column_writers) {
+ RETURN_IF_ERROR(column_writer->append_nulls(num_rows));
+ }
+ if (is_nullable()) {
+ std::vector<vectorized::UInt8> null_signs(num_rows, 1);
+ const uint8_t* null_sign_ptr = null_signs.data();
+ RETURN_IF_ERROR(_null_writer->append_data(&null_sign_ptr, num_rows));
+ }
+ return Status::OK();
}
Status StructColumnWriter::finish_current_page() {
diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h
index c1329f2404..274c9be6f3 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.h
+++ b/be/src/olap/rowset/segment_v2/column_writer.h
@@ -113,7 +113,7 @@ public:
Status append_nullable(const uint8_t* nullmap, const void* data, size_t num_rows);
// use only in vectorized load
- Status append_nullable(const uint8_t* null_map, const uint8_t** data, size_t num_rows);
+ virtual Status append_nullable(const uint8_t* null_map, const uint8_t** data, size_t num_rows);
virtual Status append_nulls(size_t num_rows) = 0;
@@ -274,6 +274,7 @@ public:
Status init() override;
+ Status append_nullable(const uint8_t* null_map, const uint8_t** data, size_t num_rows) override;
Status append_data(const uint8_t** ptr, size_t num_rows) override;
uint64_t estimate_buffer_size() override;
diff --git a/be/src/vec/data_types/data_type_struct.cpp b/be/src/vec/data_types/data_type_struct.cpp
index a3a54966b0..7249977295 100644
--- a/be/src/vec/data_types/data_type_struct.cpp
+++ b/be/src/vec/data_types/data_type_struct.cpp
@@ -272,7 +272,7 @@ bool DataTypeStruct::equals(const IDataType& rhs) const {
}
for (size_t i = 0; i < size; ++i) {
- if (!elems[i]->equals(*rhs_tuple.elems[i]) || names[i] != rhs_tuple.names[i]) {
+ if (!elems[i]->equals(*rhs_tuple.elems[i])) {
return false;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java
index a9209f1c5a..95362a37ab 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java
@@ -58,7 +58,7 @@ public class StructLiteral extends LiteralExpr {
@Override
protected String toSqlImpl() {
List<String> list = new ArrayList<>(children.size());
- children.forEach(v -> list.add(v.toDigestImpl()));
+ children.forEach(v -> list.add(v.toSqlImpl()));
return "STRUCT(" + StringUtils.join(list, ", ") + ")";
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/18: save
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5e9a6e40ba4c39275a30e965624eec0ce7b8bc89
Author: xy720 <xu...@baidu.com>
AuthorDate: Thu Nov 17 19:15:50 2022 +0800
save
---
test_file | 1 -
1 file changed, 1 deletion(-)
diff --git a/test_file b/test_file
deleted file mode 100644
index 63d3c2d75e..0000000000
--- a/test_file
+++ /dev/null
@@ -1 +0,0 @@
-just for test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/18: test
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6628baae9322ad77bd7e2b0e32a5cd17a9dfa012
Author: xy720 <xu...@baidu.com>
AuthorDate: Thu Nov 17 19:01:17 2022 +0800
test
---
test_file | 1 +
1 file changed, 1 insertion(+)
diff --git a/test_file b/test_file
new file mode 100644
index 0000000000..63d3c2d75e
--- /dev/null
+++ b/test_file
@@ -0,0 +1 @@
+just for test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 16/18: [fix](map-type) fix compile error (#16413)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 454ff4c69ab199c55d965b6d6b0ca98ac6ec7c72
Author: xy720 <22...@users.noreply.github.com>
AuthorDate: Fri Feb 3 21:54:53 2023 +0800
[fix](map-type) fix compile error (#16413)
---
be/src/runtime/types.cpp | 1 +
be/src/vec/exprs/vexpr.cpp | 1 +
2 files changed, 2 insertions(+)
diff --git a/be/src/runtime/types.cpp b/be/src/runtime/types.cpp
index 386f3b57f0..2945bfd33f 100644
--- a/be/src/runtime/types.cpp
+++ b/be/src/runtime/types.cpp
@@ -234,6 +234,7 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
children.push_back(TypeDescriptor(types, idx));
++(*idx);
children.push_back(TypeDescriptor(types, idx));
+ }
case TTypeNodeType::STRUCT: {
type = TYPE_STRUCT;
size_t children_size = node.struct_fields_size();
diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp
index c83e524cb7..573a2b705c 100644
--- a/be/src/vec/exprs/vexpr.cpp
+++ b/be/src/vec/exprs/vexpr.cpp
@@ -127,6 +127,7 @@ Status VExpr::create_expr(doris::ObjectPool* pool, const doris::TExprNode& texpr
}
case TExprNodeType::MAP_LITERAL: {
*expr = pool->add(new VMapLiteral(texpr_node));
+ }
case TExprNodeType::STRUCT_LITERAL: {
*expr = pool->add(new VStructLiteral(texpr_node));
return Status::OK();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 12/18: [hotfix](struct-type) Fix type descriptor make array load core (#16020)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 7a573c940443950c929b91ab3b71b74602315574
Author: xy720 <22...@users.noreply.github.com>
AuthorDate: Tue Jan 17 18:52:49 2023 +0800
[hotfix](struct-type) Fix type descriptor make array load core (#16020)
---
be/src/vec/sink/vtablet_sink.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/be/src/vec/sink/vtablet_sink.cpp b/be/src/vec/sink/vtablet_sink.cpp
index 27f06fbdbe..0ad8458f13 100644
--- a/be/src/vec/sink/vtablet_sink.cpp
+++ b/be/src/vec/sink/vtablet_sink.cpp
@@ -1398,7 +1398,7 @@ Status VOlapTableSink::_validate_column(RuntimeState* state, const TypeDescripto
}
fmt::format_to(error_prefix, "ARRAY type failed: ");
RETURN_IF_ERROR(_validate_column(
- state, nested_type, nested_type.contains_nulls[0], column_array->get_data_ptr(),
+ state, nested_type, type.contains_nulls[0], column_array->get_data_ptr(),
slot_index, filter_bitmap, stop_processing, error_prefix, &permutation));
}
break;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/18: [feature](struct-type) add the class ColumnStruct and class DataTypeStruct implement (#14545)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 80f6a7b0abc1d3a004e612ebf3d99aea1af8d9db
Author: carlvinhust2012 <hu...@126.com>
AuthorDate: Thu Dec 8 09:26:12 2022 +0800
[feature](struct-type) add the class ColumnStruct and class DataTypeStruct implement (#14545)
Co-authored-by: hucheng01 <hu...@baidu.com>
---
be/src/vec/CMakeLists.txt | 2 +
be/src/vec/columns/column_struct.cpp | 618 +++++++++++++++++++++++++++++
be/src/vec/columns/column_struct.h | 232 +++++++++++
be/src/vec/core/types.h | 3 +
be/src/vec/data_types/data_type_struct.cpp | 361 +++++++++++++++++
be/src/vec/data_types/data_type_struct.h | 115 ++++++
6 files changed, 1331 insertions(+)
diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt
index 4c3c387c99..d4b200bfe7 100644
--- a/be/src/vec/CMakeLists.txt
+++ b/be/src/vec/CMakeLists.txt
@@ -49,6 +49,7 @@ set(VEC_FILES
aggregate_functions/aggregate_function_histogram.cpp
columns/column.cpp
columns/column_array.cpp
+ columns/column_struct.cpp
columns/column_const.cpp
columns/column_decimal.cpp
columns/column_nullable.cpp
@@ -76,6 +77,7 @@ set(VEC_FILES
core/materialize_block.cpp
data_types/data_type.cpp
data_types/data_type_array.cpp
+ data_types/data_type_struct.cpp
data_types/data_type_bitmap.cpp
data_types/data_type_factory.cpp
data_types/data_type_fixed_length_object.cpp
diff --git a/be/src/vec/columns/column_struct.cpp b/be/src/vec/columns/column_struct.cpp
new file mode 100644
index 0000000000..05d5df0fd4
--- /dev/null
+++ b/be/src/vec/columns/column_struct.cpp
@@ -0,0 +1,618 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnStruct.cpp
+// and modified by Doris
+
+#include "vec/columns/column_struct.h"
+
+namespace doris::vectorized {
+
+namespace ErrorCodes {
+extern const int ILLEGAL_COLUMN;
+extern const int NOT_IMPLEMENTED;
+extern const int CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE;
+extern const int LOGICAL_ERROR;
+} // namespace ErrorCodes
+
+std::string ColumnStruct::get_name() const {
+ std::stringstream res;
+ res << "Struct(";
+ bool is_first = true;
+ for (const auto& column : columns) {
+ if (!is_first) {
+ res << ", ";
+ }
+ is_first = false;
+ res << column->get_name();
+ }
+ res << ")";
+ return res.str();
+}
+
+ColumnStruct::ColumnStruct(MutableColumns&& mutable_columns) {
+ columns.reserve(mutable_columns.size());
+ for (auto& column : mutable_columns) {
+ if (is_column_const(*column)) {
+ throw Exception {"ColumnStruct cannot have ColumnConst as its element",
+ ErrorCodes::ILLEGAL_COLUMN};
+ }
+ columns.push_back(std::move(column));
+ }
+}
+
+ColumnStruct::ColumnStruct(Columns&& columns) {
+ columns.reserve(columns.size());
+ for (auto& column : columns) {
+ if (is_column_const(*column)) {
+ throw Exception {"ColumnStruct cannot have ColumnConst as its element",
+ ErrorCodes::ILLEGAL_COLUMN};
+ }
+ columns.push_back(std::move(column));
+ }
+}
+
+ColumnStruct::ColumnStruct(TupleColumns&& tuple_columns) {
+ columns.reserve(tuple_columns.size());
+ for (auto& column : tuple_columns) {
+ if (is_column_const(*column)) {
+ throw Exception {"ColumnStruct cannot have ColumnConst as its element",
+ ErrorCodes::ILLEGAL_COLUMN};
+ }
+ columns.push_back(std::move(column));
+ }
+}
+
+ColumnStruct::Ptr ColumnStruct::create(Columns& columns) {
+ for (const auto& column : columns) {
+ if (is_column_const(*column))
+ throw Exception {"ColumnStruct cannot have ColumnConst as its element",
+ ErrorCodes::ILLEGAL_COLUMN};
+ }
+ auto column_struct = ColumnStruct::create(columns);
+ return column_struct;
+}
+
+ColumnStruct::Ptr ColumnStruct::create(TupleColumns& tuple_columns) {
+ for (const auto& column : tuple_columns) {
+ if (is_column_const(*column)) {
+ throw Exception {"ColumnStruct cannot have ColumnConst as its element",
+ ErrorCodes::ILLEGAL_COLUMN};
+ }
+ }
+ auto column_struct = ColumnStruct::create(tuple_columns);
+ return column_struct;
+}
+
+MutableColumnPtr ColumnStruct::clone_empty() const {
+ const size_t tuple_size = columns.size();
+ MutableColumns new_columns(tuple_size);
+ for (size_t i = 0; i < tuple_size; ++i) {
+ new_columns[i] = columns[i]->clone_empty();
+ }
+ return ColumnStruct::create(std::move(new_columns));
+}
+
+MutableColumnPtr ColumnStruct::clone_resized(size_t new_size) const {
+ const size_t tuple_size = columns.size();
+ MutableColumns new_columns(tuple_size);
+ for (size_t i = 0; i < tuple_size; ++i) {
+ new_columns[i] = columns[i]->clone_resized(new_size);
+ }
+ return ColumnStruct::create(std::move(new_columns));
+}
+
+Field ColumnStruct::operator[](size_t n) const {
+ Field res;
+ get(n, res);
+ return res;
+}
+
+void ColumnStruct::get(size_t n, Field& res) const {
+ const size_t tuple_size = columns.size();
+
+ res = Tuple();
+ Tuple& res_tuple = res.get<Tuple&>();
+ res_tuple.reserve(tuple_size);
+
+ for (size_t i = 0; i < tuple_size; ++i) {
+ res_tuple.push_back((*columns[i])[n]);
+ }
+}
+
+bool ColumnStruct::is_default_at(size_t n) const {
+ const size_t tuple_size = columns.size();
+ for (size_t i = 0; i < tuple_size; ++i) {
+ if (!columns[i]->is_default_at(n)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+StringRef ColumnStruct::get_data_at(size_t) const {
+ throw Exception("Method get_data_at is not supported for " + get_name(),
+ ErrorCodes::NOT_IMPLEMENTED);
+}
+
+void ColumnStruct::insert_data(const char*, size_t) {
+ throw Exception("Method insert_data is not supported for " + get_name(),
+ ErrorCodes::NOT_IMPLEMENTED);
+}
+
+void ColumnStruct::insert(const Field& x) {
+ const auto& tuple = x.get<const Tuple&>();
+ const size_t tuple_size = columns.size();
+ if (tuple.size() != tuple_size) {
+ throw Exception("Cannot insert value of different size into tuple",
+ ErrorCodes::CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE);
+ }
+
+ for (size_t i = 0; i < tuple_size; ++i) {
+ columns[i]->insert(tuple[i]);
+ }
+}
+
+void ColumnStruct::insert_from(const IColumn& src_, size_t n) {
+ const ColumnStruct& src = assert_cast<const ColumnStruct&>(src_);
+
+ const size_t tuple_size = columns.size();
+ if (src.columns.size() != tuple_size) {
+ throw Exception("Cannot insert value of different size into tuple",
+ ErrorCodes::CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE);
+ }
+
+ for (size_t i = 0; i < tuple_size; ++i) {
+ columns[i]->insert_from(*src.columns[i], n);
+ }
+}
+
+void ColumnStruct::insert_default() {
+ for (auto& column : columns) {
+ column->insert_default();
+ }
+}
+
+void ColumnStruct::pop_back(size_t n) {
+ for (auto& column : columns) {
+ column->pop_back(n);
+ }
+}
+
+StringRef ColumnStruct::serialize_value_into_arena(size_t n, Arena& arena,
+ char const*& begin) const {
+ StringRef res(begin, 0);
+ for (const auto& column : columns) {
+ auto value_ref = column->serialize_value_into_arena(n, arena, begin);
+ res.data = value_ref.data - res.size;
+ res.size += value_ref.size;
+ }
+
+ return res;
+}
+
+const char* ColumnStruct::deserialize_and_insert_from_arena(const char* pos) {
+ for (auto& column : columns) {
+ pos = column->deserialize_and_insert_from_arena(pos);
+ }
+
+ return pos;
+}
+
+void ColumnStruct::update_hash_with_value(size_t n, SipHash& hash) const {
+ for (const auto& column : columns) {
+ column->update_hash_with_value(n, hash);
+ }
+}
+
+// void ColumnStruct::update_weak_hash32(WeakHash32 & hash) const {
+// auto s = size();
+// if (hash.get_data().size() != s) {
+// throw Exception("Size of WeakHash32 does not match size of column: column size is " + std::to_string(s) +
+// ", hash size is " + std::to_string(hash.getData().size()), ErrorCodes::LOGICAL_ERROR);
+// }
+
+// for (const auto & column : columns) {
+// column->update_weak_hash32(hash);
+// }
+// }
+
+// void ColumnStruct::update_hash_fast(SipHash & hash) const {
+// for (const auto & column : columns) {
+// column->update_hash_fast(hash);
+// }
+// }
+
+// const char * ColumnStruct::skip_serialized_in_arena(const char * pos) const {
+// for (const auto & column : columns) {
+// pos = column->skip_serialized_in_arena(pos);
+// }
+// return pos;
+// }
+
+// void ColumnStruct::expand(const Filter & mask, bool inverted)
+// {
+// for (auto & column : columns) {
+// column->expand(mask, inverted);
+// }
+// }
+
+// ColumnPtr ColumnStruct::index(const IColumn & indexes, size_t limit) const
+// {
+// const size_t tuple_size = columns.size();
+// Columns new_columns(tuple_size);
+
+// for (size_t i = 0; i < tuple_size; ++i) {
+// new_columns[i] = columns[i]->index(indexes, limit);
+// }
+
+// return ColumnStruct::create(new_columns);
+// }
+
+void ColumnStruct::insert_range_from(const IColumn& src, size_t start, size_t length) {
+ const size_t tuple_size = columns.size();
+ for (size_t i = 0; i < tuple_size; ++i) {
+ columns[i]->insert_range_from(*assert_cast<const ColumnStruct&>(src).columns[i], start,
+ length);
+ }
+}
+
+ColumnPtr ColumnStruct::filter(const Filter& filt, ssize_t result_size_hint) const {
+ const size_t tuple_size = columns.size();
+ Columns new_columns(tuple_size);
+
+ for (size_t i = 0; i < tuple_size; ++i) {
+ new_columns[i] = columns[i]->filter(filt, result_size_hint);
+ }
+ return ColumnStruct::create(new_columns);
+}
+
+ColumnPtr ColumnStruct::permute(const Permutation& perm, size_t limit) const {
+ const size_t tuple_size = columns.size();
+ Columns new_columns(tuple_size);
+
+ for (size_t i = 0; i < tuple_size; ++i) {
+ new_columns[i] = columns[i]->permute(perm, limit);
+ }
+
+ return ColumnStruct::create(new_columns);
+}
+
+ColumnPtr ColumnStruct::replicate(const Offsets& offsets) const {
+ const size_t tuple_size = columns.size();
+ Columns new_columns(tuple_size);
+
+ for (size_t i = 0; i < tuple_size; ++i) {
+ new_columns[i] = columns[i]->replicate(offsets);
+ }
+
+ return ColumnStruct::create(new_columns);
+}
+
+MutableColumns ColumnStruct::scatter(ColumnIndex num_columns, const Selector& selector) const {
+ const size_t tuple_size = columns.size();
+ std::vector<MutableColumns> scattered_tuple_elements(tuple_size);
+
+ for (size_t tuple_element_idx = 0; tuple_element_idx < tuple_size; ++tuple_element_idx) {
+ scattered_tuple_elements[tuple_element_idx] =
+ columns[tuple_element_idx]->scatter(num_columns, selector);
+ }
+
+ MutableColumns res(num_columns);
+
+ for (size_t scattered_idx = 0; scattered_idx < num_columns; ++scattered_idx) {
+ MutableColumns new_columns(tuple_size);
+ for (size_t tuple_element_idx = 0; tuple_element_idx < tuple_size; ++tuple_element_idx) {
+ new_columns[tuple_element_idx] =
+ std::move(scattered_tuple_elements[tuple_element_idx][scattered_idx]);
+ }
+ res[scattered_idx] = ColumnStruct::create(std::move(new_columns));
+ }
+
+ return res;
+}
+
+// int ColumnStruct::compare_at_impl(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint,
+// const Collator* collator) const {
+// const size_t tuple_size = columns.size();
+// for (size_t i = 0; i < tuple_size; ++i) {
+// int res = 0;
+// if (collator && columns[i]->is_collation_supported()) {
+// res = columns[i]->compare_at_with_collation(
+// n, m, *assert_cast<const ColumnStruct&>(rhs).columns[i], nan_direction_hint,
+// *collator);
+// } else {
+// res = columns[i]->compare_at(n, m, *assert_cast<const ColumnStruct&>(rhs).columns[i],
+// nan_direction_hint);
+// }
+
+// if (res) {
+// return res;
+// }
+// }
+// return 0;
+// }
+
+// int ColumnStruct::compare_at(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const {
+// return compare_at_impl(n, m, rhs, nan_direction_hint);
+// }
+
+// void ColumnStruct::compare_column(const IColumn& rhs, size_t rhs_row_num,
+// PaddedPODArray<UInt64>* row_indexes,
+// PaddedPODArray<Int8>& compare_results, int direction,
+// int nan_direction_hint) const {
+// return do_compare_column<ColumnStruct>(assert_cast<const ColumnStruct&>(rhs), rhs_row_num,
+// row_indexes, compare_results, direction,
+// nan_direction_hint);
+// }
+
+// int ColumnStruct::compare_at_with_collation(size_t n, size_t m, const IColumn& rhs,
+// int nan_direction_hint,
+// const Collator& collator) const {
+// return compare_at_impl(n, m, rhs, nan_direction_hint, &collator);
+// }
+
+// bool ColumnStruct::has_equal_values() const {
+// return has_equal_values_impl<ColumnStruct>();
+// }
+
+// template <bool positive>
+// struct ColumnStruct::Less {
+// TupleColumns columns;
+// int nan_direction_hint;
+// const Collator* collator;
+
+// Less(const TupleColumns& columns_, int nan_direction_hint_, const Collator* collator_ = nullptr)
+// : columns(columns_), nan_direction_hint(nan_direction_hint_), collator(collator_) {}
+
+// bool operator()(size_t a, size_t b) const {
+// for (const auto& column : columns) {
+// int res;
+// if (collator && column->isCollationSupported()) {
+// res = column->compareAtWithCollation(a, b, *column, nan_direction_hint, *collator);
+// } else {
+// res = column->compareAt(a, b, *column, nan_direction_hint);
+// }
+// if (res < 0) {
+// return positive;
+// } else if (res > 0) {
+// return !positive;
+// }
+// }
+// return false;
+// }
+// };
+
+// void ColumnStruct::get_permutation_impl(IColumn::PermutationSortDirection direction,
+// IColumn::PermutationSortStability stability, size_t limit,
+// int nan_direction_hint, Permutation& res,
+// const Collator* collator) const {
+// size_t rows = size();
+// res.resize(rows);
+// for (size_t i = 0; i < rows; ++i) {
+// res[i] = i;
+// }
+
+// if (limit >= rows) {
+// limit = 0;
+// }
+
+// EqualRange ranges;
+// ranges.emplace_back(0, rows);
+// update_permutation_impl(direction, stability, limit, nan_direction_hint, res, ranges, collator);
+// }
+
+// void ColumnStruct::update_permutation_impl(IColumn::PermutationSortDirection direction,
+// IColumn::PermutationSortStability stability,
+// size_t limit, int nan_direction_hint,
+// IColumn::Permutation& res, EqualRanges& equal_ranges,
+// const Collator* collator) const {
+// if (equal_ranges.empty()) {
+// return;
+// }
+
+// for (const auto& column : columns) {
+// while (!equal_ranges.empty() && limit && limit <= equal_ranges.back().first) {
+// equal_ranges.pop_back();
+// }
+
+// if (collator && column->isCollationSupported()) {
+// column->update_permutation_with_collation(*collator, direction, stability, limit,
+// nan_direction_hint, res, equal_ranges);
+// } else {
+// column->update_permutation(direction, stability, limit, nan_direction_hint, res,
+// equal_ranges);
+// }
+// if (equal_ranges.empty()) {
+// break;
+// }
+// }
+// }
+
+// void ColumnStruct::get_permutation(IColumn::PermutationSortDirection direction,
+// IColumn::PermutationSortStability stability, size_t limit,
+// int nan_direction_hint, Permutation& res) const {
+// get_permutation_impl(direction, stability, limit, nan_direction_hint, res, nullptr);
+// }
+
+// void ColumnStruct::update_permutation(IColumn::PermutationSortDirection direction,
+// IColumn::PermutationSortStability stability, size_t limit,
+// int nan_direction_hint, IColumn::Permutation& res,
+// EqualRanges& equal_ranges) const {
+// update_permutation_impl(direction, stability, limit, nan_direction_hint, res, equal_ranges);
+// }
+
+// void ColumnStruct::get_permutation_with_collation(const Collator& collator,
+// IColumn::PermutationSortDirection direction,
+// IColumn::PermutationSortStability stability,
+// size_t limit, int nan_direction_hint,
+// Permutation& res) const {
+// get_permutation_impl(direction, stability, limit, nan_direction_hint, res, &collator);
+// }
+
+// void ColumnStruct::update_permutation_with_collation(const Collator& collator,
+// IColumn::PermutationSortDirection direction,
+// IColumn::PermutationSortStability stability,
+// size_t limit, int nan_direction_hint,
+// Permutation& res,
+// EqualRanges& equal_ranges) const {
+// update_permutation_impl(direction, stability, limit, nan_direction_hint, res, equal_ranges,
+// &collator);
+// }
+
+// void ColumnStruct::gather(ColumnGathererStream& gatherer) {
+// gatherer.gather(*this);
+// }
+
+void ColumnStruct::reserve(size_t n) {
+ const size_t tuple_size = columns.size();
+ for (size_t i = 0; i < tuple_size; ++i) {
+ get_column(i).reserve(n);
+ }
+}
+
+size_t ColumnStruct::byte_size() const {
+ size_t res = 0;
+ for (const auto& column : columns) {
+ res += column->byte_size();
+ }
+ return res;
+}
+
+// size_t ColumnStruct::byte_size_at(size_t n) const {
+// size_t res = 0;
+// for (const auto& column : columns) {
+// res += column->byte_size_at(n);
+// }
+// return res;
+// }
+
+// void ColumnStruct::ensure_ownership() {
+// const size_t tuple_size = columns.size();
+// for (size_t i = 0; i < tuple_size; ++i) {
+// get_column(i).ensure_ownership();
+// }
+// }
+
+size_t ColumnStruct::allocated_bytes() const {
+ size_t res = 0;
+ for (const auto& column : columns) {
+ res += column->allocated_bytes();
+ }
+ return res;
+}
+
+void ColumnStruct::protect() {
+ for (auto& column : columns) {
+ column->protect();
+ }
+}
+
+void ColumnStruct::get_extremes(Field& min, Field& max) const {
+ const size_t tuple_size = columns.size();
+
+ Tuple min_tuple(tuple_size);
+ Tuple max_tuple(tuple_size);
+
+ for (size_t i = 0; i < tuple_size; ++i) {
+ columns[i]->get_extremes(min_tuple[i], max_tuple[i]);
+ }
+
+ min = min_tuple;
+ max = max_tuple;
+}
+
+void ColumnStruct::for_each_subcolumn(ColumnCallback callback) {
+ for (auto& column : columns) {
+ callback(column);
+ }
+}
+
+bool ColumnStruct::structure_equals(const IColumn& rhs) const {
+ if (const auto* rhs_tuple = typeid_cast<const ColumnStruct*>(&rhs)) {
+ const size_t tuple_size = columns.size();
+ if (tuple_size != rhs_tuple->columns.size()) {
+ return false;
+ }
+
+ for (size_t i = 0; i < tuple_size; ++i) {
+ if (!columns[i]->structure_equals(*rhs_tuple->columns[i])) {
+ return false;
+ }
+ }
+ return true;
+ } else {
+ return false;
+ }
+}
+
+// void ColumnStruct::for_each_subcolumn_recursively(ColumnCallback callback) {
+// for (auto& column : columns) {
+// callback(column);
+// column->for_each_subcolumn_recursively(callback);
+// }
+// }
+
+// bool ColumnStruct::is_collation_supported() const {
+// for (const auto& column : columns) {
+// if (column->is_collation_supported()) {
+// return true;
+// }
+// }
+// return false;
+// }
+
+// ColumnPtr ColumnStruct::compress() const {
+// size_t byte_size = 0;
+// Columns compressed;
+// compressed.reserve(columns.size());
+// for (const auto& column : columns) {
+// auto compressed_column = column->compress();
+// byte_size += compressed_column->byteSize();
+// compressed.emplace_back(std::move(compressed_column));
+// }
+
+// return ColumnCompressed::create(size(), byte_size,
+// [compressed = std::move(compressed)]() mutable {
+// for (auto& column : compressed) {
+// column = column->decompress();
+// }
+// return ColumnStruct::create(compressed);
+// });
+// }
+
+// double ColumnStruct::get_ratio_of_default_rows(double sample_ratio) const {
+// return get_ratio_of_default_rows_impl<ColumnStruct>(sample_ratio);
+// }
+
+// void ColumnStruct::get_indices_of_nondefault_rows(Offsets& indices, size_t from,
+// size_t limit) const {
+// return get_indices_of_nondefault_rows_impl<ColumnStruct>(indices, from, limit);
+// }
+
+// void ColumnStruct::finalize() {
+// for (auto& column : columns) {
+// column->finalize();
+// }
+// }
+
+// bool ColumnStruct::is_finalized() const {
+// return std::all_of(columns.begin(), columns.end(),
+// [](const auto& column) { return column->is_finalized(); });
+// }
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/columns/column_struct.h b/be/src/vec/columns/column_struct.h
new file mode 100644
index 0000000000..895a2796b7
--- /dev/null
+++ b/be/src/vec/columns/column_struct.h
@@ -0,0 +1,232 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnTuple.h
+// and modified by Doris
+
+/********************************************************************************
+// doris/core/be/src/vec/core/field.h
+class Field;
+using FieldVector = std::vector<Field>;
+
+/// Array and Tuple use the same storage type -- FieldVector, but we declare
+/// distinct types for them, so that the caller can choose whether it wants to
+/// construct a Field of Array or a Tuple type. An alternative approach would be
+/// to construct both of these types from FieldVector, and have the caller
+/// specify the desired Field type explicitly.
+
+#define DEFINE_FIELD_VECTOR(X) \
+ struct X : public FieldVector { \
+ using FieldVector::FieldVector; \
+ }
+
+DEFINE_FIELD_VECTOR(Array);
+DEFINE_FIELD_VECTOR(Tuple);
+
+#undef DEFINE_FIELD_VECTOR
+
+// defination of some pointer
+using WrappedPtr = chameleon_ptr<Derived>;
+
+using Ptr = immutable_ptr<Derived>;
+using ColumnPtr = IColumn::Ptr;
+using Columns = std::vector<ColumnPtr>;
+using MutablePtr = mutable_ptr<Derived>;
+using MutableColumnPtr = IColumn::MutablePtr;
+using MutableColumns = std::vector<MutableColumnPtr>;
+****************************************************************************/
+
+#pragma once
+
+#include "vec/columns/column.h"
+#include "vec/columns/column_impl.h"
+#include "vec/columns/column_vector.h"
+#include "vec/common/arena.h"
+#include "vec/common/assert_cast.h"
+#include "vec/common/typeid_cast.h"
+#include "vec/core/field.h"
+#include "vec/core/types.h"
+
+namespace doris::vectorized {
+
+/** Column, that is just group of few another columns.
+ *
+ * For constant Tuples, see ColumnConst.
+ * Mixed constant/non-constant columns is prohibited in tuple
+ * for implementation simplicity.
+ */
+class ColumnStruct final : public COWHelper<IColumn, ColumnStruct> {
+private:
+ friend class COWHelper<IColumn, ColumnStruct>;
+
+ using TupleColumns = std::vector<WrappedPtr>;
+ TupleColumns columns;
+
+ template <bool positive>
+ struct Less;
+
+ ColumnStruct(Columns&& columns);
+ ColumnStruct(TupleColumns&& tuple_columns);
+ explicit ColumnStruct(MutableColumns&& mutable_columns);
+ ColumnStruct(const ColumnStruct&) = default;
+
+public:
+ /** Create immutable column using immutable arguments. This arguments may be shared with other columns.
+ * Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
+ */
+ using Base = COWHelper<IColumn, ColumnStruct>;
+ static Ptr create(Columns& columns);
+ static Ptr create(MutableColumns& columns);
+ static Ptr create(TupleColumns& columns);
+ static Ptr create(Columns&& arg) { return create(arg); }
+
+ template <typename... Args>
+ static MutablePtr create(Args&&... args) {
+ return Base::create(std::forward<Args>(args)...);
+ }
+
+ std::string get_name() const override;
+ const char* get_family_name() const override { return "Struct"; }
+ TypeIndex get_data_type() const { return TypeIndex::Struct; }
+
+ MutableColumnPtr clone_empty() const override;
+ MutableColumnPtr clone_resized(size_t size) const override;
+
+ size_t size() const override { return columns.at(0)->size(); }
+
+ Field operator[](size_t n) const override;
+ void get(size_t n, Field& res) const override;
+
+ bool is_default_at(size_t n) const override;
+ StringRef get_data_at(size_t n) const override;
+ void insert_data(const char* pos, size_t length) override;
+ void insert(const Field& x) override;
+ void insert_from(const IColumn& src_, size_t n) override;
+ void insert_default() override;
+ void pop_back(size_t n) override;
+ StringRef serialize_value_into_arena(size_t n, Arena& arena, char const*& begin) const override;
+ const char* deserialize_and_insert_from_arena(const char* pos) override;
+ void update_hash_with_value(size_t n, SipHash& hash) const override;
+
+ // const char * skip_serialized_in_arena(const char * pos) const override;
+ // void update_weak_hash32(WeakHash32 & hash) const override;
+ // void update_hash_fast(SipHash & hash) const override;
+
+ void insert_indices_from(const IColumn& src, const int* indices_begin,
+ const int* indices_end) override {
+ LOG(FATAL) << "insert_indices_from not implemented";
+ }
+
+ void get_permutation(bool reverse, size_t limit, int nan_direction_hint,
+ Permutation& res) const override {
+ LOG(FATAL) << "get_permutation not implemented";
+ }
+ void append_data_by_selector(MutableColumnPtr& res, const Selector& selector) const override {
+ return append_data_by_selector_impl<ColumnStruct>(res, selector);
+ }
+ void replace_column_data(const IColumn&, size_t row, size_t self_row = 0) override {
+ LOG(FATAL) << "replace_column_data not implemented";
+ }
+ void replace_column_data_default(size_t self_row = 0) override {
+ LOG(FATAL) << "replace_column_data_default not implemented";
+ }
+
+ void insert_range_from(const IColumn& src, size_t start, size_t length) override;
+ ColumnPtr filter(const Filter& filt, ssize_t result_size_hint) const override;
+ ColumnPtr permute(const Permutation& perm, size_t limit) const override;
+ ColumnPtr replicate(const Offsets& offsets) const override;
+ MutableColumns scatter(ColumnIndex num_columns, const Selector& selector) const override;
+
+ // ColumnPtr index(const IColumn & indexes, size_t limit) const override;
+ // void expand(const Filter & mask, bool inverted) override;
+ // void gather(ColumnGathererStream & gatherer_stream) override;
+ // bool has_equal_values() const override;
+
+ // void compare_column(const IColumn& rhs, size_t rhs_row_num, PaddedPODArray<UInt64>* row_indexes,
+ // PaddedPODArray<Int8>& compare_results, int direction,
+ // int nan_direction_hint) const override;
+ // int compare_at_with_collation(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint,
+ // const Collator& collator) const override;
+
+ int compare_at(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const override;
+ void get_extremes(Field& min, Field& max) const override;
+
+ // void get_permutation(IColumn::PermutationSortDirection direction,
+ // IColumn::PermutationSortStability stability, size_t limit,
+ // int nan_direction_hint, IColumn::Permutation& res) const override;
+ // void update_permutation(IColumn::PermutationSortDirection direction,
+ // IColumn::PermutationSortStability stability, size_t limit,
+ // int nan_direction_hint, IColumn::Permutation& res,
+ // EqualRanges& equal_ranges) const override;
+ // void get_permutation_with_collation(const Collator& collator,
+ // IColumn::PermutationSortDirection direction,
+ // IColumn::PermutationSortStability stability, size_t limit,
+ // int nan_direction_hint,
+ // IColumn::Permutation& res) const override;
+ // void update_permutation_with_collation(const Collator& collator,
+ // IColumn::PermutationSortDirection direction,
+ // IColumn::PermutationSortStability stability,
+ // size_t limit, int nan_direction_hint,
+ // IColumn::Permutation& res,
+ // EqualRanges& equal_ranges) const override;
+
+ void reserve(size_t n) override;
+ size_t byte_size() const override;
+
+ // size_t byte_size_at(size_t n) const override;
+ // void ensure_ownership() override;
+
+ size_t allocated_bytes() const override;
+ void protect() override;
+ void for_each_subcolumn(ColumnCallback callback) override;
+ bool structure_equals(const IColumn& rhs) const override;
+
+ // void for_each_subcolumn_recursively(ColumnCallback callback) override;
+ // bool is_collation_supported() const override;
+ // ColumnPtr compress() const override;
+ // double get_ratio_of_default_rows(double sample_ratio) const override;
+ // void get_indices_of_nondefault_rows(Offsets & indices, size_t from, size_t limit) const override;
+ // void finalize() override;
+ // bool is_finalized() const override;
+
+ size_t tuple_size() const { return columns.size(); }
+
+ const IColumn& get_column(size_t idx) const { return *columns[idx]; }
+ IColumn& get_column(size_t idx) { return *columns[idx]; }
+
+ const TupleColumns& get_columns() const { return columns; }
+ Columns get_columns_copy() const { return {columns.begin(), columns.end()}; }
+
+ const ColumnPtr& get_column_ptr(size_t idx) const { return columns[idx]; }
+ ColumnPtr& get_column_ptr(size_t idx) { return columns[idx]; }
+
+private:
+ int compare_at_impl(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const;
+
+ // void get_permutation_impl(IColumn::PermutationSortDirection direction,
+ // IColumn::PermutationSortStability stability, size_t limit,
+ // int nan_direction_hint, Permutation& res,
+ // const Collator* collator) const;
+
+ // void update_permutation_impl(IColumn::PermutationSortDirection direction,
+ // IColumn::PermutationSortStability stability, size_t limit,
+ // int nan_direction_hint, IColumn::Permutation& res,
+ // EqualRanges& equal_ranges,
+ // const Collator* collator = nullptr) const;
+};
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/core/types.h b/be/src/vec/core/types.h
index 7f0fefc0ad..ccbdbbebd8 100644
--- a/be/src/vec/core/types.h
+++ b/be/src/vec/core/types.h
@@ -80,6 +80,7 @@ enum class TypeIndex {
FixedLengthObject,
JSONB,
Decimal128I,
+ Struct,
};
struct Consted {
@@ -525,6 +526,8 @@ inline const char* getTypeName(TypeIndex idx) {
return "FixedLengthObject";
case TypeIndex::JSONB:
return "JSONB";
+ case TypeIndex::Struct:
+ return "Struct";
}
__builtin_unreachable();
diff --git a/be/src/vec/data_types/data_type_struct.cpp b/be/src/vec/data_types/data_type_struct.cpp
new file mode 100644
index 0000000000..91aff67a40
--- /dev/null
+++ b/be/src/vec/data_types/data_type_struct.cpp
@@ -0,0 +1,361 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypeTuple.cpp
+// and modified by Doris
+
+#include "vec/data_types/data_type_struct.h"
+
+namespace doris::vectorized {
+
+namespace ErrorCodes {
+extern const int BAD_ARGUMENTS;
+extern const int DUPLICATE_COLUMN;
+extern const int EMPTY_DATA_PASSED;
+extern const int NOT_FOUND_COLUMN_IN_BLOCK;
+extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+extern const int SIZES_OF_COLUMNS_IN_TUPLE_DOESNT_MATCH;
+extern const int ILLEGAL_INDEX;
+extern const int LOGICAL_ERROR;
+} // namespace ErrorCodes
+
+DataTypeStruct::DataTypeStruct(const DataTypes& elems_)
+ : elems(elems_), have_explicit_names(false) {
+ /// Automatically assigned names in form of '1', '2', ...
+ size_t size = elems.size();
+ names.resize(size);
+ for (size_t i = 0; i < size; ++i) {
+ names[i] = std::to_string(i + 1);
+ }
+}
+
+static std::optional<Exception> check_tuple_names(const Strings& names) {
+ std::unordered_set<String> names_set;
+ for (const auto& name : names) {
+ if (name.empty()) {
+ return Exception("Names of tuple elements cannot be empty", ErrorCodes::BAD_ARGUMENTS);
+ }
+
+ if (!names_set.insert(name).second) {
+ return Exception("Names of tuple elements must be unique",
+ ErrorCodes::DUPLICATE_COLUMN);
+ }
+ }
+
+ return {};
+}
+
+DataTypeStruct::DataTypeStruct(const DataTypes& elems_, const Strings& names_)
+ : elems(elems_), names(names_), have_explicit_names(true) {
+ size_t size = elems.size();
+ if (names.size() != size) {
+ throw Exception("Wrong number of names passed to constructor of DataTypeStruct",
+ ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+ }
+
+ if (auto exception = check_tuple_names(names)) {
+ throw std::move(*exception);
+ }
+}
+
+std::string DataTypeStruct::do_get_name() const {
+ size_t size = elems.size();
+ std::stringstream s;
+
+ s << "Struct(";
+ for (size_t i = 0; i < size; ++i) {
+ if (i != 0) {
+ s << ", ";
+ }
+
+ // if (have_explicit_names) {
+ // s << back_quote_if_need(names[i]) << ' ';
+ // }
+
+ s << elems[i]->get_name();
+ }
+ s << ")";
+
+ return s.str();
+}
+
+static inline IColumn& extract_element_column(IColumn& column, size_t idx) {
+ return assert_cast<ColumnStruct&>(column).get_column(idx);
+}
+
+template <typename F>
+static void add_element_safe(const DataTypes& elems, IColumn& column, F&& impl) {
+ /// We use the assumption that tuples of zero size do not exist.
+ size_t old_size = column.size();
+
+ try {
+ impl();
+
+ // Check that all columns now have the same size.
+ size_t new_size = column.size();
+
+ // for (auto i : collections::range(0, elems.size())) {
+ for (auto i = 0; i < elems.size(); i++) {
+ const auto& element_column = extract_element_column(column, i);
+ if (element_column.size() != new_size) {
+ // This is not a logical error because it may work with
+ // user-supplied data.
+ throw Exception("Cannot read a tuple because not all elements are present",
+ ErrorCodes::SIZES_OF_COLUMNS_IN_TUPLE_DOESNT_MATCH);
+ }
+ }
+ } catch (...) {
+ // for (const auto& i : collections::range(0, elems.size())) {
+ for (auto i = 0; i < elems.size(); i++) {
+ auto& element_column = extract_element_column(column, i);
+
+ if (element_column.size() > old_size) {
+ element_column.pop_back(1);
+ }
+ }
+
+ throw;
+ }
+}
+
+MutableColumnPtr DataTypeStruct::create_column() const {
+ size_t size = elems.size();
+ MutableColumns tuple_columns(size);
+ for (size_t i = 0; i < size; ++i) {
+ tuple_columns[i] = elems[i]->create_column();
+ }
+ return ColumnStruct::create(std::move(tuple_columns));
+}
+
+// MutableColumnPtr DataTypeStruct::create_column(const ISerialization& serialization) const {
+// /// If we read subcolumn of nested Tuple, it may be wrapped to SerializationNamed
+// /// several times to allow to reconstruct the substream path name.
+// /// Here we don't need substream path name, so we drop first several wrapper serializations.
+
+// const auto* current_serialization = &serialization;
+// while (const auto* serialization_named =
+// typeid_cast<const SerializationNamed*>(current_serialization))
+// current_serialization = serialization_named->get_nested().get();
+
+// const auto* serialization_tuple = typeid_cast<const SerializationTuple*>(current_serialization);
+// if (!serialization_tuple)
+// throw Exception(ErrorCodes::LOGICAL_ERROR,
+// "Unexpected serialization to create column of type Tuple");
+
+// const auto& element_serializations = serialization_tuple->getElementsSerializations();
+
+// size_t size = elems.size();
+// assert(element_serializations.size() == size);
+// MutableColumns tuple_columns(size);
+// for (size_t i = 0; i < size; ++i) {
+// tuple_columns[i] = elems[i]->create_column(*element_serializations[i]->get_nested());
+// }
+
+// return ColumnStruct::create(std::move(tuple_columns));
+// }
+
+// Field DataTypeStruct::get_default() const {
+// return Tuple(collections::map<Tuple>(
+// elems, [](const DataTypePtr& elem) { return elem->get_default(); }));
+// }
+
+void DataTypeStruct::insert_default_into(IColumn& column) const {
+ add_element_safe(elems, column, [&] {
+ // for (const auto& i : collections::range(0, elems.size()))
+ for (auto i = 0; i < elems.size(); i++) {
+ elems[i]->insert_default_into(extract_element_column(column, i));
+ }
+ });
+}
+
+bool DataTypeStruct::equals(const IDataType& rhs) const {
+ if (typeid(rhs) != typeid(*this)) {
+ return false;
+ }
+
+ const DataTypeStruct& rhs_tuple = static_cast<const DataTypeStruct&>(rhs);
+
+ size_t size = elems.size();
+ if (size != rhs_tuple.elems.size()) {
+ return false;
+ }
+
+ for (size_t i = 0; i < size; ++i) {
+ if (!elems[i]->equals(*rhs_tuple.elems[i]) || names[i] != rhs_tuple.names[i]) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+size_t DataTypeStruct::get_position_by_name(const String& name) const {
+ size_t size = elems.size();
+ for (size_t i = 0; i < size; ++i) {
+ if (names[i] == name) {
+ return i;
+ }
+ }
+ throw Exception("Struct doesn't have element with name '" + name + "'",
+ ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK);
+}
+
+std::optional<size_t> DataTypeStruct::try_get_position_by_name(const String& name) const {
+ size_t size = elems.size();
+ for (size_t i = 0; i < size; ++i) {
+ if (names[i] == name) {
+ return std::optional<size_t>(i);
+ }
+ }
+ return std::nullopt;
+}
+
+String DataTypeStruct::get_name_by_position(size_t i) const {
+ if (i == 0 || i > names.size()) {
+ fmt::memory_buffer error_msg;
+ fmt::format_to(error_msg, "Index of tuple element ({}) if out range ([1, {}])", i,
+ names.size());
+ throw Exception(fmt::to_string(error_msg), ErrorCodes::ILLEGAL_INDEX);
+ }
+
+ return names[i - 1];
+}
+
+bool DataTypeStruct::text_can_contain_only_valid_utf8() const {
+ return std::all_of(elems.begin(), elems.end(),
+ [](auto&& elem) { return elem->text_can_contain_only_valid_utf8(); });
+}
+
+bool DataTypeStruct::have_maximum_size_of_value() const {
+ return std::all_of(elems.begin(), elems.end(),
+ [](auto&& elem) { return elem->have_maximum_size_of_value(); });
+}
+
+bool DataTypeStruct::is_comparable() const {
+ return std::all_of(elems.begin(), elems.end(),
+ [](auto&& elem) { return elem->is_comparable(); });
+}
+
+size_t DataTypeStruct::get_maximum_size_of_value_in_memory() const {
+ size_t res = 0;
+ for (const auto& elem : elems) {
+ res += elem->get_maximum_size_of_value_in_memory();
+ }
+ return res;
+}
+
+size_t DataTypeStruct::get_size_of_value_in_memory() const {
+ size_t res = 0;
+ for (const auto& elem : elems) {
+ res += elem->get_size_of_value_in_memory();
+ }
+ return res;
+}
+
+// bool DataTypeStruct::has_dynamic_subcolumns() const {
+// return std::any_of(elems.begin(), elems.end(),
+// [](auto&& elem) { return elem->has_dynamic_subcolumns(); });
+// }
+
+// SerializationPtr DataTypeStruct::do_get_default_serialization() const {
+// SerializationTuple::ElementSerializations serializations(elems.size());
+
+// for (size_t i = 0; i < elems.size(); ++i) {
+// String elem_name = have_explicit_names ? names[i] : toString(i + 1);
+// auto serialization = elems[i]->get_default_serialization();
+// serializations[i] = std::make_shared<SerializationNamed>(serialization, elem_name);
+// }
+
+// return std::make_shared<SerializationTuple>(std::move(serializations), have_explicit_names);
+// }
+
+// SerializationPtr DataTypeStruct::get_serialization(const SerializationInfo& info) const {
+// SerializationTuple::ElementSerializations serializations(elems.size());
+// const auto& info_tuple = assert_cast<const SerializationInfoTuple&>(info);
+
+// for (size_t i = 0; i < elems.size(); ++i) {
+// String elem_name = have_explicit_names ? names[i] : toString(i + 1);
+// auto serialization = elems[i]->get_serialization(*info_tuple.get_element_info(i));
+// serializations[i] = std::make_shared<SerializationNamed>(serialization, elem_name);
+// }
+
+// return std::make_shared<SerializationTuple>(std::move(serializations), have_explicit_names);
+// }
+
+// MutableSerializationInfoPtr DataTypeStruct::create_serialization_info(
+// const SerializationInfo::Settings& settings) const {
+// MutableSerializationInfos infos;
+// infos.reserve(elems.size());
+// for (const auto& elem : elems) {
+// infos.push_back(elem->create_serializationInfo(settings));
+// }
+
+// return std::make_shared<SerializationInfoTuple>(std::move(infos), names, settings);
+// }
+
+// SerializationInfoPtr DataTypeStruct::get_serialization_info(const IColumn& column) const {
+// if (const auto* column_const = check_and_get_column<ColumnConst>(&column)) {
+// return get_serialization_info(column_const->get_data_column());
+// }
+
+// MutableSerializationInfos infos;
+// infos.reserve(elems.size());
+
+// const auto& column_tuple = assert_cast<const ColumnStruct&>(column);
+// assert(elems.size() == column_tuple.get_columns().size());
+
+// for (size_t i = 0; i < elems.size(); ++i) {
+// auto element_info = elems[i]->get_serialization_info(column_tuple.getColumn(i));
+// infos.push_back(const_pointer_cast<SerializationInfo>(element_info));
+// }
+
+// return std::make_shared<SerializationInfoTuple>(std::move(infos), names,
+// SerializationInfo::Settings {});
+// }
+
+// static DataTypePtr create(const ASTPtr& arguments) {
+// if (!arguments || arguments->children.empty())
+// throw Exception("Struct cannot be empty", ErrorCodes::EMPTY_DATA_PASSED);
+
+// DataTypes nested_types;
+// nested_types.reserve(arguments->children.size());
+
+// Strings names;
+// names.reserve(arguments->children.size());
+
+// for (const ASTPtr& child : arguments->children) {
+// if (const auto* name_and_type_pair = child->as<ASTNameTypePair>()) {
+// nested_types.emplace_back(DataTypeFactory::instance().get(name_and_type_pair->type));
+// names.emplace_back(name_and_type_pair->name);
+// } else
+// nested_types.emplace_back(DataTypeFactory::instance().get(child));
+// }
+
+// if (names.empty())
+// return std::make_shared<DataTypeStruct>(nested_types);
+// else if (names.size() != nested_types.size())
+// throw Exception("Names are specified not for all elements of Struct type",
+// ErrorCodes::BAD_ARGUMENTS);
+// else
+// return std::make_shared<DataTypeStruct>(nested_types, names);
+// }
+
+// void registerDataTypeStruct(DataTypeFactory& factory) {
+// factory.registerDataType("Struct", create);
+// }
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/data_types/data_type_struct.h b/be/src/vec/data_types/data_type_struct.h
new file mode 100644
index 0000000000..4201583ef4
--- /dev/null
+++ b/be/src/vec/data_types/data_type_struct.h
@@ -0,0 +1,115 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypeTuple.h
+// and modified by Doris
+
+#pragma once
+
+#include <exception>
+
+#include "gen_cpp/data.pb.h"
+#include "util/stack_util.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_nullable.h"
+#include "vec/columns/column_struct.h"
+#include "vec/data_types/data_type.h"
+#include "vec/data_types/data_type_nullable.h"
+
+namespace doris::vectorized {
+
+/** Struct data type.
+ * Used as an intermediate result when evaluating expressions.
+ * Also can be used as a column - the result of the query execution.
+ *
+ * Struct elements can have names.
+ * If an element is unnamed, it will have automatically assigned name like '1', '2', '3' corresponding to its position.
+ * Manually assigned names must not begin with digit. Names must be unique.
+ *
+ * All tuples with same size and types of elements are equivalent for expressions, regardless to names of elements.
+ */
+class DataTypeStruct final : public IDataType {
+private:
+ // using DataTypePtr = std::shared_ptr<const IDataType>;
+ // using DataTypes = std::vector<DataTypePtr>;
+ // using Strings = std::vector<std::string>;
+
+ DataTypes elems;
+ Strings names;
+ bool have_explicit_names;
+
+public:
+ // static constexpr bool is_parametric = true;
+
+ explicit DataTypeStruct(const DataTypes& elems);
+ DataTypeStruct(const DataTypes& elems, const Strings& names);
+
+ TypeIndex get_type_id() const override { return TypeIndex::Struct; }
+ std::string do_get_name() const override;
+ const char* get_family_name() const override { return "Struct"; }
+
+ bool can_be_inside_nullable() const override { return false; }
+ bool supports_sparse_serialization() const { return true; }
+
+ MutableColumnPtr create_column() const override;
+ // MutableColumnPtr create_column(const ISerialization& serialization) const override;
+
+ Field get_default() const override;
+ void insert_default_into(IColumn& column) const override;
+
+ bool equals(const IDataType& rhs) const override;
+
+ bool get_is_parametric() const override { return true; }
+ bool have_subtypes() const override { return !elems.empty(); }
+ bool is_comparable() const override;
+ bool text_can_contain_only_valid_utf8() const override;
+ bool have_maximum_size_of_value() const override;
+ bool has_dynamic_subcolumns() const;
+ size_t get_maximum_size_of_value_in_memory() const override;
+ size_t get_size_of_value_in_memory() const override;
+
+ const DataTypePtr& get_element(size_t i) const { return elems[i]; }
+ const DataTypes& get_elements() const { return elems; }
+ const Strings& get_element_names() const { return names; }
+
+ size_t get_position_by_name(const String& name) const;
+ std::optional<size_t> try_get_position_by_name(const String& name) const;
+ String get_name_by_position(size_t i) const;
+
+ int64_t get_uncompressed_serialized_bytes(const IColumn& column,
+ int be_exec_version) const override {
+ LOG(FATAL) << "get_uncompressed_serialized_bytes not implemented";
+ }
+
+ char* serialize(const IColumn& column, char* buf, int be_exec_version) const override {
+ LOG(FATAL) << "serialize not implemented";
+ }
+
+ const char* deserialize(const char* buf, IColumn* column, int be_exec_version) const override {
+ LOG(FATAL) << "serialize not implemented";
+ }
+
+ // bool is_parametric() const { return true; }
+ // SerializationPtr do_get_default_serialization() const override;
+ // SerializationPtr get_serialization(const SerializationInfo& info) const override;
+ // MutableSerializationInfoPtr create_serialization_info(
+ // const SerializationInfo::Settings& settings) const override;
+ // SerializationInfoPtr get_serialization_info(const IColumn& column) const override;
+ // bool have_explicit_names() const { return have_explicit_names; }
+};
+
+} // namespace doris::vectorized
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 06/18: [fix](struct-type) struct create fix and add more struct codes (#15879)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5137ca6da0dd71ec9d5cd5be2a69184a3134c030
Author: camby <10...@qq.com>
AuthorDate: Fri Jan 13 11:24:09 2023 +0800
[fix](struct-type) struct create fix and add more struct codes (#15879)
Fix the dead loop when create column struct
Co-authored-by: cambyzju <zh...@baidu.com>
---
be/src/runtime/types.cpp | 29 +++++++++++++++++--
be/src/vec/columns/column_struct.cpp | 38 ++++++++++---------------
be/src/vec/columns/column_struct.h | 18 +++++-------
be/src/vec/data_types/data_type.cpp | 2 ++
be/src/vec/data_types/data_type_factory.cpp | 14 ++++++++++
be/src/vec/data_types/data_type_struct.cpp | 43 +++++++++++++++++++++++++++++
be/src/vec/data_types/data_type_struct.h | 19 ++++---------
7 files changed, 111 insertions(+), 52 deletions(-)
diff --git a/be/src/runtime/types.cpp b/be/src/runtime/types.cpp
index f26b1dcbce..45b8f8e5f5 100644
--- a/be/src/runtime/types.cpp
+++ b/be/src/runtime/types.cpp
@@ -150,7 +150,7 @@ void TypeDescriptor::to_thrift(TTypeDesc* thrift_type) const {
}
void TypeDescriptor::to_protobuf(PTypeDesc* ptype) const {
- DCHECK(!is_complex_type() || type == TYPE_ARRAY)
+ DCHECK(!is_complex_type() || type == TYPE_ARRAY || type == TYPE_STRUCT)
<< "Don't support complex type now, type=" << type;
auto node = ptype->add_types();
node->set_type(TTypeNodeType::SCALAR);
@@ -170,8 +170,18 @@ void TypeDescriptor::to_protobuf(PTypeDesc* ptype) const {
for (const TypeDescriptor& child : children) {
child.to_protobuf(ptype);
}
+ } else if (type == TYPE_STRUCT) {
+ node->set_type(TTypeNodeType::STRUCT);
+ DCHECK_EQ(field_names.size(), contains_nulls.size());
+ for (size_t i = 0; i < field_names.size(); ++i) {
+ auto field = node->add_struct_fields();
+ field->set_name(field_names[i]);
+ field->set_contains_null(contains_nulls[i]);
+ }
+ for (const TypeDescriptor& child : children) {
+ child.to_protobuf(ptype);
+ }
}
- // TODO(xy): support struct
}
TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNode>& types, int* idx)
@@ -213,7 +223,20 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
children.push_back(TypeDescriptor(types, idx));
break;
}
- // TODO(xy): support struct
+ case TTypeNodeType::STRUCT: {
+ type = TYPE_STRUCT;
+ size_t children_size = node.struct_fields_size();
+ for (size_t i = 0; i < children_size; ++i) {
+ const auto& field = node.struct_fields(i);
+ field_names.push_back(field.name());
+ contains_nulls.push_back(field.contains_null());
+ }
+ for (size_t i = 0; i < children_size; ++i) {
+ ++(*idx);
+ children.push_back(TypeDescriptor(types, idx));
+ }
+ break;
+ }
default:
DCHECK(false) << node.type();
}
diff --git a/be/src/vec/columns/column_struct.cpp b/be/src/vec/columns/column_struct.cpp
index c7e5c23d2a..d51bebb536 100644
--- a/be/src/vec/columns/column_struct.cpp
+++ b/be/src/vec/columns/column_struct.cpp
@@ -54,43 +54,25 @@ ColumnStruct::ColumnStruct(MutableColumns&& mutable_columns) {
}
}
-ColumnStruct::ColumnStruct(Columns&& columns) {
- columns.reserve(columns.size());
- for (auto& column : columns) {
- if (is_column_const(*column)) {
- LOG(FATAL) << "ColumnStruct cannot have ColumnConst as its element";
- }
- columns.push_back(std::move(column));
- }
-}
-
-ColumnStruct::ColumnStruct(TupleColumns&& tuple_columns) {
- columns.reserve(tuple_columns.size());
- for (auto& column : tuple_columns) {
- if (is_column_const(*column)) {
- LOG(FATAL) << "ColumnStruct cannot have ColumnConst as its element";
- }
- columns.push_back(std::move(column));
- }
-}
-
-ColumnStruct::Ptr ColumnStruct::create(Columns& columns) {
+ColumnStruct::Ptr ColumnStruct::create(const Columns& columns) {
for (const auto& column : columns) {
if (is_column_const(*column)) {
LOG(FATAL) << "ColumnStruct cannot have ColumnConst as its element";
}
}
- auto column_struct = ColumnStruct::create(columns);
+ auto column_struct = ColumnStruct::create(MutableColumns());
+ column_struct->columns.assign(columns.begin(), columns.end());
return column_struct;
}
-ColumnStruct::Ptr ColumnStruct::create(TupleColumns& tuple_columns) {
+ColumnStruct::Ptr ColumnStruct::create(const TupleColumns& tuple_columns) {
for (const auto& column : tuple_columns) {
if (is_column_const(*column)) {
LOG(FATAL) << "ColumnStruct cannot have ColumnConst as its element";
}
}
- auto column_struct = ColumnStruct::create(tuple_columns);
+ auto column_struct = ColumnStruct::create(MutableColumns());
+ column_struct->columns = tuple_columns;
return column_struct;
}
@@ -221,6 +203,14 @@ void ColumnStruct::update_hash_with_value(size_t n, SipHash& hash) const {
// }
// }
+void ColumnStruct::insert_indices_from(const IColumn& src, const int* indices_begin,
+ const int* indices_end) {
+ const ColumnStruct& src_concrete = assert_cast<const ColumnStruct&>(src);
+ for (size_t i = 0; i < columns.size(); ++i) {
+ columns[i]->insert_indices_from(src_concrete.get_column(i), indices_begin, indices_end);
+ }
+}
+
// const char * ColumnStruct::skip_serialized_in_arena(const char * pos) const {
// for (const auto & column : columns) {
// pos = column->skip_serialized_in_arena(pos);
diff --git a/be/src/vec/columns/column_struct.h b/be/src/vec/columns/column_struct.h
index 75dade874d..a66d91f3e0 100644
--- a/be/src/vec/columns/column_struct.h
+++ b/be/src/vec/columns/column_struct.h
@@ -79,8 +79,6 @@ private:
template <bool positive>
struct Less;
- ColumnStruct(Columns&& columns);
- ColumnStruct(TupleColumns&& tuple_columns);
explicit ColumnStruct(MutableColumns&& mutable_columns);
ColumnStruct(const ColumnStruct&) = default;
@@ -89,14 +87,14 @@ public:
* Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
*/
using Base = COWHelper<IColumn, ColumnStruct>;
- static Ptr create(Columns& columns);
- static Ptr create(MutableColumns& columns);
- static Ptr create(TupleColumns& columns);
+ static Ptr create(const Columns& columns);
+ static Ptr create(const TupleColumns& columns);
static Ptr create(Columns&& arg) { return create(arg); }
- template <typename... Args>
- static MutablePtr create(Args&&... args) {
- return Base::create(std::forward<Args>(args)...);
+ template <typename Arg,
+ typename = typename std::enable_if<std::is_rvalue_reference<Arg&&>::value>::type>
+ static MutablePtr create(Arg&& arg) {
+ return Base::create(std::forward<Arg>(arg));
}
std::string get_name() const override;
@@ -131,9 +129,7 @@ public:
// void update_hash_fast(SipHash & hash) const override;
void insert_indices_from(const IColumn& src, const int* indices_begin,
- const int* indices_end) override {
- LOG(FATAL) << "insert_indices_from not implemented";
- }
+ const int* indices_end) override;
void get_permutation(bool reverse, size_t limit, int nan_direction_hint,
Permutation& res) const override {
diff --git a/be/src/vec/data_types/data_type.cpp b/be/src/vec/data_types/data_type.cpp
index 7a0d67d4a4..360fa48a98 100644
--- a/be/src/vec/data_types/data_type.cpp
+++ b/be/src/vec/data_types/data_type.cpp
@@ -145,6 +145,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) {
return PGenericType::HLL;
case TypeIndex::Array:
return PGenericType::LIST;
+ case TypeIndex::Struct:
+ return PGenericType::STRUCT;
case TypeIndex::FixedLengthObject:
return PGenericType::FIXEDLENGTHOBJECT;
case TypeIndex::JSONB:
diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp
index 7eec1eb2fd..553f117a48 100644
--- a/be/src/vec/data_types/data_type_factory.cpp
+++ b/be/src/vec/data_types/data_type_factory.cpp
@@ -338,6 +338,20 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) {
case PGenericType::FIXEDLENGTHOBJECT:
nested = std::make_shared<DataTypeFixedLengthObject>();
break;
+ case PGenericType::STRUCT: {
+ size_t col_size = pcolumn.children_size();
+ DCHECK(col_size >= 1);
+ DataTypes dataTypes;
+ Strings names;
+ dataTypes.reserve(col_size);
+ names.reserve(col_size);
+ for (size_t i = 0; i < col_size; i++) {
+ dataTypes.push_back(create_data_type(pcolumn.children(i)));
+ names.push_back(pcolumn.name());
+ }
+ nested = std::make_shared<DataTypeStruct>(dataTypes, names);
+ break;
+ }
default: {
LOG(FATAL) << fmt::format("Unknown data type: {}", pcolumn.type());
return nullptr;
diff --git a/be/src/vec/data_types/data_type_struct.cpp b/be/src/vec/data_types/data_type_struct.cpp
index 8fd2179bfa..4364f5c34f 100644
--- a/be/src/vec/data_types/data_type_struct.cpp
+++ b/be/src/vec/data_types/data_type_struct.cpp
@@ -232,6 +232,49 @@ String DataTypeStruct::get_name_by_position(size_t i) const {
return names[i - 1];
}
+int64_t DataTypeStruct::get_uncompressed_serialized_bytes(const IColumn& column,
+ int be_exec_version) const {
+ auto ptr = column.convert_to_full_column_if_const();
+ const auto& struct_column = assert_cast<const ColumnStruct&>(*ptr.get());
+ DCHECK(elems.size() == struct_column.tuple_size());
+
+ int64_t bytes = 0;
+ for (size_t i = 0; i < elems.size(); ++i) {
+ bytes += elems[i]->get_uncompressed_serialized_bytes(struct_column.get_column(i),
+ be_exec_version);
+ }
+ return bytes;
+}
+
+char* DataTypeStruct::serialize(const IColumn& column, char* buf, int be_exec_version) const {
+ auto ptr = column.convert_to_full_column_if_const();
+ const auto& struct_column = assert_cast<const ColumnStruct&>(*ptr.get());
+ DCHECK(elems.size() == struct_column.tuple_size());
+
+ for (size_t i = 0; i < elems.size(); ++i) {
+ buf = elems[i]->serialize(struct_column.get_column(i), buf, be_exec_version);
+ }
+ return buf;
+}
+
+const char* DataTypeStruct::deserialize(const char* buf, IColumn* column,
+ int be_exec_version) const {
+ auto* struct_column = assert_cast<ColumnStruct*>(column);
+ DCHECK(elems.size() == struct_column->tuple_size());
+
+ for (size_t i = 0; i < elems.size(); ++i) {
+ buf = elems[i]->deserialize(buf, &struct_column->get_column(i), be_exec_version);
+ }
+ return buf;
+}
+
+void DataTypeStruct::to_pb_column_meta(PColumnMeta* col_meta) const {
+ IDataType::to_pb_column_meta(col_meta);
+ for (size_t i = 0; i < elems.size(); ++i) {
+ elems[i]->to_pb_column_meta(col_meta->add_children());
+ }
+}
+
bool DataTypeStruct::text_can_contain_only_valid_utf8() const {
return std::all_of(elems.begin(), elems.end(),
[](auto&& elem) { return elem->text_can_contain_only_valid_utf8(); });
diff --git a/be/src/vec/data_types/data_type_struct.h b/be/src/vec/data_types/data_type_struct.h
index 9405544b76..88255de38c 100644
--- a/be/src/vec/data_types/data_type_struct.h
+++ b/be/src/vec/data_types/data_type_struct.h
@@ -90,20 +90,11 @@ public:
std::optional<size_t> try_get_position_by_name(const String& name) const;
String get_name_by_position(size_t i) const;
- [[noreturn]] int64_t get_uncompressed_serialized_bytes(const IColumn& column,
- int be_exec_version) const override {
- LOG(FATAL) << "get_uncompressed_serialized_bytes not implemented";
- }
-
- [[noreturn]] char* serialize(const IColumn& column, char* buf,
- int be_exec_version) const override {
- LOG(FATAL) << "serialize not implemented";
- }
-
- [[noreturn]] const char* deserialize(const char* buf, IColumn* column,
- int be_exec_version) const override {
- LOG(FATAL) << "serialize not implemented";
- }
+ int64_t get_uncompressed_serialized_bytes(const IColumn& column,
+ int be_exec_version) const override;
+ char* serialize(const IColumn& column, char* buf, int be_exec_version) const override;
+ const char* deserialize(const char* buf, IColumn* column, int be_exec_version) const override;
+ void to_pb_column_meta(PColumnMeta* col_meta) const override;
// bool is_parametric() const { return true; }
// SerializationPtr do_get_default_serialization() const override;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 13/18: [fix](struct-type) select of struct return wrong result (#16032)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e6c298e9c33862a473c2b5d6c71784563303883e
Author: camby <10...@qq.com>
AuthorDate: Tue Jan 17 23:22:00 2023 +0800
[fix](struct-type) select of struct return wrong result (#16032)
Co-authored-by: cambyzju <zh...@baidu.com>
---
be/src/vec/columns/column_struct.h | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/be/src/vec/columns/column_struct.h b/be/src/vec/columns/column_struct.h
index 393b2c275f..627a6736f2 100644
--- a/be/src/vec/columns/column_struct.h
+++ b/be/src/vec/columns/column_struct.h
@@ -216,6 +216,12 @@ public:
const ColumnPtr& get_column_ptr(size_t idx) const { return columns[idx]; }
ColumnPtr& get_column_ptr(size_t idx) { return columns[idx]; }
+ void clear() override {
+ for (auto col : columns) {
+ col->clear();
+ }
+ }
+
private:
int compare_at_impl(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 15/18: [Feature](map) add map type to doris (#15966)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ebfd14ca2c3916dfac76610ec3e98fb07e418933
Author: amory <wa...@selectdb.com>
AuthorDate: Fri Feb 3 13:26:29 2023 +0800
[Feature](map) add map type to doris (#15966)
Add complex type map to doris on vectorized engine
---
be/src/olap/field.h | 33 +-
be/src/olap/page_cache.cpp | 1 -
be/src/olap/rowset/segment_v2/column_reader.cpp | 105 +++++++
be/src/olap/rowset/segment_v2/column_reader.h | 35 +++
be/src/olap/rowset/segment_v2/column_writer.cpp | 188 +++++++++++-
be/src/olap/rowset/segment_v2/column_writer.h | 55 ++++
be/src/olap/rowset/segment_v2/segment_writer.cpp | 9 +
be/src/olap/tablet_meta.cpp | 6 +
be/src/olap/tablet_schema.cpp | 18 ++
be/src/olap/types.cpp | 48 ++-
be/src/olap/types.h | 92 ++++++
be/src/runtime/CMakeLists.txt | 1 +
be/src/runtime/map_value.cpp | 29 ++
be/src/runtime/map_value.h | 61 ++++
be/src/runtime/primitive_type.cpp | 14 +
be/src/runtime/types.cpp | 144 +++++----
be/src/udf/udf.h | 1 +
be/src/vec/CMakeLists.txt | 3 +
be/src/vec/columns/column.h | 2 +
be/src/vec/columns/column_map.cpp | 186 ++++++++++++
be/src/vec/columns/column_map.h | 153 ++++++++++
be/src/vec/core/field.h | 32 ++
be/src/vec/core/types.h | 3 +
be/src/vec/data_types/data_type.cpp | 2 +
be/src/vec/data_types/data_type.h | 5 +-
be/src/vec/data_types/data_type_factory.cpp | 29 ++
be/src/vec/data_types/data_type_factory.hpp | 1 +
be/src/vec/data_types/data_type_map.cpp | 202 +++++++++++++
be/src/vec/data_types/data_type_map.h | 81 +++++
be/src/vec/exprs/vexpr.cpp | 336 +++++++++++----------
be/src/vec/exprs/vmap_literal.cpp | 52 ++++
be/src/vec/exprs/vmap_literal.h | 33 ++
.../vec/functions/array/function_array_element.h | 104 ++++++-
be/src/vec/olap/olap_data_convertor.cpp | 62 +++-
be/src/vec/olap/olap_data_convertor.h | 25 ++
be/src/vec/sink/vmysql_result_writer.cpp | 29 ++
.../java/org/apache/doris/catalog/MapType.java | 48 +++
.../org/apache/doris/catalog/MysqlColType.java | 3 +-
.../org/apache/doris/catalog/PrimitiveType.java | 6 +
.../main/java/org/apache/doris/catalog/Type.java | 2 +
fe/fe-core/src/main/cup/sql_parser.cup | 32 +-
.../java/org/apache/doris/analysis/CastExpr.java | 5 +
.../main/java/org/apache/doris/analysis/Expr.java | 9 +-
.../java/org/apache/doris/analysis/MapLiteral.java | 177 +++++++++++
.../main/java/org/apache/doris/catalog/Column.java | 14 +
.../java/org/apache/doris/catalog/FunctionSet.java | 3 +-
.../java/org/apache/doris/common/util/Util.java | 1 +
fe/fe-core/src/main/jflex/sql_scanner.flex | 2 +
gensrc/script/doris_builtins_functions.py | 3 +
gensrc/script/gen_builtins_functions.py | 6 +
gensrc/thrift/Exprs.thrift | 3 +
51 files changed, 2224 insertions(+), 270 deletions(-)
diff --git a/be/src/olap/field.h b/be/src/olap/field.h
index e3f5ede607..dd3c455b90 100644
--- a/be/src/olap/field.h
+++ b/be/src/olap/field.h
@@ -28,6 +28,7 @@
#include "olap/types.h"
#include "olap/utils.h"
#include "runtime/collection_value.h"
+#include "runtime/map_value.h"
#include "runtime/mem_pool.h"
#include "util/hash_util.hpp"
#include "util/mem_util.hpp"
@@ -355,10 +356,9 @@ uint32_t Field::hash_code(const CellType& cell, uint32_t seed) const {
return _type_info->hash_code(cell.cell_ptr(), seed);
}
-class StructField : public Field {
+class MapField : public Field {
public:
- explicit StructField(const TabletColumn& column) : Field(column) {}
-
+ explicit MapField(const TabletColumn& column) : Field(column) {}
void consume(RowCursorCell* dst, const char* src, bool src_null, MemPool* mem_pool,
ObjectPool* agg_pool) const override {
dst->set_is_null(src_null);
@@ -367,7 +367,17 @@ public:
}
_type_info->deep_copy(dst->mutable_cell_ptr(), src, mem_pool);
}
+ // make variable_ptr memory allocate to cell_ptr as MapValue
+ char* allocate_memory(char* cell_ptr, char* variable_ptr) const override {
+ return variable_ptr + _length;
+ }
+
+ size_t get_variable_len() const override { return _length; }
+};
+class StructField : public Field {
+public:
+ explicit StructField(const TabletColumn& column) : Field(column) {}
char* allocate_memory(char* cell_ptr, char* variable_ptr) const override {
auto struct_v = (StructValue*)cell_ptr;
struct_v->set_values(reinterpret_cast<void**>(variable_ptr));
@@ -656,6 +666,14 @@ public:
local->add_sub_field(std::move(item_field));
return local;
}
+ case OLAP_FIELD_TYPE_MAP: {
+ std::unique_ptr<Field> key_field(FieldFactory::create(column.get_sub_column(0)));
+ std::unique_ptr<Field> val_field(FieldFactory::create(column.get_sub_column(1)));
+ auto* local = new MapField(column);
+ local->add_sub_field(std::move(key_field));
+ local->add_sub_field(std::move(val_field));
+ return local;
+ }
case OLAP_FIELD_TYPE_DECIMAL:
[[fallthrough]];
case OLAP_FIELD_TYPE_DECIMAL32:
@@ -705,6 +723,15 @@ public:
local->add_sub_field(std::move(item_field));
return local;
}
+ case OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column.get_subtype_count() == 2);
+ auto* local = new MapField(column);
+ std::unique_ptr<Field> key_field(FieldFactory::create(column.get_sub_column(0)));
+ std::unique_ptr<Field> value_field(FieldFactory::create(column.get_sub_column(1)));
+ local->add_sub_field(std::move(key_field));
+ local->add_sub_field(std::move(value_field));
+ return local;
+ }
case OLAP_FIELD_TYPE_DECIMAL:
[[fallthrough]];
case OLAP_FIELD_TYPE_DECIMAL32:
diff --git a/be/src/olap/page_cache.cpp b/be/src/olap/page_cache.cpp
index 2813f85dd3..a49043aebb 100644
--- a/be/src/olap/page_cache.cpp
+++ b/be/src/olap/page_cache.cpp
@@ -65,7 +65,6 @@ bool StoragePageCache::lookup(const CacheKey& key, PageCacheHandle* handle,
void StoragePageCache::insert(const CacheKey& key, const Slice& data, PageCacheHandle* handle,
segment_v2::PageTypePB page_type, bool in_memory) {
auto deleter = [](const doris::CacheKey& key, void* value) { delete[] (uint8_t*)value; };
-
CachePriority priority = CachePriority::NORMAL;
if (in_memory) {
priority = CachePriority::DURABLE;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 07feb00c20..3bc4895da1 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -30,6 +30,7 @@
#include "util/rle_encoding.h" // for RleDecoder
#include "vec/columns/column.h"
#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
#include "vec/columns/column_struct.h"
#include "vec/core/types.h"
#include "vec/runtime/vdatetime_value.h" //for VecDateTime
@@ -100,6 +101,32 @@ Status ColumnReader::create(const ColumnReaderOptions& opts, const ColumnMetaPB&
*reader = std::move(array_reader);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ // map reader now has 3 sub readers for key(arr), value(arr), null(scala)
+ std::unique_ptr<ColumnReader> map_reader(
+ new ColumnReader(opts, meta, num_rows, file_reader));
+ std::unique_ptr<ColumnReader> key_reader;
+ RETURN_IF_ERROR(ColumnReader::create(opts, meta.children_columns(0), num_rows,
+ file_reader, &key_reader));
+ std::unique_ptr<ColumnReader> val_reader;
+ RETURN_IF_ERROR(ColumnReader::create(opts, meta.children_columns(1), num_rows,
+ file_reader, &val_reader));
+ std::unique_ptr<ColumnReader> null_reader;
+ if (meta.is_nullable()) {
+ RETURN_IF_ERROR(ColumnReader::create(opts, meta.children_columns(2),
+ meta.children_columns(2).num_rows(),
+ file_reader, &null_reader));
+ }
+ map_reader->_sub_readers.resize(meta.children_columns_size());
+
+ map_reader->_sub_readers[0] = std::move(key_reader);
+ map_reader->_sub_readers[1] = std::move(val_reader);
+ if (meta.is_nullable()) {
+ map_reader->_sub_readers[2] = std::move(null_reader);
+ }
+ *reader = std::move(map_reader);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type for ColumnReader: {}",
std::to_string(type));
@@ -494,6 +521,18 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
null_iterator);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ ColumnIterator* key_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&key_iterator));
+ ColumnIterator* val_iterator = nullptr;
+ RETURN_IF_ERROR(_sub_readers[1]->new_iterator(&val_iterator));
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[2]->new_iterator(&null_iterator));
+ }
+ *iterator = new MapFileColumnIterator(this, null_iterator, key_iterator, val_iterator);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type to create iterator: {}",
std::to_string(type));
@@ -501,6 +540,72 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
}
}
+///====================== MapFileColumnIterator ============================////
+MapFileColumnIterator::MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator,
+ ColumnIterator* val_iterator)
+ : _map_reader(reader) {
+ _key_iterator.reset(key_iterator);
+ _val_iterator.reset(val_iterator);
+ if (_map_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status MapFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ RETURN_IF_ERROR(_key_iterator->init(opts));
+ RETURN_IF_ERROR(_val_iterator->init(opts));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) {
+ return Status::NotSupported("Not support next_batch");
+}
+
+Status MapFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ RETURN_IF_ERROR(_key_iterator->seek_to_ordinal(ord));
+ RETURN_IF_ERROR(_val_iterator->seek_to_ordinal(ord));
+ if (_map_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+ const auto* column_map = vectorized::check_and_get_column<vectorized::ColumnMap>(
+ dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
+ : *dst);
+ size_t num_read = *n;
+ auto column_key_ptr = column_map->get_keys().assume_mutable();
+ auto column_val_ptr = column_map->get_values().assume_mutable();
+ RETURN_IF_ERROR(_key_iterator->next_batch(&num_read, column_key_ptr, has_null));
+ RETURN_IF_ERROR(_val_iterator->next_batch(&num_read, column_val_ptr, has_null));
+
+ if (dst->is_nullable()) {
+ auto null_map_ptr =
+ static_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column_ptr();
+ bool null_signs_has_null = false;
+ RETURN_IF_ERROR(_null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
+ DCHECK(num_read == *n);
+ }
+ return Status::OK();
+}
+
+Status MapFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
+ vectorized::MutableColumnPtr& dst) {
+ for (size_t i = 0; i < count; ++i) {
+ RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
+ size_t num_read = 1;
+ RETURN_IF_ERROR(next_batch(&num_read, dst, nullptr));
+ DCHECK(num_read == 1);
+ }
+ return Status::OK();
+}
+
////////////////////////////////////////////////////////////////////////////////
StructFileColumnIterator::StructFileColumnIterator(
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 6e699027cc..5a0d4e8bdd 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -376,6 +376,41 @@ public:
ordinal_t get_current_ordinal() const override { return 0; }
};
+// This iterator is used to read map value column
+class MapFileColumnIterator final : public ColumnIterator {
+public:
+ explicit MapFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ ColumnIterator* key_iterator, ColumnIterator* val_iterator);
+
+ ~MapFileColumnIterator() override = default;
+
+ Status init(const ColumnIteratorOptions& opts) override;
+
+ Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) override;
+
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst, bool* has_null) override;
+
+ Status read_by_rowids(const rowid_t* rowids, const size_t count,
+ vectorized::MutableColumnPtr& dst) override;
+
+ Status seek_to_first() override {
+ RETURN_IF_ERROR(_key_iterator->seek_to_first());
+ RETURN_IF_ERROR(_val_iterator->seek_to_first());
+ RETURN_IF_ERROR(_null_iterator->seek_to_first());
+ return Status::OK();
+ }
+
+ Status seek_to_ordinal(ordinal_t ord) override;
+
+ ordinal_t get_current_ordinal() const override { return _key_iterator->get_current_ordinal(); }
+
+private:
+ ColumnReader* _map_reader;
+ std::unique_ptr<ColumnIterator> _null_iterator;
+ std::unique_ptr<ColumnIterator> _key_iterator; // ArrayFileColumnIterator
+ std::unique_ptr<ColumnIterator> _val_iterator; // ArrayFileColumnIterator
+};
+
class StructFileColumnIterator final : public ColumnIterator {
public:
explicit StructFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp
index 7949ed02ce..050d113f21 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -245,6 +245,79 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
*writer = std::move(writer_local);
return Status::OK();
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ DCHECK(column->get_subtype_count() == 2);
+ ScalarColumnWriter* null_writer = nullptr;
+ // create null writer
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(3);
+ null_options.meta->set_unique_id(3);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column =
+ TabletColumn(OLAP_FIELD_AGGREGATION_NONE, null_type, false,
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ // create key & value writer
+ std::vector<std::unique_ptr<ColumnWriter>> inner_writer_list;
+ for (int i = 0; i < 2; ++i) {
+ std::unique_ptr<ColumnWriter> inner_array_writer;
+ ColumnWriterOptions arr_opts;
+ TabletColumn array_column(OLAP_FIELD_AGGREGATION_NONE, OLAP_FIELD_TYPE_ARRAY);
+
+ array_column.set_index_length(-1);
+ arr_opts.meta = opts.meta->mutable_children_columns(i);
+ ColumnMetaPB* child_meta = arr_opts.meta->add_children_columns();
+ // inner column meta from actual opts meta
+ const TabletColumn& inner_column =
+ column->get_sub_column(i); // field_type is true key and value
+ array_column.add_sub_column(const_cast<TabletColumn&>(inner_column));
+ array_column.set_name("map.arr");
+ child_meta->set_type(inner_column.type());
+ child_meta->set_length(inner_column.length());
+ child_meta->set_column_id(arr_opts.meta->column_id() + 1);
+ child_meta->set_unique_id(arr_opts.meta->unique_id() + 1);
+ child_meta->set_compression(arr_opts.meta->compression());
+ child_meta->set_encoding(arr_opts.meta->encoding());
+ child_meta->set_is_nullable(true);
+
+ // set array column meta
+ arr_opts.meta->set_type(OLAP_FIELD_TYPE_ARRAY);
+ arr_opts.meta->set_encoding(opts.meta->encoding());
+ arr_opts.meta->set_compression(opts.meta->compression());
+ arr_opts.need_zone_map = false;
+ // no need inner array's null map
+ arr_opts.meta->set_is_nullable(false);
+ RETURN_IF_ERROR(ColumnWriter::create(arr_opts, &array_column, file_writer,
+ &inner_array_writer));
+ inner_writer_list.push_back(std::move(inner_array_writer));
+ }
+ // create map writer
+ std::unique_ptr<ColumnWriter> sub_column_writer;
+ std::unique_ptr<ColumnWriter> writer_local = std::unique_ptr<ColumnWriter>(
+ new MapColumnWriter(opts, std::move(field), null_writer, inner_writer_list));
+
+ *writer = std::move(writer_local);
+ return Status::OK();
+ }
default:
return Status::NotSupported("unsupported type for ColumnWriter: {}",
std::to_string(field->type()));
@@ -868,7 +941,7 @@ Status ArrayColumnWriter::append_nulls(size_t num_rows) {
Status ArrayColumnWriter::write_null_column(size_t num_rows, bool is_null) {
uint8_t null_sign = is_null ? 1 : 0;
- while (num_rows > 0) {
+ while (is_nullable() && num_rows > 0) {
// TODO llj bulk write
const uint8_t* null_sign_ptr = &null_sign;
RETURN_IF_ERROR(_null_writer->append_data(&null_sign_ptr, 1));
@@ -881,5 +954,118 @@ Status ArrayColumnWriter::finish_current_page() {
return Status::NotSupported("array writer has no data, can not finish_current_page");
}
+/// ============================= MapColumnWriter =====================////
+MapColumnWriter::MapColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
+ ScalarColumnWriter* null_writer,
+ std::vector<std::unique_ptr<ColumnWriter>>& kv_writers)
+ : ColumnWriter(std::move(field), opts.meta->is_nullable()), _opts(opts) {
+ CHECK_EQ(kv_writers.size(), 2);
+ if (is_nullable()) {
+ _null_writer.reset(null_writer);
+ }
+ for (auto& sub_writers : kv_writers) {
+ _kv_writers.push_back(std::move(sub_writers));
+ }
+}
+
+Status MapColumnWriter::init() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->init());
+ }
+ for (auto& sub_writer : _kv_writers) {
+ RETURN_IF_ERROR(sub_writer->init());
+ }
+ return Status::OK();
+}
+
+uint64_t MapColumnWriter::estimate_buffer_size() {
+ size_t estimate = 0;
+ for (auto& sub_writer : _kv_writers) {
+ estimate += sub_writer->estimate_buffer_size();
+ }
+ if (is_nullable()) {
+ estimate += _null_writer->estimate_buffer_size();
+ }
+ return estimate;
+}
+
+Status MapColumnWriter::finish() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->finish());
+ }
+ for (auto& sub_writer : _kv_writers) {
+ RETURN_IF_ERROR(sub_writer->finish());
+ }
+ return Status::OK();
+}
+
+// todo. make keys and values write
+Status MapColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
+ auto kv_ptr = reinterpret_cast<const uint64_t*>(*ptr);
+ for (size_t i = 0; i < 2; ++i) {
+ auto data = *(kv_ptr + i);
+ const uint8_t* val_ptr = (const uint8_t*)data;
+ RETURN_IF_ERROR(_kv_writers[i]->append_data(&val_ptr, num_rows));
+ }
+ if (is_nullable()) {
+ return write_null_column(num_rows, false);
+ }
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_data() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_data());
+ }
+ for (auto& sub_writer : _kv_writers) {
+ RETURN_IF_ERROR(sub_writer->write_data());
+ }
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_ordinal_index() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_ordinal_index());
+ }
+ for (auto& sub_writer : _kv_writers) {
+ RETURN_IF_ERROR(sub_writer->write_ordinal_index());
+ }
+ return Status::OK();
+}
+
+Status MapColumnWriter::append_nulls(size_t num_rows) {
+ for (auto& sub_writer : _kv_writers) {
+ RETURN_IF_ERROR(sub_writer->append_nulls(num_rows));
+ }
+ return write_null_column(num_rows, true);
+}
+
+Status MapColumnWriter::write_null_column(size_t num_rows, bool is_null) {
+ if (is_nullable()) {
+ uint8_t null_sign = is_null ? 1 : 0;
+ std::vector<vectorized::UInt8> null_signs(num_rows, null_sign);
+ const uint8_t* null_sign_ptr = null_signs.data();
+ RETURN_IF_ERROR(_null_writer->append_data(&null_sign_ptr, num_rows));
+ }
+ return Status::OK();
+}
+
+Status MapColumnWriter::finish_current_page() {
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->finish_current_page());
+ }
+ for (auto& sub_writer : _kv_writers) {
+ RETURN_IF_ERROR(sub_writer->finish_current_page());
+ }
+ return Status::OK();
+}
+
+Status MapColumnWriter::write_inverted_index() {
+ if (_opts.inverted_index) {
+ return _inverted_index_builder->finish();
+ }
+ return Status::OK();
+}
+
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h
index 274c9be6f3..7d140324dd 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.h
+++ b/be/src/olap/rowset/segment_v2/column_writer.h
@@ -371,5 +371,60 @@ private:
ColumnWriterOptions _opts;
};
+class MapColumnWriter final : public ColumnWriter, public FlushPageCallback {
+public:
+ explicit MapColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
+ ScalarColumnWriter* null_writer,
+ std::vector<std::unique_ptr<ColumnWriter>>& _kv_writers);
+
+ ~MapColumnWriter() override = default;
+
+ Status init() override;
+
+ Status append_data(const uint8_t** ptr, size_t num_rows) override;
+
+ uint64_t estimate_buffer_size() override;
+
+ Status finish() override;
+ Status write_data() override;
+ Status write_ordinal_index() override;
+ Status write_inverted_index() override;
+ Status append_nulls(size_t num_rows) override;
+
+ Status finish_current_page() override;
+
+ Status write_zone_map() override {
+ if (_opts.need_zone_map) {
+ return Status::NotSupported("map not support zone map");
+ }
+ return Status::OK();
+ }
+
+ Status write_bitmap_index() override {
+ if (_opts.need_bitmap_index) {
+ return Status::NotSupported("map not support bitmap index");
+ }
+ return Status::OK();
+ }
+ Status write_bloom_filter_index() override {
+ if (_opts.need_bloom_filter) {
+ return Status::NotSupported("map not support bloom filter index");
+ }
+ return Status::OK();
+ }
+
+ // according key writer to get next rowid
+ ordinal_t get_next_rowid() const override { return _kv_writers[0]->get_next_rowid(); }
+
+private:
+ Status write_null_column(size_t num_rows, bool is_null);
+
+ std::vector<std::unique_ptr<ColumnWriter>> _kv_writers;
+ // we need null writer to make sure a row is null or not
+ std::unique_ptr<ScalarColumnWriter> _null_writer;
+ std::unique_ptr<InvertedIndexColumnWriter> _inverted_index_builder;
+ ColumnWriterOptions _opts;
+};
+
} // namespace segment_v2
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 4195d798a1..9a90cb314f 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -163,6 +163,15 @@ Status SegmentWriter::init(const std::vector<uint32_t>& col_ids, bool has_key) {
return Status::NotSupported("Do not support bitmap index for jsonb type");
}
}
+ if (column.type() == FieldType::OLAP_FIELD_TYPE_MAP) {
+ opts.need_zone_map = false;
+ if (opts.need_bloom_filter) {
+ return Status::NotSupported("Do not support bloom filter for map type");
+ }
+ if (opts.need_bitmap_index) {
+ return Status::NotSupported("Do not support bitmap index for map type");
+ }
+ }
if (column.is_row_store_column()) {
// smaller page size for row store column
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 4fdefb2bb0..bd53ea6713 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -306,6 +306,12 @@ void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tco
ColumnPB* children_column = column->add_children_columns();
init_column_from_tcolumn(0, tcolumn.children_column[0], children_column);
}
+ if (tcolumn.column_type.type == TPrimitiveType::MAP) {
+ ColumnPB* key_column = column->add_children_columns();
+ init_column_from_tcolumn(0, tcolumn.children_column[0], key_column);
+ ColumnPB* val_column = column->add_children_columns();
+ init_column_from_tcolumn(0, tcolumn.children_column[1], val_column);
+ }
}
Status TabletMeta::create_from_file(const string& file_path) {
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 39a45c3422..b4d6cf76f6 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -317,6 +317,8 @@ uint32_t TabletColumn::get_field_length_by_type(TPrimitiveType::type type, uint3
return OLAP_STRUCT_MAX_LENGTH;
case TPrimitiveType::ARRAY:
return OLAP_ARRAY_MAX_LENGTH;
+ case TPrimitiveType::MAP:
+ return OLAP_ARRAY_MAX_LENGTH;
case TPrimitiveType::DECIMAL32:
return 4;
case TPrimitiveType::DECIMAL64:
@@ -419,6 +421,15 @@ void TabletColumn::init_from_pb(const ColumnPB& column) {
child_column.init_from_pb(column.children_columns(0));
add_sub_column(child_column);
}
+ if (_type == FieldType::OLAP_FIELD_TYPE_MAP) {
+ DCHECK(column.children_columns_size() == 2) << "MAP type has more than 2 children types.";
+ TabletColumn key_column;
+ TabletColumn value_column;
+ key_column.init_from_pb(column.children_columns(0));
+ value_column.init_from_pb(column.children_columns(1));
+ add_sub_column(key_column);
+ add_sub_column(value_column);
+ }
}
void TabletColumn::to_schema_pb(ColumnPB* column) const {
@@ -455,6 +466,13 @@ void TabletColumn::to_schema_pb(ColumnPB* column) const {
ColumnPB* child = column->add_children_columns();
_sub_columns[0].to_schema_pb(child);
}
+ if (_type == OLAP_FIELD_TYPE_MAP) {
+ DCHECK(_sub_columns.size() == 2) << "MAP type has more than 2 children types.";
+ ColumnPB* child_key = column->add_children_columns();
+ _sub_columns[0].to_schema_pb(child_key);
+ ColumnPB* child_val = column->add_children_columns();
+ _sub_columns[1].to_schema_pb(child_val);
+ }
}
void TabletColumn::add_sub_column(TabletColumn& sub_column) {
diff --git a/be/src/olap/types.cpp b/be/src/olap/types.cpp
index 916f7768af..bee790e066 100644
--- a/be/src/olap/types.cpp
+++ b/be/src/olap/types.cpp
@@ -201,6 +201,15 @@ TypeInfoPtr get_type_info(segment_v2::ColumnMetaPB* column_meta_pb) {
}
return create_static_type_info_ptr(
get_array_type_info((FieldType)child_column->type(), iterations));
+ } else if (UNLIKELY(type == OLAP_FIELD_TYPE_MAP)) {
+ segment_v2::ColumnMetaPB key_meta = column_meta_pb->children_columns(0);
+ TypeInfoPtr key_type_info = get_type_info(&key_meta);
+ segment_v2::ColumnMetaPB value_meta = column_meta_pb->children_columns(1);
+ TypeInfoPtr value_type_info = get_type_info(&value_meta);
+
+ MapTypeInfo* map_type_info =
+ new MapTypeInfo(std::move(key_type_info), std::move(value_type_info));
+ return create_static_type_info_ptr(map_type_info);
} else {
return create_static_type_info_ptr(get_scalar_type_info(type));
}
@@ -240,6 +249,13 @@ TypeInfoPtr get_type_info(const TabletColumn* col) {
child_column = &child_column->get_sub_column(0);
}
return create_static_type_info_ptr(get_array_type_info(child_column->type(), iterations));
+ } else if (UNLIKELY(type == OLAP_FIELD_TYPE_MAP)) {
+ const auto* key_column = &col->get_sub_column(0);
+ TypeInfoPtr key_type = get_type_info(key_column);
+ const auto* val_column = &col->get_sub_column(1);
+ TypeInfoPtr value_type = get_type_info(val_column);
+ MapTypeInfo* map_type_info = new MapTypeInfo(std::move(key_type), std::move(value_type));
+ return create_static_type_info_ptr(map_type_info);
} else {
return create_static_type_info_ptr(get_scalar_type_info(type));
}
@@ -248,22 +264,24 @@ TypeInfoPtr get_type_info(const TabletColumn* col) {
TypeInfoPtr clone_type_info(const TypeInfo* type_info) {
if (is_scalar_type(type_info->type())) {
return create_static_type_info_ptr(type_info);
- } else {
- auto type = type_info->type();
- if (type == OLAP_FIELD_TYPE_STRUCT) {
- const auto struct_type_info = dynamic_cast<const StructTypeInfo*>(type_info);
- std::vector<TypeInfoPtr> clone_type_infos;
- const std::vector<TypeInfoPtr>* sub_type_infos = struct_type_info->type_infos();
- clone_type_infos.reserve(sub_type_infos->size());
- for (size_t i = 0; i < sub_type_infos->size(); i++) {
- clone_type_infos.push_back(clone_type_info((*sub_type_infos)[i].get()));
- }
- return create_dynamic_type_info_ptr(new StructTypeInfo(clone_type_infos));
- } else {
- const auto array_type_info = dynamic_cast<const ArrayTypeInfo*>(type_info);
- return create_dynamic_type_info_ptr(
- new ArrayTypeInfo(clone_type_info(array_type_info->item_type_info())));
+ } else if (type_info->type() == OLAP_FIELD_TYPE_MAP) {
+ const auto map_type_info = dynamic_cast<const MapTypeInfo*>(type_info);
+ return create_dynamic_type_info_ptr(
+ new MapTypeInfo(clone_type_info(map_type_info->get_key_type_info()),
+ clone_type_info(map_type_info->get_value_type_info())));
+ } else if (type_info->type() == OLAP_FIELD_TYPE_STRUCT) {
+ const auto struct_type_info = dynamic_cast<const StructTypeInfo*>(type_info);
+ std::vector<TypeInfoPtr> clone_type_infos;
+ const std::vector<TypeInfoPtr>* sub_type_infos = struct_type_info->type_infos();
+ clone_type_infos.reserve(sub_type_infos->size());
+ for (size_t i = 0; i < sub_type_infos->size(); i++) {
+ clone_type_infos.push_back(clone_type_info((*sub_type_infos)[i].get()));
}
+ return create_dynamic_type_info_ptr(new StructTypeInfo(clone_type_infos));
+ } else if (type_info->type() == OLAP_FIELD_TYPE_ARRAY) {
+ const auto array_type_info = dynamic_cast<const ArrayTypeInfo*>(type_info);
+ return create_dynamic_type_info_ptr(
+ new ArrayTypeInfo(clone_type_info(array_type_info->item_type_info())));
}
}
diff --git a/be/src/olap/types.h b/be/src/olap/types.h
index 540f781a69..73a0e4c080 100644
--- a/be/src/olap/types.h
+++ b/be/src/olap/types.h
@@ -31,6 +31,7 @@
#include "olap/olap_define.h"
#include "runtime/collection_value.h"
#include "runtime/jsonb_value.h"
+#include "runtime/map_value.h"
#include "runtime/mem_pool.h"
#include "runtime/struct_value.h"
#include "util/jsonb_document.h"
@@ -323,6 +324,93 @@ private:
TypeInfoPtr _item_type_info;
const size_t _item_size;
};
+///====================== MapType Info ==========================///
+class MapTypeInfo : public TypeInfo {
+public:
+ explicit MapTypeInfo(TypeInfoPtr key_type_info, TypeInfoPtr value_type_info)
+ : _key_type_info(std::move(key_type_info)),
+ _value_type_info(std::move(value_type_info)) {}
+ ~MapTypeInfo() override = default;
+
+ inline bool equal(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ return l_value->size() == r_value->size();
+ }
+
+ int cmp(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const MapValue*>(left);
+ auto r_value = reinterpret_cast<const MapValue*>(right);
+ uint32_t l_size = l_value->size();
+ uint32_t r_size = r_value->size();
+ if (l_size < r_size) {
+ return -1;
+ } else if (l_size > r_size) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ void shallow_copy(void* dest, const void* src) const override {
+ auto dest_value = reinterpret_cast<MapValue*>(dest);
+ auto src_value = reinterpret_cast<const MapValue*>(src);
+ dest_value->shallow_copy(src_value);
+ }
+
+ void deep_copy(void* dest, const void* src, MemPool* mem_pool) const override { DCHECK(false); }
+
+ void copy_object(void* dest, const void* src, MemPool* mem_pool) const override {
+ deep_copy(dest, src, mem_pool);
+ }
+
+ void direct_copy(void* dest, const void* src) const override { CHECK(false); }
+
+ void direct_copy(uint8_t** base, void* dest, const void* src) const { CHECK(false); }
+
+ void direct_copy_may_cut(void* dest, const void* src) const override { direct_copy(dest, src); }
+
+ Status convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool,
+ size_t variable_len = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ Status from_string(void* buf, const std::string& scan_key, const int precision = 0,
+ const int scale = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ std::string to_string(const void* src) const override { return "{}"; }
+
+ void set_to_max(void* buf) const override {
+ DCHECK(false) << "set_to_max of list is not implemented.";
+ }
+
+ void set_to_min(void* buf) const override {
+ DCHECK(false) << "set_to_min of list is not implemented.";
+ }
+
+ uint32_t hash_code(const void* data, uint32_t seed) const override {
+ auto map_value = reinterpret_cast<const MapValue*>(data);
+ auto size = map_value->size();
+ uint32_t result = HashUtil::hash(&size, sizeof(size), seed);
+ result = seed * result + _key_type_info->hash_code(map_value->key_data(), seed) +
+ _value_type_info->hash_code(map_value->value_data(), seed);
+ return result;
+ }
+
+ // todo . is here only to need return 16 for two ptr?
+ const size_t size() const override { return sizeof(MapValue); }
+
+ FieldType type() const override { return OLAP_FIELD_TYPE_MAP; }
+
+ inline const TypeInfo* get_key_type_info() const { return _key_type_info.get(); }
+ inline const TypeInfo* get_value_type_info() const { return _value_type_info.get(); }
+
+private:
+ TypeInfoPtr _key_type_info;
+ TypeInfoPtr _value_type_info;
+};
class StructTypeInfo : public TypeInfo {
public:
@@ -710,6 +798,10 @@ template <>
struct CppTypeTraits<OLAP_FIELD_TYPE_ARRAY> {
using CppType = CollectionValue;
};
+template <>
+struct CppTypeTraits<OLAP_FIELD_TYPE_MAP> {
+ using CppType = MapValue;
+};
template <FieldType field_type>
struct BaseFieldtypeTraits : public CppTypeTraits<field_type> {
using CppType = typename CppTypeTraits<field_type>::CppType;
diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt
index 97e8de4aa8..3f262c5f61 100644
--- a/be/src/runtime/CMakeLists.txt
+++ b/be/src/runtime/CMakeLists.txt
@@ -47,6 +47,7 @@ set(RUNTIME_FILES
large_int_value.cpp
struct_value.cpp
collection_value.cpp
+ map_value.cpp
fragment_mgr.cpp
load_path_mgr.cpp
types.cpp
diff --git a/be/src/runtime/map_value.cpp b/be/src/runtime/map_value.cpp
new file mode 100644
index 0000000000..1828fd27b3
--- /dev/null
+++ b/be/src/runtime/map_value.cpp
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "map_value.h"
+
+namespace doris {
+
+///====================== map-value funcs ======================///
+void MapValue::shallow_copy(const MapValue* value) {
+ _length = value->_length;
+ _key_data = value->_key_data;
+ _value_data = value->_value_data;
+}
+
+} // namespace doris
diff --git a/be/src/runtime/map_value.h b/be/src/runtime/map_value.h
new file mode 100644
index 0000000000..d275316cf6
--- /dev/null
+++ b/be/src/runtime/map_value.h
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <type_traits>
+
+#include "runtime/primitive_type.h"
+
+namespace doris {
+
+/**
+ * MapValue is for map type in memory
+ */
+class MapValue {
+public:
+ MapValue() = default;
+
+ explicit MapValue(int32_t length) : _key_data(nullptr), _value_data(nullptr), _length(length) {}
+
+ MapValue(void* k_data, void* v_data, int32_t length)
+ : _key_data(k_data), _value_data(v_data), _length(length) {}
+
+ int32_t size() const { return _length; }
+
+ int32_t length() const { return _length; }
+
+ void shallow_copy(const MapValue* other);
+
+ const void* key_data() const { return _key_data; }
+ void* mutable_key_data() const { return _key_data; }
+ const void* value_data() const { return _value_data; }
+ void* mutable_value_data() const { return _value_data; }
+
+ void set_length(int32_t length) { _length = length; }
+ void set_key(void* data) { _key_data = data; }
+ void set_value(void* data) { _value_data = data; }
+
+private:
+ // child column data pointer
+ void* _key_data;
+ void* _value_data;
+ // length for map size
+ int32_t _length;
+
+}; //map-value
+} // namespace doris
diff --git a/be/src/runtime/primitive_type.cpp b/be/src/runtime/primitive_type.cpp
index 792b80a238..caa6e093f8 100644
--- a/be/src/runtime/primitive_type.cpp
+++ b/be/src/runtime/primitive_type.cpp
@@ -22,6 +22,7 @@
#include "runtime/define_primitive_type.h"
#include "runtime/jsonb_value.h"
#include "vec/common/string_ref.h"
+#include "runtime/map_value.h"
#include "runtime/struct_value.h"
namespace doris {
@@ -54,6 +55,8 @@ PrimitiveType convert_type_to_primitive(FunctionContext::Type type) {
return PrimitiveType::TYPE_BOOLEAN;
case FunctionContext::Type::TYPE_ARRAY:
return PrimitiveType::TYPE_ARRAY;
+ case FunctionContext::Type::TYPE_MAP:
+ return PrimitiveType::TYPE_MAP;
case FunctionContext::Type::TYPE_STRUCT:
return PrimitiveType::TYPE_STRUCT;
case FunctionContext::Type::TYPE_OBJECT:
@@ -215,6 +218,9 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) {
case TPrimitiveType::ARRAY:
return TYPE_ARRAY;
+ case TPrimitiveType::MAP:
+ return TYPE_MAP;
+
case TPrimitiveType::STRUCT:
return TYPE_STRUCT;
@@ -312,6 +318,9 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) {
case TYPE_ARRAY:
return TPrimitiveType::ARRAY;
+ case TYPE_MAP:
+ return TPrimitiveType::MAP;
+
case TYPE_STRUCT:
return TPrimitiveType::STRUCT;
@@ -409,6 +418,9 @@ std::string type_to_string(PrimitiveType t) {
case TYPE_ARRAY:
return "ARRAY";
+ case TYPE_MAP:
+ return "MAP";
+
case TYPE_STRUCT:
return "STRUCT";
@@ -551,6 +563,8 @@ int get_slot_size(PrimitiveType type) {
return sizeof(JsonBinaryValue);
case TYPE_ARRAY:
return sizeof(CollectionValue);
+ case TYPE_MAP:
+ return sizeof(MapValue);
case TYPE_STRUCT:
return sizeof(StructValue);
diff --git a/be/src/runtime/types.cpp b/be/src/runtime/types.cpp
index 45b8f8e5f5..386f3b57f0 100644
--- a/be/src/runtime/types.cpp
+++ b/be/src/runtime/types.cpp
@@ -95,15 +95,17 @@ TypeDescriptor::TypeDescriptor(const std::vector<TTypeNode>& types, int* idx)
// ++(*idx);
// children.push_back(TypeDescriptor(types, idx));
// break;
- // case TTypeNodeType::MAP:
- // DCHECK(!node.__isset.scalar_type);
- // DCHECK_LT(*idx, types.size() - 2);
- // type = TYPE_MAP;
- // ++(*idx);
- // children.push_back(TypeDescriptor(types, idx));
- // ++(*idx);
- // children.push_back(TypeDescriptor(types, idx));
- // break;
+ case TTypeNodeType::MAP: {
+ DCHECK(!node.__isset.scalar_type);
+ DCHECK_LT(*idx, types.size() - 2);
+ DCHECK(!node.__isset.contains_null);
+ type = TYPE_MAP;
+ ++(*idx);
+ children.push_back(TypeDescriptor(types, idx));
+ ++(*idx);
+ children.push_back(TypeDescriptor(types, idx));
+ break;
+ }
default:
DCHECK(false) << node.type;
}
@@ -150,8 +152,6 @@ void TypeDescriptor::to_thrift(TTypeDesc* thrift_type) const {
}
void TypeDescriptor::to_protobuf(PTypeDesc* ptype) const {
- DCHECK(!is_complex_type() || type == TYPE_ARRAY || type == TYPE_STRUCT)
- << "Don't support complex type now, type=" << type;
auto node = ptype->add_types();
node->set_type(TTypeNodeType::SCALAR);
auto scalar_type = node->mutable_scalar_type();
@@ -181,6 +181,11 @@ void TypeDescriptor::to_protobuf(PTypeDesc* ptype) const {
for (const TypeDescriptor& child : children) {
child.to_protobuf(ptype);
}
+ } else if (type == TYPE_MAP) {
+ node->set_type(TTypeNodeType::MAP);
+ for (const TypeDescriptor& child : children) {
+ child.to_protobuf(ptype);
+ }
}
}
@@ -223,6 +228,12 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
children.push_back(TypeDescriptor(types, idx));
break;
}
+ case TTypeNodeType::MAP: {
+ type = TYPE_MAP;
+ ++(*idx);
+ children.push_back(TypeDescriptor(types, idx));
+ ++(*idx);
+ children.push_back(TypeDescriptor(types, idx));
case TTypeNodeType::STRUCT: {
type = TYPE_STRUCT;
size_t children_size = node.struct_fields_size();
@@ -240,64 +251,67 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
default:
DCHECK(false) << node.type();
}
-}
-
-std::string TypeDescriptor::debug_string() const {
- std::stringstream ss;
- switch (type) {
- case TYPE_CHAR:
- ss << "CHAR(" << len << ")";
- return ss.str();
- case TYPE_DECIMALV2:
- ss << "DECIMALV2(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL32:
- ss << "DECIMAL32(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL64:
- ss << "DECIMAL64(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_DECIMAL128I:
- ss << "DECIMAL128(" << precision << ", " << scale << ")";
- return ss.str();
- case TYPE_ARRAY: {
- ss << "ARRAY<" << children[0].debug_string() << ">";
- return ss.str();
}
- case TYPE_STRUCT: {
- ss << "STRUCT<";
- for (size_t i = 0; i < children.size(); i++) {
- ss << field_names[i];
- ss << ":";
- ss << children[i].debug_string();
- if (i != children.size() - 1) {
- ss << ",";
+
+ std::string TypeDescriptor::debug_string() const {
+ std::stringstream ss;
+ switch (type) {
+ case TYPE_CHAR:
+ ss << "CHAR(" << len << ")";
+ return ss.str();
+ case TYPE_DECIMALV2:
+ ss << "DECIMALV2(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL32:
+ ss << "DECIMAL32(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL64:
+ ss << "DECIMAL64(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_DECIMAL128I:
+ ss << "DECIMAL128(" << precision << ", " << scale << ")";
+ return ss.str();
+ case TYPE_ARRAY: {
+ ss << "ARRAY<" << children[0].debug_string() << ">";
+ return ss.str();
+ }
+ case TYPE_MAP:
+ ss << "MAP<" << children[0].debug_string() << ", " << children[1].debug_string() << ">";
+ return ss.str();
+ case TYPE_STRUCT: {
+ ss << "STRUCT<";
+ for (size_t i = 0; i < children.size(); i++) {
+ ss << field_names[i];
+ ss << ":";
+ ss << children[i].debug_string();
+ if (i != children.size() - 1) {
+ ss << ",";
+ }
}
+ ss << ">";
+ return ss.str();
+ }
+ default:
+ return type_to_string(type);
}
- ss << ">";
- return ss.str();
- }
- default:
- return type_to_string(type);
}
-}
-std::ostream& operator<<(std::ostream& os, const TypeDescriptor& type) {
- os << type.debug_string();
- return os;
-}
+ std::ostream& operator<<(std::ostream& os, const TypeDescriptor& type) {
+ os << type.debug_string();
+ return os;
+ }
-TTypeDesc create_type_desc(PrimitiveType type, int precision, int scale) {
- TTypeDesc type_desc;
- std::vector<TTypeNode> node_type;
- node_type.emplace_back();
- TScalarType scalarType;
- scalarType.__set_type(to_thrift(type));
- scalarType.__set_len(-1);
- scalarType.__set_precision(precision);
- scalarType.__set_scale(scale);
- node_type.back().__set_scalar_type(scalarType);
- type_desc.__set_types(node_type);
- return type_desc;
-}
+ TTypeDesc create_type_desc(PrimitiveType type, int precision, int scale) {
+ TTypeDesc type_desc;
+ std::vector<TTypeNode> node_type;
+ node_type.emplace_back();
+ TScalarType scalarType;
+ scalarType.__set_type(to_thrift(type));
+ scalarType.__set_len(-1);
+ scalarType.__set_precision(precision);
+ scalarType.__set_scale(scale);
+ node_type.back().__set_scalar_type(scalarType);
+ type_desc.__set_types(node_type);
+ return type_desc;
+ }
} // namespace doris
diff --git a/be/src/udf/udf.h b/be/src/udf/udf.h
index 1c4517e967..8b62898e45 100644
--- a/be/src/udf/udf.h
+++ b/be/src/udf/udf.h
@@ -81,6 +81,7 @@ public:
TYPE_DECIMALV2,
TYPE_OBJECT,
TYPE_ARRAY,
+ TYPE_MAP,
TYPE_STRUCT,
TYPE_QUANTILE_STATE,
TYPE_DATEV2,
diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt
index 7584b46e5e..ede1bdfae4 100644
--- a/be/src/vec/CMakeLists.txt
+++ b/be/src/vec/CMakeLists.txt
@@ -55,6 +55,7 @@ set(VEC_FILES
columns/column_nullable.cpp
columns/column_string.cpp
columns/column_vector.cpp
+ columns/column_map.cpp
columns/columns_common.cpp
common/demangle.cpp
common/exception.cpp
@@ -89,6 +90,7 @@ set(VEC_FILES
data_types/data_type_number_base.cpp
data_types/data_type_string.cpp
data_types/data_type_decimal.cpp
+ data_types/data_type_map.cpp
data_types/get_least_supertype.cpp
data_types/nested_utils.cpp
data_types/data_type_date.cpp
@@ -133,6 +135,7 @@ set(VEC_FILES
exprs/vexpr_context.cpp
exprs/vliteral.cpp
exprs/varray_literal.cpp
+ exprs/vmap_literal.cpp
exprs/vstruct_literal.cpp
exprs/vin_predicate.cpp
exprs/vbloom_predicate.cpp
diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h
index 6452c8f3ff..ef78f10468 100644
--- a/be/src/vec/columns/column.h
+++ b/be/src/vec/columns/column.h
@@ -590,6 +590,8 @@ public:
virtual bool is_column_array() const { return false; }
+ virtual bool is_column_map() const { return false; }
+
/// If the only value column can contain is NULL.
/// Does not imply type of object, because it can be ColumnNullable(ColumnNothing) or ColumnConst(ColumnNullable(ColumnNothing))
virtual bool only_null() const { return false; }
diff --git a/be/src/vec/columns/column_map.cpp b/be/src/vec/columns/column_map.cpp
new file mode 100644
index 0000000000..f477b9f336
--- /dev/null
+++ b/be/src/vec/columns/column_map.cpp
@@ -0,0 +1,186 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnMap.cpp
+// and modified by Doris
+
+#include "vec/columns/column_map.h"
+
+namespace doris::vectorized {
+
+/** A column of map values.
+ */
+std::string ColumnMap::get_name() const {
+ return "Map(" + keys->get_name() + ", " + values->get_name() + ")";
+}
+
+ColumnMap::ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values)
+ : keys(std::move(keys)), values(std::move(values)) {
+ check_size();
+}
+
+ColumnArray::Offsets64& ColumnMap::get_offsets() const {
+ const ColumnArray& column_keys = assert_cast<const ColumnArray&>(get_keys());
+ // todo . did here check size ?
+ return const_cast<Offsets64&>(column_keys.get_offsets());
+}
+
+void ColumnMap::check_size() const {
+ const auto* key_array = typeid_cast<const ColumnArray*>(keys.get());
+ const auto* value_array = typeid_cast<const ColumnArray*>(values.get());
+ CHECK(key_array) << "ColumnMap keys can be created only from array";
+ CHECK(value_array) << "ColumnMap values can be created only from array";
+ CHECK_EQ(get_keys_ptr()->size(), get_values_ptr()->size());
+}
+
+// todo. here to resize every row map
+MutableColumnPtr ColumnMap::clone_resized(size_t to_size) const {
+ auto res = ColumnMap::create(keys->clone_resized(to_size), values->clone_resized(to_size));
+ return res;
+}
+
+// to support field functions
+Field ColumnMap::operator[](size_t n) const {
+ // Map is FieldVector , see in field.h
+ Map res(2);
+ keys->get(n, res[0]);
+ values->get(n, res[1]);
+
+ return res;
+}
+
+// here to compare to below
+void ColumnMap::get(size_t n, Field& res) const {
+ Map map(2);
+ keys->get(n, map[0]);
+ values->get(n, map[1]);
+
+ res = map;
+}
+
+StringRef ColumnMap::get_data_at(size_t n) const {
+ LOG(FATAL) << "Method get_data_at is not supported for " << get_name();
+}
+
+void ColumnMap::insert_data(const char*, size_t) {
+ LOG(FATAL) << "Method insert_data is not supported for " << get_name();
+}
+
+void ColumnMap::insert(const Field& x) {
+ const auto& map = doris::vectorized::get<const Map&>(x);
+ CHECK_EQ(map.size(), 2);
+ keys->insert(map[0]);
+ values->insert(map[1]);
+}
+
+void ColumnMap::insert_default() {
+ keys->insert_default();
+ values->insert_default();
+}
+
+void ColumnMap::pop_back(size_t n) {
+ keys->pop_back(n);
+ values->pop_back(n);
+}
+
+StringRef ColumnMap::serialize_value_into_arena(size_t n, Arena& arena, char const*& begin) const {
+ StringRef res(begin, 0);
+ auto keys_ref = keys->serialize_value_into_arena(n, arena, begin);
+ res.data = keys_ref.data - res.size;
+ res.size += keys_ref.size;
+ auto value_ref = values->serialize_value_into_arena(n, arena, begin);
+ res.data = value_ref.data - res.size;
+ res.size += value_ref.size;
+
+ return res;
+}
+
+void ColumnMap::insert_from(const IColumn& src_, size_t n) {
+ const ColumnMap& src = assert_cast<const ColumnMap&>(src_);
+
+ if ((!get_keys().is_nullable() && src.get_keys().is_nullable()) ||
+ (!get_values().is_nullable() && src.get_values().is_nullable())) {
+ DCHECK(false);
+ } else if ((get_keys().is_nullable() && !src.get_keys().is_nullable()) ||
+ (get_values().is_nullable() && !src.get_values().is_nullable())) {
+ DCHECK(false);
+ } else {
+ keys->insert_from(*assert_cast<const ColumnMap&>(src_).keys, n);
+ values->insert_from(*assert_cast<const ColumnMap&>(src_).values, n);
+ }
+}
+
+void ColumnMap::insert_indices_from(const IColumn& src, const int* indices_begin,
+ const int* indices_end) {
+ for (auto x = indices_begin; x != indices_end; ++x) {
+ if (*x == -1) {
+ ColumnMap::insert_default();
+ } else {
+ ColumnMap::insert_from(src, *x);
+ }
+ }
+}
+
+const char* ColumnMap::deserialize_and_insert_from_arena(const char* pos) {
+ pos = keys->deserialize_and_insert_from_arena(pos);
+ pos = values->deserialize_and_insert_from_arena(pos);
+
+ return pos;
+}
+
+void ColumnMap::update_hash_with_value(size_t n, SipHash& hash) const {
+ keys->update_hash_with_value(n, hash);
+ values->update_hash_with_value(n, hash);
+}
+
+void ColumnMap::insert_range_from(const IColumn& src, size_t start, size_t length) {
+ keys->insert_range_from(*assert_cast<const ColumnMap&>(src).keys, start, length);
+ values->insert_range_from(*assert_cast<const ColumnMap&>(src).values, start, length);
+}
+
+ColumnPtr ColumnMap::filter(const Filter& filt, ssize_t result_size_hint) const {
+ return ColumnMap::create(keys->filter(filt, result_size_hint),
+ values->filter(filt, result_size_hint));
+}
+
+ColumnPtr ColumnMap::permute(const Permutation& perm, size_t limit) const {
+ return ColumnMap::create(keys->permute(perm, limit), values->permute(perm, limit));
+}
+
+ColumnPtr ColumnMap::replicate(const Offsets& offsets) const {
+ return ColumnMap::create(keys->replicate(offsets), values->replicate(offsets));
+}
+
+void ColumnMap::reserve(size_t n) {
+ get_keys().reserve(n);
+ get_values().reserve(n);
+}
+
+size_t ColumnMap::byte_size() const {
+ return get_keys().byte_size() + get_values().byte_size();
+}
+
+size_t ColumnMap::allocated_bytes() const {
+ return get_keys().allocated_bytes() + get_values().allocated_bytes();
+}
+
+void ColumnMap::protect() {
+ get_keys().protect();
+ get_values().protect();
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/columns/column_map.h b/be/src/vec/columns/column_map.h
new file mode 100644
index 0000000000..4840ee4f69
--- /dev/null
+++ b/be/src/vec/columns/column_map.h
@@ -0,0 +1,153 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/Columns/ColumnMap.cpp
+// and modified by Doris
+
+#pragma once
+
+#include "vec/columns/column.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_impl.h"
+#include "vec/common/arena.h"
+#include "vec/core/field.h"
+#include "vec/core/types.h"
+
+namespace doris::vectorized {
+
+/** A column of map values.
+ */
+class ColumnMap final : public COWHelper<IColumn, ColumnMap> {
+public:
+ /** Create immutable column using immutable arguments. This arguments may be shared with other columns.
+ * Use IColumn::mutate in order to make mutable column and mutate shared nested columns.
+ */
+ using Base = COWHelper<IColumn, ColumnMap>;
+
+ static Ptr create(const ColumnPtr& keys, const ColumnPtr& values) {
+ return ColumnMap::create(keys->assume_mutable(), values->assume_mutable());
+ }
+
+ template <typename... Args,
+ typename = typename std::enable_if<IsMutableColumns<Args...>::value>::type>
+ static MutablePtr create(Args&&... args) {
+ return Base::create(std::forward<Args>(args)...);
+ }
+
+ std::string get_name() const override;
+ const char* get_family_name() const override { return "Map"; }
+ TypeIndex get_data_type() const { return TypeIndex::Map; }
+
+ void for_each_subcolumn(ColumnCallback callback) override {
+ callback(keys);
+ callback(values);
+ }
+
+ void clear() override {
+ keys->clear();
+ values->clear();
+ }
+
+ MutableColumnPtr clone_resized(size_t size) const override;
+
+ bool can_be_inside_nullable() const override { return true; }
+ size_t size() const override { return keys->size(); }
+ Field operator[](size_t n) const override;
+ void get(size_t n, Field& res) const override;
+ StringRef get_data_at(size_t n) const override;
+
+ void insert_data(const char* pos, size_t length) override;
+ void insert_range_from(const IColumn& src, size_t start, size_t length) override;
+ void insert_from(const IColumn& src_, size_t n) override;
+ void insert(const Field& x) override;
+ void insert_default() override;
+
+ void pop_back(size_t n) override;
+ bool is_column_map() const override { return true; }
+ StringRef serialize_value_into_arena(size_t n, Arena& arena, char const*& begin) const override;
+ const char* deserialize_and_insert_from_arena(const char* pos) override;
+
+ void update_hash_with_value(size_t n, SipHash& hash) const override;
+
+ ColumnPtr filter(const Filter& filt, ssize_t result_size_hint) const override;
+ ColumnPtr permute(const Permutation& perm, size_t limit) const override;
+ ColumnPtr replicate(const Offsets& offsets) const override;
+ MutableColumns scatter(ColumnIndex num_columns, const Selector& selector) const override {
+ return scatter_impl<ColumnMap>(num_columns, selector);
+ }
+ void get_extremes(Field& min, Field& max) const override {
+ LOG(FATAL) << "get_extremes not implemented";
+ };
+ [[noreturn]] int compare_at(size_t n, size_t m, const IColumn& rhs_,
+ int nan_direction_hint) const override {
+ LOG(FATAL) << "compare_at not implemented";
+ }
+ void get_permutation(bool reverse, size_t limit, int nan_direction_hint,
+ Permutation& res) const override {
+ LOG(FATAL) << "get_permutation not implemented";
+ }
+ void insert_indices_from(const IColumn& src, const int* indices_begin,
+ const int* indices_end) override;
+
+ void append_data_by_selector(MutableColumnPtr& res,
+ const IColumn::Selector& selector) const override {
+ return append_data_by_selector_impl<ColumnMap>(res, selector);
+ }
+
+ void replace_column_data(const IColumn&, size_t row, size_t self_row = 0) override {
+ LOG(FATAL) << "replace_column_data not implemented";
+ }
+ void replace_column_data_default(size_t self_row = 0) override {
+ LOG(FATAL) << "replace_column_data_default not implemented";
+ }
+ void check_size() const;
+ ColumnArray::Offsets64& get_offsets() const;
+ void reserve(size_t n) override;
+ size_t byte_size() const override;
+ size_t allocated_bytes() const override;
+ void protect() override;
+
+ /******************** keys and values ***************/
+ const ColumnPtr& get_keys_ptr() const { return keys; }
+ ColumnPtr& get_keys_ptr() { return keys; }
+
+ const IColumn& get_keys() const { return *keys; }
+ IColumn& get_keys() { return *keys; }
+
+ const ColumnPtr& get_values_ptr() const { return values; }
+ ColumnPtr& get_values_ptr() { return values; }
+
+ const IColumn& get_values() const { return *values; }
+ IColumn& get_values() { return *values; }
+
+private:
+ friend class COWHelper<IColumn, ColumnMap>;
+
+ WrappedPtr keys; // nullable
+ WrappedPtr values; // nullable
+
+ size_t ALWAYS_INLINE offset_at(ssize_t i) const { return get_offsets()[i - 1]; }
+ size_t ALWAYS_INLINE size_at(ssize_t i) const {
+ return get_offsets()[i] - get_offsets()[i - 1];
+ }
+
+ explicit ColumnMap(MutableColumnPtr&& keys, MutableColumnPtr&& values);
+
+ ColumnMap(const ColumnMap&) = default;
+};
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/core/field.h b/be/src/vec/core/field.h
index c71f2e4971..78ac881a19 100644
--- a/be/src/vec/core/field.h
+++ b/be/src/vec/core/field.h
@@ -85,6 +85,7 @@ using FieldVector = std::vector<Field>;
DEFINE_FIELD_VECTOR(Array);
DEFINE_FIELD_VECTOR(Tuple);
+DEFINE_FIELD_VECTOR(Map);
#undef DEFINE_FIELD_VECTOR
@@ -308,6 +309,7 @@ public:
AggregateFunctionState = 22,
JSONB = 23,
Decimal128I = 24,
+ Map = 25,
};
static const int MIN_NON_POD = 16;
@@ -334,6 +336,8 @@ public:
return "Array";
case Tuple:
return "Tuple";
+ case Map:
+ return "Map";
case Decimal32:
return "Decimal32";
case Decimal64:
@@ -508,6 +512,8 @@ public:
return get<Array>() < rhs.get<Array>();
case Types::Tuple:
return get<Tuple>() < rhs.get<Tuple>();
+ case Types::Map:
+ return get<Map>() < rhs.get<Map>();
case Types::Decimal32:
return get<DecimalField<Decimal32>>() < rhs.get<DecimalField<Decimal32>>();
case Types::Decimal64:
@@ -553,6 +559,8 @@ public:
return get<Array>() <= rhs.get<Array>();
case Types::Tuple:
return get<Tuple>() <= rhs.get<Tuple>();
+ case Types::Map:
+ return get<Map>() < rhs.get<Map>();
case Types::Decimal32:
return get<DecimalField<Decimal32>>() <= rhs.get<DecimalField<Decimal32>>();
case Types::Decimal64:
@@ -590,6 +598,8 @@ public:
return get<Array>() == rhs.get<Array>();
case Types::Tuple:
return get<Tuple>() == rhs.get<Tuple>();
+ case Types::Map:
+ return get<Map>() < rhs.get<Map>();
case Types::UInt128:
return get<UInt128>() == rhs.get<UInt128>();
case Types::Int128:
@@ -680,6 +690,9 @@ private:
case Types::Tuple:
f(field.template get<Tuple>());
return;
+ case Types::Map:
+ f(field.template get<Map>());
+ return;
case Types::Decimal32:
f(field.template get<DecimalField<Decimal32>>());
return;
@@ -752,6 +765,9 @@ private:
case Types::Tuple:
destroy<Tuple>();
break;
+ case Types::Map:
+ destroy<Map>();
+ break;
case Types::AggregateFunctionState:
destroy<AggregateFunctionStateData>();
break;
@@ -813,6 +829,10 @@ struct Field::TypeToEnum<Tuple> {
static const Types::Which value = Types::Tuple;
};
template <>
+struct Field::TypeToEnum<Map> {
+ static const Types::Which value = Types::Map;
+};
+template <>
struct Field::TypeToEnum<DecimalField<Decimal32>> {
static const Types::Which value = Types::Decimal32;
};
@@ -874,6 +894,10 @@ struct Field::EnumToType<Field::Types::Tuple> {
using Type = Tuple;
};
template <>
+struct Field::EnumToType<Field::Types::Map> {
+ using Type = Map;
+};
+template <>
struct Field::EnumToType<Field::Types::Decimal32> {
using Type = DecimalField<Decimal32>;
};
@@ -923,6 +947,10 @@ struct TypeName<Tuple> {
static std::string get() { return "Tuple"; }
};
template <>
+struct TypeName<Map> {
+ static std::string get() { return "Map"; }
+};
+template <>
struct TypeName<AggregateFunctionStateData> {
static std::string get() { return "AggregateFunctionState"; }
};
@@ -1050,6 +1078,10 @@ struct NearestFieldTypeImpl<Tuple> {
using Type = Tuple;
};
template <>
+struct NearestFieldTypeImpl<Map> {
+ using Type = Map;
+};
+template <>
struct NearestFieldTypeImpl<bool> {
using Type = UInt64;
};
diff --git a/be/src/vec/core/types.h b/be/src/vec/core/types.h
index ccbdbbebd8..a46d5233ea 100644
--- a/be/src/vec/core/types.h
+++ b/be/src/vec/core/types.h
@@ -80,6 +80,7 @@ enum class TypeIndex {
FixedLengthObject,
JSONB,
Decimal128I,
+ Map,
Struct,
};
@@ -506,6 +507,8 @@ inline const char* getTypeName(TypeIndex idx) {
return "Array";
case TypeIndex::Tuple:
return "Tuple";
+ case TypeIndex::Map:
+ return "Map";
case TypeIndex::Set:
return "Set";
case TypeIndex::Interval:
diff --git a/be/src/vec/data_types/data_type.cpp b/be/src/vec/data_types/data_type.cpp
index 360fa48a98..3fa53aa49c 100644
--- a/be/src/vec/data_types/data_type.cpp
+++ b/be/src/vec/data_types/data_type.cpp
@@ -151,6 +151,8 @@ PGenericType_TypeId IDataType::get_pdata_type(const IDataType* data_type) {
return PGenericType::FIXEDLENGTHOBJECT;
case TypeIndex::JSONB:
return PGenericType::JSONB;
+ case TypeIndex::Map:
+ return PGenericType::MAP;
default:
return PGenericType::UNKNOWN;
}
diff --git a/be/src/vec/data_types/data_type.h b/be/src/vec/data_types/data_type.h
index 900d7b7ac1..e223f4a2e2 100644
--- a/be/src/vec/data_types/data_type.h
+++ b/be/src/vec/data_types/data_type.h
@@ -313,6 +313,7 @@ struct WhichDataType {
bool is_uuid() const { return idx == TypeIndex::UUID; }
bool is_array() const { return idx == TypeIndex::Array; }
bool is_tuple() const { return idx == TypeIndex::Tuple; }
+ bool is_map() const { return idx == TypeIndex::Map; }
bool is_set() const { return idx == TypeIndex::Set; }
bool is_interval() const { return idx == TypeIndex::Interval; }
@@ -354,7 +355,9 @@ inline bool is_tuple(const DataTypePtr& data_type) {
inline bool is_array(const DataTypePtr& data_type) {
return WhichDataType(data_type).is_array();
}
-
+inline bool is_map(const DataTypePtr& data_type) {
+ return WhichDataType(data_type).is_map();
+}
inline bool is_nothing(const DataTypePtr& data_type) {
return WhichDataType(data_type).is_nothing();
}
diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp
index 78c5fae6b6..f1cc28003e 100644
--- a/be/src/vec/data_types/data_type_factory.cpp
+++ b/be/src/vec/data_types/data_type_factory.cpp
@@ -29,6 +29,11 @@ DataTypePtr DataTypeFactory::create_data_type(const doris::Field& col_desc) {
if (col_desc.type() == OLAP_FIELD_TYPE_ARRAY) {
DCHECK(col_desc.get_sub_field_count() == 1);
nested = std::make_shared<DataTypeArray>(create_data_type(*col_desc.get_sub_field(0)));
+ } else if (col_desc.type() == OLAP_FIELD_TYPE_MAP) {
+ DCHECK(col_desc.get_sub_field_count() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(*col_desc.get_sub_field(0)),
+ create_data_type(*col_desc.get_sub_field(1)));
} else if (col_desc.type() == OLAP_FIELD_TYPE_STRUCT) {
DCHECK(col_desc.get_sub_field_count() >= 1);
size_t field_size = col_desc.get_sub_field_count();
@@ -57,6 +62,11 @@ DataTypePtr DataTypeFactory::create_data_type(const TabletColumn& col_desc, bool
if (col_desc.type() == OLAP_FIELD_TYPE_ARRAY) {
DCHECK(col_desc.get_subtype_count() == 1);
nested = std::make_shared<DataTypeArray>(create_data_type(col_desc.get_sub_column(0)));
+ } else if (col_desc.type() == OLAP_FIELD_TYPE_MAP) {
+ DCHECK(col_desc.get_subtype_count() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.get_sub_column(0)),
+ create_data_type(col_desc.get_sub_column(1)));
} else if (col_desc.type() == OLAP_FIELD_TYPE_STRUCT) {
DCHECK(col_desc.get_subtype_count() >= 1);
size_t col_size = col_desc.get_subtype_count();
@@ -155,6 +165,12 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo
nested = std::make_shared<vectorized::DataTypeArray>(
create_data_type(col_desc.children[0], col_desc.contains_nulls[0]));
break;
+ case TYPE_MAP:
+ DCHECK(col_desc.children.size() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(col_desc.children[0], col_desc.contains_nulls[0]),
+ create_data_type(col_desc.children[1], col_desc.contains_nulls[1]));
+ break;
case TYPE_STRUCT: {
DCHECK(col_desc.children.size() >= 1);
size_t child_size = col_desc.children.size();
@@ -338,6 +354,13 @@ DataTypePtr DataTypeFactory::create_data_type(const PColumnMeta& pcolumn) {
case PGenericType::FIXEDLENGTHOBJECT:
nested = std::make_shared<DataTypeFixedLengthObject>();
break;
+ case PGenericType::MAP:
+ DCHECK(pcolumn.children_size() == 2);
+ // here to check pcolumn is list?
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(pcolumn.children(0).children(0)),
+ create_data_type(pcolumn.children(1).children(0)));
+ break;
case PGenericType::STRUCT: {
size_t col_size = pcolumn.children_size();
DCHECK(col_size >= 1);
@@ -421,6 +444,12 @@ DataTypePtr DataTypeFactory::create_data_type(const arrow::DataType* type, bool
nested = std::make_shared<vectorized::DataTypeArray>(
create_data_type(type->field(0)->type().get(), true));
break;
+ case ::arrow::Type::MAP:
+ DCHECK(type->num_fields() == 2);
+ nested = std::make_shared<vectorized::DataTypeMap>(
+ create_data_type(type->field(0)->type().get(), true),
+ create_data_type(type->field(1)->type().get(), true));
+ break;
default:
DCHECK(false) << "invalid arrow type:" << (int)(type->id());
break;
diff --git a/be/src/vec/data_types/data_type_factory.hpp b/be/src/vec/data_types/data_type_factory.hpp
index 9bc5e20d5c..879418a326 100644
--- a/be/src/vec/data_types/data_type_factory.hpp
+++ b/be/src/vec/data_types/data_type_factory.hpp
@@ -37,6 +37,7 @@
#include "vec/data_types/data_type_fixed_length_object.h"
#include "vec/data_types/data_type_hll.h"
#include "vec/data_types/data_type_jsonb.h"
+#include "vec/data_types/data_type_map.h"
#include "vec/data_types/data_type_nothing.h"
#include "vec/data_types/data_type_nullable.h"
#include "vec/data_types/data_type_number.h"
diff --git a/be/src/vec/data_types/data_type_map.cpp b/be/src/vec/data_types/data_type_map.cpp
new file mode 100644
index 0000000000..c40e0362c5
--- /dev/null
+++ b/be/src/vec/data_types/data_type_map.cpp
@@ -0,0 +1,202 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "data_type_map.h"
+
+#include "gen_cpp/data.pb.h"
+#include "vec/columns/column_array.h"
+#include "vec/columns/column_map.h"
+#include "vec/common/assert_cast.h"
+#include "vec/data_types/data_type_factory.hpp"
+
+namespace doris::vectorized {
+
+DataTypeMap::DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_) {
+ key_type = keys_;
+ value_type = values_;
+
+ keys = std::make_shared<DataTypeArray>(key_type);
+ values = std::make_shared<DataTypeArray>(value_type);
+}
+
+std::string DataTypeMap::to_string(const IColumn& column, size_t row_num) const {
+ const ColumnMap& map_column = assert_cast<const ColumnMap&>(column);
+ const ColumnArray::Offsets64& offsets = map_column.get_offsets();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ auto& keys_arr = assert_cast<const ColumnArray&>(map_column.get_keys());
+ auto& values_arr = assert_cast<const ColumnArray&>(map_column.get_values());
+
+ const IColumn& nested_keys_column = keys_arr.get_data();
+ const IColumn& nested_values_column = values_arr.get_data();
+
+ std::stringstream ss;
+ ss << "{";
+ for (size_t i = offset; i < next_offset; ++i) {
+ if (i != offset) {
+ ss << ", ";
+ }
+ if (nested_keys_column.is_null_at(i)) {
+ ss << "NULL";
+ } else if (WhichDataType(remove_nullable(key_type)).is_string_or_fixed_string()) {
+ ss << "'" << key_type->to_string(nested_keys_column, i) << "'";
+ } else {
+ ss << key_type->to_string(nested_keys_column, i);
+ }
+ ss << ":";
+ if (nested_values_column.is_null_at(i)) {
+ ss << "NULL";
+ } else if (WhichDataType(remove_nullable(value_type)).is_string_or_fixed_string()) {
+ ss << "'" << value_type->to_string(nested_values_column, i) << "'";
+ } else {
+ ss << value_type->to_string(nested_values_column, i);
+ }
+ }
+ ss << "}";
+ return ss.str();
+}
+
+void DataTypeMap::to_string(const class doris::vectorized::IColumn& column, size_t row_num,
+ class doris::vectorized::BufferWritable& ostr) const {
+ std::string ss = to_string(column, row_num);
+ ostr.write(ss.c_str(), strlen(ss.c_str()));
+}
+
+Status DataTypeMap::from_string(ReadBuffer& rb, IColumn* column) const {
+ DCHECK(!rb.eof());
+ auto* map_column = assert_cast<ColumnMap*>(column);
+
+ if (*rb.position() != '{') {
+ return Status::InvalidArgument("map does not start with '{' character, found '{}'",
+ *rb.position());
+ }
+ if (*(rb.end() - 1) != '}') {
+ return Status::InvalidArgument("map does not end with '}' character, found '{}'",
+ *(rb.end() - 1));
+ }
+
+ std::stringstream keyCharset;
+ std::stringstream valCharset;
+
+ if (rb.count() == 2) {
+ // empty map {} , need to make empty array to add offset
+ keyCharset << "[]";
+ valCharset << "[]";
+ } else {
+ // {"aaa": 1, "bbb": 20}, need to handle key and value to make key column arr and value arr
+ // skip "{"
+ ++rb.position();
+ keyCharset << "[";
+ valCharset << "[";
+ while (!rb.eof()) {
+ size_t kv_len = 0;
+ auto start = rb.position();
+ while (!rb.eof() && *start != ',' && *start != '}') {
+ kv_len++;
+ start++;
+ }
+ if (kv_len >= rb.count()) {
+ return Status::InvalidArgument("Invalid Length");
+ }
+
+ size_t k_len = 0;
+ auto k_rb = rb.position();
+ while (kv_len > 0 && *k_rb != ':') {
+ k_len++;
+ k_rb++;
+ }
+ ReadBuffer key_rb(rb.position(), k_len);
+ ReadBuffer val_rb(k_rb + 1, kv_len - k_len - 1);
+
+ // handle key
+ keyCharset << key_rb.to_string();
+ keyCharset << ",";
+
+ // handle value
+ valCharset << val_rb.to_string();
+ valCharset << ",";
+
+ rb.position() += kv_len + 1;
+ }
+ keyCharset << ']';
+ valCharset << ']';
+ }
+
+ ReadBuffer kb(keyCharset.str().data(), keyCharset.str().length());
+ ReadBuffer vb(valCharset.str().data(), valCharset.str().length());
+ keys->from_string(kb, &map_column->get_keys());
+ values->from_string(vb, &map_column->get_values());
+ return Status::OK();
+}
+
+MutableColumnPtr DataTypeMap::create_column() const {
+ return ColumnMap::create(keys->create_column(), values->create_column());
+}
+
+void DataTypeMap::to_pb_column_meta(PColumnMeta* col_meta) const {
+ IDataType::to_pb_column_meta(col_meta);
+ auto key_children = col_meta->add_children();
+ auto value_children = col_meta->add_children();
+ keys->to_pb_column_meta(key_children);
+ values->to_pb_column_meta(value_children);
+}
+
+bool DataTypeMap::equals(const IDataType& rhs) const {
+ if (typeid(rhs) != typeid(*this)) {
+ return false;
+ }
+
+ const DataTypeMap& rhs_map = static_cast<const DataTypeMap&>(rhs);
+
+ if (!keys->equals(*rhs_map.keys)) {
+ return false;
+ }
+
+ if (!values->equals(*rhs_map.values)) {
+ return false;
+ }
+
+ return true;
+}
+
+int64_t DataTypeMap::get_uncompressed_serialized_bytes(const IColumn& column,
+ int data_version) const {
+ auto ptr = column.convert_to_full_column_if_const();
+ const auto& data_column = assert_cast<const ColumnMap&>(*ptr.get());
+ return get_keys()->get_uncompressed_serialized_bytes(data_column.get_keys(), data_version) +
+ get_values()->get_uncompressed_serialized_bytes(data_column.get_values(), data_version);
+}
+
+// serialize to binary
+char* DataTypeMap::serialize(const IColumn& column, char* buf, int data_version) const {
+ auto ptr = column.convert_to_full_column_if_const();
+ const auto& map_column = assert_cast<const ColumnMap&>(*ptr.get());
+
+ buf = get_keys()->serialize(map_column.get_keys(), buf, data_version);
+ return get_values()->serialize(map_column.get_values(), buf, data_version);
+}
+
+const char* DataTypeMap::deserialize(const char* buf, IColumn* column, int data_version) const {
+ const auto* map_column = assert_cast<const ColumnMap*>(column);
+ buf = get_keys()->deserialize(buf, map_column->get_keys_ptr()->assume_mutable(), data_version);
+ return get_values()->deserialize(buf, map_column->get_values_ptr()->assume_mutable(),
+ data_version);
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/data_types/data_type_map.h b/be/src/vec/data_types/data_type_map.h
new file mode 100644
index 0000000000..58261b0b3d
--- /dev/null
+++ b/be/src/vec/data_types/data_type_map.h
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/ClickHouse/ClickHouse/blob/master/src/DataTypes/DataTypeMap.h
+// and modified by Doris
+
+#pragma once
+
+#include "vec/data_types/data_type.h"
+
+namespace doris::vectorized {
+/** Map data type.
+ *
+ * Map's key and value only have types.
+ * If only one type is set, then key's type is "String" in default.
+ */
+class DataTypeMap final : public IDataType {
+private:
+ DataTypePtr key_type;
+ DataTypePtr value_type;
+ DataTypePtr keys; // array
+ DataTypePtr values; // array
+
+public:
+ static constexpr bool is_parametric = true;
+
+ DataTypeMap(const DataTypePtr& keys_, const DataTypePtr& values_);
+
+ TypeIndex get_type_id() const override { return TypeIndex::Map; }
+ std::string do_get_name() const override {
+ return "Map(" + key_type->get_name() + ", " + value_type->get_name() + ")";
+ }
+ const char* get_family_name() const override { return "Map"; }
+
+ bool can_be_inside_nullable() const override { return true; }
+ MutableColumnPtr create_column() const override;
+ Field get_default() const override { return Map(); };
+ bool equals(const IDataType& rhs) const override;
+ bool get_is_parametric() const override { return true; }
+ bool have_subtypes() const override { return true; }
+ bool is_comparable() const override {
+ return key_type->is_comparable() && value_type->is_comparable();
+ }
+ bool can_be_compared_with_collation() const override { return false; }
+ bool is_value_unambiguously_represented_in_contiguous_memory_region() const override {
+ return true;
+ }
+
+ const DataTypePtr& get_keys() const { return keys; }
+ const DataTypePtr& get_values() const { return values; }
+
+ const DataTypePtr& get_key_type() const { return key_type; }
+ const DataTypePtr& get_value_type() const { return value_type; }
+
+ int64_t get_uncompressed_serialized_bytes(const IColumn& column,
+ int be_exec_version) const override;
+ char* serialize(const IColumn& column, char* buf, int be_exec_version) const override;
+ const char* deserialize(const char* buf, IColumn* column, int be_exec_version) const override;
+
+ void to_pb_column_meta(PColumnMeta* col_meta) const override;
+
+ std::string to_string(const IColumn& column, size_t row_num) const override;
+ void to_string(const IColumn& column, size_t row_num, BufferWritable& ostr) const override;
+ Status from_string(ReadBuffer& rb, IColumn* column) const override;
+};
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp
index 79eaa13b46..c83e524cb7 100644
--- a/be/src/vec/exprs/vexpr.cpp
+++ b/be/src/vec/exprs/vexpr.cpp
@@ -32,6 +32,7 @@
#include "vec/exprs/vin_predicate.h"
#include "vec/exprs/vinfo_func.h"
#include "vec/exprs/vliteral.h"
+#include "vec/exprs/vmap_literal.h"
#include "vec/exprs/vruntimefilter_wrapper.h"
#include "vec/exprs/vslot_ref.h"
#include "vec/exprs/vstruct_literal.h"
@@ -124,6 +125,8 @@ Status VExpr::create_expr(doris::ObjectPool* pool, const doris::TExprNode& texpr
*expr = pool->add(new VArrayLiteral(texpr_node));
return Status::OK();
}
+ case TExprNodeType::MAP_LITERAL: {
+ *expr = pool->add(new VMapLiteral(texpr_node));
case TExprNodeType::STRUCT_LITERAL: {
*expr = pool->add(new VStructLiteral(texpr_node));
return Status::OK();
@@ -176,107 +179,90 @@ Status VExpr::create_expr(doris::ObjectPool* pool, const doris::TExprNode& texpr
return Status::OK();
}
-Status VExpr::create_tree_from_thrift(doris::ObjectPool* pool,
- const std::vector<doris::TExprNode>& nodes, VExpr* parent,
- int* node_idx, VExpr** root_expr, VExprContext** ctx) {
- // propagate error case
- if (*node_idx >= nodes.size()) {
- return Status::InternalError("Failed to reconstruct expression tree from thrift.");
- }
- int num_children = nodes[*node_idx].num_children;
- VExpr* expr = nullptr;
- RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
- DCHECK(expr != nullptr);
- if (parent != nullptr) {
- parent->add_child(expr);
- } else {
- DCHECK(root_expr != nullptr);
- DCHECK(ctx != nullptr);
- *root_expr = expr;
- *ctx = pool->add(new VExprContext(expr));
- }
- for (int i = 0; i < num_children; i++) {
- *node_idx += 1;
- RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
- // we are expecting a child, but have used all nodes
- // this means we have been given a bad tree and must fail
+ Status VExpr::create_tree_from_thrift(doris::ObjectPool * pool,
+ const std::vector<doris::TExprNode>& nodes, VExpr* parent,
+ int* node_idx, VExpr** root_expr, VExprContext** ctx) {
+ // propagate error case
if (*node_idx >= nodes.size()) {
return Status::InternalError("Failed to reconstruct expression tree from thrift.");
}
- }
- return Status::OK();
-}
-
-Status VExpr::create_expr_tree(doris::ObjectPool* pool, const doris::TExpr& texpr,
- VExprContext** ctx) {
- if (texpr.nodes.size() == 0) {
- *ctx = nullptr;
+ int num_children = nodes[*node_idx].num_children;
+ VExpr* expr = nullptr;
+ RETURN_IF_ERROR(create_expr(pool, nodes[*node_idx], &expr));
+ DCHECK(expr != nullptr);
+ if (parent != nullptr) {
+ parent->add_child(expr);
+ } else {
+ DCHECK(root_expr != nullptr);
+ DCHECK(ctx != nullptr);
+ *root_expr = expr;
+ *ctx = pool->add(new VExprContext(expr));
+ }
+ for (int i = 0; i < num_children; i++) {
+ *node_idx += 1;
+ RETURN_IF_ERROR(create_tree_from_thrift(pool, nodes, expr, node_idx, nullptr, nullptr));
+ // we are expecting a child, but have used all nodes
+ // this means we have been given a bad tree and must fail
+ if (*node_idx >= nodes.size()) {
+ return Status::InternalError("Failed to reconstruct expression tree from thrift.");
+ }
+ }
return Status::OK();
}
- int node_idx = 0;
- VExpr* e = nullptr;
- Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
- if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
- status = Status::InternalError(
- "Expression tree only partially reconstructed. Not all thrift nodes were used.");
- }
- if (!status.ok()) {
- LOG(ERROR) << "Could not construct expr tree.\n"
- << status << "\n"
- << apache::thrift::ThriftDebugString(texpr);
- }
- return status;
-}
-Status VExpr::create_expr_trees(ObjectPool* pool, const std::vector<doris::TExpr>& texprs,
- std::vector<VExprContext*>* ctxs) {
- ctxs->clear();
- for (int i = 0; i < texprs.size(); ++i) {
- VExprContext* ctx = nullptr;
- RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
- ctxs->push_back(ctx);
+ Status VExpr::create_expr_tree(doris::ObjectPool * pool, const doris::TExpr& texpr,
+ VExprContext** ctx) {
+ if (texpr.nodes.size() == 0) {
+ *ctx = nullptr;
+ return Status::OK();
+ }
+ int node_idx = 0;
+ VExpr* e = nullptr;
+ Status status = create_tree_from_thrift(pool, texpr.nodes, nullptr, &node_idx, &e, ctx);
+ if (status.ok() && node_idx + 1 != texpr.nodes.size()) {
+ status = Status::InternalError(
+ "Expression tree only partially reconstructed. Not all thrift nodes were "
+ "used.");
+ }
+ if (!status.ok()) {
+ LOG(ERROR) << "Could not construct expr tree.\n"
+ << status << "\n"
+ << apache::thrift::ThriftDebugString(texpr);
+ }
+ return status;
}
- return Status::OK();
-}
-Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- const RowDescriptor& row_desc) {
- for (auto ctx : ctxs) {
- RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ Status VExpr::create_expr_trees(ObjectPool * pool, const std::vector<doris::TExpr>& texprs,
+ std::vector<VExprContext*>* ctxs) {
+ ctxs->clear();
+ for (int i = 0; i < texprs.size(); ++i) {
+ VExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(create_expr_tree(pool, texprs[i], &ctx));
+ ctxs->push_back(ctx);
+ }
+ return Status::OK();
}
- return Status::OK();
-}
-void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (auto ctx : ctxs) {
- ctx->close(state);
+ Status VExpr::prepare(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ const RowDescriptor& row_desc) {
+ for (auto ctx : ctxs) {
+ RETURN_IF_ERROR(ctx->prepare(state, row_desc));
+ }
+ return Status::OK();
}
-}
-Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->open(state));
+ void VExpr::close(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (auto ctx : ctxs) {
+ ctx->close(state);
+ }
}
- return Status::OK();
-}
-Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
- std::vector<VExprContext*>* new_ctxs) {
- DCHECK(new_ctxs != nullptr);
- if (!new_ctxs->empty()) {
- // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
- DCHECK_EQ(new_ctxs->size(), ctxs.size());
- for (int i = 0; i < new_ctxs->size(); ++i) {
- DCHECK((*new_ctxs)[i]->_is_clone);
+ Status VExpr::open(const std::vector<VExprContext*>& ctxs, RuntimeState* state) {
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->open(state));
}
return Status::OK();
}
- new_ctxs->resize(ctxs.size());
- for (int i = 0; i < ctxs.size(); ++i) {
- RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
- }
- return Status::OK();
-}
FunctionContext::TypeDesc VExpr::column_type_to_type_desc(const TypeDescriptor& type) {
FunctionContext::TypeDesc out;
@@ -380,114 +366,132 @@ FunctionContext::TypeDesc VExpr::column_type_to_type_desc(const TypeDescriptor&
return out;
}
-std::string VExpr::debug_string() const {
- // TODO: implement partial debug string for member vars
- std::stringstream out;
- out << " type=" << _type.debug_string();
- out << " codegen="
- << "false";
-
- if (!_children.empty()) {
- out << " children=" << debug_string(_children);
+ Status VExpr::clone_if_not_exists(const std::vector<VExprContext*>& ctxs, RuntimeState* state,
+ std::vector<VExprContext*>* new_ctxs) {
+ DCHECK(new_ctxs != nullptr);
+ if (!new_ctxs->empty()) {
+ // 'ctxs' was already cloned into '*new_ctxs', nothing to do.
+ DCHECK_EQ(new_ctxs->size(), ctxs.size());
+ for (int i = 0; i < new_ctxs->size(); ++i) {
+ DCHECK((*new_ctxs)[i]->_is_clone);
+ }
+ return Status::OK();
+ }
+ new_ctxs->resize(ctxs.size());
+ for (int i = 0; i < ctxs.size(); ++i) {
+ RETURN_IF_ERROR(ctxs[i]->clone(state, &(*new_ctxs)[i]));
+ }
+ return Status::OK();
}
+ std::string VExpr::debug_string() const {
+ // TODO: implement partial debug string for member vars
+ std::stringstream out;
+ out << " type=" << _type.debug_string();
+ out << " codegen="
+ << "false";
- return out.str();
-}
-
-std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
- std::stringstream out;
- out << "[";
+ if (!_children.empty()) {
+ out << " children=" << debug_string(_children);
+ }
- for (int i = 0; i < exprs.size(); ++i) {
- out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
+ return out.str();
}
- out << "]";
- return out.str();
-}
+ std::string VExpr::debug_string(const std::vector<VExpr*>& exprs) {
+ std::stringstream out;
+ out << "[";
+
+ for (int i = 0; i < exprs.size(); ++i) {
+ out << (i == 0 ? "" : " ") << exprs[i]->debug_string();
+ }
-std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
- std::vector<VExpr*> exprs;
- for (int i = 0; i < ctxs.size(); ++i) {
- exprs.push_back(ctxs[i]->root());
+ out << "]";
+ return out.str();
}
- return debug_string(exprs);
-}
-bool VExpr::is_constant() const {
- for (int i = 0; i < _children.size(); ++i) {
- if (!_children[i]->is_constant()) {
- return false;
+ std::string VExpr::debug_string(const std::vector<VExprContext*>& ctxs) {
+ std::vector<VExpr*> exprs;
+ for (int i = 0; i < ctxs.size(); ++i) {
+ exprs.push_back(ctxs[i]->root());
}
+ return debug_string(exprs);
}
- return true;
-}
+ bool VExpr::is_constant() const {
+ for (int i = 0; i < _children.size(); ++i) {
+ if (!_children[i]->is_constant()) {
+ return false;
+ }
+ }
-Status VExpr::get_const_col(VExprContext* context, ColumnPtrWrapper** output) {
- *output = nullptr;
- if (!is_constant()) {
- return Status::OK();
+ return true;
}
- if (_constant_col != nullptr) {
+ Status VExpr::get_const_col(VExprContext * context, ColumnPtrWrapper * *output) {
+ *output = nullptr;
+ if (!is_constant()) {
+ return Status::OK();
+ }
+
+ if (_constant_col != nullptr) {
+ *output = _constant_col.get();
+ return Status::OK();
+ }
+
+ int result = -1;
+ Block block;
+ // If block is empty, some functions will produce no result. So we insert a column with
+ // single value here.
+ block.insert({ColumnUInt8::create(1), std::make_shared<DataTypeUInt8>(), ""});
+ RETURN_IF_ERROR(execute(context, &block, &result));
+ DCHECK(result != -1);
+ const auto& column = block.get_by_position(result).column;
+ _constant_col = std::make_shared<ColumnPtrWrapper>(column);
*output = _constant_col.get();
return Status::OK();
}
- int result = -1;
- Block block;
- // If block is empty, some functions will produce no result. So we insert a column with
- // single value here.
- block.insert({ColumnUInt8::create(1), std::make_shared<DataTypeUInt8>(), ""});
- RETURN_IF_ERROR(execute(context, &block, &result));
- DCHECK(result != -1);
- const auto& column = block.get_by_position(result).column;
- _constant_col = std::make_shared<ColumnPtrWrapper>(column);
- *output = _constant_col.get();
- return Status::OK();
-}
+ void VExpr::register_function_context(doris::RuntimeState * state, VExprContext * context) {
+ FunctionContext::TypeDesc return_type = VExpr::column_type_to_type_desc(_type);
+ std::vector<FunctionContext::TypeDesc> arg_types;
+ for (int i = 0; i < _children.size(); ++i) {
+ arg_types.push_back(VExpr::column_type_to_type_desc(_children[i]->type()));
+ }
-void VExpr::register_function_context(doris::RuntimeState* state, VExprContext* context) {
- FunctionContext::TypeDesc return_type = VExpr::column_type_to_type_desc(_type);
- std::vector<FunctionContext::TypeDesc> arg_types;
- for (int i = 0; i < _children.size(); ++i) {
- arg_types.push_back(VExpr::column_type_to_type_desc(_children[i]->type()));
+ _fn_context_index = context->register_func(state, return_type, arg_types, 0);
}
- _fn_context_index = context->register_func(state, return_type, arg_types, 0);
-}
-
-Status VExpr::init_function_context(VExprContext* context,
- FunctionContext::FunctionStateScope scope,
- const FunctionBasePtr& function) const {
- FunctionContext* fn_ctx = context->fn_context(_fn_context_index);
- if (scope == FunctionContext::FRAGMENT_LOCAL) {
- std::vector<ColumnPtrWrapper*> constant_cols;
- for (auto c : _children) {
- ColumnPtrWrapper* const_col_wrapper = nullptr;
- RETURN_IF_ERROR(c->get_const_col(context, &const_col_wrapper));
- constant_cols.push_back(const_col_wrapper);
+ Status VExpr::init_function_context(VExprContext * context,
+ FunctionContext::FunctionStateScope scope,
+ const FunctionBasePtr& function) const {
+ FunctionContext* fn_ctx = context->fn_context(_fn_context_index);
+ if (scope == FunctionContext::FRAGMENT_LOCAL) {
+ std::vector<ColumnPtrWrapper*> constant_cols;
+ for (auto c : _children) {
+ ColumnPtrWrapper* const_col_wrapper = nullptr;
+ RETURN_IF_ERROR(c->get_const_col(context, &const_col_wrapper));
+ constant_cols.push_back(const_col_wrapper);
+ }
+ fn_ctx->impl()->set_constant_cols(constant_cols);
}
- fn_ctx->impl()->set_constant_cols(constant_cols);
- }
- if (scope == FunctionContext::FRAGMENT_LOCAL) {
- RETURN_IF_ERROR(function->prepare(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
+ if (scope == FunctionContext::FRAGMENT_LOCAL) {
+ RETURN_IF_ERROR(function->prepare(fn_ctx, FunctionContext::FRAGMENT_LOCAL));
+ }
+ RETURN_IF_ERROR(function->prepare(fn_ctx, FunctionContext::THREAD_LOCAL));
+ return Status::OK();
}
- RETURN_IF_ERROR(function->prepare(fn_ctx, FunctionContext::THREAD_LOCAL));
- return Status::OK();
-}
-void VExpr::close_function_context(VExprContext* context, FunctionContext::FunctionStateScope scope,
- const FunctionBasePtr& function) const {
- if (_fn_context_index != -1 && !context->_stale) {
- FunctionContext* fn_ctx = context->fn_context(_fn_context_index);
- function->close(fn_ctx, FunctionContext::THREAD_LOCAL);
- if (scope == FunctionContext::FRAGMENT_LOCAL) {
- function->close(fn_ctx, FunctionContext::FRAGMENT_LOCAL);
+ void VExpr::close_function_context(VExprContext * context,
+ FunctionContext::FunctionStateScope scope,
+ const FunctionBasePtr& function) const {
+ if (_fn_context_index != -1 && !context->_stale) {
+ FunctionContext* fn_ctx = context->fn_context(_fn_context_index);
+ function->close(fn_ctx, FunctionContext::THREAD_LOCAL);
+ if (scope == FunctionContext::FRAGMENT_LOCAL) {
+ function->close(fn_ctx, FunctionContext::FRAGMENT_LOCAL);
+ }
}
}
-}
} // namespace doris::vectorized
diff --git a/be/src/vec/exprs/vmap_literal.cpp b/be/src/vec/exprs/vmap_literal.cpp
new file mode 100644
index 0000000000..954142f04d
--- /dev/null
+++ b/be/src/vec/exprs/vmap_literal.cpp
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exprs/vmap_literal.h"
+
+//insert into table_map values ({'name':'zhangsan', 'gender':'male'}), ({'name':'lisi', 'gender':'female'});
+namespace doris::vectorized {
+
+Status VMapLiteral::prepare(RuntimeState* state, const RowDescriptor& row_desc,
+ VExprContext* context) {
+ DCHECK_EQ(type().children.size(), 2) << "map children type not 2";
+
+ RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, row_desc, context));
+ // map-field should contain two vector field for keys and values
+ Field map = Map();
+ Field keys = Array();
+ Field values = Array();
+ // each child is slot with key1, value1, key2, value2...
+ for (int idx = 0; idx < _children.size(); ++idx) {
+ Field item;
+ ColumnPtrWrapper* const_col_wrapper = nullptr;
+ RETURN_IF_ERROR(_children[idx]->get_const_col(context, &const_col_wrapper));
+ const_col_wrapper->column_ptr->get(0, item);
+
+ if ((idx & 1) == 0) {
+ keys.get<Array>().push_back(item);
+ } else {
+ values.get<Array>().push_back(item);
+ }
+ }
+ map.get<Map>().push_back(keys);
+ map.get<Map>().push_back(values);
+
+ _column_ptr = _data_type->create_column_const(1, map);
+ return Status::OK();
+}
+
+} // namespace doris::vectorized
diff --git a/be/src/vec/exprs/vmap_literal.h b/be/src/vec/exprs/vmap_literal.h
new file mode 100644
index 0000000000..6206d4c58f
--- /dev/null
+++ b/be/src/vec/exprs/vmap_literal.h
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include "vec/exprs/vliteral.h"
+
+namespace doris {
+
+namespace vectorized {
+class VMapLiteral : public VLiteral {
+public:
+ VMapLiteral(const TExprNode& node) : VLiteral(node, false) {}
+ ~VMapLiteral() override = default;
+ Status prepare(RuntimeState* state, const RowDescriptor& row_desc,
+ VExprContext* context) override;
+};
+} // namespace vectorized
+
+} // namespace doris
diff --git a/be/src/vec/functions/array/function_array_element.h b/be/src/vec/functions/array/function_array_element.h
index 6722e09e9c..ac685b172b 100644
--- a/be/src/vec/functions/array/function_array_element.h
+++ b/be/src/vec/functions/array/function_array_element.h
@@ -22,9 +22,10 @@
#include <string_view>
#include "vec/columns/column_array.h"
-#include "vec/columns/column_const.h"
+#include "vec/columns/column_map.h"
#include "vec/columns/column_string.h"
#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_map.h"
#include "vec/data_types/data_type_number.h"
#include "vec/functions/function.h"
#include "vec/functions/function_helpers.h"
@@ -44,12 +45,21 @@ public:
size_t get_number_of_arguments() const override { return 2; }
DataTypePtr get_return_type_impl(const DataTypes& arguments) const override {
- DCHECK(is_array(arguments[0]))
- << "first argument for function: " << name << " should be DataTypeArray";
- DCHECK(is_integer(arguments[1]))
- << "second argument for function: " << name << " should be Integer";
- return make_nullable(
- check_and_get_data_type<DataTypeArray>(arguments[0].get())->get_nested_type());
+ DCHECK(is_array(arguments[0]) || is_map(arguments[0]))
+ << "first argument for function: " << name
+ << " should be DataTypeArray or DataTypeMap";
+ if (is_array(arguments[0])) {
+ DCHECK(is_integer(arguments[1])) << "second argument for function: " << name
+ << " should be Integer for array element";
+ return make_nullable(
+ check_and_get_data_type<DataTypeArray>(arguments[0].get())->get_nested_type());
+ } else if (is_map(arguments[0])) {
+ return make_nullable(
+ check_and_get_data_type<DataTypeMap>(arguments[0].get())->get_value_type());
+ } else {
+ LOG(ERROR) << "element_at only support array and map so far.";
+ return nullptr;
+ }
}
Status execute_impl(FunctionContext* context, Block& block, const ColumnNumbers& arguments,
@@ -68,19 +78,64 @@ public:
} else {
args = {col_left, block.get_by_position(arguments[1])};
}
-
- auto res_column = _execute_non_nullable(args, input_rows_count, src_null_map, dst_null_map);
+ ColumnPtr res_column = nullptr;
+ if (args[0].column->is_column_map()) {
+ res_column = _execute_map(args, input_rows_count, src_null_map, dst_null_map);
+ } else {
+ res_column = _execute_non_nullable(args, input_rows_count, src_null_map, dst_null_map);
+ }
if (!res_column) {
return Status::RuntimeError("unsupported types for function {}({}, {})", get_name(),
block.get_by_position(arguments[0]).type->get_name(),
block.get_by_position(arguments[1]).type->get_name());
}
- block.replace_by_position(
- result, ColumnNullable::create(std::move(res_column), std::move(dst_null_column)));
+ block.replace_by_position(result,
+ ColumnNullable::create(res_column, std::move(dst_null_column)));
return Status::OK();
}
private:
+ //=========================== map element===========================//
+ ColumnPtr _get_mapped_idx(const ColumnArray& key_column,
+ const ColumnWithTypeAndName& argument) {
+ return _mapped_key(key_column, argument);
+ }
+
+ ColumnPtr _mapped_key(const ColumnArray& column, const ColumnWithTypeAndName& argument) {
+ auto right_column = argument.column->convert_to_full_column_if_const();
+ const ColumnArray::Offsets64& offsets = column.get_offsets();
+ ColumnPtr nested_ptr = nullptr;
+ if (is_column_nullable(column.get_data())) {
+ nested_ptr = reinterpret_cast<const ColumnNullable&>(column.get_data())
+ .get_nested_column_ptr();
+ } else {
+ nested_ptr = column.get_data_ptr();
+ }
+ size_t rows = offsets.size();
+ // prepare return data
+ auto matched_indices = ColumnVector<Int8>::create();
+ matched_indices->reserve(rows);
+
+ for (size_t i = 0; i < rows; i++) {
+ bool matched = false;
+ size_t begin = offsets[i - 1];
+ size_t end = offsets[i];
+ for (size_t j = begin; j < end; j++) {
+ if (nested_ptr->compare_at(j, i, *right_column, -1) == 0) {
+ matched_indices->insert_value(j - begin + 1);
+ matched = true;
+ break;
+ }
+ }
+
+ if (!matched) {
+ matched_indices->insert_value(end - begin + 1); // make indices for null
+ }
+ }
+
+ return matched_indices;
+ }
+
template <typename ColumnType>
ColumnPtr _execute_number(const ColumnArray::Offsets64& offsets, const IColumn& nested_column,
const UInt8* arr_null_map, const IColumn& indices,
@@ -176,6 +231,33 @@ private:
return dst_column;
}
+ ColumnPtr _execute_map(const ColumnsWithTypeAndName& arguments, size_t input_rows_count,
+ const UInt8* src_null_map, UInt8* dst_null_map) {
+ auto left_column = arguments[0].column->convert_to_full_column_if_const();
+ DataTypePtr val_type =
+ reinterpret_cast<const DataTypeMap&>(*arguments[0].type).get_values();
+ const auto& map_column = reinterpret_cast<const ColumnMap&>(*left_column);
+
+ const ColumnArray& column_keys = assert_cast<const ColumnArray&>(map_column.get_keys());
+
+ const auto& offsets = column_keys.get_offsets();
+ const size_t rows = offsets.size();
+
+ if (rows <= 0) {
+ return nullptr;
+ }
+
+ ColumnPtr matched_indices = _get_mapped_idx(column_keys, arguments[1]);
+ if (!matched_indices) {
+ return nullptr;
+ }
+ DataTypePtr indices_type(std::make_shared<vectorized::DataTypeInt8>());
+ ColumnWithTypeAndName indices(matched_indices, indices_type, "indices");
+ ColumnWithTypeAndName data(map_column.get_values_ptr(), val_type, "value");
+ ColumnsWithTypeAndName args = {data, indices};
+ return _execute_non_nullable(args, input_rows_count, src_null_map, dst_null_map);
+ }
+
ColumnPtr _execute_non_nullable(const ColumnsWithTypeAndName& arguments,
size_t input_rows_count, const UInt8* src_null_map,
UInt8* dst_null_map) {
diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp
index da0910ca2c..431dda727d 100644
--- a/be/src/vec/olap/olap_data_convertor.cpp
+++ b/be/src/vec/olap/olap_data_convertor.cpp
@@ -60,7 +60,6 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co
case FieldType::OLAP_FIELD_TYPE_CHAR: {
return std::make_unique<OlapColumnDataConvertorChar>(column.length());
}
- case FieldType::OLAP_FIELD_TYPE_MAP:
case FieldType::OLAP_FIELD_TYPE_VARCHAR: {
return std::make_unique<OlapColumnDataConvertorVarChar>(false);
}
@@ -133,6 +132,15 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co
return std::make_unique<OlapColumnDataConvertorArray>(
create_olap_column_data_convertor(sub_column));
}
+ case FieldType::OLAP_FIELD_TYPE_MAP: {
+ const auto& key_column = column.get_sub_column(0);
+ const auto& value_column = column.get_sub_column(1);
+ return std::make_unique<OlapColumnDataConvertorMap>(
+ std::make_unique<OlapColumnDataConvertorArray>(
+ create_olap_column_data_convertor(key_column)),
+ std::make_unique<OlapColumnDataConvertorArray>(
+ create_olap_column_data_convertor(value_column)));
+ }
default: {
DCHECK(false) << "Invalid type in RowBlockV2:" << column.type();
return nullptr;
@@ -778,4 +786,56 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap(
return Status::OK();
}
+Status OlapBlockDataConvertor::OlapColumnDataConvertorMap::convert_to_olap() {
+ const ColumnMap* column_map = nullptr;
+ const DataTypeMap* data_type_map = nullptr;
+ if (_nullmap) {
+ const auto* nullable_column =
+ assert_cast<const ColumnNullable*>(_typed_column.column.get());
+ column_map = assert_cast<const ColumnMap*>(nullable_column->get_nested_column_ptr().get());
+ data_type_map = assert_cast<const DataTypeMap*>(
+ (assert_cast<const DataTypeNullable*>(_typed_column.type.get())->get_nested_type())
+ .get());
+ } else {
+ column_map = assert_cast<const ColumnMap*>(_typed_column.column.get());
+ data_type_map = assert_cast<const DataTypeMap*>(_typed_column.type.get());
+ }
+ assert(column_map);
+ assert(data_type_map);
+
+ return convert_to_olap(column_map, data_type_map);
+}
+
+Status OlapBlockDataConvertor::OlapColumnDataConvertorMap::convert_to_olap(
+ const ColumnMap* column_map, const DataTypeMap* data_type_map) {
+ ColumnPtr key_data = column_map->get_keys_ptr();
+ ColumnPtr value_data = column_map->get_values_ptr();
+ if (column_map->get_keys().is_nullable()) {
+ const auto& key_nullable_column =
+ assert_cast<const ColumnNullable&>(column_map->get_keys());
+ key_data = key_nullable_column.get_nested_column_ptr();
+ }
+
+ if (column_map->get_values().is_nullable()) {
+ const auto& val_nullable_column =
+ assert_cast<const ColumnNullable&>(column_map->get_values());
+ value_data = val_nullable_column.get_nested_column_ptr();
+ }
+
+ ColumnWithTypeAndName key_typed_column = {key_data, remove_nullable(data_type_map->get_keys()),
+ "map.key"};
+ _key_convertor->set_source_column(key_typed_column, _row_pos, _num_rows);
+ _key_convertor->convert_to_olap();
+
+ ColumnWithTypeAndName value_typed_column = {
+ value_data, remove_nullable(data_type_map->get_values()), "map.value"};
+ _value_convertor->set_source_column(value_typed_column, _row_pos, _num_rows);
+ _value_convertor->convert_to_olap();
+
+ _results[0] = _key_convertor->get_data();
+ _results[1] = _value_convertor->get_data();
+
+ return Status::OK();
+}
+
} // namespace doris::vectorized
diff --git a/be/src/vec/olap/olap_data_convertor.h b/be/src/vec/olap/olap_data_convertor.h
index 12cc0fba55..6b648ea947 100644
--- a/be/src/vec/olap/olap_data_convertor.h
+++ b/be/src/vec/olap/olap_data_convertor.h
@@ -19,9 +19,11 @@
#include "olap/types.h"
#include "runtime/mem_pool.h"
+#include "vec/columns/column_map.h"
#include "vec/columns/column_nullable.h"
#include "vec/core/column_with_type_and_name.h"
#include "vec/core/types.h"
+#include "vec/data_types/data_type_map.h"
namespace doris {
@@ -395,6 +397,29 @@ private:
OlapColumnDataConvertorBaseUPtr _item_convertor;
};
+ class OlapColumnDataConvertorMap : public OlapColumnDataConvertorBase {
+ public:
+ OlapColumnDataConvertorMap(OlapColumnDataConvertorBaseUPtr key_convertor,
+ OlapColumnDataConvertorBaseUPtr value_convertor)
+ : _key_convertor(std::move(key_convertor)),
+ _value_convertor(std::move(value_convertor)) {
+ _results.resize(2);
+ }
+
+ Status convert_to_olap() override;
+ const void* get_data() const override { return _results.data(); };
+
+ const void* get_data_at(size_t offset) const override {
+ LOG(FATAL) << "now not support get_data_at for OlapColumnDataConvertorMap";
+ };
+
+ private:
+ Status convert_to_olap(const ColumnMap* column_map, const DataTypeMap* data_type_map);
+ OlapColumnDataConvertorBaseUPtr _key_convertor;
+ OlapColumnDataConvertorBaseUPtr _value_convertor;
+ std::vector<const void*> _results;
+ }; //OlapColumnDataConvertorMap
+
private:
std::vector<OlapColumnDataConvertorBaseUPtr> _convertors;
};
diff --git a/be/src/vec/sink/vmysql_result_writer.cpp b/be/src/vec/sink/vmysql_result_writer.cpp
index 78fa7d4ccb..6ec225f961 100644
--- a/be/src/vec/sink/vmysql_result_writer.cpp
+++ b/be/src/vec/sink/vmysql_result_writer.cpp
@@ -29,6 +29,7 @@
#include "vec/common/assert_cast.h"
#include "vec/data_types/data_type_array.h"
#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_map.h"
#include "vec/data_types/data_type_struct.h"
#include "vec/exprs/vexpr.h"
#include "vec/exprs/vexpr_context.h"
@@ -186,6 +187,22 @@ Status VMysqlResultWriter<is_binary_format>::_add_one_column(
buf_ret = rows_buffer[i].push_string("]", 1);
rows_buffer[i].close_dynamic_mode();
}
+ } else if constexpr (type == TYPE_MAP) {
+ DCHECK_GE(sub_types.size(), 1);
+ auto& map_type = assert_cast<const DataTypeMap&>(*sub_types[0]);
+ for (ssize_t i = 0; i < row_size; ++i) {
+ if (0 != buf_ret) {
+ return Status::InternalError("pack mysql buffer failed.");
+ }
+ _buffer.reset();
+
+ _buffer.open_dynamic_mode();
+ std::string cell_str = map_type.to_string(*column, i);
+ buf_ret = _buffer.push_string(cell_str.c_str(), strlen(cell_str.c_str()));
+
+ _buffer.close_dynamic_mode();
+ result->result_batch.rows[i].append(_buffer.buf(), _buffer.length());
+ }
} else if constexpr (type == TYPE_STRUCT) {
DCHECK_GE(sub_types.size(), 1);
auto& column_struct = assert_cast<const ColumnStruct&>(*column);
@@ -750,6 +767,18 @@ Status VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
}
break;
}
+ case TYPE_MAP: {
+ if (type_ptr->is_nullable()) {
+ auto& nested_type =
+ assert_cast<const DataTypeNullable&>(*type_ptr).get_nested_type(); //for map
+ status = _add_one_column<PrimitiveType::TYPE_MAP, true>(column_ptr, result, scale,
+ {nested_type});
+ } else {
+ status = _add_one_column<PrimitiveType::TYPE_MAP, false>(column_ptr, result, scale,
+ {type_ptr});
+ }
+ break;
+ }
default: {
LOG(WARNING) << "can't convert this type to mysql type. type = "
<< _output_vexpr_ctxs[i]->root()->type();
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
index adf74f5c6a..986fb3658a 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
@@ -17,18 +17,25 @@
package org.apache.doris.catalog;
+import org.apache.doris.thrift.TColumnType;
import org.apache.doris.thrift.TTypeDesc;
import org.apache.doris.thrift.TTypeNode;
import org.apache.doris.thrift.TTypeNodeType;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.gson.annotations.SerializedName;
+
+import java.util.Objects;
/**
* Describes a MAP type. MAP types have a scalar key and an arbitrarily-typed value.
*/
public class MapType extends Type {
+
+ @SerializedName(value = "keyType")
private final Type keyType;
+ @SerializedName(value = "valueType")
private final Type valueType;
public MapType() {
@@ -75,6 +82,30 @@ public class MapType extends Type {
keyType.toSql(depth + 1), valueType.toSql(depth + 1));
}
+ @Override
+ public boolean matchesType(Type t) {
+ if (equals(t)) {
+ return true;
+ }
+
+ if (!t.isMapType()) {
+ return false;
+ }
+
+ if ((keyType.isNull() || ((MapType) t).getKeyType().isNull())
+ && (valueType.isNull() || ((MapType) t).getKeyType().isNull())) {
+ return true;
+ }
+
+ return keyType.matchesType(((MapType) t).keyType)
+ && (valueType.matchesType(((MapType) t).valueType));
+ }
+
+ @Override
+ public String toString() {
+ return toSql(0).toUpperCase();
+ }
+
@Override
protected String prettyPrint(int lpad) {
String leftPadding = Strings.repeat(" ", lpad);
@@ -88,6 +119,11 @@ public class MapType extends Type {
return String.format("%sMAP<%s,%s>", leftPadding, keyType.toSql(), structStr);
}
+ public static boolean canCastTo(MapType type, MapType targetType) {
+ return Type.canCastTo(type.getKeyType(), targetType.getKeyType())
+ && Type.canCastTo(type.getValueType(), targetType.getValueType());
+ }
+
@Override
public boolean supportSubType(Type subType) {
return true;
@@ -103,4 +139,16 @@ public class MapType extends Type {
keyType.toThrift(container);
valueType.toThrift(container);
}
+
+ @Override
+ public TColumnType toColumnTypeThrift() {
+ TColumnType thrift = new TColumnType();
+ thrift.type = PrimitiveType.MAP.toThrift();
+ return thrift;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(keyType, valueType);
+ }
}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MysqlColType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/MysqlColType.java
index ff6d81d022..0bdf85384d 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MysqlColType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MysqlColType.java
@@ -52,7 +52,8 @@ public enum MysqlColType {
MYSQL_TYPE_BLOB(252, "BLOB"),
MYSQL_TYPE_VARSTRING(253, "VAR STRING"),
MYSQL_TYPE_STRING(254, "STRING"),
- MYSQL_TYPE_GEOMETRY(255, "GEOMETRY");
+ MYSQL_TYPE_GEOMETRY(255, "GEOMETRY"),
+ MYSQL_TYPE_MAP(400, "MAP");
private MysqlColType(int code, String desc) {
this.code = code;
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
index d64565c465..cdc24225a2 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
@@ -1105,6 +1105,10 @@ public enum PrimitiveType {
return this == ARRAY;
}
+ public boolean isMapType() {
+ return this == MAP;
+ }
+
public boolean isComplexType() {
return this == HLL || this == BITMAP;
}
@@ -1166,6 +1170,8 @@ public enum PrimitiveType {
return MysqlColType.MYSQL_TYPE_BLOB;
case JSONB:
return MysqlColType.MYSQL_TYPE_JSON;
+ case MAP:
+ return MysqlColType.MYSQL_TYPE_MAP;
default:
return MysqlColType.MYSQL_TYPE_STRING;
}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index 551b85e4b6..63d9965d51 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -579,6 +579,8 @@ public abstract class Type {
return ScalarType.canCastTo((ScalarType) sourceType, (ScalarType) targetType);
} else if (sourceType.isArrayType() && targetType.isArrayType()) {
return ArrayType.canCastTo((ArrayType) sourceType, (ArrayType) targetType);
+ } else if (sourceType.isMapType() && targetType.isMapType()) {
+ return MapType.canCastTo((MapType) sourceType, (MapType) targetType);
} else if (targetType.isArrayType() && !((ArrayType) targetType).getItemType().isScalarType()
&& !sourceType.isNull()) {
// TODO: current not support cast any non-array type(except for null) to nested array type.
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index 1d1d392105..f286618ef2 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -698,7 +698,7 @@ nonterminal SelectList select_clause, select_list, select_sublist;
nonterminal SelectListItem select_list_item, star_expr;
nonterminal Expr expr, non_pred_expr, arithmetic_expr, timestamp_arithmetic_expr, expr_or_default;
nonterminal Expr set_expr_or_default;
-nonterminal ArrayList<Expr> expr_list, values, row_value, opt_values;
+nonterminal ArrayList<Expr> expr_list, values, row_value, opt_values, kv_list;
nonterminal ArrayList<LiteralExpr> literal_values, args_list;
nonterminal ArrayList<Expr> func_arg_list;
nonterminal ArrayList<Expr> expr_pipe_list;
@@ -737,6 +737,7 @@ nonterminal ArrayList<CaseWhenClause> case_when_clause_list;
nonterminal FunctionParams function_params;
nonterminal Expr function_call_expr, array_expr;
nonterminal ArrayLiteral array_literal;
+nonterminal MapLiteral map_literal;
nonterminal StructField struct_field;
nonterminal ArrayList<StructField> struct_field_list;
nonterminal StructLiteral struct_literal;
@@ -5950,6 +5951,33 @@ array_expr ::=
:}
;
+kv_list ::=
+ expr:k COLON expr:v
+ {:
+ ArrayList<Expr> list = new ArrayList<Expr>();
+ list.add(k);
+ list.add(v);
+ RESULT = list ;
+ :}
+ |kv_list:list COMMA expr:k COLON expr:v
+ {:
+ list.add(k);
+ list.add(v);
+ RESULT = list;
+ :}
+ ;
+
+map_literal ::=
+ LBRACE RBRACE
+ {:
+ RESULT = new MapLiteral();
+ :}
+ | LBRACE kv_list:list RBRACE
+ {:
+ RESULT = new MapLiteral(list.toArray(new LiteralExpr[0]));
+ :}
+ ;
+
struct_field ::=
ident:name COLON type:type
{: RESULT = new StructField(name, type); :}
@@ -5996,6 +6024,8 @@ non_pred_expr ::=
{: RESULT = a; :}
| array_literal:a
{: RESULT = a; :}
+ | map_literal:a
+ {: RESULT = a; :}
| struct_literal:s
{: RESULT = s; :}
| function_call_expr:e
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
index 87dc3d680b..63f5dbf3cd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
@@ -319,6 +319,11 @@ public class CastExpr extends Expr {
type, Function.NullableMode.ALWAYS_NULLABLE,
Lists.newArrayList(Type.VARCHAR), false,
"doris::CastFunctions::cast_to_array_val", null, null, true);
+ } else if (type.isMapType()) {
+ fn = ScalarFunction.createBuiltin(getFnName(Type.MAP),
+ type, Function.NullableMode.ALWAYS_NULLABLE,
+ Lists.newArrayList(Type.VARCHAR), false,
+ "doris::CastFunctions::cast_to_map_val", null, null, true);
} else if (type.isStructType()) {
fn = ScalarFunction.createBuiltin(getFnName(Type.STRUCT),
type, Function.NullableMode.ALWAYS_NULLABLE,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
old mode 100755
new mode 100644
index f1b0ca936e..0f3b0b0dc9
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -1354,7 +1354,7 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
}
public Expr checkTypeCompatibility(Type targetType) throws AnalysisException {
- if (targetType.getPrimitiveType() != PrimitiveType.ARRAY
+ if (targetType.getPrimitiveType() != PrimitiveType.ARRAY && targetType.getPrimitiveType() != PrimitiveType.MAP
&& targetType.getPrimitiveType() == type.getPrimitiveType()) {
if (targetType.isDecimalV2() && type.isDecimalV2()) {
return this;
@@ -1826,7 +1826,8 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
CAST_EXPR(14),
JSON_LITERAL(15),
ARITHMETIC_EXPR(16),
- STRUCT_LITERAL(17);
+ STRUCT_LITERAL(17),
+ MAP_LITERAL(18);
private static Map<Integer, ExprSerCode> codeMap = Maps.newHashMap();
@@ -1878,6 +1879,8 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
output.writeInt(ExprSerCode.FUNCTION_CALL.getCode());
} else if (expr instanceof ArrayLiteral) {
output.writeInt(ExprSerCode.ARRAY_LITERAL.getCode());
+ } else if (expr instanceof MapLiteral) {
+ output.writeInt(ExprSerCode.MAP_LITERAL.getCode());
} else if (expr instanceof StructLiteral) {
output.writeInt(ExprSerCode.STRUCT_LITERAL.getCode());
} else if (expr instanceof CastExpr) {
@@ -1929,6 +1932,8 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
return FunctionCallExpr.read(in);
case ARRAY_LITERAL:
return ArrayLiteral.read(in);
+ case MAP_LITERAL:
+ return MapLiteral.read(in);
case STRUCT_LITERAL:
return StructLiteral.read(in);
case CAST_EXPR:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java
new file mode 100644
index 0000000000..92f558641a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/MapLiteral.java
@@ -0,0 +1,177 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.MapType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+import org.apache.doris.thrift.TTypeDesc;
+import org.apache.doris.thrift.TTypeNode;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+
+// INSERT INTO table_map VALUES ({'key1':1, 'key2':10, 'k3':100}), ({'key1':2,'key2':20}), ({'key1':3,'key2':30});
+// MapLiteral is one row-based literal
+public class MapLiteral extends LiteralExpr {
+
+ public MapLiteral() {
+ type = new MapType(Type.NULL, Type.NULL);
+ children = new ArrayList<>();
+ }
+
+ public MapLiteral(LiteralExpr... exprs) throws AnalysisException {
+ Type keyType = Type.NULL;
+ Type valueType = Type.NULL;
+ children = new ArrayList<>();
+ int idx = 0;
+ for (LiteralExpr expr : exprs) {
+ if (idx % 2 == 0) {
+ if (keyType == Type.NULL) {
+ keyType = expr.getType();
+ } else {
+ keyType = Type.getAssignmentCompatibleType(keyType, expr.getType(), true);
+ }
+ if (keyType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ } else {
+ if (valueType == Type.NULL) {
+ valueType = expr.getType();
+ } else {
+ valueType = Type.getAssignmentCompatibleType(valueType, expr.getType(), true);
+ }
+ if (valueType == Type.INVALID) {
+ throw new AnalysisException("Invalid element type in Map");
+ }
+ }
+ children.add(expr);
+ ++ idx;
+ }
+
+ type = new MapType(keyType, valueType);
+ }
+
+ protected MapLiteral(MapLiteral other) {
+ super(other);
+ }
+
+ @Override
+ public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
+ if (!targetType.isMapType()) {
+ return super.uncheckedCastTo(targetType);
+ }
+ MapLiteral literal = new MapLiteral(this);
+ Type keyType = ((MapType) targetType).getKeyType();
+ Type valueType = ((MapType) targetType).getValueType();
+
+ for (int i = 0; i < children.size(); ++ i) {
+ Expr child = children.get(i);
+ if ((i % 2) == 0) {
+ literal.children.set(i, child.uncheckedCastTo(keyType));
+ } else {
+ literal.children.set(i, child.uncheckedCastTo(valueType));
+ }
+ }
+ literal.setType(targetType);
+ return literal;
+ }
+
+ @Override
+ public void checkValueValid() throws AnalysisException {
+ for (Expr e : children) {
+ e.checkValueValid();
+ }
+ }
+
+ @Override
+ protected String toSqlImpl() {
+ List<String> list = new ArrayList<>(children.size());
+ for (int i = 0; i < children.size(); i += 2) {
+ list.add(children.get(i).toSqlImpl() + ":" + children.get(i + 1).toSqlImpl());
+ }
+ return "MAP{" + StringUtils.join(list, ", ") + "}";
+ }
+
+ @Override
+ protected void toThrift(TExprNode msg) {
+ msg.node_type = TExprNodeType.MAP_LITERAL;
+ TTypeDesc container = new TTypeDesc();
+ container.setTypes(new ArrayList<TTypeNode>());
+ type.toThrift(container);
+ msg.setType(container);
+ }
+
+ @Override
+ public Expr clone() {
+ return new MapLiteral(this);
+ }
+
+ @Override
+ public boolean isMinValue() {
+ return false;
+ }
+
+ @Override
+ public int compareLiteral(LiteralExpr expr) {
+ return 0;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ int size = in.readInt();
+ children = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ children.add(Expr.readIn(in));
+ }
+ }
+
+ public static MapLiteral read(DataInput in) throws IOException {
+ MapLiteral literal = new MapLiteral();
+ literal.readFields(in);
+ return literal;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ super.write(out);
+ out.writeInt(children.size());
+ for (Expr e : children) {
+ Expr.writeTo(e, out);
+ }
+ }
+
+ @Override
+ public String getStringValue() {
+ return toSqlImpl();
+ }
+
+ @Override
+ public String getStringValueForArray() {
+ return null;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 3551908d7a..a4f00f4bf3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -63,6 +63,8 @@ public class Column implements Writable, GsonPostProcessable {
private static final String COLUMN_ARRAY_CHILDREN = "item";
private static final String COLUMN_STRUCT_CHILDREN = "field";
public static final int COLUMN_UNIQUE_ID_INIT_VALUE = -1;
+ private static final String COLUMN_MAP_KEY = "key";
+ private static final String COLUMN_MAP_VALUE = "value";
public static final Column UNSUPPORTED_COLUMN = new Column("unknown",
Type.UNSUPPORTED, true, null, true, null, "invalid", true, null, -1, null);
@@ -211,6 +213,11 @@ public class Column implements Writable, GsonPostProcessable {
Column c = new Column(COLUMN_ARRAY_CHILDREN, ((ArrayType) type).getItemType());
c.setIsAllowNull(((ArrayType) type).getContainsNull());
column.addChildrenColumn(c);
+ } else if (type.isMapType()) {
+ Column k = new Column(COLUMN_MAP_KEY, ((MapType) type).getKeyType());
+ Column v = new Column(COLUMN_MAP_VALUE, ((MapType) type).getValueType());
+ column.addChildrenColumn(k);
+ column.addChildrenColumn(v);
} else if (type.isStructType()) {
ArrayList<StructField> fields = ((StructType) type).getFields();
for (StructField field : fields) {
@@ -467,6 +474,12 @@ public class Column implements Writable, GsonPostProcessable {
Column children = column.getChildren().get(0);
tColumn.setChildrenColumn(new ArrayList<>());
setChildrenTColumn(children, tColumn);
+ } else if (column.type.isMapType()) {
+ Column k = column.getChildren().get(0);
+ Column v = column.getChildren().get(1);
+ tColumn.setChildrenColumn(new ArrayList<>());
+ setChildrenTColumn(k, tColumn);
+ setChildrenTColumn(v, tColumn);
} else if (column.type.isStructType()) {
List<Column> childrenColumns = column.getChildren();
tColumn.setChildrenColumn(new ArrayList<>());
@@ -476,6 +489,7 @@ public class Column implements Writable, GsonPostProcessable {
}
}
+
public void checkSchemaChangeAllowed(Column other) throws DdlException {
if (Strings.isNullOrEmpty(other.name)) {
throw new DdlException("Dest column name is empty");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
index a3b1e36c08..35c0d40b2a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
@@ -1275,10 +1275,9 @@ public class FunctionSet<T> {
final Type[] candicateArgTypes = candicate.getArgs();
if (!(descArgTypes[0] instanceof ScalarType)
|| !(candicateArgTypes[0] instanceof ScalarType)) {
- if (candicateArgTypes[0] instanceof ArrayType) {
+ if (candicateArgTypes[0] instanceof ArrayType || candicateArgTypes[0] instanceof MapType) {
return descArgTypes[0].matchesType(candicateArgTypes[0]);
}
-
return false;
}
final ScalarType descArgType = (ScalarType) descArgTypes[0];
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
index 4dcb130087..877f4c28d0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
@@ -87,6 +87,7 @@ public class Util {
TYPE_STRING_MAP.put(PrimitiveType.BITMAP, "bitmap");
TYPE_STRING_MAP.put(PrimitiveType.QUANTILE_STATE, "quantile_state");
TYPE_STRING_MAP.put(PrimitiveType.ARRAY, "Array<%s>");
+ TYPE_STRING_MAP.put(PrimitiveType.MAP, "Map<%s,%s>");
TYPE_STRING_MAP.put(PrimitiveType.NULL_TYPE, "null");
}
diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex
index bd59b63e61..e974156574 100644
--- a/fe/fe-core/src/main/jflex/sql_scanner.flex
+++ b/fe/fe-core/src/main/jflex/sql_scanner.flex
@@ -670,6 +670,8 @@ EndOfLineComment = "--" !({HintContent}|{ContainsLineTerminator}) {LineTerminato
"!" { return newToken(SqlParserSymbols.NOT, null); }
"<" { return newToken(SqlParserSymbols.LESSTHAN, null); }
">" { return newToken(SqlParserSymbols.GREATERTHAN, null); }
+"{" { return newToken(SqlParserSymbols.LBRACE, null); }
+"}" { return newToken(SqlParserSymbols.RBRACE, null); }
"\"" { return newToken(SqlParserSymbols.UNMATCHED_STRING_LITERAL, null); }
"'" { return newToken(SqlParserSymbols.UNMATCHED_STRING_LITERAL, null); }
"`" { return newToken(SqlParserSymbols.UNMATCHED_STRING_LITERAL, null); }
diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py
index 26e93be7b5..a4f83e9fbd 100644
--- a/gensrc/script/doris_builtins_functions.py
+++ b/gensrc/script/doris_builtins_functions.py
@@ -99,6 +99,9 @@ visible_functions = [
[['element_at', '%element_extract%'], 'VARCHAR', ['ARRAY_VARCHAR', 'BIGINT'], 'ALWAYS_NULLABLE'],
[['element_at', '%element_extract%'], 'STRING', ['ARRAY_STRING', 'BIGINT'], 'ALWAYS_NULLABLE'],
+ # map element
+ [['element_at', '%element_extract%'], 'INT', ['MAP_STRING_INT', 'STRING'], 'ALWAYS_NULLABLE'],
+
[['arrays_overlap'], 'BOOLEAN', ['ARRAY_BOOLEAN', 'ARRAY_BOOLEAN'], 'ALWAYS_NULLABLE'],
[['arrays_overlap'], 'BOOLEAN', ['ARRAY_TINYINT', 'ARRAY_TINYINT'], 'ALWAYS_NULLABLE'],
[['arrays_overlap'], 'BOOLEAN', ['ARRAY_SMALLINT', 'ARRAY_SMALLINT'], 'ALWAYS_NULLABLE'],
diff --git a/gensrc/script/gen_builtins_functions.py b/gensrc/script/gen_builtins_functions.py
index e47020a3a2..28b8812142 100755
--- a/gensrc/script/gen_builtins_functions.py
+++ b/gensrc/script/gen_builtins_functions.py
@@ -53,6 +53,7 @@ java_registry_preamble = '\
package org.apache.doris.builtins;\n\
\n\
import org.apache.doris.catalog.ArrayType;\n\
+import org.apache.doris.catalog.MapType;\n\
import org.apache.doris.catalog.Type;\n\
import org.apache.doris.catalog.Function;\n\
import org.apache.doris.catalog.FunctionSet;\n\
@@ -100,12 +101,17 @@ for example:
in[TINYINT] --> out[Type.TINYINT]
in[INT] --> out[Type.INT]
in[ARRAY_INT] --> out[new ArrayType(Type.INT)]
+ in[MAP_STRING_INT] --> out[new MapType(Type.STRING,Type.INT)]
"""
def generate_fe_datatype(str_type):
if str_type.startswith("ARRAY_"):
vec_type = str_type.split('_', 1);
if len(vec_type) > 1 and vec_type[0] == "ARRAY":
return "new ArrayType(" + generate_fe_datatype(vec_type[1]) + ")"
+ if str_type.startswith("MAP_"):
+ vec_type = str_type.split('_', 2)
+ if len(vec_type) > 2 and vec_type[0] == "MAP":
+ return "new MapType(" + generate_fe_datatype(vec_type[1]) + "," + generate_fe_datatype(vec_type[2])+")"
if str_type == "DECIMALV2":
return "Type.MAX_DECIMALV2_TYPE"
if str_type == "DECIMAL32":
diff --git a/gensrc/thrift/Exprs.thrift b/gensrc/thrift/Exprs.thrift
index e507a8e7aa..0db378f037 100644
--- a/gensrc/thrift/Exprs.thrift
+++ b/gensrc/thrift/Exprs.thrift
@@ -61,6 +61,9 @@ enum TExprNodeType {
// for fulltext search
MATCH_PRED,
+
+ // for map
+ MAP_LITERAL,
}
//enum TAggregationOp {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 05/18: [WIP](struct-type) support struct-type in vectorize engine (#15665)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3d68b650eb9d82a8d7881f772150f4abe6d629b5
Author: xy720 <22...@users.noreply.github.com>
AuthorDate: Thu Jan 12 14:40:08 2023 +0800
[WIP](struct-type) support struct-type in vectorize engine (#15665)
---
be/src/olap/field.h | 150 +++++++++++++
be/src/olap/olap_define.h | 3 +
be/src/olap/rowset/segment_v2/column_reader.cpp | 112 ++++++++++
be/src/olap/rowset/segment_v2/column_reader.h | 38 ++++
be/src/olap/rowset/segment_v2/column_writer.cpp | 174 +++++++++++++++
be/src/olap/rowset/segment_v2/column_writer.h | 51 +++++
be/src/olap/rowset/segment_v2/segment_writer.cpp | 9 +
be/src/olap/tablet_meta.cpp | 8 +-
be/src/olap/tablet_schema.cpp | 20 +-
be/src/olap/types.cpp | 52 ++++-
be/src/olap/types.h | 245 ++++++++++++++++++++-
be/src/runtime/CMakeLists.txt | 1 +
be/src/runtime/primitive_type.cpp | 14 ++
be/src/runtime/primitive_type.h | 1 +
be/src/runtime/struct_value.cpp | 27 +++
be/src/runtime/struct_value.h | 67 ++++++
be/src/runtime/types.cpp | 48 +++-
be/src/runtime/types.h | 2 +-
be/src/udf/udf.h | 1 +
be/src/vec/CMakeLists.txt | 1 +
be/src/vec/columns/column_struct.cpp | 34 +--
be/src/vec/columns/column_struct.h | 15 +-
be/src/vec/data_types/data_type_factory.cpp | 41 +++-
be/src/vec/data_types/data_type_factory.hpp | 4 +
be/src/vec/data_types/data_type_struct.cpp | 33 ++-
be/src/vec/data_types/data_type_struct.h | 12 +-
be/src/vec/exprs/vcast_expr.cpp | 1 +
be/src/vec/exprs/vexpr.cpp | 5 +
be/src/vec/exprs/vstruct_literal.cpp | 37 ++++
be/src/vec/exprs/vstruct_literal.h | 34 +++
be/src/vec/functions/function_cast.h | 1 +
be/src/vec/olap/olap_data_convertor.cpp | 62 ++++++
be/src/vec/olap/olap_data_convertor.h | 21 ++
be/src/vec/sink/vmysql_result_writer.cpp | 100 +++++++--
be/src/vec/sink/vmysql_result_writer.h | 6 +-
be/src/vec/sink/vtablet_sink.cpp | 3 +-
.../java/org/apache/doris/catalog/ArrayType.java | 10 +
.../java/org/apache/doris/catalog/MapType.java | 5 +
.../org/apache/doris/catalog/PrimitiveType.java | 5 +-
.../java/org/apache/doris/catalog/StructField.java | 55 ++++-
.../java/org/apache/doris/catalog/StructType.java | 94 +++++++-
.../main/java/org/apache/doris/catalog/Type.java | 24 +-
fe/fe-core/src/main/cup/sql_parser.cup | 33 ++-
.../java/org/apache/doris/analysis/ColumnDef.java | 4 +
.../org/apache/doris/analysis/CreateTableStmt.java | 2 +-
.../main/java/org/apache/doris/analysis/Expr.java | 7 +-
.../org/apache/doris/analysis/StructLiteral.java | 150 +++++++++++++
.../java/org/apache/doris/analysis/Subquery.java | 2 +-
.../java/org/apache/doris/analysis/TypeDef.java | 47 ++--
.../main/java/org/apache/doris/catalog/Column.java | 66 +++---
fe/fe-core/src/main/jflex/sql_scanner.flex | 4 +
gensrc/proto/types.proto | 1 +
gensrc/thrift/Exprs.thrift | 1 +
gensrc/thrift/Types.thrift | 1 +
54 files changed, 1775 insertions(+), 169 deletions(-)
diff --git a/be/src/olap/field.h b/be/src/olap/field.h
index 2db058d14a..e3f5ede607 100644
--- a/be/src/olap/field.h
+++ b/be/src/olap/field.h
@@ -211,6 +211,9 @@ protected:
TypeInfoPtr _type_info;
// unit : byte
// except for strings, other types have fixed lengths
+ // Note that, the struct type itself has fixed length, but due to
+ // its number of subfields is a variable, so the actual length of
+ // a struct field is not fixed.
uint32_t _length;
// Since the length of the STRING type cannot be determined,
// only dynamic memory can be used. Mempool cannot realize realloc.
@@ -256,6 +259,135 @@ private:
int32_t _unique_id;
};
+<<<<<<< HEAD
+=======
+template <typename LhsCellType, typename RhsCellType>
+int Field::index_cmp(const LhsCellType& lhs, const RhsCellType& rhs) const {
+ bool l_null = lhs.is_null();
+ bool r_null = rhs.is_null();
+ if (l_null != r_null) {
+ return l_null ? -1 : 1;
+ } else if (l_null) {
+ return 0;
+ }
+
+ int32_t res = 0;
+ if (type() == OLAP_FIELD_TYPE_VARCHAR || type() == OLAP_FIELD_TYPE_STRING) {
+ const Slice* l_slice = reinterpret_cast<const Slice*>(lhs.cell_ptr());
+ const Slice* r_slice = reinterpret_cast<const Slice*>(rhs.cell_ptr());
+ uint32_t max_bytes =
+ type() == OLAP_FIELD_TYPE_VARCHAR ? OLAP_VARCHAR_MAX_BYTES : OLAP_STRING_MAX_BYTES;
+ if (r_slice->size + max_bytes > _index_size || l_slice->size + max_bytes > _index_size) {
+ // if the actual length of the field is longer than the short key, only the prefix is compared,
+ // make sure all blocks with the same short key are scanned
+ // Otherwise, the short key and field can be compared directly
+ int compare_size = _index_size - max_bytes;
+ // l_slice size and r_slice size may be less than compare_size
+ // so calculate the min of the three size as new compare_size
+ compare_size = std::min(std::min(compare_size, (int)l_slice->size), (int)r_slice->size);
+
+ // This function is used to compare prefix index.
+ // Only the fixed length of prefix index should be compared.
+ // If r_slice->size > l_slice->size, ignore the extra parts directly.
+ res = strncmp(l_slice->data, r_slice->data, compare_size);
+ if (res == 0 && compare_size != (_index_size - max_bytes)) {
+ if (l_slice->size < r_slice->size) {
+ res = -1;
+ } else if (l_slice->size > r_slice->size) {
+ res = 1;
+ } else {
+ res = 0;
+ }
+ }
+ } else {
+ res = l_slice->compare(*r_slice);
+ }
+ } else {
+ res = _type_info->cmp(lhs.cell_ptr(), rhs.cell_ptr());
+ }
+
+ return res;
+}
+
+template <typename DstCellType, typename SrcCellType>
+void Field::to_index(DstCellType* dst, const SrcCellType& src) const {
+ bool is_null = src.is_null();
+ dst->set_is_null(is_null);
+ if (is_null) {
+ return;
+ }
+
+ if (type() == OLAP_FIELD_TYPE_VARCHAR) {
+ // clear before copy
+ memset(dst->mutable_cell_ptr(), 0, _index_size);
+ const Slice* slice = reinterpret_cast<const Slice*>(src.cell_ptr());
+ size_t copy_size = slice->size < _index_size - OLAP_VARCHAR_MAX_BYTES
+ ? slice->size
+ : _index_size - OLAP_VARCHAR_MAX_BYTES;
+ *reinterpret_cast<VarcharLengthType*>(dst->mutable_cell_ptr()) = copy_size;
+ memory_copy((char*)dst->mutable_cell_ptr() + OLAP_VARCHAR_MAX_BYTES, slice->data,
+ copy_size);
+ } else if (type() == OLAP_FIELD_TYPE_STRING) {
+ // clear before copy
+ memset(dst->mutable_cell_ptr(), 0, _index_size);
+ const Slice* slice = reinterpret_cast<const Slice*>(src.cell_ptr());
+ size_t copy_size = slice->size < _index_size - OLAP_STRING_MAX_BYTES
+ ? slice->size
+ : _index_size - OLAP_STRING_MAX_BYTES;
+ *reinterpret_cast<StringLengthType*>(dst->mutable_cell_ptr()) = copy_size;
+ memory_copy((char*)dst->mutable_cell_ptr() + OLAP_STRING_MAX_BYTES, slice->data, copy_size);
+ } else if (type() == OLAP_FIELD_TYPE_CHAR) {
+ // clear before copy
+ memset(dst->mutable_cell_ptr(), 0, _index_size);
+ const Slice* slice = reinterpret_cast<const Slice*>(src.cell_ptr());
+ memory_copy(dst->mutable_cell_ptr(), slice->data, _index_size);
+ } else {
+ memory_copy(dst->mutable_cell_ptr(), src.cell_ptr(), size());
+ }
+}
+
+template <typename CellType>
+uint32_t Field::hash_code(const CellType& cell, uint32_t seed) const {
+ bool is_null = cell.is_null();
+ if (is_null) {
+ return HashUtil::hash(&is_null, sizeof(is_null), seed);
+ }
+ return _type_info->hash_code(cell.cell_ptr(), seed);
+}
+
+class StructField : public Field {
+public:
+ explicit StructField(const TabletColumn& column) : Field(column) {}
+
+ void consume(RowCursorCell* dst, const char* src, bool src_null, MemPool* mem_pool,
+ ObjectPool* agg_pool) const override {
+ dst->set_is_null(src_null);
+ if (src_null) {
+ return;
+ }
+ _type_info->deep_copy(dst->mutable_cell_ptr(), src, mem_pool);
+ }
+
+ char* allocate_memory(char* cell_ptr, char* variable_ptr) const override {
+ auto struct_v = (StructValue*)cell_ptr;
+ struct_v->set_values(reinterpret_cast<void**>(variable_ptr));
+ variable_ptr += _length;
+ for (size_t i = 0; i < get_sub_field_count(); i++) {
+ variable_ptr += get_sub_field(i)->get_variable_len();
+ }
+ return variable_ptr;
+ }
+
+ size_t get_variable_len() const override {
+ size_t variable_len = _length;
+ for (size_t i = 0; i < get_sub_field_count(); i++) {
+ variable_len += get_sub_field(i)->get_variable_len();
+ }
+ return variable_len;
+ }
+};
+
+>>>>>>> [WIP](struct-type) support struct-type in vectorize engine (#15665)
class ArrayField : public Field {
public:
explicit ArrayField(const TabletColumn& column) : Field(column) {}
@@ -509,6 +641,15 @@ public:
return new VarcharField(column);
case OLAP_FIELD_TYPE_STRING:
return new StringField(column);
+ case OLAP_FIELD_TYPE_STRUCT: {
+ auto* local = new StructField(column);
+ for (uint32_t i = 0; i < column.get_subtype_count(); i++) {
+ std::unique_ptr<Field> sub_field(
+ FieldFactory::create(column.get_sub_column(i)));
+ local->add_sub_field(std::move(sub_field));
+ }
+ return local;
+ }
case OLAP_FIELD_TYPE_ARRAY: {
std::unique_ptr<Field> item_field(FieldFactory::create(column.get_sub_column(0)));
auto* local = new ArrayField(column);
@@ -549,6 +690,15 @@ public:
return new VarcharField(column);
case OLAP_FIELD_TYPE_STRING:
return new StringField(column);
+ case OLAP_FIELD_TYPE_STRUCT: {
+ auto* local = new StructField(column);
+ for (uint32_t i = 0; i < column.get_subtype_count(); i++) {
+ std::unique_ptr<Field> sub_field(
+ FieldFactory::create(column.get_sub_column(i)));
+ local->add_sub_field(std::move(sub_field));
+ }
+ return local;
+ }
case OLAP_FIELD_TYPE_ARRAY: {
std::unique_ptr<Field> item_field(FieldFactory::create(column.get_sub_column(0)));
auto* local = new ArrayField(column);
diff --git a/be/src/olap/olap_define.h b/be/src/olap/olap_define.h
index a9c92c69e2..fadbea6834 100644
--- a/be/src/olap/olap_define.h
+++ b/be/src/olap/olap_define.h
@@ -60,6 +60,9 @@ static const uint32_t OLAP_STRING_MAX_LENGTH = 2147483647;
// the max length supported for jsonb type 2G
static const uint32_t OLAP_JSONB_MAX_LENGTH = 2147483647;
+// the max length supported for struct, but excluding the length of its subtypes.
+static const uint16_t OLAP_STRUCT_MAX_LENGTH = 65535;
+
// the max length supported for array
static const uint16_t OLAP_ARRAY_MAX_LENGTH = 65535;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index eaf1c251aa..07feb00c20 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -30,6 +30,7 @@
#include "util/rle_encoding.h" // for RleDecoder
#include "vec/columns/column.h"
#include "vec/columns/column_array.h"
+#include "vec/columns/column_struct.h"
#include "vec/core/types.h"
#include "vec/runtime/vdatetime_value.h" //for VecDateTime
@@ -48,6 +49,23 @@ Status ColumnReader::create(const ColumnReaderOptions& opts, const ColumnMetaPB&
} else {
auto type = (FieldType)meta.type();
switch (type) {
+ case FieldType::OLAP_FIELD_TYPE_STRUCT: {
+ // not support empty struct
+ DCHECK(meta.children_columns_size() >= 1);
+ // create struct column reader
+ std::unique_ptr<ColumnReader> struct_reader(
+ new ColumnReader(opts, meta, num_rows, file_reader));
+ struct_reader->_sub_readers.reserve(meta.children_columns_size());
+ for (size_t i = 0; i < meta.children_columns_size(); i++) {
+ std::unique_ptr<ColumnReader> sub_reader;
+ RETURN_IF_ERROR(ColumnReader::create(opts, meta.children_columns(i),
+ meta.children_columns(i).num_rows(),
+ file_reader, &sub_reader));
+ struct_reader->_sub_readers.push_back(std::move(sub_reader));
+ }
+ *reader = std::move(struct_reader);
+ return Status::OK();
+ }
case FieldType::OLAP_FIELD_TYPE_ARRAY: {
DCHECK(meta.children_columns_size() == 2 || meta.children_columns_size() == 3);
@@ -442,6 +460,24 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
} else {
auto type = (FieldType)_meta.type();
switch (type) {
+ case FieldType::OLAP_FIELD_TYPE_STRUCT: {
+ std::vector<ColumnIterator*> sub_column_iterators;
+ size_t child_size = is_nullable() ? _sub_readers.size() - 1 : _sub_readers.size();
+ sub_column_iterators.reserve(child_size);
+
+ ColumnIterator* sub_column_iterator;
+ for (size_t i = 0; i < child_size; i++) {
+ RETURN_IF_ERROR(_sub_readers[i]->new_iterator(&sub_column_iterator));
+ sub_column_iterators.push_back(sub_column_iterator);
+ }
+
+ ColumnIterator* null_iterator = nullptr;
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_sub_readers[child_size]->new_iterator(&null_iterator));
+ }
+ *iterator = new StructFileColumnIterator(this, null_iterator, sub_column_iterators);
+ return Status::OK();
+ }
case FieldType::OLAP_FIELD_TYPE_ARRAY: {
ColumnIterator* item_iterator = nullptr;
RETURN_IF_ERROR(_sub_readers[0]->new_iterator(&item_iterator));
@@ -467,6 +503,82 @@ Status ColumnReader::new_iterator(ColumnIterator** iterator) {
////////////////////////////////////////////////////////////////////////////////
+StructFileColumnIterator::StructFileColumnIterator(
+ ColumnReader* reader, ColumnIterator* null_iterator,
+ std::vector<ColumnIterator*>& sub_column_iterators)
+ : _struct_reader(reader) {
+ _sub_column_iterators.resize(sub_column_iterators.size());
+ for (size_t i = 0; i < sub_column_iterators.size(); i++) {
+ _sub_column_iterators[i].reset(sub_column_iterators[i]);
+ }
+ if (_struct_reader->is_nullable()) {
+ _null_iterator.reset(null_iterator);
+ }
+}
+
+Status StructFileColumnIterator::init(const ColumnIteratorOptions& opts) {
+ for (auto& column_iterator : _sub_column_iterators) {
+ RETURN_IF_ERROR(column_iterator->init(opts));
+ }
+ if (_struct_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->init(opts));
+ }
+ return Status::OK();
+}
+
+Status StructFileColumnIterator::next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) {
+ return Status::NotSupported("not supported");
+}
+
+Status StructFileColumnIterator::next_batch(size_t* n, vectorized::MutableColumnPtr& dst,
+ bool* has_null) {
+ const auto* column_struct = vectorized::check_and_get_column<vectorized::ColumnStruct>(
+ dst->is_nullable() ? static_cast<vectorized::ColumnNullable&>(*dst).get_nested_column()
+ : *dst);
+ for (size_t i = 0; i < column_struct->tuple_size(); i++) {
+ size_t num_read = *n;
+ auto sub_column_ptr = column_struct->get_column(i).assume_mutable();
+ bool column_has_null = false;
+ RETURN_IF_ERROR(
+ _sub_column_iterators[i]->next_batch(&num_read, sub_column_ptr, &column_has_null));
+ DCHECK(num_read == *n);
+ }
+
+ if (dst->is_nullable()) {
+ size_t num_read = *n;
+ auto null_map_ptr =
+ static_cast<vectorized::ColumnNullable&>(*dst).get_null_map_column_ptr();
+ bool null_signs_has_null = false;
+ RETURN_IF_ERROR(_null_iterator->next_batch(&num_read, null_map_ptr, &null_signs_has_null));
+ DCHECK(num_read == *n);
+ }
+
+ return Status::OK();
+}
+
+Status StructFileColumnIterator::seek_to_ordinal(ordinal_t ord) {
+ for (auto& column_iterator : _sub_column_iterators) {
+ RETURN_IF_ERROR(column_iterator->seek_to_ordinal(ord));
+ }
+ if (_struct_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->seek_to_ordinal(ord));
+ }
+ return Status::OK();
+}
+
+Status StructFileColumnIterator::read_by_rowids(const rowid_t* rowids, const size_t count,
+ vectorized::MutableColumnPtr& dst) {
+ for (size_t i = 0; i < count; ++i) {
+ RETURN_IF_ERROR(seek_to_ordinal(rowids[i]));
+ size_t num_read = 1;
+ RETURN_IF_ERROR(next_batch(&num_read, dst, nullptr));
+ DCHECK(num_read == 1);
+ }
+ return Status::OK();
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
ArrayFileColumnIterator::ArrayFileColumnIterator(ColumnReader* reader,
FileColumnIterator* offset_reader,
ColumnIterator* item_iterator,
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 7c03f4537a..6e699027cc 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -376,6 +376,44 @@ public:
ordinal_t get_current_ordinal() const override { return 0; }
};
+class StructFileColumnIterator final : public ColumnIterator {
+public:
+ explicit StructFileColumnIterator(ColumnReader* reader, ColumnIterator* null_iterator,
+ std::vector<ColumnIterator*>& sub_column_iterators);
+
+ ~StructFileColumnIterator() override = default;
+
+ Status init(const ColumnIteratorOptions& opts) override;
+
+ Status next_batch(size_t* n, ColumnBlockView* dst, bool* has_null) override;
+
+ Status next_batch(size_t* n, vectorized::MutableColumnPtr& dst, bool* has_null) override;
+
+ Status read_by_rowids(const rowid_t* rowids, const size_t count,
+ vectorized::MutableColumnPtr& dst) override;
+
+ Status seek_to_first() override {
+ for (auto& column_iterator : _sub_column_iterators) {
+ RETURN_IF_ERROR(column_iterator->seek_to_first());
+ }
+ if (_struct_reader->is_nullable()) {
+ RETURN_IF_ERROR(_null_iterator->seek_to_first());
+ }
+ return Status::OK();
+ }
+
+ Status seek_to_ordinal(ordinal_t ord) override;
+
+ ordinal_t get_current_ordinal() const override {
+ return _sub_column_iterators[0]->get_current_ordinal();
+ }
+
+private:
+ ColumnReader* _struct_reader;
+ std::unique_ptr<ColumnIterator> _null_iterator;
+ std::vector<std::unique_ptr<ColumnIterator>> _sub_column_iterators;
+};
+
class ArrayFileColumnIterator final : public ColumnIterator {
public:
explicit ArrayFileColumnIterator(ColumnReader* reader, FileColumnIterator* offset_reader,
diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp
index 9a76e32c91..5b37d48fa6 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -88,6 +88,78 @@ Status ColumnWriter::create(const ColumnWriterOptions& opts, const TabletColumn*
return Status::OK();
} else {
switch (column->type()) {
+ case FieldType::OLAP_FIELD_TYPE_STRUCT: {
+ // not support empty struct
+ DCHECK(column->get_subtype_count() >= 1);
+ std::vector<std::unique_ptr<ColumnWriter>> sub_column_writers;
+ sub_column_writers.reserve(column->get_subtype_count());
+ for (uint32_t i = 0; i < column->get_subtype_count(); i++) {
+ const TabletColumn& sub_column = column->get_sub_column(i);
+
+ // create sub writer
+ ColumnWriterOptions column_options;
+ column_options.meta = opts.meta->mutable_children_columns(i);
+ column_options.need_zone_map = false;
+ column_options.need_bloom_filter = sub_column.is_bf_column();
+ column_options.need_bitmap_index = sub_column.has_bitmap_index();
+ column_options.inverted_index = nullptr;
+ if (sub_column.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ if (column_options.need_bloom_filter) {
+ return Status::NotSupported("Do not support bloom filter for struct type");
+ }
+ if (column_options.need_bitmap_index) {
+ return Status::NotSupported("Do not support bitmap index for struct type");
+ }
+ }
+ if (sub_column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) {
+ if (column_options.need_bloom_filter) {
+ return Status::NotSupported("Do not support bloom filter for array type");
+ }
+ if (column_options.need_bitmap_index) {
+ return Status::NotSupported("Do not support bitmap index for array type");
+ }
+ }
+ std::unique_ptr<ColumnWriter> sub_column_writer;
+ RETURN_IF_ERROR(ColumnWriter::create(column_options, &sub_column, file_writer,
+ &sub_column_writer));
+ sub_column_writers.push_back(std::move(sub_column_writer));
+ }
+
+ // if nullable, create null writer
+ ScalarColumnWriter* null_writer = nullptr;
+ if (opts.meta->is_nullable()) {
+ FieldType null_type = FieldType::OLAP_FIELD_TYPE_TINYINT;
+ ColumnWriterOptions null_options;
+ null_options.meta = opts.meta->add_children_columns();
+ null_options.meta->set_column_id(column->get_subtype_count() + 1);
+ null_options.meta->set_unique_id(column->get_subtype_count() + 1);
+ null_options.meta->set_type(null_type);
+ null_options.meta->set_is_nullable(false);
+ null_options.meta->set_length(
+ get_scalar_type_info<OLAP_FIELD_TYPE_TINYINT>()->size());
+ null_options.meta->set_encoding(DEFAULT_ENCODING);
+ null_options.meta->set_compression(opts.meta->compression());
+
+ null_options.need_zone_map = false;
+ null_options.need_bloom_filter = false;
+ null_options.need_bitmap_index = false;
+
+ TabletColumn null_column = TabletColumn(
+ OLAP_FIELD_AGGREGATION_NONE, null_type, null_options.meta->is_nullable(),
+ null_options.meta->unique_id(), null_options.meta->length());
+ null_column.set_name("nullable");
+ null_column.set_index_length(-1); // no short key index
+ std::unique_ptr<Field> null_field(FieldFactory::create(null_column));
+ null_writer =
+ new ScalarColumnWriter(null_options, std::move(null_field), file_writer);
+ }
+
+ std::unique_ptr<ColumnWriter> writer_local =
+ std::unique_ptr<ColumnWriter>(new StructColumnWriter(
+ opts, std::move(field), null_writer, sub_column_writers));
+ *writer = std::move(writer_local);
+ return Status::OK();
+ }
case FieldType::OLAP_FIELD_TYPE_ARRAY: {
DCHECK(column->get_subtype_count() == 1);
const TabletColumn& item_column = column->get_sub_column(0);
@@ -541,6 +613,108 @@ Status ScalarColumnWriter::finish_current_page() {
////////////////////////////////////////////////////////////////////////////////
+StructColumnWriter::StructColumnWriter(
+ const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
+ ScalarColumnWriter* null_writer,
+ std::vector<std::unique_ptr<ColumnWriter>>& sub_column_writers)
+ : ColumnWriter(std::move(field), opts.meta->is_nullable()), _opts(opts) {
+ for (auto& sub_column_writer : sub_column_writers) {
+ _sub_column_writers.push_back(std::move(sub_column_writer));
+ }
+ _num_sub_column_writers = _sub_column_writers.size();
+ DCHECK(_num_sub_column_writers >= 1);
+ if (is_nullable()) {
+ _null_writer.reset(null_writer);
+ }
+}
+
+Status StructColumnWriter::init() {
+ for (auto& column_writer : _sub_column_writers) {
+ RETURN_IF_ERROR(column_writer->init());
+ }
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->init());
+ }
+ return Status::OK();
+}
+
+Status StructColumnWriter::write_inverted_index() {
+ if (_opts.inverted_index) {
+ for (auto& column_writer : _sub_column_writers) {
+ RETURN_IF_ERROR(column_writer->write_inverted_index());
+ }
+ }
+ return Status::OK();
+}
+
+Status StructColumnWriter::append_nullable(const uint8_t* null_map, const uint8_t** ptr,
+ size_t num_rows) {
+ RETURN_IF_ERROR(append_data(ptr, num_rows));
+ RETURN_IF_ERROR(_null_writer->append_data(&null_map, num_rows));
+ return Status::OK();
+}
+
+Status StructColumnWriter::append_data(const uint8_t** ptr, size_t num_rows) {
+ auto data_cursor = reinterpret_cast<const void**>(ptr);
+ auto null_map_cursor = data_cursor + _num_sub_column_writers;
+ for (auto& column_writer : _sub_column_writers) {
+ RETURN_IF_ERROR(column_writer->append(reinterpret_cast<const uint8_t*>(*null_map_cursor),
+ *data_cursor, num_rows));
+ data_cursor++;
+ null_map_cursor++;
+ }
+ return Status::OK();
+}
+
+uint64_t StructColumnWriter::estimate_buffer_size() {
+ uint64_t size = 0;
+ for (auto& column_writer : _sub_column_writers) {
+ size += column_writer->estimate_buffer_size();
+ }
+ size += is_nullable() ? _null_writer->estimate_buffer_size() : 0;
+ return size;
+}
+
+Status StructColumnWriter::finish() {
+ for (auto& column_writer : _sub_column_writers) {
+ RETURN_IF_ERROR(column_writer->finish());
+ }
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->finish());
+ }
+ return Status::OK();
+}
+
+Status StructColumnWriter::write_data() {
+ for (auto& column_writer : _sub_column_writers) {
+ RETURN_IF_ERROR(column_writer->write_data());
+ }
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_data());
+ }
+ return Status::OK();
+}
+
+Status StructColumnWriter::write_ordinal_index() {
+ for (auto& column_writer : _sub_column_writers) {
+ RETURN_IF_ERROR(column_writer->write_ordinal_index());
+ }
+ if (is_nullable()) {
+ RETURN_IF_ERROR(_null_writer->write_ordinal_index());
+ }
+ return Status::OK();
+}
+
+Status StructColumnWriter::append_nulls(size_t num_rows) {
+ return Status::NotSupported("struct writer not support append nulls");
+}
+
+Status StructColumnWriter::finish_current_page() {
+ return Status::NotSupported("struct writer has no data, can not finish_current_page");
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
ArrayColumnWriter::ArrayColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
ScalarColumnWriter* offset_writer,
ScalarColumnWriter* null_writer,
diff --git a/be/src/olap/rowset/segment_v2/column_writer.h b/be/src/olap/rowset/segment_v2/column_writer.h
index 5ea7ae654c..007a69a6d9 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.h
+++ b/be/src/olap/rowset/segment_v2/column_writer.h
@@ -265,6 +265,57 @@ private:
FlushPageCallback* _new_page_callback = nullptr;
};
+class StructColumnWriter final : public ColumnWriter {
+public:
+ explicit StructColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
+ ScalarColumnWriter* null_writer,
+ std::vector<std::unique_ptr<ColumnWriter>>& sub_column_writers);
+ ~StructColumnWriter() override = default;
+
+ Status init() override;
+
+ Status append_nullable(const uint8_t* null_map, const uint8_t** data, size_t num_rows);
+ Status append_data(const uint8_t** ptr, size_t num_rows) override;
+
+ uint64_t estimate_buffer_size() override;
+
+ Status finish() override;
+ Status write_data() override;
+ Status write_ordinal_index() override;
+ Status append_nulls(size_t num_rows) override;
+
+ Status finish_current_page() override;
+
+ Status write_zone_map() override {
+ if (_opts.need_zone_map) {
+ return Status::NotSupported("struct not support zone map");
+ }
+ return Status::OK();
+ }
+
+ Status write_bitmap_index() override {
+ if (_opts.need_bitmap_index) {
+ return Status::NotSupported("struct not support bitmap index");
+ }
+ return Status::OK();
+ }
+ Status write_inverted_index() override;
+ Status write_bloom_filter_index() override {
+ if (_opts.need_bloom_filter) {
+ return Status::NotSupported("struct not support bloom filter index");
+ }
+ return Status::OK();
+ }
+
+ ordinal_t get_next_rowid() const override { return _sub_column_writers[0]->get_next_rowid(); }
+
+private:
+ size_t _num_sub_column_writers;
+ std::unique_ptr<ScalarColumnWriter> _null_writer;
+ std::vector<std::unique_ptr<ColumnWriter>> _sub_column_writers;
+ ColumnWriterOptions _opts;
+};
+
class ArrayColumnWriter final : public ColumnWriter, public FlushPageCallback {
public:
explicit ArrayColumnWriter(const ColumnWriterOptions& opts, std::unique_ptr<Field> field,
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index 91806dc0ba..4195d798a1 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -136,6 +136,15 @@ Status SegmentWriter::init(const std::vector<uint32_t>& col_ids, bool has_key) {
break;
}
}
+ if (column.type() == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ opts.need_zone_map = false;
+ if (opts.need_bloom_filter) {
+ return Status::NotSupported("Do not support bloom filter for struct type");
+ }
+ if (opts.need_bitmap_index) {
+ return Status::NotSupported("Do not support bitmap index for struct type");
+ }
+ }
if (column.type() == FieldType::OLAP_FIELD_TYPE_ARRAY) {
opts.need_zone_map = false;
if (opts.need_bloom_filter) {
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 756835f1b7..4fdefb2bb0 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -297,8 +297,12 @@ void TabletMeta::init_column_from_tcolumn(uint32_t unique_id, const TColumn& tco
if (tcolumn.__isset.is_bloom_filter_column) {
column->set_is_bf_column(tcolumn.is_bloom_filter_column);
}
-
- if (tcolumn.column_type.type == TPrimitiveType::ARRAY) {
+ if (tcolumn.column_type.type == TPrimitiveType::STRUCT) {
+ for (size_t i = 0; i < tcolumn.children_column.size(); i++) {
+ ColumnPB* children_column = column->add_children_columns();
+ init_column_from_tcolumn(i, tcolumn.children_column[i], children_column);
+ }
+ } else if (tcolumn.column_type.type == TPrimitiveType::ARRAY) {
ColumnPB* children_column = column->add_children_columns();
init_column_from_tcolumn(0, tcolumn.children_column[0], children_column);
}
diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp
index 787ed8c834..39a45c3422 100644
--- a/be/src/olap/tablet_schema.cpp
+++ b/be/src/olap/tablet_schema.cpp
@@ -311,6 +311,10 @@ uint32_t TabletColumn::get_field_length_by_type(TPrimitiveType::type type, uint3
return string_length + sizeof(OLAP_STRING_MAX_LENGTH);
case TPrimitiveType::JSONB:
return string_length + sizeof(OLAP_JSONB_MAX_LENGTH);
+ case TPrimitiveType::STRUCT:
+ // Note that(xy): this is the length of struct type itself,
+ // the length of its subtypes are not included.
+ return OLAP_STRUCT_MAX_LENGTH;
case TPrimitiveType::ARRAY:
return OLAP_ARRAY_MAX_LENGTH;
case TPrimitiveType::DECIMAL32:
@@ -403,8 +407,13 @@ void TabletColumn::init_from_pb(const ColumnPB& column) {
if (column.has_visible()) {
_visible = column.visible();
}
-
- if (_type == FieldType::OLAP_FIELD_TYPE_ARRAY) {
+ if (_type == FieldType::OLAP_FIELD_TYPE_STRUCT) {
+ for (size_t i = 0; i < column.children_columns_size(); i++) {
+ TabletColumn child_column;
+ child_column.init_from_pb(column.children_columns(i));
+ add_sub_column(child_column);
+ }
+ } else if (_type == FieldType::OLAP_FIELD_TYPE_ARRAY) {
DCHECK(column.children_columns_size() == 1) << "ARRAY type has more than 1 children types.";
TabletColumn child_column;
child_column.init_from_pb(column.children_columns(0));
@@ -436,7 +445,12 @@ void TabletColumn::to_schema_pb(ColumnPB* column) const {
}
column->set_visible(_visible);
- if (_type == OLAP_FIELD_TYPE_ARRAY) {
+ if (_type == OLAP_FIELD_TYPE_STRUCT) {
+ for (size_t i = 0; i < _sub_columns.size(); i++) {
+ ColumnPB* child = column->add_children_columns();
+ _sub_columns[i].to_schema_pb(child);
+ }
+ } else if (_type == OLAP_FIELD_TYPE_ARRAY) {
DCHECK(_sub_columns.size() == 1) << "ARRAY type has more than 1 children types.";
ColumnPB* child = column->add_children_columns();
_sub_columns[0].to_schema_pb(child);
diff --git a/be/src/olap/types.cpp b/be/src/olap/types.cpp
index aa2226a84a..916f7768af 100644
--- a/be/src/olap/types.cpp
+++ b/be/src/olap/types.cpp
@@ -166,10 +166,33 @@ const TypeInfo* get_array_type_info(FieldType leaf_type, int32_t iterations) {
return array_type_Info_arr[leaf_type][iterations];
}
+// Produce a struct type info
+// TODO(xy): Need refactor to this produce method
+const TypeInfo* get_struct_type_info(std::vector<FieldType> field_types) {
+ std::vector<TypeInfoPtr> type_infos;
+ type_infos.reserve(field_types.size());
+ for (FieldType& type : field_types) {
+ if (is_scalar_type(type)) {
+ type_infos.push_back(create_static_type_info_ptr(get_scalar_type_info(type)));
+ } else {
+ // TODO(xy): Not supported nested complex type now
+ }
+ }
+ return new StructTypeInfo(type_infos);
+}
+
// TODO: Support the type info of the nested array with more than 9 depths.
+// TODO(xy): Support the type info of the nested struct
TypeInfoPtr get_type_info(segment_v2::ColumnMetaPB* column_meta_pb) {
FieldType type = (FieldType)column_meta_pb->type();
- if (UNLIKELY(type == OLAP_FIELD_TYPE_ARRAY)) {
+ if (UNLIKELY(type == OLAP_FIELD_TYPE_STRUCT)) {
+ std::vector<FieldType> field_types;
+ for (uint32_t i = 0; i < column_meta_pb->children_columns_size(); i++) {
+ const auto* child_column = &column_meta_pb->children_columns(i);
+ field_types.push_back((FieldType)child_column->type());
+ }
+ return create_dynamic_type_info_ptr(get_struct_type_info(field_types));
+ } else if (UNLIKELY(type == OLAP_FIELD_TYPE_ARRAY)) {
int32_t iterations = 0;
const auto* child_column = &column_meta_pb->children_columns(0);
while (child_column->type() == OLAP_FIELD_TYPE_ARRAY) {
@@ -202,7 +225,14 @@ TypeInfoPtr create_type_info_ptr(const TypeInfo* type_info, bool should_reclaim_
// TODO: Support the type info of the nested array with more than 9 depths.
TypeInfoPtr get_type_info(const TabletColumn* col) {
auto type = col->type();
- if (UNLIKELY(type == OLAP_FIELD_TYPE_ARRAY)) {
+ if (UNLIKELY(type == OLAP_FIELD_TYPE_STRUCT)) {
+ std::vector<FieldType> field_types;
+ for (uint32_t i = 0; i < col->get_subtype_count(); i++) {
+ const auto* child_column = &col->get_sub_column(i);
+ field_types.push_back(child_column->type());
+ }
+ return create_dynamic_type_info_ptr(get_struct_type_info(field_types));
+ } else if (UNLIKELY(type == OLAP_FIELD_TYPE_ARRAY)) {
int32_t iterations = 0;
const auto* child_column = &col->get_sub_column(0);
while (child_column->type() == OLAP_FIELD_TYPE_ARRAY) {
@@ -219,9 +249,21 @@ TypeInfoPtr clone_type_info(const TypeInfo* type_info) {
if (is_scalar_type(type_info->type())) {
return create_static_type_info_ptr(type_info);
} else {
- const auto array_type_info = dynamic_cast<const ArrayTypeInfo*>(type_info);
- return create_dynamic_type_info_ptr(
- new ArrayTypeInfo(clone_type_info(array_type_info->item_type_info())));
+ auto type = type_info->type();
+ if (type == OLAP_FIELD_TYPE_STRUCT) {
+ const auto struct_type_info = dynamic_cast<const StructTypeInfo*>(type_info);
+ std::vector<TypeInfoPtr> clone_type_infos;
+ const std::vector<TypeInfoPtr>* sub_type_infos = struct_type_info->type_infos();
+ clone_type_infos.reserve(sub_type_infos->size());
+ for (size_t i = 0; i < sub_type_infos->size(); i++) {
+ clone_type_infos.push_back(clone_type_info((*sub_type_infos)[i].get()));
+ }
+ return create_dynamic_type_info_ptr(new StructTypeInfo(clone_type_infos));
+ } else {
+ const auto array_type_info = dynamic_cast<const ArrayTypeInfo*>(type_info);
+ return create_dynamic_type_info_ptr(
+ new ArrayTypeInfo(clone_type_info(array_type_info->item_type_info())));
+ }
}
}
diff --git a/be/src/olap/types.h b/be/src/olap/types.h
index 97566adb69..540f781a69 100644
--- a/be/src/olap/types.h
+++ b/be/src/olap/types.h
@@ -32,6 +32,7 @@
#include "runtime/collection_value.h"
#include "runtime/jsonb_value.h"
#include "runtime/mem_pool.h"
+#include "runtime/struct_value.h"
#include "util/jsonb_document.h"
#include "util/jsonb_utils.h"
#include "util/mem_util.hpp"
@@ -323,6 +324,245 @@ private:
const size_t _item_size;
};
+class StructTypeInfo : public TypeInfo {
+public:
+ explicit StructTypeInfo(std::vector<TypeInfoPtr>& type_infos) {
+ for (TypeInfoPtr& type_info : type_infos) {
+ _type_infos.push_back(std::move(type_info));
+ }
+ }
+ ~StructTypeInfo() override = default;
+
+ bool equal(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const StructValue*>(left);
+ auto r_value = reinterpret_cast<const StructValue*>(right);
+ if (l_value->size() != r_value->size()) {
+ return false;
+ }
+ uint32_t size = l_value->size();
+
+ if (!l_value->has_null() && !r_value->has_null()) {
+ for (size_t i = 0; i < size; ++i) {
+ if (!_type_infos[i]->equal(l_value->child_value(i), r_value->child_value(i))) {
+ return false;
+ }
+ }
+ } else {
+ for (size_t i = 0; i < size; ++i) {
+ if (l_value->is_null_at(i)) {
+ if (r_value->is_null_at(i)) { // both are null
+ continue;
+ } else { // left is null & right is not null
+ return false;
+ }
+ } else if (r_value->is_null_at(i)) { // left is not null & right is null
+ return false;
+ }
+ if (!_type_infos[i]->equal(l_value->child_value(i), r_value->child_value(i))) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ int cmp(const void* left, const void* right) const override {
+ auto l_value = reinterpret_cast<const StructValue*>(left);
+ auto r_value = reinterpret_cast<const StructValue*>(right);
+ uint32_t l_size = l_value->size();
+ uint32_t r_size = r_value->size();
+ size_t cur = 0;
+
+ if (!l_value->has_null() && !r_value->has_null()) {
+ while (cur < l_size && cur < r_size) {
+ int result =
+ _type_infos[cur]->cmp(l_value->child_value(cur), r_value->child_value(cur));
+ if (result != 0) {
+ return result;
+ }
+ ++cur;
+ }
+ } else {
+ while (cur < l_size && cur < r_size) {
+ if (l_value->is_null_at(cur)) {
+ if (!r_value->is_null_at(cur)) { // left is null & right is not null
+ return -1;
+ }
+ } else if (r_value->is_null_at(cur)) { // left is not null & right is null
+ return 1;
+ } else { // both are not null
+ int result = _type_infos[cur]->cmp(l_value->child_value(cur),
+ r_value->child_value(cur));
+ if (result != 0) {
+ return result;
+ }
+ }
+ ++cur;
+ }
+ }
+
+ if (l_size < r_size) {
+ return -1;
+ } else if (l_size > r_size) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+
+ void shallow_copy(void* dest, const void* src) const override {
+ auto dest_value = reinterpret_cast<StructValue*>(dest);
+ auto src_value = reinterpret_cast<const StructValue*>(src);
+ dest_value->shallow_copy(src_value);
+ }
+
+ void deep_copy(void* dest, const void* src, MemPool* mem_pool) const override {
+ auto dest_value = reinterpret_cast<StructValue*>(dest);
+ auto src_value = reinterpret_cast<const StructValue*>(src);
+
+ if (src_value->size() == 0) {
+ new (dest_value) StructValue(src_value->size());
+ return;
+ }
+
+ dest_value->set_size(src_value->size());
+ dest_value->set_has_null(src_value->has_null());
+
+ size_t allocate_size = src_value->size() * sizeof(*src_value->values());
+ // allocate memory for children value
+ for (size_t i = 0; i < src_value->size(); ++i) {
+ if (src_value->is_null_at(i)) continue;
+ allocate_size += _type_infos[i]->size();
+ }
+
+ dest_value->set_values((void**)mem_pool->allocate(allocate_size));
+ auto ptr = reinterpret_cast<uint8_t*>(dest_value->mutable_values());
+ ptr += dest_value->size() * sizeof(*dest_value->values());
+
+ for (size_t i = 0; i < src_value->size(); ++i) {
+ dest_value->set_child_value(nullptr, i);
+ if (src_value->is_null_at(i)) continue;
+ dest_value->set_child_value(ptr, i);
+ ptr += _type_infos[i]->size();
+ }
+
+ // copy children value
+ for (size_t i = 0; i < src_value->size(); ++i) {
+ if (src_value->is_null_at(i)) continue;
+ _type_infos[i]->deep_copy(dest_value->mutable_child_value(i), src_value->child_value(i),
+ mem_pool);
+ }
+ }
+
+ void copy_object(void* dest, const void* src, MemPool* mem_pool) const override {
+ deep_copy(dest, src, mem_pool);
+ }
+
+ void direct_copy(void* dest, const void* src) const override {
+ auto dest_value = static_cast<StructValue*>(dest);
+ auto base = reinterpret_cast<uint8_t*>(dest_value->mutable_values());
+ direct_copy(&base, dest, src);
+ }
+
+ void direct_copy(uint8_t** base, void* dest, const void* src) const {
+ auto dest_value = static_cast<StructValue*>(dest);
+ auto src_value = static_cast<const StructValue*>(src);
+
+ dest_value->set_size(src_value->size());
+ dest_value->set_has_null(src_value->has_null());
+ *base += src_value->size() * sizeof(*src_value->values());
+
+ for (size_t i = 0; i < src_value->size(); ++i) {
+ dest_value->set_child_value(nullptr, i);
+ if (src_value->is_null_at(i)) continue;
+ dest_value->set_child_value(*base, i);
+ *base += _type_infos[i]->size();
+ }
+
+ for (size_t i = 0; i < src_value->size(); ++i) {
+ if (dest_value->is_null_at(i)) {
+ continue;
+ }
+ auto dest_address = dest_value->mutable_child_value(i);
+ auto src_address = src_value->child_value(i);
+ if (_type_infos[i]->type() == OLAP_FIELD_TYPE_STRUCT) {
+ dynamic_cast<const StructTypeInfo*>(_type_infos[i].get())
+ ->direct_copy(base, dest_address, src_address);
+ } else if (_type_infos[i]->type() == OLAP_FIELD_TYPE_ARRAY) {
+ dynamic_cast<const ArrayTypeInfo*>(_type_infos[i].get())
+ ->direct_copy(base, dest_address, src_address);
+ } else {
+ if (is_olap_string_type(_type_infos[i]->type())) {
+ auto dest_slice = reinterpret_cast<Slice*>(dest_address);
+ auto src_slice = reinterpret_cast<const Slice*>(src_address);
+ dest_slice->data = reinterpret_cast<char*>(*base);
+ dest_slice->size = src_slice->size;
+ *base += src_slice->size;
+ }
+ _type_infos[i]->direct_copy(dest_address, src_address);
+ }
+ }
+ }
+
+ void direct_copy_may_cut(void* dest, const void* src) const override { direct_copy(dest, src); }
+
+ Status convert_from(void* dest, const void* src, const TypeInfo* src_type, MemPool* mem_pool,
+ size_t variable_len = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ Status from_string(void* buf, const std::string& scan_key, const int precision = 0,
+ const int scale = 0) const override {
+ return Status::Error<ErrorCode::NOT_IMPLEMENTED_ERROR>();
+ }
+
+ std::string to_string(const void* src) const override {
+ auto src_value = reinterpret_cast<const StructValue*>(src);
+ std::string result = "{";
+
+ for (size_t i = 0; i < src_value->size(); ++i) {
+ std::string field_value = _type_infos[i]->to_string(src_value->child_value(i));
+ result += field_value;
+ if (i < src_value->size() - 1) {
+ result += ", ";
+ }
+ }
+ result += "}";
+ return result;
+ }
+
+ void set_to_max(void* buf) const override {
+ DCHECK(false) << "set_to_max of list is not implemented.";
+ }
+
+ void set_to_min(void* buf) const override {
+ DCHECK(false) << "set_to_min of list is not implemented.";
+ }
+
+ uint32_t hash_code(const void* data, uint32_t seed) const override {
+ auto struct_value = reinterpret_cast<const StructValue*>(data);
+ auto size = struct_value->size();
+ uint32_t result = HashUtil::hash(&size, sizeof(size), seed);
+ for (size_t i = 0; i < size; ++i) {
+ if (struct_value->is_null_at(i)) {
+ result = seed * result;
+ } else {
+ result = seed * result + _type_infos[i]->hash_code(struct_value->values()[i], seed);
+ }
+ }
+ return result;
+ }
+
+ const size_t size() const override { return sizeof(StructValue); }
+
+ FieldType type() const override { return OLAP_FIELD_TYPE_STRUCT; }
+
+ inline const std::vector<TypeInfoPtr>* type_infos() const { return &_type_infos; }
+
+private:
+ std::vector<TypeInfoPtr> _type_infos;
+};
+
bool is_scalar_type(FieldType field_type);
const TypeInfo* get_scalar_type_info(FieldType field_type);
@@ -458,12 +698,15 @@ template <>
struct CppTypeTraits<OLAP_FIELD_TYPE_OBJECT> {
using CppType = Slice;
};
-
template <>
struct CppTypeTraits<OLAP_FIELD_TYPE_QUANTILE_STATE> {
using CppType = Slice;
};
template <>
+struct CppTypeTraits<OLAP_FIELD_TYPE_STRUCT> {
+ using CppType = StructValue;
+};
+template <>
struct CppTypeTraits<OLAP_FIELD_TYPE_ARRAY> {
using CppType = CollectionValue;
};
diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt
index 94b95a6fe6..97e8de4aa8 100644
--- a/be/src/runtime/CMakeLists.txt
+++ b/be/src/runtime/CMakeLists.txt
@@ -45,6 +45,7 @@ set(RUNTIME_FILES
threadlocal.cc
decimalv2_value.cpp
large_int_value.cpp
+ struct_value.cpp
collection_value.cpp
fragment_mgr.cpp
load_path_mgr.cpp
diff --git a/be/src/runtime/primitive_type.cpp b/be/src/runtime/primitive_type.cpp
index b2fbb8db8f..792b80a238 100644
--- a/be/src/runtime/primitive_type.cpp
+++ b/be/src/runtime/primitive_type.cpp
@@ -22,6 +22,7 @@
#include "runtime/define_primitive_type.h"
#include "runtime/jsonb_value.h"
#include "vec/common/string_ref.h"
+#include "runtime/struct_value.h"
namespace doris {
@@ -53,6 +54,8 @@ PrimitiveType convert_type_to_primitive(FunctionContext::Type type) {
return PrimitiveType::TYPE_BOOLEAN;
case FunctionContext::Type::TYPE_ARRAY:
return PrimitiveType::TYPE_ARRAY;
+ case FunctionContext::Type::TYPE_STRUCT:
+ return PrimitiveType::TYPE_STRUCT;
case FunctionContext::Type::TYPE_OBJECT:
return PrimitiveType::TYPE_OBJECT;
case FunctionContext::Type::TYPE_HLL:
@@ -212,6 +215,9 @@ PrimitiveType thrift_to_type(TPrimitiveType::type ttype) {
case TPrimitiveType::ARRAY:
return TYPE_ARRAY;
+ case TPrimitiveType::STRUCT:
+ return TYPE_STRUCT;
+
default:
return INVALID_TYPE;
}
@@ -306,6 +312,9 @@ TPrimitiveType::type to_thrift(PrimitiveType ptype) {
case TYPE_ARRAY:
return TPrimitiveType::ARRAY;
+ case TYPE_STRUCT:
+ return TPrimitiveType::STRUCT;
+
default:
return TPrimitiveType::INVALID_TYPE;
}
@@ -400,6 +409,9 @@ std::string type_to_string(PrimitiveType t) {
case TYPE_ARRAY:
return "ARRAY";
+ case TYPE_STRUCT:
+ return "STRUCT";
+
default:
return "";
};
@@ -539,6 +551,8 @@ int get_slot_size(PrimitiveType type) {
return sizeof(JsonBinaryValue);
case TYPE_ARRAY:
return sizeof(CollectionValue);
+ case TYPE_STRUCT:
+ return sizeof(StructValue);
case TYPE_NULL:
case TYPE_BOOLEAN:
diff --git a/be/src/runtime/primitive_type.h b/be/src/runtime/primitive_type.h
index bae0d8c1f5..d3a2a9f9e9 100644
--- a/be/src/runtime/primitive_type.h
+++ b/be/src/runtime/primitive_type.h
@@ -54,6 +54,7 @@ constexpr bool is_enumeration_type(PrimitiveType type) {
case TYPE_DECIMAL128I:
case TYPE_BOOLEAN:
case TYPE_ARRAY:
+ case TYPE_STRUCT:
case TYPE_HLL:
return false;
case TYPE_TINYINT:
diff --git a/be/src/runtime/struct_value.cpp b/be/src/runtime/struct_value.cpp
new file mode 100644
index 0000000000..ff0b9cc62a
--- /dev/null
+++ b/be/src/runtime/struct_value.cpp
@@ -0,0 +1,27 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "runtime/struct_value.h"
+
+namespace doris {
+
+void StructValue::shallow_copy(const StructValue* other) {
+ _size = other->_size;
+ _values = other->_values;
+ _has_null = other->_has_null;
+}
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/struct_value.h b/be/src/runtime/struct_value.h
new file mode 100644
index 0000000000..ec243d729c
--- /dev/null
+++ b/be/src/runtime/struct_value.h
@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <type_traits>
+
+#include "runtime/primitive_type.h"
+
+namespace doris {
+
+class StructValue {
+public:
+ StructValue() = default;
+
+ explicit StructValue(uint32_t size) : _values(nullptr), _size(size), _has_null(false) {}
+ StructValue(void** values, uint32_t size) : _values(values), _size(size), _has_null(false) {}
+ StructValue(void** values, uint32_t size, bool has_null)
+ : _values(values), _size(size), _has_null(has_null) {}
+
+ //void to_struct_val(StructVal* val) const;
+ //static StructValue from_struct_val(const StructVal& val);
+
+ uint32_t size() const { return _size; }
+ void set_size(uint32_t size) { _size = size; }
+ bool has_null() const { return _has_null; }
+ void set_has_null(bool has_null) { _has_null = has_null; }
+ bool is_null_at(uint32_t index) const {
+ return this->_has_null && this->_values[index] == nullptr;
+ }
+
+ void shallow_copy(const StructValue* other);
+
+ // size_t get_byte_size(const TypeDescriptor& type) const;
+
+ const void** values() const { return const_cast<const void**>(_values); }
+ void** mutable_values() { return _values; }
+ void set_values(void** values) { _values = values; }
+ const void* child_value(uint32_t index) const { return _values[index]; }
+ void* mutable_child_value(uint32_t index) { return _values[index]; }
+ void set_child_value(void* value, uint32_t index) { _values[index] = value; }
+
+private:
+ // pointer to the start of the vector of children pointers. These pointers are
+ // point to children values where a null pointer means that this child is NULL.
+ void** _values;
+ // the number of values in this struct value.
+ uint32_t _size;
+ // child has no null value if has_null is false.
+ // child may has null value if has_null is true.
+ bool _has_null;
+};
+} // namespace doris
\ No newline at end of file
diff --git a/be/src/runtime/types.cpp b/be/src/runtime/types.cpp
index 37f4933569..f26b1dcbce 100644
--- a/be/src/runtime/types.cpp
+++ b/be/src/runtime/types.cpp
@@ -55,14 +55,31 @@ TypeDescriptor::TypeDescriptor(const std::vector<TTypeNode>& types, int* idx)
case TTypeNodeType::ARRAY: {
DCHECK(!node.__isset.scalar_type);
DCHECK_LT(*idx, types.size() - 1);
+ // contains_null should be always set in ArrayType
+ DCHECK(node.__isset.contains_null);
type = TYPE_ARRAY;
- if (node.__isset.contains_null) {
- contains_null = node.contains_null;
- }
+ contains_nulls.reserve(1);
+ contains_nulls.push_back(node.contains_null);
++(*idx);
children.push_back(TypeDescriptor(types, idx));
break;
}
+ case TTypeNodeType::STRUCT: {
+ DCHECK(!node.__isset.scalar_type);
+ DCHECK_LT(*idx, types.size() - 1);
+ DCHECK(!node.__isset.contains_null);
+ DCHECK(node.__isset.struct_fields);
+ DCHECK_GE(node.struct_fields.size(), 1);
+ type = TYPE_STRUCT;
+ contains_nulls.reserve(node.struct_fields.size());
+ for (size_t i = 0; i < node.struct_fields.size(); i++) {
+ ++(*idx);
+ children.push_back(TypeDescriptor(types, idx));
+ field_names.push_back(node.struct_fields[i].name);
+ contains_nulls.push_back(node.struct_fields[i].contains_null);
+ }
+ break;
+ }
// case TTypeNodeType::STRUCT:
// type = TYPE_STRUCT;
// for (int i = 0; i < node.struct_fields.size(); ++i) {
@@ -98,15 +115,17 @@ void TypeDescriptor::to_thrift(TTypeDesc* thrift_type) const {
if (is_complex_type()) {
if (type == TYPE_ARRAY) {
node.type = TTypeNodeType::ARRAY;
+ node.contains_null = contains_nulls[0];
} else if (type == TYPE_MAP) {
node.type = TTypeNodeType::MAP;
} else {
DCHECK_EQ(type, TYPE_STRUCT);
node.type = TTypeNodeType::STRUCT;
node.__set_struct_fields(std::vector<TStructField>());
- for (auto& field_name : field_names) {
+ for (size_t i = 0; i < field_names.size(); i++) {
node.struct_fields.push_back(TStructField());
- node.struct_fields.back().name = field_name;
+ node.struct_fields.back().name = field_names[i];
+ node.struct_fields.back().contains_null = contains_nulls[i];
}
}
for (const TypeDescriptor& child : children) {
@@ -147,10 +166,12 @@ void TypeDescriptor::to_protobuf(PTypeDesc* ptype) const {
scalar_type->set_scale(scale);
} else if (type == TYPE_ARRAY) {
node->set_type(TTypeNodeType::ARRAY);
+ node->set_contains_null(contains_nulls[0]);
for (const TypeDescriptor& child : children) {
child.to_protobuf(ptype);
}
}
+ // TODO(xy): support struct
}
TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNode>& types, int* idx)
@@ -184,13 +205,15 @@ TypeDescriptor::TypeDescriptor(const google::protobuf::RepeatedPtrField<PTypeNod
}
case TTypeNodeType::ARRAY: {
type = TYPE_ARRAY;
+ contains_nulls.push_back(true);
if (node.has_contains_null()) {
- contains_null = node.contains_null();
+ contains_nulls[0] = node.contains_null();
}
++(*idx);
children.push_back(TypeDescriptor(types, idx));
break;
}
+ // TODO(xy): support struct
default:
DCHECK(false) << node.type();
}
@@ -218,6 +241,19 @@ std::string TypeDescriptor::debug_string() const {
ss << "ARRAY<" << children[0].debug_string() << ">";
return ss.str();
}
+ case TYPE_STRUCT: {
+ ss << "STRUCT<";
+ for (size_t i = 0; i < children.size(); i++) {
+ ss << field_names[i];
+ ss << ":";
+ ss << children[i].debug_string();
+ if (i != children.size() - 1) {
+ ss << ",";
+ }
+ }
+ ss << ">";
+ return ss.str();
+ }
default:
return type_to_string(type);
}
diff --git a/be/src/runtime/types.h b/be/src/runtime/types.h
index 32d2dd486b..3a5f4f14dd 100644
--- a/be/src/runtime/types.h
+++ b/be/src/runtime/types.h
@@ -64,7 +64,7 @@ struct TypeDescriptor {
std::vector<std::string> field_names;
// Used for complex types only.
- bool contains_null = true;
+ std::vector<bool> contains_nulls;
TypeDescriptor() : type(INVALID_TYPE), len(-1), precision(-1), scale(-1) {}
diff --git a/be/src/udf/udf.h b/be/src/udf/udf.h
index 7de1667241..1c4517e967 100644
--- a/be/src/udf/udf.h
+++ b/be/src/udf/udf.h
@@ -81,6 +81,7 @@ public:
TYPE_DECIMALV2,
TYPE_OBJECT,
TYPE_ARRAY,
+ TYPE_STRUCT,
TYPE_QUANTILE_STATE,
TYPE_DATEV2,
TYPE_DATETIMEV2,
diff --git a/be/src/vec/CMakeLists.txt b/be/src/vec/CMakeLists.txt
index d4b200bfe7..7584b46e5e 100644
--- a/be/src/vec/CMakeLists.txt
+++ b/be/src/vec/CMakeLists.txt
@@ -133,6 +133,7 @@ set(VEC_FILES
exprs/vexpr_context.cpp
exprs/vliteral.cpp
exprs/varray_literal.cpp
+ exprs/vstruct_literal.cpp
exprs/vin_predicate.cpp
exprs/vbloom_predicate.cpp
exprs/vbitmap_predicate.cpp
diff --git a/be/src/vec/columns/column_struct.cpp b/be/src/vec/columns/column_struct.cpp
index 05d5df0fd4..c7e5c23d2a 100644
--- a/be/src/vec/columns/column_struct.cpp
+++ b/be/src/vec/columns/column_struct.cpp
@@ -48,8 +48,7 @@ ColumnStruct::ColumnStruct(MutableColumns&& mutable_columns) {
columns.reserve(mutable_columns.size());
for (auto& column : mutable_columns) {
if (is_column_const(*column)) {
- throw Exception {"ColumnStruct cannot have ColumnConst as its element",
- ErrorCodes::ILLEGAL_COLUMN};
+ LOG(FATAL) << "ColumnStruct cannot have ColumnConst as its element";
}
columns.push_back(std::move(column));
}
@@ -59,8 +58,7 @@ ColumnStruct::ColumnStruct(Columns&& columns) {
columns.reserve(columns.size());
for (auto& column : columns) {
if (is_column_const(*column)) {
- throw Exception {"ColumnStruct cannot have ColumnConst as its element",
- ErrorCodes::ILLEGAL_COLUMN};
+ LOG(FATAL) << "ColumnStruct cannot have ColumnConst as its element";
}
columns.push_back(std::move(column));
}
@@ -70,8 +68,7 @@ ColumnStruct::ColumnStruct(TupleColumns&& tuple_columns) {
columns.reserve(tuple_columns.size());
for (auto& column : tuple_columns) {
if (is_column_const(*column)) {
- throw Exception {"ColumnStruct cannot have ColumnConst as its element",
- ErrorCodes::ILLEGAL_COLUMN};
+ LOG(FATAL) << "ColumnStruct cannot have ColumnConst as its element";
}
columns.push_back(std::move(column));
}
@@ -79,9 +76,9 @@ ColumnStruct::ColumnStruct(TupleColumns&& tuple_columns) {
ColumnStruct::Ptr ColumnStruct::create(Columns& columns) {
for (const auto& column : columns) {
- if (is_column_const(*column))
- throw Exception {"ColumnStruct cannot have ColumnConst as its element",
- ErrorCodes::ILLEGAL_COLUMN};
+ if (is_column_const(*column)) {
+ LOG(FATAL) << "ColumnStruct cannot have ColumnConst as its element";
+ }
}
auto column_struct = ColumnStruct::create(columns);
return column_struct;
@@ -90,8 +87,7 @@ ColumnStruct::Ptr ColumnStruct::create(Columns& columns) {
ColumnStruct::Ptr ColumnStruct::create(TupleColumns& tuple_columns) {
for (const auto& column : tuple_columns) {
if (is_column_const(*column)) {
- throw Exception {"ColumnStruct cannot have ColumnConst as its element",
- ErrorCodes::ILLEGAL_COLUMN};
+ LOG(FATAL) << "ColumnStruct cannot have ColumnConst as its element";
}
}
auto column_struct = ColumnStruct::create(tuple_columns);
@@ -144,22 +140,11 @@ bool ColumnStruct::is_default_at(size_t n) const {
return true;
}
-StringRef ColumnStruct::get_data_at(size_t) const {
- throw Exception("Method get_data_at is not supported for " + get_name(),
- ErrorCodes::NOT_IMPLEMENTED);
-}
-
-void ColumnStruct::insert_data(const char*, size_t) {
- throw Exception("Method insert_data is not supported for " + get_name(),
- ErrorCodes::NOT_IMPLEMENTED);
-}
-
void ColumnStruct::insert(const Field& x) {
const auto& tuple = x.get<const Tuple&>();
const size_t tuple_size = columns.size();
if (tuple.size() != tuple_size) {
- throw Exception("Cannot insert value of different size into tuple",
- ErrorCodes::CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE);
+ LOG(FATAL) << "Cannot insert value of different size into tuple.";
}
for (size_t i = 0; i < tuple_size; ++i) {
@@ -172,8 +157,7 @@ void ColumnStruct::insert_from(const IColumn& src_, size_t n) {
const size_t tuple_size = columns.size();
if (src.columns.size() != tuple_size) {
- throw Exception("Cannot insert value of different size into tuple",
- ErrorCodes::CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE);
+ LOG(FATAL) << "Cannot insert value of different size into tuple.";
}
for (size_t i = 0; i < tuple_size; ++i) {
diff --git a/be/src/vec/columns/column_struct.h b/be/src/vec/columns/column_struct.h
index 895a2796b7..75dade874d 100644
--- a/be/src/vec/columns/column_struct.h
+++ b/be/src/vec/columns/column_struct.h
@@ -102,7 +102,7 @@ public:
std::string get_name() const override;
const char* get_family_name() const override { return "Struct"; }
TypeIndex get_data_type() const { return TypeIndex::Struct; }
-
+ bool can_be_inside_nullable() const override { return true; }
MutableColumnPtr clone_empty() const override;
MutableColumnPtr clone_resized(size_t size) const override;
@@ -112,8 +112,12 @@ public:
void get(size_t n, Field& res) const override;
bool is_default_at(size_t n) const override;
- StringRef get_data_at(size_t n) const override;
- void insert_data(const char* pos, size_t length) override;
+ [[noreturn]] StringRef get_data_at(size_t n) const override {
+ LOG(FATAL) << "Method get_data_at is not supported for " + get_name();
+ }
+ [[noreturn]] void insert_data(const char* pos, size_t length) override {
+ LOG(FATAL) << "Method insert_data is not supported for " + get_name();
+ }
void insert(const Field& x) override;
void insert_from(const IColumn& src_, size_t n) override;
void insert_default() override;
@@ -162,7 +166,10 @@ public:
// int compare_at_with_collation(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint,
// const Collator& collator) const override;
- int compare_at(size_t n, size_t m, const IColumn& rhs, int nan_direction_hint) const override;
+ [[noreturn]] int compare_at(size_t n, size_t m, const IColumn& rhs_,
+ int nan_direction_hint) const override {
+ LOG(FATAL) << "compare_at not implemented";
+ }
void get_extremes(Field& min, Field& max) const override;
// void get_permutation(IColumn::PermutationSortDirection direction,
diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp
index e622d979d5..7eec1eb2fd 100644
--- a/be/src/vec/data_types/data_type_factory.cpp
+++ b/be/src/vec/data_types/data_type_factory.cpp
@@ -29,6 +29,18 @@ DataTypePtr DataTypeFactory::create_data_type(const doris::Field& col_desc) {
if (col_desc.type() == OLAP_FIELD_TYPE_ARRAY) {
DCHECK(col_desc.get_sub_field_count() == 1);
nested = std::make_shared<DataTypeArray>(create_data_type(*col_desc.get_sub_field(0)));
+ } else if (col_desc.type() == OLAP_FIELD_TYPE_STRUCT) {
+ DCHECK(col_desc.get_sub_field_count() >= 1);
+ size_t field_size = col_desc.get_sub_field_count();
+ DataTypes dataTypes;
+ Strings names;
+ dataTypes.reserve(field_size);
+ names.reserve(field_size);
+ for (size_t i = 0; i < field_size; i++) {
+ dataTypes.push_back(create_data_type(*col_desc.get_sub_field(i)));
+ names.push_back(col_desc.name());
+ }
+ nested = std::make_shared<DataTypeStruct>(dataTypes, names);
} else {
nested = _create_primitive_data_type(col_desc.type(), col_desc.get_precision(),
col_desc.get_scale());
@@ -45,6 +57,18 @@ DataTypePtr DataTypeFactory::create_data_type(const TabletColumn& col_desc, bool
if (col_desc.type() == OLAP_FIELD_TYPE_ARRAY) {
DCHECK(col_desc.get_subtype_count() == 1);
nested = std::make_shared<DataTypeArray>(create_data_type(col_desc.get_sub_column(0)));
+ } else if (col_desc.type() == OLAP_FIELD_TYPE_STRUCT) {
+ DCHECK(col_desc.get_subtype_count() >= 1);
+ size_t col_size = col_desc.get_subtype_count();
+ DataTypes dataTypes;
+ Strings names;
+ dataTypes.reserve(col_size);
+ names.reserve(col_size);
+ for (size_t i = 0; i < col_size; i++) {
+ dataTypes.push_back(create_data_type(col_desc.get_sub_column(i)));
+ names.push_back(col_desc.name());
+ }
+ nested = std::make_shared<DataTypeStruct>(dataTypes, names);
} else {
nested =
_create_primitive_data_type(col_desc.type(), col_desc.precision(), col_desc.frac());
@@ -129,8 +153,23 @@ DataTypePtr DataTypeFactory::create_data_type(const TypeDescriptor& col_desc, bo
case TYPE_ARRAY:
DCHECK(col_desc.children.size() == 1);
nested = std::make_shared<vectorized::DataTypeArray>(
- create_data_type(col_desc.children[0], col_desc.contains_null));
+ create_data_type(col_desc.children[0], col_desc.contains_nulls[0]));
+ break;
+ case TYPE_STRUCT: {
+ DCHECK(col_desc.children.size() >= 1);
+ size_t child_size = col_desc.children.size();
+ DCHECK_EQ(col_desc.field_names.size(), child_size);
+ DataTypes dataTypes;
+ Strings names;
+ dataTypes.reserve(child_size);
+ names.reserve(child_size);
+ for (size_t i = 0; i < child_size; i++) {
+ dataTypes.push_back(create_data_type(col_desc.children[i], col_desc.contains_nulls[i]));
+ names.push_back(col_desc.field_names[i]);
+ }
+ nested = std::make_shared<DataTypeStruct>(dataTypes, names);
break;
+ }
case INVALID_TYPE:
default:
DCHECK(false) << "invalid PrimitiveType:" << (int)col_desc.type;
diff --git a/be/src/vec/data_types/data_type_factory.hpp b/be/src/vec/data_types/data_type_factory.hpp
index ed270b40ea..d46dd3435d 100644
--- a/be/src/vec/data_types/data_type_factory.hpp
+++ b/be/src/vec/data_types/data_type_factory.hpp
@@ -41,6 +41,7 @@
#include "vec/data_types/data_type_nullable.h"
#include "vec/data_types/data_type_number.h"
#include "vec/data_types/data_type_string.h"
+#include "vec/data_types/data_type_struct.h"
namespace doris::vectorized {
@@ -95,7 +96,10 @@ public:
});
return instance;
}
+
+ // TODO(xy): support creator to create dynamic struct type
DataTypePtr get(const std::string& name) { return _data_type_map[name]; }
+ // TODO(xy): support creator to create dynamic struct type
const std::string& get(const DataTypePtr& data_type) const {
auto type_ptr = data_type->is_nullable()
? ((DataTypeNullable*)(data_type.get()))->get_nested_type()
diff --git a/be/src/vec/data_types/data_type_struct.cpp b/be/src/vec/data_types/data_type_struct.cpp
index 91aff67a40..8fd2179bfa 100644
--- a/be/src/vec/data_types/data_type_struct.cpp
+++ b/be/src/vec/data_types/data_type_struct.cpp
@@ -43,16 +43,15 @@ DataTypeStruct::DataTypeStruct(const DataTypes& elems_)
}
}
-static std::optional<Exception> check_tuple_names(const Strings& names) {
+static Status check_tuple_names(const Strings& names) {
std::unordered_set<String> names_set;
for (const auto& name : names) {
if (name.empty()) {
- return Exception("Names of tuple elements cannot be empty", ErrorCodes::BAD_ARGUMENTS);
+ return Status::InvalidArgument("Names of tuple elements cannot be empty");
}
if (!names_set.insert(name).second) {
- return Exception("Names of tuple elements must be unique",
- ErrorCodes::DUPLICATE_COLUMN);
+ return Status::InvalidArgument("Names of tuple elements must be unique");
}
}
@@ -63,13 +62,12 @@ DataTypeStruct::DataTypeStruct(const DataTypes& elems_, const Strings& names_)
: elems(elems_), names(names_), have_explicit_names(true) {
size_t size = elems.size();
if (names.size() != size) {
- throw Exception("Wrong number of names passed to constructor of DataTypeStruct",
- ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
+ LOG(FATAL) << "Wrong number of names passed to constructor of DataTypeStruct";
}
- if (auto exception = check_tuple_names(names)) {
- throw std::move(*exception);
- }
+ Status st = check_tuple_names(names);
+ //if (!st.ok()) {
+ //}
}
std::string DataTypeStruct::do_get_name() const {
@@ -114,8 +112,7 @@ static void add_element_safe(const DataTypes& elems, IColumn& column, F&& impl)
if (element_column.size() != new_size) {
// This is not a logical error because it may work with
// user-supplied data.
- throw Exception("Cannot read a tuple because not all elements are present",
- ErrorCodes::SIZES_OF_COLUMNS_IN_TUPLE_DOESNT_MATCH);
+ LOG(FATAL) << "Cannot read a tuple because not all elements are present";
}
}
} catch (...) {
@@ -168,10 +165,11 @@ MutableColumnPtr DataTypeStruct::create_column() const {
// return ColumnStruct::create(std::move(tuple_columns));
// }
-// Field DataTypeStruct::get_default() const {
-// return Tuple(collections::map<Tuple>(
-// elems, [](const DataTypePtr& elem) { return elem->get_default(); }));
-// }
+Field DataTypeStruct::get_default() const {
+ return Tuple();
+ //return Tuple(collections::map<Tuple>(
+ // elems, [](const DataTypePtr& elem) { return elem->get_default(); }));
+}
void DataTypeStruct::insert_default_into(IColumn& column) const {
add_element_safe(elems, column, [&] {
@@ -210,8 +208,7 @@ size_t DataTypeStruct::get_position_by_name(const String& name) const {
return i;
}
}
- throw Exception("Struct doesn't have element with name '" + name + "'",
- ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK);
+ LOG(FATAL) << "Struct doesn't have element with name '" + name + "'";
}
std::optional<size_t> DataTypeStruct::try_get_position_by_name(const String& name) const {
@@ -229,7 +226,7 @@ String DataTypeStruct::get_name_by_position(size_t i) const {
fmt::memory_buffer error_msg;
fmt::format_to(error_msg, "Index of tuple element ({}) if out range ([1, {}])", i,
names.size());
- throw Exception(fmt::to_string(error_msg), ErrorCodes::ILLEGAL_INDEX);
+ LOG(FATAL) << fmt::to_string(error_msg);
}
return names[i - 1];
diff --git a/be/src/vec/data_types/data_type_struct.h b/be/src/vec/data_types/data_type_struct.h
index 4201583ef4..9405544b76 100644
--- a/be/src/vec/data_types/data_type_struct.h
+++ b/be/src/vec/data_types/data_type_struct.h
@@ -62,7 +62,7 @@ public:
std::string do_get_name() const override;
const char* get_family_name() const override { return "Struct"; }
- bool can_be_inside_nullable() const override { return false; }
+ bool can_be_inside_nullable() const override { return true; }
bool supports_sparse_serialization() const { return true; }
MutableColumnPtr create_column() const override;
@@ -90,16 +90,18 @@ public:
std::optional<size_t> try_get_position_by_name(const String& name) const;
String get_name_by_position(size_t i) const;
- int64_t get_uncompressed_serialized_bytes(const IColumn& column,
- int be_exec_version) const override {
+ [[noreturn]] int64_t get_uncompressed_serialized_bytes(const IColumn& column,
+ int be_exec_version) const override {
LOG(FATAL) << "get_uncompressed_serialized_bytes not implemented";
}
- char* serialize(const IColumn& column, char* buf, int be_exec_version) const override {
+ [[noreturn]] char* serialize(const IColumn& column, char* buf,
+ int be_exec_version) const override {
LOG(FATAL) << "serialize not implemented";
}
- const char* deserialize(const char* buf, IColumn* column, int be_exec_version) const override {
+ [[noreturn]] const char* deserialize(const char* buf, IColumn* column,
+ int be_exec_version) const override {
LOG(FATAL) << "serialize not implemented";
}
diff --git a/be/src/vec/exprs/vcast_expr.cpp b/be/src/vec/exprs/vcast_expr.cpp
index a31d5ec3fa..86e942b24b 100644
--- a/be/src/vec/exprs/vcast_expr.cpp
+++ b/be/src/vec/exprs/vcast_expr.cpp
@@ -37,6 +37,7 @@ doris::Status VCastExpr::prepare(doris::RuntimeState* state, const doris::RowDes
// create a const string column
_target_data_type = _data_type;
+ // TODO(xy): support return struct type name
_target_data_type_name = DataTypeFactory::instance().get(_target_data_type);
_cast_param_data_type = std::make_shared<DataTypeString>();
_cast_param = _cast_param_data_type->create_column_const(1, _target_data_type_name);
diff --git a/be/src/vec/exprs/vexpr.cpp b/be/src/vec/exprs/vexpr.cpp
index dad8a33579..79eaa13b46 100644
--- a/be/src/vec/exprs/vexpr.cpp
+++ b/be/src/vec/exprs/vexpr.cpp
@@ -34,6 +34,7 @@
#include "vec/exprs/vliteral.h"
#include "vec/exprs/vruntimefilter_wrapper.h"
#include "vec/exprs/vslot_ref.h"
+#include "vec/exprs/vstruct_literal.h"
#include "vec/exprs/vtuple_is_null_predicate.h"
namespace doris::vectorized {
@@ -123,6 +124,10 @@ Status VExpr::create_expr(doris::ObjectPool* pool, const doris::TExprNode& texpr
*expr = pool->add(new VArrayLiteral(texpr_node));
return Status::OK();
}
+ case TExprNodeType::STRUCT_LITERAL: {
+ *expr = pool->add(new VStructLiteral(texpr_node));
+ return Status::OK();
+ }
case doris::TExprNodeType::SLOT_REF: {
*expr = pool->add(new VSlotRef(texpr_node));
break;
diff --git a/be/src/vec/exprs/vstruct_literal.cpp b/be/src/vec/exprs/vstruct_literal.cpp
new file mode 100644
index 0000000000..9ae647b481
--- /dev/null
+++ b/be/src/vec/exprs/vstruct_literal.cpp
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "vec/exprs/vstruct_literal.h"
+
+namespace doris::vectorized {
+
+Status VStructLiteral::prepare(RuntimeState* state, const RowDescriptor& row_desc,
+ VExprContext* context) {
+ RETURN_IF_ERROR_OR_PREPARED(VExpr::prepare(state, row_desc, context));
+ Field struct_field = Tuple();
+ for (const auto child : _children) {
+ Field item;
+ ColumnPtrWrapper* const_col_wrapper = nullptr;
+ RETURN_IF_ERROR(child->get_const_col(context, &const_col_wrapper));
+ const_col_wrapper->column_ptr->get(0, item);
+ struct_field.get<Tuple>().push_back(item);
+ }
+ _column_ptr = _data_type->create_column_const(1, struct_field);
+ return Status::OK();
+}
+
+} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/exprs/vstruct_literal.h b/be/src/vec/exprs/vstruct_literal.h
new file mode 100644
index 0000000000..4cc4fe78d6
--- /dev/null
+++ b/be/src/vec/exprs/vstruct_literal.h
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "vec/exprs/vliteral.h"
+
+namespace doris {
+
+namespace vectorized {
+class VStructLiteral : public VLiteral {
+public:
+ VStructLiteral(const TExprNode& node) : VLiteral(node, false) {}
+ ~VStructLiteral() override = default;
+ Status prepare(RuntimeState* state, const RowDescriptor& row_desc,
+ VExprContext* context) override;
+};
+} // namespace vectorized
+
+} // namespace doris
diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h
index 59f58be84c..9c2878db5b 100644
--- a/be/src/vec/functions/function_cast.h
+++ b/be/src/vec/functions/function_cast.h
@@ -1756,6 +1756,7 @@ protected:
LOG(FATAL) << fmt::format(
"Second argument to {} must be a constant string describing type", get_name());
}
+ // TODO(xy): support return struct type for factory
auto type = DataTypeFactory::instance().get(type_col->get_value<String>());
DCHECK(type != nullptr);
diff --git a/be/src/vec/olap/olap_data_convertor.cpp b/be/src/vec/olap/olap_data_convertor.cpp
index c140cfc36f..cd683e5283 100644
--- a/be/src/vec/olap/olap_data_convertor.cpp
+++ b/be/src/vec/olap/olap_data_convertor.cpp
@@ -20,8 +20,10 @@
#include "olap/tablet_schema.h"
#include "vec/columns/column_array.h"
#include "vec/columns/column_complex.h"
+#include "vec/columns/column_struct.h"
#include "vec/columns/column_vector.h"
#include "vec/data_types/data_type_array.h"
+#include "vec/data_types/data_type_struct.h"
namespace doris::vectorized {
@@ -118,6 +120,14 @@ OlapBlockDataConvertor::create_olap_column_data_convertor(const TabletColumn& co
case FieldType::OLAP_FIELD_TYPE_DOUBLE: {
return std::make_unique<OlapColumnDataConvertorSimple<vectorized::Float64>>();
}
+ case FieldType::OLAP_FIELD_TYPE_STRUCT: {
+ std::vector<OlapColumnDataConvertorBaseUPtr> sub_convertors;
+ for (uint32_t i = 0; i < column.get_subtype_count(); i++) {
+ const TabletColumn& sub_column = column.get_sub_column(i);
+ sub_convertors.emplace_back(create_olap_column_data_convertor(sub_column));
+ }
+ return std::make_unique<OlapColumnDataConvertorStruct>(sub_convertors);
+ }
case FieldType::OLAP_FIELD_TYPE_ARRAY: {
const auto& sub_column = column.get_sub_column(0);
return std::make_unique<OlapColumnDataConvertorArray>(
@@ -643,6 +653,58 @@ Status OlapBlockDataConvertor::OlapColumnDataConvertorDecimal::convert_to_olap()
return Status::OK();
}
+void OlapBlockDataConvertor::OlapColumnDataConvertorStruct::set_source_column(
+ const ColumnWithTypeAndName& typed_column, size_t row_pos, size_t num_rows) {
+ OlapBlockDataConvertor::OlapColumnDataConvertorBase::set_source_column(typed_column, row_pos,
+ num_rows);
+}
+
+const void* OlapBlockDataConvertor::OlapColumnDataConvertorStruct::get_data() const {
+ return _results[0];
+}
+
+const void* OlapBlockDataConvertor::OlapColumnDataConvertorStruct::get_data_at(
+ size_t offset) const {
+ // Todo(xy): struct not supported
+ return nullptr;
+}
+
+Status OlapBlockDataConvertor::OlapColumnDataConvertorStruct::convert_to_olap() {
+ assert(_typed_column.column);
+ const vectorized::ColumnStruct* column_struct = nullptr;
+ const vectorized::DataTypeStruct* data_type_struct = nullptr;
+ if (_nullmap) {
+ auto nullable_column =
+ assert_cast<const vectorized::ColumnNullable*>(_typed_column.column.get());
+ column_struct = assert_cast<const vectorized::ColumnStruct*>(
+ nullable_column->get_nested_column_ptr().get());
+ data_type_struct = assert_cast<const DataTypeStruct*>(
+ (assert_cast<const DataTypeNullable*>(_typed_column.type.get())->get_nested_type())
+ .get());
+ } else {
+ column_struct = assert_cast<const vectorized::ColumnStruct*>(_typed_column.column.get());
+ data_type_struct = assert_cast<const DataTypeStruct*>(_typed_column.type.get());
+ }
+ assert(column_struct);
+ assert(data_type_struct);
+
+ size_t data_size = column_struct->tuple_size();
+ size_t data_cursor = 0;
+ size_t null_map_cursor = data_cursor + data_size;
+ for (size_t i = 0; i < data_size; i++) {
+ ColumnPtr sub_column = column_struct->get_column_ptr(i);
+ DataTypePtr sub_type = data_type_struct->get_element(i);
+ ColumnWithTypeAndName sub_typed_column = {sub_column, sub_type, ""};
+ _sub_convertors[i]->set_source_column(sub_typed_column, _row_pos, _num_rows);
+ _sub_convertors[i]->convert_to_olap();
+ _results[data_cursor] = _sub_convertors[i]->get_data();
+ _results[null_map_cursor] = _sub_convertors[i]->get_nullmap();
+ data_cursor++;
+ null_map_cursor++;
+ }
+ return Status::OK();
+}
+
Status OlapBlockDataConvertor::OlapColumnDataConvertorArray::convert_to_olap() {
const ColumnArray* column_array = nullptr;
const DataTypeArray* data_type_array = nullptr;
diff --git a/be/src/vec/olap/olap_data_convertor.h b/be/src/vec/olap/olap_data_convertor.h
index c86b08bb05..12cc0fba55 100644
--- a/be/src/vec/olap/olap_data_convertor.h
+++ b/be/src/vec/olap/olap_data_convertor.h
@@ -360,6 +360,27 @@ private:
}
};
+ class OlapColumnDataConvertorStruct : public OlapColumnDataConvertorBase {
+ public:
+ OlapColumnDataConvertorStruct(
+ std::vector<OlapColumnDataConvertorBaseUPtr>& sub_convertors) {
+ for (auto& sub_convertor : sub_convertors) {
+ _sub_convertors.push_back(std::move(sub_convertor));
+ }
+ size_t allocate_size = _sub_convertors.size() * 2;
+ _results.resize(allocate_size);
+ }
+ void set_source_column(const ColumnWithTypeAndName& typed_column, size_t row_pos,
+ size_t num_rows) override;
+ const void* get_data() const override;
+ const void* get_data_at(size_t offset) const override;
+ Status convert_to_olap() override;
+
+ private:
+ std::vector<OlapColumnDataConvertorBaseUPtr> _sub_convertors;
+ std::vector<const void*> _results;
+ };
+
class OlapColumnDataConvertorArray
: public OlapColumnDataConvertorPaddedPODArray<CollectionValue> {
public:
diff --git a/be/src/vec/sink/vmysql_result_writer.cpp b/be/src/vec/sink/vmysql_result_writer.cpp
index 31871b6c36..4d30e17227 100644
--- a/be/src/vec/sink/vmysql_result_writer.cpp
+++ b/be/src/vec/sink/vmysql_result_writer.cpp
@@ -29,6 +29,7 @@
#include "vec/common/assert_cast.h"
#include "vec/data_types/data_type_array.h"
#include "vec/data_types/data_type_decimal.h"
+#include "vec/data_types/data_type_struct.h"
#include "vec/exprs/vexpr.h"
#include "vec/exprs/vexpr_context.h"
#include "vec/runtime/vdatetime_value.h"
@@ -67,8 +68,8 @@ template <bool is_binary_format>
template <PrimitiveType type, bool is_nullable>
Status VMysqlResultWriter<is_binary_format>::_add_one_column(
const ColumnPtr& column_ptr, std::unique_ptr<TFetchDataResult>& result,
- std::vector<MysqlRowBuffer<is_binary_format>>& rows_buffer,
- const DataTypePtr& nested_type_ptr, int scale) {
+ std::vector<MysqlRowBuffer<is_binary_format>>& rows_buffer, int scale,
+ const DataTypes& sub_types) {
SCOPED_TIMER(_convert_tuple_timer);
const auto row_size = column_ptr->size();
@@ -146,6 +147,7 @@ Status VMysqlResultWriter<is_binary_format>::_add_one_column(
}
}
} else if constexpr (type == TYPE_ARRAY) {
+ DCHECK_EQ(sub_types.size(), 1);
auto& column_array = assert_cast<const ColumnArray&>(*column);
auto& offsets = column_array.get_offsets();
for (ssize_t i = 0; i < row_size; ++i) {
@@ -171,12 +173,12 @@ Status VMysqlResultWriter<is_binary_format>::_add_one_column(
if (data->is_null_at(j)) {
buf_ret = rows_buffer[i].push_string("NULL", strlen("NULL"));
} else {
- if (WhichDataType(remove_nullable(nested_type_ptr)).is_string()) {
- buf_ret = rows_buffer[i].push_string("'", 1);
- buf_ret = _add_one_cell(data, j, nested_type_ptr, rows_buffer[i], scale);
- buf_ret = rows_buffer[i].push_string("'", 1);
+ if (WhichDataType(remove_nullable(sub_types[0])).is_string()) {
+ buf_ret = _buffer.push_string("'", 1);
+ buf_ret = _add_one_cell(data, j, sub_types[0], _buffer);
+ buf_ret = _buffer.push_string("'", 1);
} else {
- buf_ret = _add_one_cell(data, j, nested_type_ptr, rows_buffer[i], scale);
+ buf_ret = _add_one_cell(data, j, sub_types[0], _buffer);
}
}
begin = false;
@@ -184,8 +186,51 @@ Status VMysqlResultWriter<is_binary_format>::_add_one_column(
buf_ret = rows_buffer[i].push_string("]", 1);
rows_buffer[i].close_dynamic_mode();
}
+ } else if constexpr (type == TYPE_STRUCT) {
+ DCHECK_GE(sub_types.size(), 1);
+ auto& column_struct = assert_cast<const ColumnStruct&>(*column);
+ for (ssize_t i = 0; i < row_size; ++i) {
+ if (0 != buf_ret) {
+ return Status::InternalError("pack mysql buffer failed.");
+ }
+ _buffer.reset();
+
+ if constexpr (is_nullable) {
+ if (column_ptr->is_null_at(i)) {
+ buf_ret = _buffer.push_null();
+ result->result_batch.rows[i].append(_buffer.buf(), _buffer.length());
+ continue;
+ }
+ }
+
+ _buffer.open_dynamic_mode();
+ buf_ret = _buffer.push_string("{", 1);
+ bool begin = true;
+ for (size_t j = 0; j < sub_types.size(); ++j) {
+ if (!begin) {
+ buf_ret = _buffer.push_string(", ", 2);
+ }
+ const auto& data = column_struct.get_column_ptr(j);
+ if (data->is_null_at(j)) {
+ buf_ret = _buffer.push_string("NULL", strlen("NULL"));
+ } else {
+ if (WhichDataType(remove_nullable(sub_types[j])).is_string()) {
+ buf_ret = _buffer.push_string("'", 1);
+ buf_ret = _add_one_cell(data, j, sub_types[j], _buffer);
+ buf_ret = _buffer.push_string("'", 1);
+ } else {
+ buf_ret = _add_one_cell(data, j, sub_types[j], _buffer);
+ }
+ }
+ begin = false;
+ }
+ buf_ret = _buffer.push_string("}", 1);
+ _buffer.close_dynamic_mode();
+ result->result_batch.rows[i].append(_buffer.buf(), _buffer.length());
+ }
} else if constexpr (type == TYPE_DECIMAL32 || type == TYPE_DECIMAL64 ||
type == TYPE_DECIMAL128I) {
+ DCHECK_EQ(sub_types.size(), 1);
for (int i = 0; i < row_size; ++i) {
if (0 != buf_ret) {
return Status::InternalError("pack mysql buffer failed.");
@@ -197,7 +242,7 @@ Status VMysqlResultWriter<is_binary_format>::_add_one_column(
continue;
}
}
- std::string decimal_str = nested_type_ptr->to_string(*column, i);
+ std::string decimal_str = sub_types[0]->to_string(*column, i);
buf_ret = rows_buffer[i].push_string(decimal_str.c_str(), decimal_str.length());
}
} else {
@@ -392,6 +437,7 @@ int VMysqlResultWriter<is_binary_format>::_add_one_cell(const ColumnPtr& column_
auto decimal_str = assert_cast<const DataTypeDecimal<Decimal128I>*>(nested_type.get())
->to_string(*column, row_idx);
return buffer.push_string(decimal_str.c_str(), decimal_str.length());
+ // TODO(xy): support nested struct
} else if (which.is_array()) {
auto& column_array = assert_cast<const ColumnArray&>(*column);
auto& offsets = column_array.get_offsets();
@@ -578,10 +624,10 @@ Status VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
auto& nested_type =
assert_cast<const DataTypeNullable&>(*type_ptr).get_nested_type();
status = _add_one_column<PrimitiveType::TYPE_DECIMALV2, true>(
- column_ptr, result, rows_buffer, nested_type, scale);
+ column_ptr, result, rows_buffer, scale, {nested_type});
} else {
status = _add_one_column<PrimitiveType::TYPE_DECIMALV2, false>(
- column_ptr, result, rows_buffer, type_ptr, scale);
+ column_ptr, result, rows_buffer, scale, {type_ptr});
}
break;
}
@@ -590,10 +636,10 @@ Status VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
auto& nested_type =
assert_cast<const DataTypeNullable&>(*type_ptr).get_nested_type();
status = _add_one_column<PrimitiveType::TYPE_DECIMAL32, true>(
- column_ptr, result, rows_buffer, nested_type, scale);
+ column_ptr, result, rows_buffer, scale, {nested_type});
} else {
status = _add_one_column<PrimitiveType::TYPE_DECIMAL32, false>(
- column_ptr, result, rows_buffer, type_ptr, scale);
+ column_ptr, result, rows_buffer, scale, {type_ptr});
}
break;
}
@@ -602,10 +648,10 @@ Status VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
auto& nested_type =
assert_cast<const DataTypeNullable&>(*type_ptr).get_nested_type();
status = _add_one_column<PrimitiveType::TYPE_DECIMAL64, true>(
- column_ptr, result, rows_buffer, nested_type, scale);
+ column_ptr, result, rows_buffer, scale, {nested_type});
} else {
status = _add_one_column<PrimitiveType::TYPE_DECIMAL64, false>(
- column_ptr, result, rows_buffer, type_ptr, scale);
+ column_ptr, result, rows_buffer, scale, {type_ptr});
}
break;
}
@@ -614,10 +660,10 @@ Status VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
auto& nested_type =
assert_cast<const DataTypeNullable&>(*type_ptr).get_nested_type();
status = _add_one_column<PrimitiveType::TYPE_DECIMAL128I, true>(
- column_ptr, result, rows_buffer, nested_type, scale);
+ column_ptr, result, rows_buffer, scale, {nested_type});
} else {
status = _add_one_column<PrimitiveType::TYPE_DECIMAL128I, false>(
- column_ptr, result, rows_buffer, type_ptr, scale);
+ column_ptr, result, rows_buffer, scale, {type_ptr});
}
break;
}
@@ -655,10 +701,10 @@ Status VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
case TYPE_DATETIMEV2: {
if (type_ptr->is_nullable()) {
status = _add_one_column<PrimitiveType::TYPE_DATETIMEV2, true>(
- column_ptr, result, rows_buffer, nullptr, scale);
+ column_ptr, result, rows_buffer, scale);
} else {
status = _add_one_column<PrimitiveType::TYPE_DATETIMEV2, false>(
- column_ptr, result, rows_buffer, nullptr, scale);
+ column_ptr, result, rows_buffer, scale);
}
break;
}
@@ -682,11 +728,25 @@ Status VMysqlResultWriter<is_binary_format>::append_block(Block& input_block) {
assert_cast<const DataTypeNullable&>(*type_ptr).get_nested_type();
auto& sub_type = assert_cast<const DataTypeArray&>(*nested_type).get_nested_type();
status = _add_one_column<PrimitiveType::TYPE_ARRAY, true>(
- column_ptr, result, rows_buffer, sub_type, scale);
+ column_ptr, result, rows_buffer, scale, {sub_type});
} else {
auto& sub_type = assert_cast<const DataTypeArray&>(*type_ptr).get_nested_type();
status = _add_one_column<PrimitiveType::TYPE_ARRAY, false>(
- column_ptr, result, rows_buffer, sub_type, scale);
+ column_ptr, result, rows_buffer, scale, {sub_type});
+ }
+ break;
+ }
+ case TYPE_STRUCT: {
+ if (type_ptr->is_nullable()) {
+ auto& nested_type =
+ assert_cast<const DataTypeNullable&>(*type_ptr).get_nested_type();
+ auto& sub_types = assert_cast<const DataTypeStruct&>(*nested_type).get_elements();
+ status = _add_one_column<PrimitiveType::TYPE_STRUCT, true>(column_ptr, result,
+ scale, sub_types);
+ } else {
+ auto& sub_types = assert_cast<const DataTypeStruct&>(*type_ptr).get_elements();
+ status = _add_one_column<PrimitiveType::TYPE_STRUCT, false>(column_ptr, result,
+ scale, sub_types);
}
break;
}
diff --git a/be/src/vec/sink/vmysql_result_writer.h b/be/src/vec/sink/vmysql_result_writer.h
index 0e6a77f00d..d87f82a9d1 100644
--- a/be/src/vec/sink/vmysql_result_writer.h
+++ b/be/src/vec/sink/vmysql_result_writer.h
@@ -53,8 +53,8 @@ private:
template <PrimitiveType type, bool is_nullable>
Status _add_one_column(const ColumnPtr& column_ptr, std::unique_ptr<TFetchDataResult>& result,
- std::vector<MysqlRowBuffer<is_binary_format>>& rows_buffer,
- const DataTypePtr& nested_type_ptr = nullptr, int scale = -1);
+ std::vector<MysqlRowBuffer<is_binary_format>>& rows_buffer, int scale = -1,
+ const DataTypes& sub_types = DataTypes());
int _add_one_cell(const ColumnPtr& column_ptr, size_t row_idx, const DataTypePtr& type,
MysqlRowBuffer<is_binary_format>& buffer, int scale = -1);
@@ -76,4 +76,4 @@ private:
ResultList _results;
};
} // namespace vectorized
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/vec/sink/vtablet_sink.cpp b/be/src/vec/sink/vtablet_sink.cpp
index 0899da3fb9..27f06fbdbe 100644
--- a/be/src/vec/sink/vtablet_sink.cpp
+++ b/be/src/vec/sink/vtablet_sink.cpp
@@ -1398,11 +1398,12 @@ Status VOlapTableSink::_validate_column(RuntimeState* state, const TypeDescripto
}
fmt::format_to(error_prefix, "ARRAY type failed: ");
RETURN_IF_ERROR(_validate_column(
- state, nested_type, nested_type.contains_null, column_array->get_data_ptr(),
+ state, nested_type, nested_type.contains_nulls[0], column_array->get_data_ptr(),
slot_index, filter_bitmap, stop_processing, error_prefix, &permutation));
}
break;
}
+ // TODO(xy): add struct type validate
default:
break;
}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
index 45d01019d1..8aa6acb31a 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
@@ -159,6 +159,16 @@ public class ArrayType extends Type {
return !itemType.isNull();
}
+ @Override
+ public boolean supportSubType(Type subType) {
+ for (Type supportedType : getArraySubTypes()) {
+ if (subType.getPrimitiveType() == supportedType.getPrimitiveType()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
@Override
public String toString() {
return toSql(0).toUpperCase();
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
index 1a746a2374..adf74f5c6a 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
@@ -88,6 +88,11 @@ public class MapType extends Type {
return String.format("%sMAP<%s,%s>", leftPadding, keyType.toSql(), structStr);
}
+ @Override
+ public boolean supportSubType(Type subType) {
+ return true;
+ }
+
@Override
public void toThrift(TTypeDesc container) {
TTypeNode node = new TTypeNode();
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
index d9e27d63c8..d64565c465 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
@@ -67,7 +67,10 @@ public enum PrimitiveType {
// sizeof(CollectionValue)
ARRAY("ARRAY", 32, TPrimitiveType.ARRAY),
MAP("MAP", 24, TPrimitiveType.MAP),
- STRUCT("STRUCT", 24, TPrimitiveType.STRUCT),
+ // sizeof(StructValue)
+ // 8-byte pointer and 4-byte size and 1 bytes has_null (13 bytes total)
+ // Aligning to 16 bytes total.
+ STRUCT("STRUCT", 16, TPrimitiveType.STRUCT),
STRING("STRING", 16, TPrimitiveType.STRING),
// Unsupported scalar types.
BINARY("BINARY", -1, TPrimitiveType.BINARY),
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructField.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructField.java
index 6e87b1be40..a084bfe0e4 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructField.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructField.java
@@ -22,25 +22,43 @@ import org.apache.doris.thrift.TTypeDesc;
import org.apache.doris.thrift.TTypeNode;
import com.google.common.base.Strings;
+import com.google.gson.annotations.SerializedName;
/**
* TODO: Support comments for struct fields. The Metastore does not properly store
* comments of struct fields. We set comment to null to avoid compatibility issues.
*/
public class StructField {
+ @SerializedName(value = "name")
protected final String name;
+
+ @SerializedName(value = "type")
protected final Type type;
+
+ @SerializedName(value = "comment")
protected final String comment;
+
+ @SerializedName(value = "position")
protected int position; // in struct
- public StructField(String name, Type type, String comment) {
- this.name = name;
+ @SerializedName(value = "containsNull")
+ private final boolean containsNull; // Now always true (nullable field)
+
+ private static final String DEFAULT_FIELD_NAME = "col";
+
+ public StructField(String name, Type type, String comment, boolean containsNull) {
+ this.name = name.toLowerCase();
this.type = type;
this.comment = comment;
+ this.containsNull = containsNull;
}
public StructField(String name, Type type) {
- this(name, type, null);
+ this(name, type, null, true);
+ }
+
+ public StructField(Type type) {
+ this(DEFAULT_FIELD_NAME, type, null, true);
}
public String getComment() {
@@ -63,11 +81,20 @@ public class StructField {
this.position = position;
}
+ public boolean getContainsNull() {
+ return containsNull;
+ }
+
public String toSql(int depth) {
- String typeSql = (depth < Type.MAX_NESTING_DEPTH) ? type.toSql(depth) : "...";
+ String typeSql;
+ if (depth < Type.MAX_NESTING_DEPTH) {
+ typeSql = !containsNull ? "not_null(" + type.toSql(depth) + ")" : type.toSql(depth);
+ } else {
+ typeSql = "...";
+ }
StringBuilder sb = new StringBuilder(name);
if (type != null) {
- sb.append(":" + typeSql);
+ sb.append(":").append(typeSql);
}
if (comment != null) {
sb.append(String.format(" COMMENT '%s'", comment));
@@ -87,7 +114,7 @@ public class StructField {
// even if we then strip the top-level padding.
String typeStr = type.prettyPrint(lpad);
typeStr = typeStr.substring(lpad);
- sb.append(":" + typeStr);
+ sb.append(":").append(typeStr);
}
if (comment != null) {
sb.append(String.format(" COMMENT '%s'", comment));
@@ -95,12 +122,25 @@ public class StructField {
return sb.toString();
}
+ public static boolean canCastTo(StructField field, StructField targetField) {
+ // TODO(xy): support cast field
+ return false;
+ }
+
+ public boolean matchesField(StructField f) {
+ if (equals(f)) {
+ return true;
+ }
+ return type.matchesType(f.getType()) && containsNull == f.getContainsNull();
+ }
+
public void toThrift(TTypeDesc container, TTypeNode node) {
TStructField field = new TStructField();
field.setName(name);
if (comment != null) {
field.setComment(comment);
}
+ field.setContainsNull(containsNull);
node.struct_fields.add(field);
type.toThrift(container);
}
@@ -111,6 +151,7 @@ public class StructField {
return false;
}
StructField otherStructField = (StructField) other;
- return otherStructField.name.equals(name) && otherStructField.type.equals(type);
+ return otherStructField.name.equals(name) && otherStructField.type.equals(type)
+ && otherStructField.containsNull == containsNull;
}
}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java
index e3ffa3a521..881d451743 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/StructType.java
@@ -17,6 +17,7 @@
package org.apache.doris.catalog;
+import org.apache.doris.thrift.TColumnType;
import org.apache.doris.thrift.TStructField;
import org.apache.doris.thrift.TTypeDesc;
import org.apache.doris.thrift.TTypeNode;
@@ -27,6 +28,7 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import com.google.gson.annotations.SerializedName;
import java.util.ArrayList;
import java.util.HashMap;
@@ -35,7 +37,14 @@ import java.util.HashMap;
* Describes a STRUCT type. STRUCT types have a list of named struct fields.
*/
public class StructType extends Type {
+
+ @SerializedName(value = "fieldMap")
private final HashMap<String, StructField> fieldMap = Maps.newHashMap();
+
+ //@SerializedName(value = "positionToField")
+ //private final HashMap<Integer, StructField> positionToField = Maps.newHashMap();
+
+ @SerializedName(value = "fields")
private final ArrayList<StructField> fields;
public StructType(ArrayList<StructField> fields) {
@@ -44,11 +53,12 @@ public class StructType extends Type {
for (int i = 0; i < this.fields.size(); ++i) {
this.fields.get(i).setPosition(i);
fieldMap.put(this.fields.get(i).getName().toLowerCase(), this.fields.get(i));
+ //positionToField.put(this.fields.get(i).getPosition(), this.fields.get(i));
}
}
public StructType() {
- fields = Lists.newArrayList();
+ this.fields = Lists.newArrayList();
}
@Override
@@ -74,10 +84,36 @@ public class StructType extends Type {
leftPadding, Joiner.on(",\n").join(fieldsSql), leftPadding);
}
+ public static boolean canCastTo(StructType type, StructType targetType) {
+ // TODO(xy) : support cast struct type
+ return false;
+ }
+
+ @Override
+ public boolean isSupported() {
+ for (StructField f : fields) {
+ if (!f.getType().isSupported()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public boolean supportSubType(Type subType) {
+ for (Type supportedType : Type.getStructSubTypes()) {
+ if (subType.getPrimitiveType() == supportedType.getPrimitiveType()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
public void addField(StructField field) {
field.setPosition(fields.size());
fields.add(field);
fieldMap.put(field.getName().toLowerCase(), field);
+ //positionToField.put(field.getPosition(), field);
}
public ArrayList<StructField> getFields() {
@@ -88,9 +124,41 @@ public class StructType extends Type {
return fieldMap.get(fieldName.toLowerCase());
}
+ //public StructField getField(int position) {
+ // return positionToField.get(position);
+ //}
+
public void clearFields() {
fields.clear();
fieldMap.clear();
+ //positionToField.clear();
+ }
+
+ @Override
+ public PrimitiveType getPrimitiveType() {
+ return PrimitiveType.STRUCT;
+ }
+
+ @Override
+ public boolean matchesType(Type t) {
+ if (equals(t)) {
+ return true;
+ }
+
+ if (t.isStructType()) {
+ return false;
+ }
+
+ if (fields.size() != ((StructType) t).getFields().size()) {
+ return false;
+ }
+
+ for (int i = 0; i < fields.size(); i++) {
+ if (!fields.get(i).matchesField(((StructType) t).getFields().get(i))) {
+ return false;
+ }
+ }
+ return true;
}
@Override
@@ -107,7 +175,7 @@ public class StructType extends Type {
TTypeNode node = new TTypeNode();
container.types.add(node);
Preconditions.checkNotNull(fields);
- Preconditions.checkNotNull(!fields.isEmpty());
+ Preconditions.checkState(!fields.isEmpty());
node.setType(TTypeNodeType.STRUCT);
node.setStructFields(new ArrayList<TStructField>());
for (StructField field : fields) {
@@ -119,4 +187,26 @@ public class StructType extends Type {
public String toString() {
return toSql(0);
}
+
+ @Override
+ public TColumnType toColumnTypeThrift() {
+ TColumnType thrift = new TColumnType();
+ thrift.type = PrimitiveType.STRUCT.toThrift();
+ return thrift;
+ }
+
+ @Override
+ public boolean isFixedLengthType() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsTablePartitioning() {
+ return false;
+ }
+
+ @Override
+ public int getSlotSize() {
+ return PrimitiveType.STRUCT.getSlotSize();
+ }
}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index ef3ec7c834..3084a3eb93 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -114,6 +114,7 @@ public abstract class Type {
private static final ArrayList<ScalarType> numericDateTimeTypes;
private static final ArrayList<ScalarType> supportedTypes;
private static final ArrayList<Type> arraySubTypes;
+ private static final ArrayList<Type> structSubTypes;
private static final ArrayList<ScalarType> trivialTypes;
static {
@@ -180,6 +181,10 @@ public abstract class Type {
arraySubTypes.add(CHAR);
arraySubTypes.add(VARCHAR);
arraySubTypes.add(STRING);
+
+ structSubTypes = Lists.newArrayList();
+ structSubTypes.add(INT);
+ structSubTypes.add(STRING);
}
public static final Set<Class> DATE_SUPPORTED_JAVA_TYPE = Sets.newHashSet(LocalDate.class, java.util.Date.class,
@@ -237,6 +242,17 @@ public abstract class Type {
return arraySubTypes;
}
+ public static ArrayList<Type> getStructSubTypes() {
+ return structSubTypes;
+ }
+
+ /**
+ * Return true if this is complex type and support subType
+ */
+ public boolean supportSubType(Type subType) {
+ return false;
+ }
+
/**
* The output of this is stored directly in the hive metastore as the column type.
* The string must match exactly.
@@ -430,7 +446,7 @@ public abstract class Type {
}
public boolean isCollectionType() {
- return isMapType() || isArrayType() || isMultiRowType();
+ return isMapType() || isArrayType() || isMultiRowType() || isStructType();
}
public boolean isMapType() {
@@ -555,6 +571,10 @@ public abstract class Type {
&& !sourceType.isNull()) {
// TODO: current not support cast any non-array type(except for null) to nested array type.
return false;
+ } else if (targetType.isStructType() && sourceType.isStringType()) {
+ return true;
+ } else if (sourceType.isStructType() && targetType.isStructType()) {
+ return StructType.canCastTo((StructType) sourceType, (StructType) targetType);
}
return sourceType.isNull() || sourceType.getPrimitiveType().isCharFamily();
}
@@ -841,7 +861,7 @@ public abstract class Type {
}
Pair<Type, Integer> res = fromThrift(col, tmpNodeIdx);
tmpNodeIdx = res.second.intValue();
- structFields.add(new StructField(name, res.first, comment));
+ structFields.add(new StructField(name, res.first, comment, true));
}
type = new StructType(structFields);
break;
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index 9e873edca4..1d1d392105 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -624,7 +624,7 @@ terminal String
KW_LINES,
KW_IGNORE;
-terminal COMMA, COLON, DOT, DOTDOTDOT, AT, STAR, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET, DIVIDE, MOD, ADD, SUBTRACT, PLACEHOLDER;
+terminal COMMA, COLON, DOT, DOTDOTDOT, AT, STAR, LPAREN, RPAREN, SEMICOLON, LBRACKET, RBRACKET, LBRACE, RBRACE, DIVIDE, MOD, ADD, SUBTRACT, PLACEHOLDER;
terminal BITAND, BITOR, BITXOR, BITNOT;
terminal EQUAL, NOT, LESSTHAN, GREATERTHAN, SET_VAR;
terminal COMMENTED_PLAN_HINT_START, COMMENTED_PLAN_HINT_END;
@@ -739,6 +739,7 @@ nonterminal Expr function_call_expr, array_expr;
nonterminal ArrayLiteral array_literal;
nonterminal StructField struct_field;
nonterminal ArrayList<StructField> struct_field_list;
+nonterminal StructLiteral struct_literal;
nonterminal AnalyticWindow opt_window_clause;
nonterminal AnalyticWindow.Type window_type;
nonterminal AnalyticWindow.Boundary window_boundary;
@@ -940,6 +941,7 @@ precedence left KW_PARTITION;
precedence left KW_PARTITIONS;
precedence right KW_TEMPORARY;
precedence right LBRACKET;
+precedence right LBRACE;
precedence left KW_ENGINE;
// unused
@@ -5954,16 +5956,23 @@ struct_field ::=
;
struct_field_list ::=
- struct_field:field
- {:
- RESULT = Lists.newArrayList(field);
- :}
- | struct_field_list:fields COMMA struct_field:field
- {:
- fields.add(field);
- RESULT = fields;
- :}
- ;
+ struct_field:field
+ {:
+ RESULT = Lists.newArrayList(field);
+ :}
+ | struct_field_list:fields COMMA struct_field:field
+ {:
+ fields.add(field);
+ RESULT = fields;
+ :}
+ ;
+
+struct_literal ::=
+ LBRACE expr_list:list RBRACE
+ {:
+ RESULT = new StructLiteral(list.toArray(new LiteralExpr[0]));
+ :}
+ ;
exists_predicate ::=
KW_EXISTS subquery:s
@@ -5987,6 +5996,8 @@ non_pred_expr ::=
{: RESULT = a; :}
| array_literal:a
{: RESULT = a; :}
+ | struct_literal:s
+ {: RESULT = s; :}
| function_call_expr:e
{: RESULT = e; :}
| KW_DATE STRING_LITERAL:l
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index 123e6e0bbc..334d91d17f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -311,6 +311,10 @@ public class ColumnDef {
}
if (type.getPrimitiveType() == PrimitiveType.STRUCT) {
+ if (isKey()) {
+ throw new AnalysisException("Struct can only be used in the non-key column of"
+ + " the duplicate table at present.");
+ }
if (defaultValue.isSet && defaultValue != DefaultValue.NULL_DEFAULT_VALUE) {
throw new AnalysisException("Struct type column default value just support null");
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 75634912cf..5e39b4352a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -370,7 +370,7 @@ public class CreateTableStmt extends DdlStmt {
// So the float and double could not be the first column in OLAP table.
if (keysColumnNames.isEmpty()) {
throw new AnalysisException("The olap table first column could not be float, double, string"
- + " or array, please use decimal or varchar instead.");
+ + " or array, struct, map, please use decimal or varchar instead.");
}
keysDesc = new KeysDesc(KeysType.DUP_KEYS, keysColumnNames);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
index 9ad70b1967..f1b0ca936e 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -1825,7 +1825,8 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
ARRAY_LITERAL(13),
CAST_EXPR(14),
JSON_LITERAL(15),
- ARITHMETIC_EXPR(16);
+ ARITHMETIC_EXPR(16),
+ STRUCT_LITERAL(17);
private static Map<Integer, ExprSerCode> codeMap = Maps.newHashMap();
@@ -1877,6 +1878,8 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
output.writeInt(ExprSerCode.FUNCTION_CALL.getCode());
} else if (expr instanceof ArrayLiteral) {
output.writeInt(ExprSerCode.ARRAY_LITERAL.getCode());
+ } else if (expr instanceof StructLiteral) {
+ output.writeInt(ExprSerCode.STRUCT_LITERAL.getCode());
} else if (expr instanceof CastExpr) {
output.writeInt(ExprSerCode.CAST_EXPR.getCode());
} else if (expr instanceof ArithmeticExpr) {
@@ -1926,6 +1929,8 @@ public abstract class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
return FunctionCallExpr.read(in);
case ARRAY_LITERAL:
return ArrayLiteral.read(in);
+ case STRUCT_LITERAL:
+ return StructLiteral.read(in);
case CAST_EXPR:
return CastExpr.read(in);
case ARITHMETIC_EXPR:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java
new file mode 100644
index 0000000000..a9209f1c5a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StructLiteral.java
@@ -0,0 +1,150 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.StructField;
+import org.apache.doris.catalog.StructType;
+import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.thrift.TExprNode;
+import org.apache.doris.thrift.TExprNodeType;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class StructLiteral extends LiteralExpr {
+ // only for persist
+ public StructLiteral() {
+ type = new StructType();
+ children = new ArrayList<>();
+ }
+
+ public StructLiteral(LiteralExpr... exprs) throws AnalysisException {
+ type = new StructType();
+ children = new ArrayList<>();
+ for (LiteralExpr expr : exprs) {
+ if (!type.supportSubType(expr.getType())) {
+ throw new AnalysisException("Invalid element type in STRUCT.");
+ }
+ ((StructType) type).addField(new StructField(expr.getType()));
+ children.add(expr);
+ }
+ }
+
+ protected StructLiteral(StructLiteral other) {
+ super(other);
+ }
+
+ @Override
+ protected String toSqlImpl() {
+ List<String> list = new ArrayList<>(children.size());
+ children.forEach(v -> list.add(v.toDigestImpl()));
+ return "STRUCT(" + StringUtils.join(list, ", ") + ")";
+ }
+
+ @Override
+ public String toDigestImpl() {
+ List<String> list = new ArrayList<>(children.size());
+ children.forEach(v -> list.add(v.toDigestImpl()));
+ return "STRUCT(" + StringUtils.join(list, ", ") + ")";
+ }
+
+ @Override
+ public String getStringValue() {
+ List<String> list = new ArrayList<>(children.size());
+ children.forEach(v -> list.add(v.getStringValue()));
+ return "{" + StringUtils.join(list, ", ") + "}";
+ }
+
+ @Override
+ public String getStringValueForArray() {
+ return null;
+ }
+
+ @Override
+ protected void toThrift(TExprNode msg) {
+ msg.node_type = TExprNodeType.STRUCT_LITERAL;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ super.write(out);
+ out.writeInt(children.size());
+ for (Expr e : children) {
+ Expr.writeTo(e, out);
+ }
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ int size = in.readInt();
+ children = new ArrayList<>(size);
+ for (int i = 0; i < size; i++) {
+ children.add(Expr.readIn(in));
+ }
+ }
+
+ public static StructLiteral read(DataInput in) throws IOException {
+ StructLiteral literal = new StructLiteral();
+ literal.readFields(in);
+ return literal;
+ }
+
+ @Override
+ public Expr clone() {
+ return new StructLiteral(this);
+ }
+
+ @Override
+ public boolean isMinValue() {
+ return false;
+ }
+
+ @Override
+ public int compareLiteral(LiteralExpr expr) {
+ return 0;
+ }
+
+ @Override
+ public Expr uncheckedCastTo(Type targetType) throws AnalysisException {
+ if (!targetType.isStructType()) {
+ return super.uncheckedCastTo(targetType);
+ }
+ ArrayList<StructField> fields = ((StructType) targetType).getFields();
+ StructLiteral literal = new StructLiteral(this);
+ for (int i = 0; i < children.size(); ++ i) {
+ Expr child = children.get(i);
+ literal.children.set(i, child.uncheckedCastTo((fields.get(i).getType())));
+ }
+ literal.setType(targetType);
+ return literal;
+ }
+
+ @Override
+ public void checkValueValid() throws AnalysisException {
+ for (Expr e : children) {
+ e.checkValueValid();
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java
index 4246edb5e4..c318e067a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java
@@ -172,7 +172,7 @@ public class Subquery extends Expr {
fieldName = "_" + Integer.toString(i);
}
Preconditions.checkNotNull(fieldName);
- structFields.add(new StructField(fieldName, expr.getType(), null));
+ structFields.add(new StructField(fieldName, expr.getType()));
}
Preconditions.checkState(structFields.size() != 0);
return new StructType(structFields);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
index df00ef62b5..5690c115c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TypeDef.java
@@ -30,6 +30,8 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Set;
/**
* Represents an anonymous type definition, e.g., used in DDL and CASTs.
@@ -94,45 +96,46 @@ public class TypeDef implements ParseNode {
if (type.isArrayType()) {
Type itemType = ((ArrayType) type).getItemType();
if (itemType instanceof ScalarType) {
- analyzeNestedType((ScalarType) itemType);
+ analyzeNestedType(type, (ScalarType) itemType);
}
}
if (type.isMapType()) {
ScalarType keyType = (ScalarType) ((MapType) type).getKeyType();
ScalarType valueType = (ScalarType) ((MapType) type).getKeyType();
- analyzeNestedType(keyType);
- analyzeNestedType(valueType);
+ analyzeNestedType(type, keyType);
+ analyzeNestedType(type, valueType);
}
if (type.isStructType()) {
ArrayList<StructField> fields = ((StructType) type).getFields();
- for (int i = 0; i < fields.size(); i++) {
- ScalarType filedType = (ScalarType) fields.get(i).getType();
- analyzeNestedType(filedType);
+ Set<String> fieldNames = new HashSet<>();
+ for (StructField field : fields) {
+ Type fieldType = field.getType();
+ if (fieldType instanceof ScalarType) {
+ analyzeNestedType(type, (ScalarType) fieldType);
+ if (!fieldNames.add(field.getName())) {
+ throw new AnalysisException("Duplicate field name "
+ + field.getName() + " in struct " + type.toSql());
+ }
+ }
}
}
}
}
- private void analyzeNestedType(ScalarType type) throws AnalysisException {
- if (type.isNull()) {
- throw new AnalysisException("Unsupported data type: " + type.toSql());
- }
- // check whether the array sub-type is supported
- Boolean isSupportType = false;
- for (Type subType : Type.getArraySubTypes()) {
- if (type.getPrimitiveType() == subType.getPrimitiveType()) {
- isSupportType = true;
- break;
- }
+ private void analyzeNestedType(Type parent, ScalarType child) throws AnalysisException {
+ if (child.isNull()) {
+ throw new AnalysisException("Unsupported data type: " + child.toSql());
}
- if (!isSupportType) {
- throw new AnalysisException("Array unsupported sub-type: " + type.toSql());
+ // check whether the sub-type is supported
+ if (!parent.supportSubType(child)) {
+ throw new AnalysisException(
+ parent.getPrimitiveType() + "unsupported sub-type: " + child.toSql());
}
- if (type.getPrimitiveType().isStringType() && !type.isLengthSet()) {
- type.setLength(1);
+ if (child.getPrimitiveType().isStringType() && !child.isLengthSet()) {
+ child.setLength(1);
}
- analyze(type);
+ analyze(child);
}
private void analyzeScalarType(ScalarType scalarType)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 316649991f..18cd580788 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -126,7 +126,7 @@ public class Column implements Writable, GsonPostProcessable {
this.stats = new ColumnStats();
this.visible = true;
this.defineExpr = null;
- this.children = new ArrayList<>(Type.MAX_NESTING_DEPTH);
+ this.children = new ArrayList<>();
this.uniqueId = -1;
}
@@ -182,7 +182,7 @@ public class Column implements Writable, GsonPostProcessable {
this.comment = comment;
this.stats = new ColumnStats();
this.visible = visible;
- this.children = new ArrayList<>(Type.MAX_NESTING_DEPTH);
+ this.children = new ArrayList<>();
createChildrenColumn(this.type, this);
this.uniqueId = colUniqueId;
}
@@ -210,6 +210,13 @@ public class Column implements Writable, GsonPostProcessable {
Column c = new Column(COLUMN_ARRAY_CHILDREN, ((ArrayType) type).getItemType());
c.setIsAllowNull(((ArrayType) type).getContainsNull());
column.addChildrenColumn(c);
+ } else if (type.isStructType()) {
+ ArrayList<StructField> fields = ((StructType) type).getFields();
+ for (StructField field : fields) {
+ Column c = new Column(field.getName(), field.getType());
+ c.setIsAllowNull(field.getContainsNull());
+ column.addChildrenColumn(c);
+ }
}
}
@@ -430,34 +437,41 @@ public class Column implements Writable, GsonPostProcessable {
return tColumn;
}
+ private void setChildrenTColumn(Column children, TColumn tColumn) {
+ TColumn childrenTColumn = new TColumn();
+ childrenTColumn.setColumnName(children.name);
+
+ TColumnType childrenTColumnType = new TColumnType();
+ childrenTColumnType.setType(children.getDataType().toThrift());
+ childrenTColumnType.setLen(children.getStrLen());
+ childrenTColumnType.setPrecision(children.getPrecision());
+ childrenTColumnType.setScale(children.getScale());
+ childrenTColumnType.setIndexLen(children.getOlapColumnIndexSize());
+
+ childrenTColumn.setColumnType(childrenTColumnType);
+ childrenTColumn.setIsAllowNull(children.isAllowNull());
+ // TODO: If we don't set the aggregate type for children, the type will be
+ // considered as TAggregationType::SUM after deserializing in BE.
+ // For now, we make children inherit the aggregate type from their parent.
+ if (tColumn.getAggregationType() != null) {
+ childrenTColumn.setAggregationType(tColumn.getAggregationType());
+ }
+
+ tColumn.children_column.add(childrenTColumn);
+ toChildrenThrift(children, childrenTColumn);
+ }
+
private void toChildrenThrift(Column column, TColumn tColumn) {
if (column.type.isArrayType()) {
Column children = column.getChildren().get(0);
-
- TColumn childrenTColumn = new TColumn();
- childrenTColumn.setColumnName(children.name);
-
- TColumnType childrenTColumnType = new TColumnType();
- childrenTColumnType.setType(children.getDataType().toThrift());
- childrenTColumnType.setType(children.getDataType().toThrift());
- childrenTColumnType.setLen(children.getStrLen());
- childrenTColumnType.setPrecision(children.getPrecision());
- childrenTColumnType.setScale(children.getScale());
-
- childrenTColumnType.setIndexLen(children.getOlapColumnIndexSize());
- childrenTColumn.setColumnType(childrenTColumnType);
- childrenTColumn.setIsAllowNull(children.isAllowNull());
- // TODO: If we don't set the aggregate type for children, the type will be
- // considered as TAggregationType::SUM after deserializing in BE.
- // For now, we make children inherit the aggregate type from their parent.
- if (tColumn.getAggregationType() != null) {
- childrenTColumn.setAggregationType(tColumn.getAggregationType());
- }
-
tColumn.setChildrenColumn(new ArrayList<>());
- tColumn.children_column.add(childrenTColumn);
-
- toChildrenThrift(children, childrenTColumn);
+ setChildrenTColumn(children, tColumn);
+ } else if (column.type.isStructType()) {
+ List<Column> childrenColumns = column.getChildren();
+ tColumn.setChildrenColumn(new ArrayList<>());
+ for (Column children : childrenColumns) {
+ setChildrenTColumn(children, tColumn);
+ }
}
}
diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex
index 85a15f0b24..bd59b63e61 100644
--- a/fe/fe-core/src/main/jflex/sql_scanner.flex
+++ b/fe/fe-core/src/main/jflex/sql_scanner.flex
@@ -504,6 +504,8 @@ import org.apache.doris.qe.SqlModeHelper;
tokenIdMap.put(new Integer(SqlParserSymbols.RPAREN), ")");
tokenIdMap.put(new Integer(SqlParserSymbols.LBRACKET), "[");
tokenIdMap.put(new Integer(SqlParserSymbols.RBRACKET), "]");
+ tokenIdMap.put(new Integer(SqlParserSymbols.LBRACE), "{");
+ tokenIdMap.put(new Integer(SqlParserSymbols.RBRACE), "}");
tokenIdMap.put(new Integer(SqlParserSymbols.COLON), ":");
tokenIdMap.put(new Integer(SqlParserSymbols.SEMICOLON), ";");
tokenIdMap.put(new Integer(SqlParserSymbols.FLOATINGPOINT_LITERAL),
@@ -653,6 +655,8 @@ EndOfLineComment = "--" !({HintContent}|{ContainsLineTerminator}) {LineTerminato
";" { return newToken(SqlParserSymbols.SEMICOLON, null); }
"[" { return newToken(SqlParserSymbols.LBRACKET, null); }
"]" { return newToken(SqlParserSymbols.RBRACKET, null); }
+"{" { return newToken(SqlParserSymbols.LBRACE, null); }
+"}" { return newToken(SqlParserSymbols.RBRACE, null); }
"/" { return newToken(SqlParserSymbols.DIVIDE, null); }
"%" { return newToken(SqlParserSymbols.MOD, null); }
"+" { return newToken(SqlParserSymbols.ADD, null); }
diff --git a/gensrc/proto/types.proto b/gensrc/proto/types.proto
index 5686a6ac9d..dea020d0b4 100644
--- a/gensrc/proto/types.proto
+++ b/gensrc/proto/types.proto
@@ -39,6 +39,7 @@ message PScalarType {
message PStructField {
required string name = 1;
optional string comment = 2;
+ optional bool contains_null = 3;
};
message PTypeNode {
diff --git a/gensrc/thrift/Exprs.thrift b/gensrc/thrift/Exprs.thrift
index 569e42c3ef..e507a8e7aa 100644
--- a/gensrc/thrift/Exprs.thrift
+++ b/gensrc/thrift/Exprs.thrift
@@ -44,6 +44,7 @@ enum TExprNodeType {
INFO_FUNC,
FUNCTION_CALL,
ARRAY_LITERAL,
+ STRUCT_LITERAL,
// TODO: old style compute functions. this will be deprecated
COMPUTE_FUNCTION_CALL,
diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift
index 8eb88e1d3d..08fe18daa2 100644
--- a/gensrc/thrift/Types.thrift
+++ b/gensrc/thrift/Types.thrift
@@ -126,6 +126,7 @@ struct TScalarType {
struct TStructField {
1: required string name
2: optional string comment
+ 3: optional bool contains_null
}
struct TTypeNode {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 10/18: [fix](struct-type) support to remove double or single quote (#15977)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 747cc04cc34d487a15635766cf75f154b5beadcf
Author: carlvinhust2012 <hu...@126.com>
AuthorDate: Mon Jan 16 18:01:16 2023 +0800
[fix](struct-type) support to remove double or single quote (#15977)
---
be/src/vec/data_types/data_type_struct.cpp | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/be/src/vec/data_types/data_type_struct.cpp b/be/src/vec/data_types/data_type_struct.cpp
index 87b5c4f110..a3a54966b0 100644
--- a/be/src/vec/data_types/data_type_struct.cpp
+++ b/be/src/vec/data_types/data_type_struct.cpp
@@ -114,6 +114,7 @@ Status DataTypeStruct::from_string(ReadBuffer& rb, IColumn* column) const {
return Status::InvalidArgument("Invalid Length");
}
ReadBuffer field_rb(rb.position(), field_len);
+
size_t len = 0;
auto start_rb = field_rb.position();
while (!field_rb.eof() && *start_rb != ':') {
@@ -122,11 +123,12 @@ Status DataTypeStruct::from_string(ReadBuffer& rb, IColumn* column) const {
}
ReadBuffer field(field_rb.position() + len + 1, field_rb.count() - len - 1);
- if (field.count() < 2 || *field.position() != '"' || *field.end() != '"') {
- field_rbs.push_back(field);
+ if (field.count() >= 2 && ((*field.position() == '"' && *(field.end() - 1) == '"') ||
+ (*field.position() == '\'' && *(field.end() - 1) == '\''))) {
+ ReadBuffer field_no_quote(field.position() + 1, field.count() - 2);
+ field_rbs.push_back(field_no_quote);
} else {
- ReadBuffer field_has_quote(field.position() + 1, field.count() - 2);
- field_rbs.push_back(field_has_quote);
+ field_rbs.push_back(field);
}
rb.position() += field_len + 1;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 09/18: save (#15908)
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch struct-map-type-rebase
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 273bc1b703843cfe59eec14b800a93ac609c7865
Author: xy720 <22...@users.noreply.github.com>
AuthorDate: Fri Jan 13 17:46:49 2023 +0800
save (#15908)
---
be/src/vec/data_types/data_type_factory.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/be/src/vec/data_types/data_type_factory.cpp b/be/src/vec/data_types/data_type_factory.cpp
index c4fbfbd61b..78c5fae6b6 100644
--- a/be/src/vec/data_types/data_type_factory.cpp
+++ b/be/src/vec/data_types/data_type_factory.cpp
@@ -38,7 +38,7 @@ DataTypePtr DataTypeFactory::create_data_type(const doris::Field& col_desc) {
names.reserve(field_size);
for (size_t i = 0; i < field_size; i++) {
dataTypes.push_back(create_data_type(*col_desc.get_sub_field(i)));
- names.push_back(col_desc.get_sub_field(i).name());
+ names.push_back(col_desc.get_sub_field(i)->name());
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
} else {
@@ -66,7 +66,7 @@ DataTypePtr DataTypeFactory::create_data_type(const TabletColumn& col_desc, bool
names.reserve(col_size);
for (size_t i = 0; i < col_size; i++) {
dataTypes.push_back(create_data_type(col_desc.get_sub_column(i)));
- names.push_back(col_desc.get_sub_field(i).name());
+ names.push_back(col_desc.get_sub_column(i).name());
}
nested = std::make_shared<DataTypeStruct>(dataTypes, names);
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org