You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2021/12/16 02:46:29 UTC
[incubator-doris] branch master updated: [feat](lateral-view) Support execution of lateral view stmt (#7255)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0499b22 [feat](lateral-view) Support execution of lateral view stmt (#7255)
0499b22 is described below
commit 0499b2211b7ad685a6aae17508b3120b3490bcec
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Thu Dec 16 10:46:15 2021 +0800
[feat](lateral-view) Support execution of lateral view stmt (#7255)
1. Add table function node
2. Add 3 table functions: explode_split, explode_bitmap and explode_json_array
---
be/src/common/daemon.cpp | 2 +
be/src/exec/CMakeLists.txt | 1 +
be/src/exec/exec_node.cpp | 5 +
be/src/exec/table_function_node.cpp | 340 +++++++++++++++++++++
be/src/exec/table_function_node.h | 77 +++++
be/src/exprs/CMakeLists.txt | 9 +-
be/src/exprs/scalar_fn_call.cpp | 53 +---
be/src/exprs/scalar_fn_call.h | 5 +
be/src/exprs/slot_ref.cpp | 5 +-
.../exprs/table_function/dummy_table_functions.cpp | 49 +++
.../exprs/table_function/dummy_table_functions.h | 47 +++
be/src/exprs/table_function/explode_bitmap.cpp | 124 ++++++++
be/src/exprs/table_function/explode_bitmap.h | 56 ++++
be/src/exprs/table_function/explode_json_array.cpp | 202 ++++++++++++
be/src/exprs/table_function/explode_json_array.h | 130 ++++++++
be/src/exprs/table_function/explode_split.cpp | 115 +++++++
be/src/exprs/table_function/explode_split.h | 56 ++++
be/src/exprs/table_function/table_function.h | 65 ++++
.../table_function/table_function_factory.cpp | 48 +++
.../exprs/table_function/table_function_factory.h | 36 +++
be/src/runtime/descriptors.h | 1 -
be/src/util/bitmap_value.h | 130 ++++++++
be/test/util/bitmap_value_test.cpp | 37 +++
docs/.vuepress/sidebar/en.js | 10 +
docs/.vuepress/sidebar/zh-CN.js | 10 +
.../table-functions/explode-bitmap.md | 157 ++++++++++
.../table-functions/explode-json-array.md | 286 +++++++++++++++++
.../sql-functions/table-functions/explode-split.md | 112 +++++++
.../Data Manipulation/lateral-view.md | 94 ++++++
.../table-functions/explode-bitmap.md | 157 ++++++++++
.../table-functions/explode-json-array.md | 286 +++++++++++++++++
.../sql-functions/table-functions/explode-split.md | 112 +++++++
.../Data Manipulation/lateral-view.md | 94 ++++++
.../apache/doris/analysis/FunctionCallExpr.java | 147 +++++----
.../org/apache/doris/analysis/LateralViewRef.java | 30 +-
.../java/org/apache/doris/analysis/SlotRef.java | 14 +
.../java/org/apache/doris/catalog/FunctionSet.java | 50 ++-
.../apache/doris/planner/DistributedPlanner.java | 1 +
.../main/java/org/apache/doris/qe/Coordinator.java | 2 +
.../doris/planner/TableFunctionPlanTest.java | 61 +++-
40 files changed, 3046 insertions(+), 170 deletions(-)
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index ac4e645..b6863b5 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -40,6 +40,7 @@
#include "exprs/new_in_predicate.h"
#include "exprs/operators.h"
#include "exprs/string_functions.h"
+#include "exprs/table_function/dummy_table_functions.h"
#include "exprs/time_operators.h"
#include "exprs/timestamp_functions.h"
#include "exprs/topn_function.h"
@@ -264,6 +265,7 @@ void Daemon::init(int argc, char** argv, const std::vector<StorePath>& paths) {
HllFunctions::init();
HashFunctions::init();
TopNFunctions::init();
+ DummyTableFunctions::init();
LOG(INFO) << CpuInfo::debug_string();
LOG(INFO) << DiskInfo::debug_string();
diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt
index e33e30d..f3856ec 100644
--- a/be/src/exec/CMakeLists.txt
+++ b/be/src/exec/CMakeLists.txt
@@ -58,6 +58,7 @@ set(EXEC_FILES
plain_text_line_reader.cpp
csv_scan_node.cpp
csv_scanner.cpp
+ table_function_node.cpp
es_scan_node.cpp
es_http_scan_node.cpp
es_http_scanner.cpp
diff --git a/be/src/exec/exec_node.cpp b/be/src/exec/exec_node.cpp
index cdd4917..d692a79 100644
--- a/be/src/exec/exec_node.cpp
+++ b/be/src/exec/exec_node.cpp
@@ -48,6 +48,7 @@
#include "exec/schema_scan_node.h"
#include "exec/select_node.h"
#include "exec/spill_sort_node.h"
+#include "exec/table_function_node.h"
#include "exec/topn_node.h"
#include "exec/union_node.h"
#include "exprs/expr_context.h"
@@ -472,6 +473,10 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN
*node = pool->add(new AssertNumRowsNode(pool, tnode, descs));
return Status::OK();
+ case TPlanNodeType::TABLE_FUNCTION_NODE:
+ *node = pool->add(new TableFunctionNode(pool, tnode, descs));
+ return Status::OK();
+
default:
map<int, const char*>::const_iterator i =
_TPlanNodeType_VALUES_TO_NAMES.find(tnode.node_type);
diff --git a/be/src/exec/table_function_node.cpp b/be/src/exec/table_function_node.cpp
new file mode 100644
index 0000000..d192354
--- /dev/null
+++ b/be/src/exec/table_function_node.cpp
@@ -0,0 +1,340 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/table_function_node.h"
+
+#include "exprs/expr.h"
+#include "exprs/expr_context.h"
+#include "runtime/descriptors.h"
+#include "runtime/raw_value.h"
+#include "runtime/row_batch.h"
+#include "runtime/runtime_state.h"
+#include "runtime/tuple_row.h"
+#include "exprs/table_function/table_function_factory.h"
+
+namespace doris {
+
+TableFunctionNode::TableFunctionNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs)
+ : ExecNode(pool, tnode, descs) {
+
+}
+
+TableFunctionNode::~TableFunctionNode() {
+
+}
+
+Status TableFunctionNode::init(const TPlanNode& tnode, RuntimeState* state) {
+ RETURN_IF_ERROR(ExecNode::init(tnode, state));
+
+ for (const TExpr& texpr : tnode.table_function_node.fnCallExprList) {
+ ExprContext* ctx = nullptr;
+ RETURN_IF_ERROR(Expr::create_expr_tree(_pool, texpr, &ctx));
+ _fn_ctxs.push_back(ctx);
+
+ Expr* root = ctx->root();
+ const std::string& tf_name = root->fn().name.function_name;
+ TableFunction* fn;
+ RETURN_IF_ERROR(TableFunctionFactory::get_fn(tf_name, _pool, &fn));
+ fn->set_expr_context(ctx);
+ _fns.push_back(fn);
+ }
+ _fn_num = _fns.size();
+ _fn_values.resize(_fn_num);
+
+ // Prepare output slot ids
+ RETURN_IF_ERROR(_prepare_output_slot_ids(tnode));
+ return Status::OK();
+}
+
+Status TableFunctionNode::_prepare_output_slot_ids(const TPlanNode& tnode) {
+ // Prepare output slot ids
+ if (tnode.table_function_node.outputSlotIds.empty()) {
+ return Status::InternalError("Output slots of table function node is empty");
+ }
+ SlotId max_id = -1;
+ for (auto slot_id : tnode.table_function_node.outputSlotIds) {
+ if (slot_id > max_id) {
+ max_id = slot_id;
+ }
+ }
+ _output_slot_ids = std::vector<bool>(max_id + 1, false);
+ for (auto slot_id : tnode.table_function_node.outputSlotIds) {
+ _output_slot_ids[slot_id] = true;
+ }
+
+ return Status::OK();
+}
+
+Status TableFunctionNode::prepare(RuntimeState* state) {
+ RETURN_IF_ERROR(ExecNode::prepare(state));
+
+ RETURN_IF_ERROR(Expr::prepare(_fn_ctxs, state, _row_descriptor, expr_mem_tracker()));
+ for (auto fn : _fns) {
+ RETURN_IF_ERROR(fn->prepare());
+ }
+ return Status::OK();
+}
+
+Status TableFunctionNode::open(RuntimeState* state) {
+ SCOPED_TIMER(_runtime_profile->total_time_counter());
+ RETURN_IF_CANCELLED(state);
+ RETURN_IF_ERROR(ExecNode::open(state));
+
+ RETURN_IF_ERROR(Expr::open(_fn_ctxs, state));
+ for (auto fn : _fns) {
+ RETURN_IF_ERROR(fn->open());
+ }
+
+ RETURN_IF_ERROR(_children[0]->open(state));
+ return Status::OK();
+}
+
+Status TableFunctionNode::_process_next_child_row() {
+ if (_cur_child_offset == _cur_child_batch->num_rows()) {
+ _child_batch_exhausted = true;
+ return Status::OK();
+ }
+ _cur_child_tuple_row = _cur_child_batch->get_row(_cur_child_offset++);
+ for (TableFunction* fn : _fns) {
+ RETURN_IF_ERROR(fn->process(_cur_child_tuple_row));
+ }
+
+ _child_batch_exhausted = false;
+ return Status::OK();
+}
+
+// Returns the index of fn of the last eos counted from back to front
+// eg: there are 3 functions in `_fns`
+// eos: false, true, true
+// return: 1
+//
+// eos: false, false, true
+// return: 2
+//
+// eos: false, false, false
+// return: -1
+//
+// eos: true, true, true
+// return: 0
+//
+// return:
+// 0: all fns are eos
+// -1: all fns are not eos
+// >0: some of fns are eos
+int TableFunctionNode::_find_last_fn_eos_idx() {
+ for (int i = _fn_num - 1; i >=0; --i) {
+ if (!_fns[i]->eos()) {
+ if (i == _fn_num - 1) {
+ return -1;
+ } else {
+ return i + 1;
+ }
+ }
+ }
+ // all eos
+ return 0;
+}
+
+// Roll to reset the table function.
+// Eg:
+// There are 3 functions f1, f2 and f3 in `_fns`.
+// If `last_eos_idx` is 1, which means f2 and f3 are eos.
+// So we need to forward f1, and reset f2 and f3.
+bool TableFunctionNode::_roll_table_functions(int last_eos_idx) {
+ bool fn_eos = false;
+ int i = last_eos_idx - 1;
+ for (; i >= 0; --i) {
+ _fns[i]->forward(&fn_eos);
+ if (!fn_eos) {
+ break;
+ }
+ }
+ if (i == -1) {
+ // after forward, all functions are eos.
+ // we should process next child row to get more table function results.
+ return false;
+ }
+
+ for (int j = i + 1; j < _fn_num; ++j) {
+ _fns[j]->reset();
+ }
+
+ return true;
+}
+
+// There are 2 while loops in this method.
+// The outer loop is to get the next batch from child node.
+// And the inner loop is to expand the row by table functions, and output row by row.
+Status TableFunctionNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eos) {
+ RETURN_IF_ERROR(exec_debug_action(TExecNodePhase::GETNEXT));
+ SCOPED_TIMER(_runtime_profile->total_time_counter());
+
+ const RowDescriptor& parent_rowdesc = row_batch->row_desc();
+ const RowDescriptor& child_rowdesc = _children[0]->row_desc();
+ if (_parent_tuple_desc_size == -1) {
+ _parent_tuple_desc_size = parent_rowdesc.tuple_descriptors().size();
+ _child_tuple_desc_size = child_rowdesc.tuple_descriptors().size();
+ for (int i = 0; i < _child_tuple_desc_size; ++i) {
+ _child_slot_sizes.push_back(child_rowdesc.tuple_descriptors()[i]->slots().size());
+ }
+ }
+
+ uint8_t* tuple_buffer = nullptr;
+ Tuple* tuple_ptr = nullptr;
+ Tuple* pre_tuple_ptr = nullptr;
+ int row_idx = 0;
+
+ while (true) {
+ RETURN_IF_CANCELLED(state);
+ RETURN_IF_ERROR(state->check_query_state("TableFunctionNode, while getting next batch."));
+
+ if (_cur_child_batch == nullptr) {
+ _cur_child_batch.reset(new RowBatch(child_rowdesc, state->batch_size(), mem_tracker().get()));
+ }
+ if (_child_batch_exhausted) {
+ // current child batch is exhausted, get next batch from child
+ RETURN_IF_ERROR(_children[0]->get_next(state, _cur_child_batch.get(), eos));
+ if (*eos) {
+ break;
+ }
+ _cur_child_offset = 0;
+ RETURN_IF_ERROR(_process_next_child_row());
+ if (_child_batch_exhausted) {
+ _cur_child_batch->reset();
+ continue;
+ }
+ }
+
+ while (true) {
+ int idx = _find_last_fn_eos_idx();
+ if (idx == 0) {
+ // all table functions' results are exhausted, process next child row
+ RETURN_IF_ERROR(_process_next_child_row());
+ if (_child_batch_exhausted) {
+ break;
+ }
+ } else if (idx < _fn_num && idx != -1) {
+ // some of table functions' results are exhausted
+ if (!_roll_table_functions(idx)) {
+ // continue to process next child row
+ continue;
+ }
+ }
+
+ // get slots from every table function
+ // Notice that _fn_values[i] may be null if the table function has empty result set.
+ for (int i = 0; i < _fn_num; i++) {
+ RETURN_IF_ERROR(_fns[i]->get_value(&_fn_values[i]));
+ }
+
+ // allocate memory for row batch for the first time
+ if (tuple_buffer == nullptr) {
+ int64_t tuple_buffer_size;
+ RETURN_IF_ERROR(
+ row_batch->resize_and_allocate_tuple_buffer(state, &tuple_buffer_size, &tuple_buffer));
+ tuple_ptr = reinterpret_cast<Tuple*>(tuple_buffer);
+ }
+
+ pre_tuple_ptr = tuple_ptr;
+ // The tuples order in parent row batch should be
+ // child1, child2, tf1, tf2, ...
+ TupleRow* parent_tuple_row = row_batch->get_row(row_idx++);
+ // 1. copy child tuples
+ int tuple_idx = 0;
+ for (int i = 0; i < _child_tuple_desc_size; tuple_idx++, i++) {
+ TupleDescriptor* child_tuple_desc = child_rowdesc.tuple_descriptors()[tuple_idx];
+ TupleDescriptor* parent_tuple_desc = parent_rowdesc.tuple_descriptors()[tuple_idx];
+
+ for (int j = 0; j < _child_slot_sizes[i]; ++j) {
+ SlotDescriptor* child_slot_desc = child_tuple_desc->slots()[j];
+ SlotDescriptor* parent_slot_desc = parent_tuple_desc->slots()[j];
+
+ if (_output_slot_ids[parent_slot_desc->id()]) {
+ Tuple* child_tuple = _cur_child_tuple_row->get_tuple(child_rowdesc.get_tuple_idx(child_tuple_desc->id()));
+ void* dest_slot = tuple_ptr->get_slot(parent_slot_desc->tuple_offset());
+ RawValue::write(child_tuple->get_slot(child_slot_desc->tuple_offset()), dest_slot, parent_slot_desc->type(), row_batch->tuple_data_pool());
+ tuple_ptr->set_not_null(parent_slot_desc->null_indicator_offset());
+ } else {
+ tuple_ptr->set_null(parent_slot_desc->null_indicator_offset());
+ }
+ }
+ parent_tuple_row->set_tuple(tuple_idx, tuple_ptr);
+ tuple_ptr = reinterpret_cast<Tuple*>(reinterpret_cast<uint8_t*>(tuple_ptr) + parent_tuple_desc->byte_size());
+ }
+
+ // 2. copy funtion result
+ for (int i = 0; tuple_idx < _parent_tuple_desc_size; tuple_idx++, i++) {
+ TupleDescriptor* parent_tuple_desc = parent_rowdesc.tuple_descriptors()[tuple_idx];
+ SlotDescriptor* parent_slot_desc = parent_tuple_desc->slots()[0];
+ void* dest_slot = tuple_ptr->get_slot(parent_slot_desc->tuple_offset());
+ // if (_fn_values[i] != nullptr && _output_slot_ids[parent_slot_desc->id()]) {
+ if (_fn_values[i] != nullptr) {
+ RawValue::write(_fn_values[i], dest_slot, parent_slot_desc->type(), row_batch->tuple_data_pool());
+ tuple_ptr->set_not_null(parent_slot_desc->null_indicator_offset());
+ } else {
+ tuple_ptr->set_null(parent_slot_desc->null_indicator_offset());
+ }
+ parent_tuple_row->set_tuple(tuple_idx, tuple_ptr);
+
+ tuple_ptr = reinterpret_cast<Tuple*>(reinterpret_cast<uint8_t*>(tuple_ptr) + parent_tuple_desc->byte_size());
+ }
+
+ // 3. eval conjuncts
+ if (eval_conjuncts(&_conjunct_ctxs[0], _conjunct_ctxs.size(), parent_tuple_row)) {
+ row_batch->commit_last_row();
+ ++_num_rows_returned;
+ } else {
+ tuple_ptr = pre_tuple_ptr;
+ }
+
+ // Forward after write success.
+ // Because data in `_fn_values` points to the data saved in functions.
+ // And `forward` will change the data in functions.
+ bool tmp;
+ _fns[_fn_num - 1]->forward(&tmp);
+
+ if (row_batch->at_capacity()) {
+ *eos = false;
+ break;
+ }
+ }
+
+ if (row_batch->at_capacity()) {
+ break;
+ }
+ }
+ if (reached_limit()) {
+ int num_rows_over = _num_rows_returned - _limit;
+ row_batch->set_num_rows(row_batch->num_rows() - num_rows_over);
+ _num_rows_returned -= num_rows_over;
+ COUNTER_SET(_rows_returned_counter, _num_rows_returned);
+ *eos = true;
+ }
+
+ return Status::OK();
+}
+
+Status TableFunctionNode::close(RuntimeState* state) {
+ if (is_closed()) {
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(exec_debug_action(TExecNodePhase::CLOSE));
+ Expr::close(_fn_ctxs, state);
+ return ExecNode::close(state);
+}
+
+}; // namespace doris
diff --git a/be/src/exec/table_function_node.h b/be/src/exec/table_function_node.h
new file mode 100644
index 0000000..f91577e
--- /dev/null
+++ b/be/src/exec/table_function_node.h
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "exec/exec_node.h"
+
+namespace doris {
+
+class MemPool;
+class RowBatch;
+class TableFunction;
+class TupleRow;
+
+// TableFunctionNode
+class TableFunctionNode : public ExecNode {
+public:
+ TableFunctionNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs);
+ ~TableFunctionNode();
+
+ virtual Status init(const TPlanNode& tnode, RuntimeState* state = nullptr);
+ virtual Status prepare(RuntimeState* state);
+ virtual Status open(RuntimeState* state);
+ virtual Status get_next(RuntimeState* state, RowBatch* row_batch, bool* eos);
+ virtual Status close(RuntimeState* state);
+
+private:
+
+ Status _prepare_output_slot_ids(const TPlanNode& tnode);
+
+ // return:
+ // 0: all fns are eos
+ // -1: all fns are not eos
+ // >0: some of fns are eos
+ int _find_last_fn_eos_idx();
+
+ Status _process_next_child_row();
+
+ bool _roll_table_functions(int last_eos_idx);
+
+private:
+
+ int64_t _cur_child_offset = 0;
+ TupleRow* _cur_child_tuple_row = nullptr;
+ std::shared_ptr<RowBatch> _cur_child_batch;
+ // true means current child batch is completely consumed.
+ // we should get next batch from child node.
+ bool _child_batch_exhausted = true;
+
+ std::vector<ExprContext*> _fn_ctxs;
+ std::vector<TableFunction*> _fns;
+ std::vector<void*> _fn_values;
+ int _fn_num = 0;
+
+ // std::unordered_set<SlotId> _output_slot_ids;
+ std::vector<bool> _output_slot_ids;
+
+ int _parent_tuple_desc_size = -1;
+ int _child_tuple_desc_size = -1;
+ std::vector<int> _child_slot_sizes;
+};
+
+}; // namespace doris
diff --git a/be/src/exprs/CMakeLists.txt b/be/src/exprs/CMakeLists.txt
index 99ebbf4..5ebf1d1 100644
--- a/be/src/exprs/CMakeLists.txt
+++ b/be/src/exprs/CMakeLists.txt
@@ -70,4 +70,11 @@ add_library(Exprs
bitmap_function.cpp
hll_function.cpp
grouping_sets_functions.cpp
- topn_function.cpp)
+ topn_function.cpp
+
+ table_function/explode_split.cpp
+ table_function/explode_bitmap.cpp
+ table_function/explode_json_array.cpp
+ table_function/table_function_factory.cpp
+ table_function/dummy_table_functions.cpp
+)
diff --git a/be/src/exprs/scalar_fn_call.cpp b/be/src/exprs/scalar_fn_call.cpp
index 2a7fc82..5e1d081 100644
--- a/be/src/exprs/scalar_fn_call.cpp
+++ b/be/src/exprs/scalar_fn_call.cpp
@@ -72,7 +72,6 @@ Status ScalarFnCall::prepare(RuntimeState* state, const RowDescriptor& desc, Exp
}
_fn_context_index = context->register_func(state, return_type, arg_types, varargs_buffer_size);
- // _scalar_fn = OpcodeRegistry::instance()->get_function_ptr(_opcode);
Status status = Status::OK();
if (_scalar_fn == nullptr) {
if (SymbolsUtil::is_mangled(_fn.scalar_fn.symbol)) {
@@ -88,61 +87,11 @@ Status ScalarFnCall::prepare(RuntimeState* state, const RowDescriptor& desc, Exp
// ret_type = ColumnType(thrift_to_type(_fn.ret_type));
std::string symbol = SymbolsUtil::mangle_user_function(_fn.scalar_fn.symbol, arg_types,
_fn.has_var_args, nullptr);
+
status = UserFunctionCache::instance()->get_function_ptr(
_fn.id, symbol, _fn.hdfs_location, _fn.checksum, &_scalar_fn, &_cache_entry);
}
}
-#if 0
- // If the codegen object hasn't been created yet and we're calling a builtin or native
- // UDF with <= 8 non-variadic arguments, we can use the interpreted path and call the
- // builtin without codegen. This saves us the overhead of creating the codegen object
- // when it's not necessary (i.e., in plan fragments with no codegen-enabled operators).
- // In addition, we can never codegen char arguments.
- // TODO: codegen for char arguments
- if (char_arg || (!state->codegen_created() && num_fixed_args() <= 8 &&
- (_fn.binary_type == TFunctionBinaryType::BUILTIN ||
- _fn.binary_type == TFunctionBinaryType::NATIVE))) {
- // Builtins with char arguments must still have <= 8 arguments.
- // TODO: delete when we have codegen for char arguments
- if (char_arg) {
- DCHECK(num_fixed_args() <= 8 && _fn.binary_type == TFunctionBinaryType::BUILTIN);
- }
- Status status = UserFunctionCache::instance()->GetSoFunctionPtr(
- _fn.hdfs_location, _fn.scalar_fn.symbol, &_scalar_fn, &cache_entry_);
- if (!status.ok()) {
- if (_fn.binary_type == TFunctionBinaryType::BUILTIN) {
- // Builtins symbols should exist unless there is a version mismatch.
- status.SetErrorMsg(ErrorMsg(TErrorCode::MISSING_BUILTIN,
- _fn.name.function_name, _fn.scalar_fn.symbol));
- return status;
- } else {
- DCHECK_EQ(_fn.binary_type, TFunctionBinaryType::NATIVE);
- return Status::InternalError(strings::Substitute("Problem loading UDF '$0':\n$1",
- _fn.name.function_name, status.GetDetail()));
- return status;
- }
- }
- } else {
- // If we got here, either codegen is enabled or we need codegen to run this function.
- LlvmCodeGen* codegen;
- RETURN_IF_ERROR(state->GetCodegen(&codegen));
-
- if (_fn.binary_type == TFunctionBinaryType::IR) {
- std::string local_path;
- RETURN_IF_ERROR(UserFunctionCache::instance()->GetLocalLibPath(
- _fn.hdfs_location, UserFunctionCache::TYPE_IR, &local_path));
- // Link the UDF module into this query's main module (essentially copy the UDF
- // module into the main module) so the UDF's functions are available in the main
- // module.
- RETURN_IF_ERROR(codegen->LinkModule(local_path));
- }
-
- Function* ir_udf_wrapper;
- RETURN_IF_ERROR(GetCodegendComputeFn(state, &ir_udf_wrapper));
- // TODO: don't do this for child exprs
- codegen->AddFunctionToJit(ir_udf_wrapper, &_scalar_fn_wrapper);
- }
-#endif
if (_fn.scalar_fn.__isset.prepare_fn_symbol) {
RETURN_IF_ERROR(get_function(state, _fn.scalar_fn.prepare_fn_symbol,
reinterpret_cast<void**>(&_prepare_fn)));
diff --git a/be/src/exprs/scalar_fn_call.h b/be/src/exprs/scalar_fn_call.h
index e94e9d0..cf8f1f7 100644
--- a/be/src/exprs/scalar_fn_call.h
+++ b/be/src/exprs/scalar_fn_call.h
@@ -54,6 +54,11 @@ public:
return pool->add(new ScalarFnCall(*this));
}
+ // TODO: just for table function.
+ // It is not good to expose this field to public.
+ // We should refactor it after implementing real table functions.
+ int get_fn_context_index() const { return _fn_context_index; }
+
protected:
friend class Expr;
diff --git a/be/src/exprs/slot_ref.cpp b/be/src/exprs/slot_ref.cpp
index 0068c03..204d20c 100644
--- a/be/src/exprs/slot_ref.cpp
+++ b/be/src/exprs/slot_ref.cpp
@@ -20,6 +20,7 @@
#include <sstream>
#include "gen_cpp/Exprs_types.h"
+#include "gutil/strings/substitute.h"
#include "runtime/runtime_state.h"
#include "util/types.h"
@@ -62,7 +63,7 @@ Status SlotRef::prepare(const SlotDescriptor* slot_desc, const RowDescriptor& ro
}
_tuple_idx = row_desc.get_tuple_idx(slot_desc->parent());
if (_tuple_idx == RowDescriptor::INVALID_IDX) {
- return Status::InternalError("can't support");
+ return Status::InternalError(strings::Substitute("failed to get tuple idx with tuple id: $0, slot id: $1", slot_desc->parent(), _slot_id));
}
_tuple_is_nullable = row_desc.tuple_is_nullable(_tuple_idx);
_slot_offset = slot_desc->tuple_offset();
@@ -94,7 +95,7 @@ Status SlotRef::prepare(RuntimeState* state, const RowDescriptor& row_desc, Expr
// TODO(marcel): get from runtime state
_tuple_idx = row_desc.get_tuple_idx(slot_desc->parent());
if (_tuple_idx == RowDescriptor::INVALID_IDX) {
- return Status::InternalError("can't support");
+ return Status::InternalError(strings::Substitute("failed to get tuple idx when prepare with tuple id: $0, slot id: $1", slot_desc->parent(), _slot_id));
}
DCHECK(_tuple_idx != RowDescriptor::INVALID_IDX);
_tuple_is_nullable = row_desc.tuple_is_nullable(_tuple_idx);
diff --git a/be/src/exprs/table_function/dummy_table_functions.cpp b/be/src/exprs/table_function/dummy_table_functions.cpp
new file mode 100644
index 0000000..bb64100
--- /dev/null
+++ b/be/src/exprs/table_function/dummy_table_functions.cpp
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exprs/table_function/dummy_table_functions.h"
+
+namespace doris {
+
+void DummyTableFunctions::init() {}
+
+StringVal DummyTableFunctions::explode_split(FunctionContext* context, const StringVal& str,
+ const StringVal& sep) {
+ return StringVal();
+}
+
+BigIntVal DummyTableFunctions::explode_bitmap(doris_udf::FunctionContext* context,
+ const doris_udf::StringVal& bitmap) {
+ return BigIntVal();
+}
+
+BigIntVal DummyTableFunctions::explode_json_array_int(doris_udf::FunctionContext* context,
+ const doris_udf::StringVal& str) {
+ return BigIntVal();
+}
+
+DoubleVal DummyTableFunctions::explode_json_array_double(doris_udf::FunctionContext* context,
+ const doris_udf::StringVal& str) {
+ return DoubleVal();
+}
+
+StringVal DummyTableFunctions::explode_json_array_string(doris_udf::FunctionContext* context,
+ const doris_udf::StringVal& str) {
+ return StringVal();
+}
+
+} // namespace doris
diff --git a/be/src/exprs/table_function/dummy_table_functions.h b/be/src/exprs/table_function/dummy_table_functions.h
new file mode 100644
index 0000000..4723a52
--- /dev/null
+++ b/be/src/exprs/table_function/dummy_table_functions.h
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "exprs/anyval_util.h"
+
+namespace doris {
+
+// Currently Doris does not support array types, so the definition of table function
+// is still using the definition of the scalar function.
+// The definition here is just to facilitate the query planning stage and the query execution preparation stage
+// to make smooth use of the existing function framework
+// But the execution logic of the table function is not here. So the function names here are prefixed with "dummy".
+// TODO: refactor here after we support real array type.
+class DummyTableFunctions {
+public:
+ static void init();
+
+ static doris_udf::StringVal explode_split(doris_udf::FunctionContext* context,
+ const doris_udf::StringVal& str,
+ const doris_udf::StringVal& sep);
+ static doris_udf::BigIntVal explode_bitmap(doris_udf::FunctionContext* context,
+ const doris_udf::StringVal& bitmap);
+ static doris_udf::BigIntVal explode_json_array_int(doris_udf::FunctionContext* context,
+ const doris_udf::StringVal& str);
+ static doris_udf::DoubleVal explode_json_array_double(doris_udf::FunctionContext* context,
+ const doris_udf::StringVal& str);
+ static doris_udf::StringVal explode_json_array_string(doris_udf::FunctionContext* context,
+ const doris_udf::StringVal& str);
+};
+} // namespace doris
+
diff --git a/be/src/exprs/table_function/explode_bitmap.cpp b/be/src/exprs/table_function/explode_bitmap.cpp
new file mode 100644
index 0000000..67fe561
--- /dev/null
+++ b/be/src/exprs/table_function/explode_bitmap.cpp
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exprs/table_function/explode_bitmap.h"
+
+#include "exprs/expr_context.h"
+#include "exprs/scalar_fn_call.h"
+
+namespace doris {
+
+ExplodeBitmapTableFunction::ExplodeBitmapTableFunction() {}
+
+ExplodeBitmapTableFunction::~ExplodeBitmapTableFunction() {
+ if (_cur_iter != nullptr) {
+ delete _cur_iter;
+ _cur_iter = nullptr;
+ }
+ if (_cur_bitmap_owned && _cur_bitmap != nullptr) {
+ delete _cur_bitmap;
+ _cur_bitmap = nullptr;
+ }
+}
+
+Status ExplodeBitmapTableFunction::prepare() {
+ return Status::OK();
+}
+
+Status ExplodeBitmapTableFunction::open() {
+ return Status::OK();
+}
+
+Status ExplodeBitmapTableFunction::process(TupleRow* tuple_row) {
+ CHECK(1 == _expr_context->root()->get_num_children()) << _expr_context->root()->get_num_children();
+ _eos = false;
+ _is_current_empty = false;
+ _cur_size = 0;
+ _cur_offset = 0;
+
+ StringVal bitmap_str = _expr_context->root()->get_child(0)->get_string_val(_expr_context, tuple_row);
+ if (bitmap_str.is_null) {
+ _is_current_empty = true;
+ } else {
+ if (bitmap_str.len == 0) {
+ _cur_bitmap = reinterpret_cast<BitmapValue*>(bitmap_str.ptr);
+ _cur_bitmap_owned = false;
+ } else {
+ _cur_bitmap = new BitmapValue((char*) bitmap_str.ptr);
+ _cur_bitmap_owned = true;
+ }
+ _cur_size = _cur_bitmap->cardinality();
+ if (_cur_size == 0) {
+ _is_current_empty = true;
+ } else {
+ _reset_iterator();
+ }
+ }
+
+ return Status::OK();
+}
+
+void ExplodeBitmapTableFunction::_reset_iterator() {
+ DCHECK(_cur_bitmap->cardinality() > 0) << _cur_bitmap->cardinality();
+ if (_cur_iter != nullptr) {
+ delete _cur_iter;
+ _cur_iter = nullptr;
+ }
+ _cur_iter = new BitmapValueIterator(*_cur_bitmap);
+ _cur_value = **_cur_iter;
+}
+
+Status ExplodeBitmapTableFunction::reset() {
+ _eos = false;
+ if (!_is_current_empty) {
+ _reset_iterator();
+ }
+ return Status::OK();
+}
+
+Status ExplodeBitmapTableFunction::get_value(void** output) {
+ if (_is_current_empty) {
+ *output = nullptr;
+ } else {
+ *output = &_cur_value;
+ }
+ return Status::OK();
+}
+
+Status ExplodeBitmapTableFunction::close() {
+ return Status::OK();
+}
+
+Status ExplodeBitmapTableFunction::forward(bool* eos) {
+ if (_is_current_empty) {
+ *eos = true;
+ _eos = true;
+ } else {
+ ++(*_cur_iter);
+ ++_cur_offset;
+ if (_cur_offset == _cur_size) {
+ *eos = true;
+ _eos = true;
+ } else {
+ _cur_value = **_cur_iter;
+ *eos = false;
+ }
+ }
+ return Status::OK();
+}
+
+} // namespace doris
diff --git a/be/src/exprs/table_function/explode_bitmap.h b/be/src/exprs/table_function/explode_bitmap.h
new file mode 100644
index 0000000..b491eea
--- /dev/null
+++ b/be/src/exprs/table_function/explode_bitmap.h
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "exprs/table_function/table_function.h"
+
+#include "util/bitmap_value.h"
+
+namespace doris {
+
+class ExplodeBitmapTableFunction : public TableFunction {
+public:
+ ExplodeBitmapTableFunction();
+ virtual ~ExplodeBitmapTableFunction();
+
+ virtual Status prepare() override;
+ virtual Status open() override;
+ virtual Status process(TupleRow* tuple_row) override;
+ virtual Status reset() override;
+ virtual Status get_value(void** output) override;
+ virtual Status close() override;
+
+ virtual Status forward(bool* eos) override;
+
+private:
+ void _reset_iterator();
+
+private:
+
+ // Read from tuple row.
+ // if _cur_bitmap_owned is true, need to delete it when deconstruction
+ BitmapValue* _cur_bitmap = nullptr;
+ bool _cur_bitmap_owned = false;
+ // iterator of _cur_bitmap
+ BitmapValueIterator* _cur_iter = nullptr;
+ // current value read from bitmap, it will be referenced by
+ // table function scan node.
+ uint64_t _cur_value = 0;
+};
+
+} // namespace doris
diff --git a/be/src/exprs/table_function/explode_json_array.cpp b/be/src/exprs/table_function/explode_json_array.cpp
new file mode 100644
index 0000000..e82c9ec
--- /dev/null
+++ b/be/src/exprs/table_function/explode_json_array.cpp
@@ -0,0 +1,202 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exprs/table_function/explode_json_array.h"
+
+#include "exprs/expr_context.h"
+#include "exprs/scalar_fn_call.h"
+
+namespace doris {
+
+std::string ParsedData::true_value = "true";
+std::string ParsedData::false_value = "false";
+
+int ParsedData::set_output(ExplodeJsonArrayType type, rapidjson::Document& document) {
+ int size = document.GetArray().Size();
+ switch (type) {
+ case ExplodeJsonArrayType::INT: {
+ _data.resize(size);
+ _backup_int.resize(size);
+ int i = 0;
+ for (auto& v : document.GetArray()) {
+ if (v.IsInt64()) {
+ _backup_int[i] = v.GetInt64();
+ _data[i] = &_backup_int[i];
+ } else {
+ _data[i] = nullptr;
+ }
+ ++i;
+ }
+ break;
+ }
+ case ExplodeJsonArrayType::DOUBLE: {
+ _data.resize(size);
+ _backup_double.resize(size);
+ int i = 0;
+ for (auto& v : document.GetArray()) {
+ if (v.IsDouble()) {
+ _backup_double[i] = v.GetDouble();
+ _data[i] = &_backup_double[i];
+ } else {
+ _data[i] = nullptr;
+ }
+ ++i;
+ }
+ break;
+ }
+ case ExplodeJsonArrayType::STRING: {
+ _data_string.clear();
+ _backup_string.clear();
+ _string_nulls.clear();
+ int32_t wbytes = 0;
+ int i = 0;
+ for (auto& v : document.GetArray()) {
+ switch (v.GetType()) {
+ case rapidjson::Type::kStringType:
+ _backup_string.emplace_back(v.GetString(), v.GetStringLength());
+ _data_string.emplace_back(_backup_string.back());
+ _string_nulls.push_back(false);
+ break;
+ case rapidjson::Type::kNumberType:
+ if (v.IsUint()) {
+ wbytes = sprintf(tmp_buf, "%u", v.GetUint());
+ } else if (v.IsInt()) {
+ wbytes = sprintf(tmp_buf, "%d", v.GetInt());
+ } else if (v.IsUint64()) {
+ wbytes = sprintf(tmp_buf, "%lu", v.GetUint64());
+ } else if (v.IsInt64()) {
+ wbytes = sprintf(tmp_buf, "%ld", v.GetInt64());
+ } else {
+ wbytes = sprintf(tmp_buf, "%f", v.GetDouble());
+ }
+ _backup_string.emplace_back(tmp_buf, wbytes);
+ _data_string.emplace_back(_backup_string.back());
+ _string_nulls.push_back(false);
+ break;
+ case rapidjson::Type::kFalseType:
+ _data_string.emplace_back(true_value);
+ _string_nulls.push_back(false);
+ break;
+ case rapidjson::Type::kTrueType:
+ _data_string.emplace_back(false_value);
+ _string_nulls.push_back(false);
+ break;
+ case rapidjson::Type::kNullType:
+ _data_string.push_back({});
+ _string_nulls.push_back(true);
+ break;
+ default:
+ _data_string.push_back({});
+ _string_nulls.push_back(true);
+ break;
+ }
+ ++i;
+ }
+ break;
+ }
+ default:
+ CHECK(false) << type;
+ break;
+ }
+ return size;
+}
+
+/////////////////////////
+ExplodeJsonArrayTableFunction::ExplodeJsonArrayTableFunction(ExplodeJsonArrayType type)
+ : _type(type) {
+
+}
+
+ExplodeJsonArrayTableFunction::~ExplodeJsonArrayTableFunction() {
+}
+
+Status ExplodeJsonArrayTableFunction::prepare() {
+ return Status::OK();
+}
+
+Status ExplodeJsonArrayTableFunction::open() {
+ return Status::OK();
+}
+
+Status ExplodeJsonArrayTableFunction::process(TupleRow* tuple_row) {
+ CHECK(1 == _expr_context->root()->get_num_children()) << _expr_context->root()->get_num_children();
+ _is_current_empty = false;
+ _eos = false;
+
+ StringVal text = _expr_context->root()->get_child(0)->get_string_val(_expr_context, tuple_row);
+ if (text.is_null || text.len == 0) {
+ // _set_null_output();
+ _is_current_empty = true;
+ } else {
+ rapidjson::Document document;
+ document.Parse((char*) text.ptr, text.len);
+ if (UNLIKELY(document.HasParseError()) || !document.IsArray() || document.GetArray().Size() == 0) {
+ // _set_null_output();
+ _is_current_empty = true;
+ } else {
+ _cur_size = _parsed_data.set_output(_type, document);
+ _cur_offset = 0;
+ // _eos = _cur_size == 0;
+ }
+ }
+ // _is_current_empty = _eos;
+ return Status::OK();
+}
+
+void ExplodeJsonArrayTableFunction::_set_null_output() {
+ _parsed_data.set_null_output(_type);
+ _cur_size = 1;
+ _cur_offset = 0;
+ _eos = false;
+}
+
+Status ExplodeJsonArrayTableFunction::reset() {
+ _eos = false;
+ _cur_offset = 0;
+ return Status::OK();
+}
+
+Status ExplodeJsonArrayTableFunction::get_value(void** output) {
+ if (_is_current_empty) {
+ *output = nullptr;
+ } else {
+ _parsed_data.get_value(_type, _cur_offset, output);
+ }
+ return Status::OK();
+}
+
+Status ExplodeJsonArrayTableFunction::close() {
+ return Status::OK();
+}
+
+Status ExplodeJsonArrayTableFunction::forward(bool* eos) {
+ if (_is_current_empty) {
+ *eos = true;
+ _eos = true;
+ } else {
+ ++_cur_offset;
+ if (_cur_offset == _cur_size) {
+ *eos = true;
+ _eos = true;
+ } else {
+ *eos = false;
+ }
+ }
+ return Status::OK();
+}
+
+} // namespace doris
diff --git a/be/src/exprs/table_function/explode_json_array.h b/be/src/exprs/table_function/explode_json_array.h
new file mode 100644
index 0000000..ae5c657
--- /dev/null
+++ b/be/src/exprs/table_function/explode_json_array.h
@@ -0,0 +1,130 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "exprs/table_function/table_function.h"
+
+#include <rapidjson/document.h>
+#include <rapidjson/stringbuffer.h>
+#include "gutil/strings/stringpiece.h"
+#include "runtime/string_value.h"
+
+namespace doris {
+
+enum ExplodeJsonArrayType {
+ INT = 0,
+ DOUBLE,
+ STRING
+};
+
+struct ParsedData {
+ static std::string true_value;
+ static std::string false_value;
+
+ // The number parsed from json array
+ // the `_backup` saved the real number entity.
+ std::vector<void*> _data;
+ std::vector<StringValue> _data_string;
+ std::vector<int64_t> _backup_int;
+ std::vector<double> _backup_double;
+ std::vector<std::string> _backup_string;
+ std::vector<bool> _string_nulls;
+ char tmp_buf[128] = {0};
+
+ void reset(ExplodeJsonArrayType type) {
+ switch (type) {
+ case ExplodeJsonArrayType::INT:
+ _data.clear();
+ _backup_int.clear();
+ break;
+ case ExplodeJsonArrayType::DOUBLE:
+ _data.clear();
+ _backup_double.clear();
+ break;
+ case ExplodeJsonArrayType::STRING:
+ _data_string.clear();
+ _backup_string.clear();
+ _string_nulls.clear();
+ break;
+ default:
+ CHECK(false) << type;
+ break;
+ }
+ }
+
+ void set_null_output(ExplodeJsonArrayType type) {
+ switch (type) {
+ case ExplodeJsonArrayType::INT:
+ case ExplodeJsonArrayType::DOUBLE:
+ _data.resize(1);
+ _data[0] = nullptr;
+ break;
+ case ExplodeJsonArrayType::STRING:
+ _string_nulls.resize(1);
+ _string_nulls[0] = true;
+ break;
+ default:
+ CHECK(false) << type;
+ break;
+ }
+ }
+
+ void get_value(ExplodeJsonArrayType type, int64_t offset, void** output) {
+ switch(type) {
+ case ExplodeJsonArrayType::INT:
+ case ExplodeJsonArrayType::DOUBLE:
+ *output = _data[offset];
+ break;
+ case ExplodeJsonArrayType::STRING:
+ // LOG(INFO) << "cmy get_value offset: " << offset << ", is null: " << _string_nulls[offset] << ", data: " << (_string_nulls[offset] ? "null2" : _backup_string[offset]);
+ *output = _string_nulls[offset] ? nullptr : &_data_string[offset];
+ break;
+ default:
+ CHECK(false) << type;
+ }
+ }
+
+ int set_output(ExplodeJsonArrayType type, rapidjson::Document& document);
+};
+
+// Input: json array: [1,2,3,4,5];
+// Output: rows: 1,2,3,4,5
+// If json array contains non-numeric type, or is not a json array, will return null
+class ExplodeJsonArrayTableFunction : public TableFunction {
+public:
+ ExplodeJsonArrayTableFunction(ExplodeJsonArrayType type);
+ virtual ~ExplodeJsonArrayTableFunction();
+
+ virtual Status prepare() override;
+ virtual Status open() override;
+ virtual Status process(TupleRow* tuple_row) override;
+ virtual Status reset() override;
+ virtual Status get_value(void** output) override;
+ virtual Status close() override;
+ virtual Status forward(bool* eos) override;
+
+private:
+ void _set_null_output();
+
+private:
+ ParsedData _parsed_data;
+
+ ExplodeJsonArrayType _type;
+};
+
+} // namespace doris
diff --git a/be/src/exprs/table_function/explode_split.cpp b/be/src/exprs/table_function/explode_split.cpp
new file mode 100644
index 0000000..959dad4
--- /dev/null
+++ b/be/src/exprs/table_function/explode_split.cpp
@@ -0,0 +1,115 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exprs/table_function/explode_split.h"
+
+#include "exprs/expr_context.h"
+#include "exprs/scalar_fn_call.h"
+#include "gutil/strings/split.h"
+
+namespace doris {
+
+ExplodeSplitTableFunction::ExplodeSplitTableFunction() {
+}
+
+ExplodeSplitTableFunction::~ExplodeSplitTableFunction() {
+}
+
+Status ExplodeSplitTableFunction::prepare() {
+ return Status::OK();
+}
+
+Status ExplodeSplitTableFunction::open() {
+ ScalarFnCall* fn_call = reinterpret_cast<ScalarFnCall*>(_expr_context->root());
+ FunctionContext* fn_ctx = _expr_context->fn_context(fn_call->get_fn_context_index());
+ CHECK(2 == fn_ctx->get_num_constant_args()) << fn_ctx->get_num_constant_args();
+ // check if the delimiter argument(the 2nd arg) is constant.
+ // if yes, cache it
+ if (fn_ctx->is_arg_constant(1)) {
+ _is_delimiter_constant = true;
+ StringVal* delimiter = reinterpret_cast<StringVal*>(fn_ctx->get_constant_arg(1));
+ _const_delimter = StringPiece((char*) delimiter->ptr, delimiter->len);
+ }
+ return Status::OK();
+}
+
+Status ExplodeSplitTableFunction::process(TupleRow* tuple_row) {
+ CHECK(2 == _expr_context->root()->get_num_children()) << _expr_context->root()->get_num_children();
+ _is_current_empty = false;
+ _eos = false;
+
+ _data.clear();
+ StringVal text = _expr_context->root()->get_child(0)->get_string_val(_expr_context, tuple_row);
+ if (text.is_null) {
+ _is_current_empty = true;
+ _cur_size = 0;
+ _cur_offset = 0;
+ } else {
+ if (_is_delimiter_constant) {
+ _backup = strings::Split(StringPiece((char*) text.ptr, text.len), _const_delimter);
+ } else {
+ StringVal delimiter = _expr_context->root()->get_child(1)->get_string_val(_expr_context, tuple_row);
+ _backup = strings::Split(StringPiece((char*) text.ptr, text.len), StringPiece((char*) delimiter.ptr, delimiter.len));
+ }
+ for (const std::string str : _backup) {
+ _data.emplace_back(str);
+ }
+ _cur_size = _backup.size();
+ _cur_offset = 0;
+ _is_current_empty = (_cur_size == 0);
+ }
+ return Status::OK();
+}
+
+Status ExplodeSplitTableFunction::reset() {
+ _eos = false;
+ if (!_is_current_empty) {
+ _cur_offset = 0;
+ }
+ return Status::OK();
+}
+
+Status ExplodeSplitTableFunction::get_value(void** output) {
+ if (_is_current_empty) {
+ *output = nullptr;
+ } else {
+ *output = &_data[_cur_offset];
+ }
+ return Status::OK();
+}
+
+Status ExplodeSplitTableFunction::close() {
+ return Status::OK();
+}
+
+Status ExplodeSplitTableFunction::forward(bool* eos) {
+ if (_is_current_empty) {
+ *eos = true;
+ _eos = true;
+ } else {
+ ++_cur_offset;
+ if (_cur_offset == _cur_size) {
+ *eos = true;
+ _eos = true;
+ } else {
+ *eos = false;
+ }
+ }
+ return Status::OK();
+}
+
+} // namespace doris
diff --git a/be/src/exprs/table_function/explode_split.h b/be/src/exprs/table_function/explode_split.h
new file mode 100644
index 0000000..ad80671
--- /dev/null
+++ b/be/src/exprs/table_function/explode_split.h
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "exprs/table_function/table_function.h"
+
+#include "gutil/strings/stringpiece.h"
+#include "runtime/string_value.h"
+
+namespace doris {
+
+class ExplodeSplitTableFunction : public TableFunction {
+public:
+ ExplodeSplitTableFunction();
+ virtual ~ExplodeSplitTableFunction();
+
+ virtual Status prepare() override;
+ virtual Status open() override;
+ virtual Status process(TupleRow* tuple_row) override;
+ virtual Status reset() override;
+ virtual Status get_value(void** output) override;
+ virtual Status close() override;
+
+ virtual Status forward(bool* eos) override;
+
+private:
+
+ // The string value splitted from source, and will be referenced by
+ // table function scan node.
+ // the `_backup` saved the real string entity.
+ std::vector<StringValue> _data;
+ std::vector<std::string> _backup;
+
+ // indicate whether the delimiter is constant.
+ // if true, the constant delimiter will be saved in `_const_delimter`
+ bool _is_delimiter_constant = false;
+ StringPiece _const_delimter;
+
+};
+
+} // namespace doris
diff --git a/be/src/exprs/table_function/table_function.h b/be/src/exprs/table_function/table_function.h
new file mode 100644
index 0000000..a2b8b00
--- /dev/null
+++ b/be/src/exprs/table_function/table_function.h
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "common/status.h"
+
+namespace doris {
+
+// TODO: think about how to manager memeory consumption of table functions.
+// Currently, the memory allocated from table function is from malloc directly.
+class TableFunctionState {
+};
+
+class ExprContext;
+class TupleRow;
+class TableFunction {
+public:
+ virtual ~TableFunction() {}
+
+ virtual Status prepare() = 0;
+ virtual Status open() = 0;
+ virtual Status process(TupleRow* tuple_row) = 0;
+ virtual Status reset() = 0;
+ virtual Status get_value(void** output) = 0;
+ virtual Status close() = 0;
+
+ virtual Status forward(bool *eos) = 0;
+
+public:
+ bool eos() const { return _eos; }
+
+ void set_expr_context(ExprContext* expr_context) {
+ _expr_context = expr_context;
+ }
+
+protected:
+ std::string _fn_name;
+ ExprContext* _expr_context;
+ // true if there is no more data can be read from this function.
+ bool _eos = false;
+ // true means the function result set from current row is empty(eg, source value is null or empty).
+ // so that when calling reset(), we can do nothing and keep eos as true.
+ bool _is_current_empty = false;
+ // the position of current cursor
+ int64_t _cur_offset = 0;
+ // the size of current result
+ int64_t _cur_size = 0;
+};
+
+} // namespace doris
diff --git a/be/src/exprs/table_function/table_function_factory.cpp b/be/src/exprs/table_function/table_function_factory.cpp
new file mode 100644
index 0000000..fc6ead5
--- /dev/null
+++ b/be/src/exprs/table_function/table_function_factory.cpp
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exprs/table_function/table_function_factory.h"
+
+#include "common/object_pool.h"
+#include "exprs/table_function/explode_bitmap.h"
+#include "exprs/table_function/explode_json_array.h"
+#include "exprs/table_function/explode_split.h"
+
+namespace doris {
+
+Status TableFunctionFactory::get_fn(const std::string& fn_name, ObjectPool* pool, TableFunction** fn) {
+ if (fn_name == "explode_split") {
+ *fn = pool->add(new ExplodeSplitTableFunction());
+ return Status::OK();
+ } else if (fn_name == "explode_bitmap") {
+ *fn = pool->add(new ExplodeBitmapTableFunction());
+ return Status::OK();
+ } else if (fn_name == "explode_json_array_int") {
+ *fn = pool->add(new ExplodeJsonArrayTableFunction(ExplodeJsonArrayType::INT));
+ return Status::OK();
+ } else if (fn_name == "explode_json_array_double") {
+ *fn = pool->add(new ExplodeJsonArrayTableFunction(ExplodeJsonArrayType::DOUBLE));
+ return Status::OK();
+ } else if (fn_name == "explode_json_array_string") {
+ *fn = pool->add(new ExplodeJsonArrayTableFunction(ExplodeJsonArrayType::STRING));
+ return Status::OK();
+ } else {
+ return Status::NotSupported("Unknown table function: " + fn_name);
+ }
+}
+
+} // namespace doris
diff --git a/be/src/exprs/table_function/table_function_factory.h b/be/src/exprs/table_function/table_function_factory.h
new file mode 100644
index 0000000..ca5bd05
--- /dev/null
+++ b/be/src/exprs/table_function/table_function_factory.h
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "exprs/table_function/table_function_factory.h"
+#include "exprs/table_function/explode_split.h"
+
+#include "common/status.h"
+
+namespace doris {
+
+class ObjectPool;
+class TableFunction;
+class TableFunctionFactory {
+public:
+ TableFunctionFactory() {}
+ ~TableFunctionFactory() {}
+ static Status get_fn(const std::string& fn_name, ObjectPool* pool, TableFunction** fn);
+};
+
+} // namespace doris
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index e201d84..d668d42 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -108,7 +108,6 @@ private:
friend class TupleDescriptor;
friend class SchemaScanner;
friend class OlapTableSchemaParam;
- friend class TupleDescriptor;
const SlotId _id;
const TypeDescriptor _type;
diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h
index a3c54e0..7259685 100644
--- a/be/src/util/bitmap_value.h
+++ b/be/src/util/bitmap_value.h
@@ -1134,6 +1134,7 @@ inline Roaring64MapSetBitForwardIterator Roaring64Map::end() const {
// Represent the in-memory and on-disk structure of Doris's BITMAP data type.
// Optimize for the case where the bitmap contains 0 or 1 element which is common
// for streaming load scenario.
+class BitmapValueIterator;
class BitmapValue {
public:
// Construct an empty bitmap.
@@ -1692,6 +1693,13 @@ public:
return count;
}
+ // Implement an iterator for convenience
+ friend class BitmapValueIterator;
+ typedef BitmapValueIterator b_iterator;
+
+ b_iterator begin() const;
+ b_iterator end() const;
+
private:
void _convert_to_smaller_type() {
if (_type == BITMAP) {
@@ -1717,6 +1725,128 @@ private:
BitmapDataType _type;
};
+// A simple implement of bitmap value iterator(Read only)
+// Usage:
+// BitmapValueIterator iter = bitmap_value.begin();
+// BitmapValueIterator end = bitmap_value.end();
+// for (; iter != end(); ++iter) {
+// uint64_t v = *iter;
+// ... do something with "v" ...
+// }
+class BitmapValueIterator {
+public:
+ BitmapValueIterator()
+ : _bitmap(BitmapValue()) {
+ }
+
+ BitmapValueIterator(const BitmapValue& bitmap, bool end = false)
+ : _bitmap(bitmap), _end(end) {
+
+ switch (_bitmap._type) {
+ case BitmapValue::BitmapDataType::EMPTY:
+ _end = true;
+ break;
+ case BitmapValue::BitmapDataType::SINGLE:
+ _sv = _bitmap._sv;
+ break;
+ case BitmapValue::BitmapDataType::BITMAP:
+ _iter = new detail::Roaring64MapSetBitForwardIterator(_bitmap._bitmap, _end);
+ break;
+ default:
+ CHECK(false) << _bitmap._type;
+ }
+ }
+
+ BitmapValueIterator(const BitmapValueIterator& other)
+ : _bitmap(other._bitmap), _iter(other._iter), _sv(other._sv), _end(other._end) {
+ }
+
+ ~BitmapValueIterator() {
+ if (_iter != nullptr) {
+ delete _iter;
+ _iter = nullptr;
+ }
+ }
+
+ uint64_t operator*() const {
+ CHECK(!_end) << "should not get value of end iterator";
+ switch (_bitmap._type) {
+ case BitmapValue::BitmapDataType::SINGLE:
+ return _sv;
+ case BitmapValue::BitmapDataType::BITMAP:
+ return *(*_iter);
+ default:
+ CHECK(false) << _bitmap._type;
+ }
+ return 0;
+ }
+
+ BitmapValueIterator& operator++() { // ++i, must returned inc. value
+ CHECK(!_end) << "should not forward when iterator ends";
+ switch (_bitmap._type) {
+ case BitmapValue::BitmapDataType::SINGLE:
+ _end = true;
+ break;
+ case BitmapValue::BitmapDataType::BITMAP:
+ ++(*_iter);
+ break;
+ default:
+ CHECK(false) << _bitmap._type;
+ }
+ return *this;
+ }
+
+ BitmapValueIterator operator++(int) { // i++, must return orig. value
+ CHECK(!_end) << "should not forward when iterator ends";
+ BitmapValueIterator orig(*this);
+ switch (_bitmap._type) {
+ case BitmapValue::BitmapDataType::SINGLE:
+ _end = true;
+ break;
+ case BitmapValue::BitmapDataType::BITMAP:
+ ++(*_iter);
+ break;
+ default:
+ CHECK(false) << _bitmap._type;
+ }
+ return orig;
+ }
+
+ bool operator==(const BitmapValueIterator& other) const {
+ if (_end && other._end) return true;
+
+ switch (_bitmap._type) {
+ case BitmapValue::BitmapDataType::EMPTY:
+ return other._bitmap._type == BitmapValue::BitmapDataType::EMPTY;
+ case BitmapValue::BitmapDataType::SINGLE:
+ return _sv == other._sv;
+ case BitmapValue::BitmapDataType::BITMAP:
+ return *_iter == *(other._iter);
+ default:
+ CHECK(false) << _bitmap._type;
+ }
+ return false;
+ }
+
+ bool operator!=(const BitmapValueIterator& other) const {
+ return !(*this == other);
+ }
+
+private:
+ const BitmapValue& _bitmap;
+ detail::Roaring64MapSetBitForwardIterator* _iter = nullptr;
+ uint64_t _sv = 0;
+ bool _end = false;
+};
+
+inline BitmapValueIterator BitmapValue::begin() const {
+ return BitmapValueIterator(*this);
+}
+
+inline BitmapValueIterator BitmapValue::end() const {
+ return BitmapValueIterator(*this, true);
+}
+
} // namespace doris
#endif //DORIS_BE_SRC_UTIL_BITMAP_VALUE_H
diff --git a/be/test/util/bitmap_value_test.cpp b/be/test/util/bitmap_value_test.cpp
index 9973205..cb2083d 100644
--- a/be/test/util/bitmap_value_test.cpp
+++ b/be/test/util/bitmap_value_test.cpp
@@ -346,6 +346,43 @@ TEST(BitmapValueTest, bitmap_single_convert) {
bitmap |= bitmap_u;
ASSERT_EQ(BitmapValue::BITMAP, bitmap._type);
}
+
+TEST(BitmapValueTest, bitmap_value_iterator_test) {
+ BitmapValue empty;
+ for (auto iter = empty.begin(); iter != empty.end(); ++iter) {
+ // should not goes here
+ ASSERT_TRUE(false);
+ }
+
+ BitmapValue single(1024);
+ for (auto iter = single.begin(); iter != single.end(); ++iter) {
+ ASSERT_EQ(1024, *iter);
+ }
+
+ int i = 0;
+ BitmapValue bitmap({0, 1025, 1026, UINT32_MAX, UINT64_MAX});
+ for (auto iter = bitmap.begin(); iter != bitmap.end(); ++iter, ++i) {
+ switch (i) {
+ case 0:
+ ASSERT_EQ(0, *iter);
+ break;
+ case 1:
+ ASSERT_EQ(1025, *iter);
+ break;
+ case 2:
+ ASSERT_EQ(1026, *iter);
+ break;
+ case 3:
+ ASSERT_EQ(UINT32_MAX, *iter);
+ break;
+ case 4:
+ ASSERT_EQ(UINT64_MAX, *iter);
+ break;
+ default:
+ ASSERT_TRUE(false);
+ }
+ }
+}
} // namespace doris
int main(int argc, char** argv) {
diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js
index 1001833..93966f2 100644
--- a/docs/.vuepress/sidebar/en.js
+++ b/docs/.vuepress/sidebar/en.js
@@ -435,6 +435,15 @@ module.exports = [
directoryPath: "hash-functions/",
children: ["murmur_hash3_32"],
},
+ {
+ title: "table functions",
+ directoryPath: "table-functions/",
+ children: [
+ "explode-bitmap",
+ "explode-split",
+ "explode-json-array"
+ ],
+ },
"window-function",
"cast",
"digital-masking",
@@ -585,6 +594,7 @@ module.exports = [
"alter-routine-load",
"insert",
"UPDATE",
+ "lateral-view",
],
},
{
diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js
index a3dda8e..9ef56f1 100644
--- a/docs/.vuepress/sidebar/zh-CN.js
+++ b/docs/.vuepress/sidebar/zh-CN.js
@@ -439,6 +439,15 @@ module.exports = [
directoryPath: "hash-functions/",
children: ["murmur_hash3_32"],
},
+ {
+ title: "table functions",
+ directoryPath: "table-functions/",
+ children: [
+ "explode-bitmap",
+ "explode-split",
+ "explode-json-array"
+ ],
+ },
"window-function",
"cast",
"digital-masking",
@@ -588,6 +597,7 @@ module.exports = [
"alter-routine-load",
"insert",
"UPDATE",
+ "lateral-view",
],
},
{
diff --git a/docs/en/sql-reference/sql-functions/table-functions/explode-bitmap.md b/docs/en/sql-reference/sql-functions/table-functions/explode-bitmap.md
new file mode 100644
index 0000000..8863928
--- /dev/null
+++ b/docs/en/sql-reference/sql-functions/table-functions/explode-bitmap.md
@@ -0,0 +1,157 @@
+---
+{
+ "title": "explode_bitmap",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# explode_bitmap
+
+## description
+
+Table functions must be used in conjunction with Lateral View.
+
+Expand a bitmap type.
+
+grammar:
+
+```
+explode_bitmap(bitmap)
+```
+
+## example
+
+Original table data:
+
+```
+mysql> select k1 from example1 order by k1;
++------+
+| k1 |
++------+
+| 1 |
+| 2 |
+| 3 |
+| 4 |
+| 5 |
+| 6 |
++------+
+```
+
+Lateral View:
+
+```
+mysql> select k1, e1 from example1 lateral view explode_bitmap(bitmap_empty()) tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
+| 4 | NULL |
+| 5 | NULL |
+| 6 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_bitmap(bitmap_from_string("1")) tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | 1 |
+| 2 | 1 |
+| 3 | 1 |
+| 4 | 1 |
+| 5 | 1 |
+| 6 | 1 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_bitmap(bitmap_from_string("1,2")) tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | 1 |
+| 1 | 2 |
+| 2 | 1 |
+| 2 | 2 |
+| 3 | 1 |
+| 3 | 2 |
+| 4 | 1 |
+| 4 | 2 |
+| 5 | 1 |
+| 5 | 2 |
+| 6 | 1 |
+| 6 | 2 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_bitmap(bitmap_from_string("1,1000")) tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | 1 |
+| 1 | 1000 |
+| 2 | 1 |
+| 2 | 1000 |
+| 3 | 1 |
+| 3 | 1000 |
+| 4 | 1 |
+| 4 | 1000 |
+| 5 | 1 |
+| 5 | 1000 |
+| 6 | 1 |
+| 6 | 1000 |
++------+------+
+
+mysql> select k1, e1, e2 from example1
+lateral view explode_bitmap(bitmap_from_string("1,1000")) tmp1 as e1
+lateral view explode_split("a,b", ",") tmp2 as e2 order by k1, e1, e2;
++------+------+------+
+| k1 | e1 | e2 |
++------+------+------+
+| 1 | 1 | a |
+| 1 | 1 | b |
+| 1 | 1000 | a |
+| 1 | 1000 | b |
+| 2 | 1 | a |
+| 2 | 1 | b |
+| 2 | 1000 | a |
+| 2 | 1000 | b |
+| 3 | 1 | a |
+| 3 | 1 | b |
+| 3 | 1000 | a |
+| 3 | 1000 | b |
+| 4 | 1 | a |
+| 4 | 1 | b |
+| 4 | 1000 | a |
+| 4 | 1000 | b |
+| 5 | 1 | a |
+| 5 | 1 | b |
+| 5 | 1000 | a |
+| 5 | 1000 | b |
+| 6 | 1 | a |
+| 6 | 1 | b |
+| 6 | 1000 | a |
+| 6 | 1000 | b |
++------+------+------+
+```
+
+## keyword
+
+ explode_bitmap
\ No newline at end of file
diff --git a/docs/en/sql-reference/sql-functions/table-functions/explode-json-array.md b/docs/en/sql-reference/sql-functions/table-functions/explode-json-array.md
new file mode 100644
index 0000000..e9c136d
--- /dev/null
+++ b/docs/en/sql-reference/sql-functions/table-functions/explode-json-array.md
@@ -0,0 +1,286 @@
+---
+{
+ "title": "explode_json_array",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# `explode_json_array`
+
+## description
+
+Table functions must be used in conjunction with Lateral View.
+
+Expand a json array. According to the array element type, there are three function names. Corresponding to integer, floating point and string arrays respectively.
+
+grammar:
+
+```
+explode_json_array_int(json_str)
+explode_json_array_double(json_str)
+explode_json_array_string(json_str)
+```
+
+## example
+
+Original table data:
+
+```
+mysql> select k1 from example1 order by k1;
++------+
+| k1 |
++------+
+| 1 |
+| 2 |
+| 3 |
+| 4 |
+| 5 |
+| 6 |
++------+
+```
+
+Lateral View:
+
+```
+mysql> select k1, e1 from example1 lateral view explode_json_array_int('[]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_int('[1,2,3]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | 1 |
+| 1 | 2 |
+| 1 | 3 |
+| 2 | 1 |
+| 2 | 2 |
+| 2 | 3 |
+| 3 | 1 |
+| 3 | 2 |
+| 3 | 3 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_int('[1,"b",3]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 1 | 1 |
+| 1 | 3 |
+| 2 | NULL |
+| 2 | 1 |
+| 2 | 3 |
+| 3 | NULL |
+| 3 | 1 |
+| 3 | 3 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_int('["a","b","c"]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 1 | NULL |
+| 1 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 3 | NULL |
+| 3 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_int('{"a": 3}') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('[]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('[1,2,3]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 1 | NULL |
+| 1 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 3 | NULL |
+| 3 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('[1,"b",3]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 1 | NULL |
+| 1 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 3 | NULL |
+| 3 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('[1.0,2.0,3.0]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | 1 |
+| 1 | 2 |
+| 1 | 3 |
+| 2 | 1 |
+| 2 | 2 |
+| 2 | 3 |
+| 3 | 1 |
+| 3 | 2 |
+| 3 | 3 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('[1,"b",3]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 1 | NULL |
+| 1 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 3 | NULL |
+| 3 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('["a","b","c"]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 1 | NULL |
+| 1 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 3 | NULL |
+| 3 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('{"a": 3}') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_string('[]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_string('[1.0,2.0,3.0]') tmp1 as e1 order by k1, e1;
++------+----------+
+| k1 | e1 |
++------+----------+
+| 1 | 1.000000 |
+| 1 | 2.000000 |
+| 1 | 3.000000 |
+| 2 | 1.000000 |
+| 2 | 2.000000 |
+| 2 | 3.000000 |
+| 3 | 1.000000 |
+| 3 | 2.000000 |
+| 3 | 3.000000 |
++------+----------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_string('[1,"b",3]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | 1 |
+| 1 | 3 |
+| 1 | b |
+| 2 | 1 |
+| 2 | 3 |
+| 2 | b |
+| 3 | 1 |
+| 3 | 3 |
+| 3 | b |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_string('["a","b","c"]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | a |
+| 1 | b |
+| 1 | c |
+| 2 | a |
+| 2 | b |
+| 2 | c |
+| 3 | a |
+| 3 | b |
+| 3 | c |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_string('{"a": 3}') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
++------+------+
+```
+
+## keyword
+
+ explode_json_array
\ No newline at end of file
diff --git a/docs/en/sql-reference/sql-functions/table-functions/explode-split.md b/docs/en/sql-reference/sql-functions/table-functions/explode-split.md
new file mode 100644
index 0000000..0557b90
--- /dev/null
+++ b/docs/en/sql-reference/sql-functions/table-functions/explode-split.md
@@ -0,0 +1,112 @@
+---
+{
+ "title": "explode_split",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# explode_split
+
+## description
+
+Table functions must be used in conjunction with Lateral View.
+
+Split a string into multiple substrings according to the specified delimiter.
+
+grammar:
+
+```
+explode_split(str, delimiter)
+```
+
+## example
+
+Original table data:
+
+```
+mysql> select * from example1 order by k1;
++------+---------+
+| k1 | k2 |
++------+---------+
+| 1 | |
+| 2 | NULL |
+| 3 | , |
+| 4 | 1 |
+| 5 | 1,2,3 |
+| 6 | a, b, c |
++------+---------+
+```
+
+Lateral View:
+
+```
+mysql> select k1, e1 from example1 lateral view explode_split(k2, ',') tmp1 as e1 where k1 = 1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_split(k2, ',') tmp1 as e1 where k1 = 2 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 2 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_split(k2, ',') tmp1 as e1 where k1 = 3 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 3 | |
+| 3 | |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_split(k2, ',') tmp1 as e1 where k1 = 4 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 4 | 1 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_split(k2, ',') tmp1 as e1 where k1 = 5 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 5 | 1 |
+| 5 | 2 |
+| 5 | 3 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_split(k2, ',') tmp1 as e1 where k1 = 6 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 6 | b |
+| 6 | c |
+| 6 | a |
++------+------+
+```
+
+## keyword
+
+ explode_split
\ No newline at end of file
diff --git a/docs/en/sql-reference/sql-statements/Data Manipulation/lateral-view.md b/docs/en/sql-reference/sql-statements/Data Manipulation/lateral-view.md
new file mode 100644
index 0000000..1adacea
--- /dev/null
+++ b/docs/en/sql-reference/sql-statements/Data Manipulation/lateral-view.md
@@ -0,0 +1,94 @@
+---
+{
+ "title": "Lateral View",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Lateral View
+
+## description
+
+Lateral view syntax can be used with Table Function to fulfill the requirement of expanding one row of data into multiple rows (column to rows).
+
+grammar:
+
+```
+...
+FROM table_name
+lateral_view_ref[ lateral_view_ref ...]
+
+lateral_view_ref:
+
+LATERAL VIEW table_function(...) view_alias as col_name
+```
+
+The Lateral view clause must follow the table name or subquery. Can contain multiple Lateral view clauses. `view_alias` is the name of the corresponding Lateral View. `col_name` is the name of the column produced by the table function `table_function`.
+
+Table functions currently supported:
+
+1. `explode_split`
+2. `explode_bitmap`
+3. `explode_json_array`
+
+For specific function descriptions, please refer to the corresponding syntax help documentation.
+
+The data in the table will be Cartesian product with the result set produced by each Lateral View and then return to the upper level.
+
+## example
+
+Here, only the syntax example of Lateral View is given. For the specific meaning and output result description, please refer to the help document of the corresponding table function.
+
+1.
+
+```
+select k1, e1 from tbl1
+lateral view explode_split(v1,',') tmp1 as e1 where e1 = "abc";
+```
+
+2.
+
+```
+select k1, e1, e2 from tbl2
+lateral view explode_split(v1,',') tmp1 as e1
+lateral view explode_bitmap(bitmap1) tmp2 as e2
+where e2> 3;
+```
+
+3.
+
+```
+select k1, e1, e2 from tbl3
+lateral view explode_json_array_int("[1,2,3]") tmp1 as e1
+lateral view explode_bitmap(bitmap_from_string("4,5,6")) tmp2 as e2;
+```
+
+4.
+
+```
+select k1, e1 from (select k1, bitmap_union(members) as x from tbl1 where k1=10000 group by k1)tmp1
+lateral view explode_bitmap(x) tmp2 as e1;
+```
+
+## keyword
+
+ lateral view
diff --git a/docs/zh-CN/sql-reference/sql-functions/table-functions/explode-bitmap.md b/docs/zh-CN/sql-reference/sql-functions/table-functions/explode-bitmap.md
new file mode 100644
index 0000000..99f7971
--- /dev/null
+++ b/docs/zh-CN/sql-reference/sql-functions/table-functions/explode-bitmap.md
@@ -0,0 +1,157 @@
+---
+{
+ "title": "explode_bitmap",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# explode_bitmap
+
+## description
+
+表函数，需配合 Lateral View 使用。
+
+展开一个bitmap类型。
+
+语法：
+
+```
+explode_bitmap(bitmap)
+```
+
+## example
+
+原表数据：
+
+```
+mysql> select k1 from example1 order by k1;
++------+
+| k1 |
++------+
+| 1 |
+| 2 |
+| 3 |
+| 4 |
+| 5 |
+| 6 |
++------+
+```
+
+Lateral View:
+
+```
+mysql> select k1, e1 from example1 lateral view explode_bitmap(bitmap_empty()) tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
+| 4 | NULL |
+| 5 | NULL |
+| 6 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_bitmap(bitmap_from_string("1")) tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | 1 |
+| 2 | 1 |
+| 3 | 1 |
+| 4 | 1 |
+| 5 | 1 |
+| 6 | 1 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_bitmap(bitmap_from_string("1,2")) tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | 1 |
+| 1 | 2 |
+| 2 | 1 |
+| 2 | 2 |
+| 3 | 1 |
+| 3 | 2 |
+| 4 | 1 |
+| 4 | 2 |
+| 5 | 1 |
+| 5 | 2 |
+| 6 | 1 |
+| 6 | 2 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_bitmap(bitmap_from_string("1,1000")) tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | 1 |
+| 1 | 1000 |
+| 2 | 1 |
+| 2 | 1000 |
+| 3 | 1 |
+| 3 | 1000 |
+| 4 | 1 |
+| 4 | 1000 |
+| 5 | 1 |
+| 5 | 1000 |
+| 6 | 1 |
+| 6 | 1000 |
++------+------+
+
+mysql> select k1, e1, e2 from example1
+lateral view explode_bitmap(bitmap_from_string("1,1000")) tmp1 as e1
+lateral view explode_split("a,b", ",") tmp2 as e2 order by k1, e1, e2;
++------+------+------+
+| k1 | e1 | e2 |
++------+------+------+
+| 1 | 1 | a |
+| 1 | 1 | b |
+| 1 | 1000 | a |
+| 1 | 1000 | b |
+| 2 | 1 | a |
+| 2 | 1 | b |
+| 2 | 1000 | a |
+| 2 | 1000 | b |
+| 3 | 1 | a |
+| 3 | 1 | b |
+| 3 | 1000 | a |
+| 3 | 1000 | b |
+| 4 | 1 | a |
+| 4 | 1 | b |
+| 4 | 1000 | a |
+| 4 | 1000 | b |
+| 5 | 1 | a |
+| 5 | 1 | b |
+| 5 | 1000 | a |
+| 5 | 1000 | b |
+| 6 | 1 | a |
+| 6 | 1 | b |
+| 6 | 1000 | a |
+| 6 | 1000 | b |
++------+------+------+
+```
+
+## keyword
+
+ explode_bitmap
\ No newline at end of file
diff --git a/docs/zh-CN/sql-reference/sql-functions/table-functions/explode-json-array.md b/docs/zh-CN/sql-reference/sql-functions/table-functions/explode-json-array.md
new file mode 100644
index 0000000..8332d6a
--- /dev/null
+++ b/docs/zh-CN/sql-reference/sql-functions/table-functions/explode-json-array.md
@@ -0,0 +1,286 @@
+---
+{
+ "title": "explode_json_array",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# `explode_json_array`
+
+## description
+
+表函数，需配合 Lateral View 使用。
+
+展开一个 json 数组。根据数组元素类型，有三种函数名称。分别对应整型、浮点和字符串数组。
+
+语法：
+
+```
+explode_json_array_int(json_str)
+explode_json_array_double(json_str)
+explode_json_array_string(json_str)
+```
+
+## example
+
+原表数据：
+
+```
+mysql> select k1 from example1 order by k1;
++------+
+| k1 |
++------+
+| 1 |
+| 2 |
+| 3 |
+| 4 |
+| 5 |
+| 6 |
++------+
+```
+
+Lateral View:
+
+```
+mysql> select k1, e1 from example1 lateral view explode_json_array_int('[]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_int('[1,2,3]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | 1 |
+| 1 | 2 |
+| 1 | 3 |
+| 2 | 1 |
+| 2 | 2 |
+| 2 | 3 |
+| 3 | 1 |
+| 3 | 2 |
+| 3 | 3 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_int('[1,"b",3]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 1 | 1 |
+| 1 | 3 |
+| 2 | NULL |
+| 2 | 1 |
+| 2 | 3 |
+| 3 | NULL |
+| 3 | 1 |
+| 3 | 3 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_int('["a","b","c"]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 1 | NULL |
+| 1 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 3 | NULL |
+| 3 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_int('{"a": 3}') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('[]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('[1,2,3]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 1 | NULL |
+| 1 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 3 | NULL |
+| 3 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('[1,"b",3]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 1 | NULL |
+| 1 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 3 | NULL |
+| 3 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('[1.0,2.0,3.0]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | 1 |
+| 1 | 2 |
+| 1 | 3 |
+| 2 | 1 |
+| 2 | 2 |
+| 2 | 3 |
+| 3 | 1 |
+| 3 | 2 |
+| 3 | 3 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('[1,"b",3]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 1 | NULL |
+| 1 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 3 | NULL |
+| 3 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('["a","b","c"]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 1 | NULL |
+| 1 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 2 | NULL |
+| 3 | NULL |
+| 3 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_double('{"a": 3}') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_string('[]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_string('[1.0,2.0,3.0]') tmp1 as e1 order by k1, e1;
++------+----------+
+| k1 | e1 |
++------+----------+
+| 1 | 1.000000 |
+| 1 | 2.000000 |
+| 1 | 3.000000 |
+| 2 | 1.000000 |
+| 2 | 2.000000 |
+| 2 | 3.000000 |
+| 3 | 1.000000 |
+| 3 | 2.000000 |
+| 3 | 3.000000 |
++------+----------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_string('[1,"b",3]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | 1 |
+| 1 | 3 |
+| 1 | b |
+| 2 | 1 |
+| 2 | 3 |
+| 2 | b |
+| 3 | 1 |
+| 3 | 3 |
+| 3 | b |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_string('["a","b","c"]') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | a |
+| 1 | b |
+| 1 | c |
+| 2 | a |
+| 2 | b |
+| 2 | c |
+| 3 | a |
+| 3 | b |
+| 3 | c |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_json_array_string('{"a": 3}') tmp1 as e1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | NULL |
+| 2 | NULL |
+| 3 | NULL |
++------+------+
+```
+
+## keyword
+
+ explode_json_array
\ No newline at end of file
diff --git a/docs/zh-CN/sql-reference/sql-functions/table-functions/explode-split.md b/docs/zh-CN/sql-reference/sql-functions/table-functions/explode-split.md
new file mode 100644
index 0000000..d7c4031
--- /dev/null
+++ b/docs/zh-CN/sql-reference/sql-functions/table-functions/explode-split.md
@@ -0,0 +1,112 @@
+---
+{
+ "title": "explode_split",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# explode_split
+
+## description
+
+表函数，需配合 Lateral View 使用。
+
+将一个字符串按指定的分隔符分割成多个子串。
+
+语法：
+
+```
+explode_split(str, delimiter)
+```
+
+## example
+
+原表数据：
+
+```
+mysql> select * from example1 order by k1;
++------+---------+
+| k1 | k2 |
++------+---------+
+| 1 | |
+| 2 | NULL |
+| 3 | , |
+| 4 | 1 |
+| 5 | 1,2,3 |
+| 6 | a, b, c |
++------+---------+
+```
+
+Lateral View:
+
+```
+mysql> select k1, e1 from example1 lateral view explode_split(k2, ',') tmp1 as e1 where k1 = 1 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 1 | |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_split(k2, ',') tmp1 as e1 where k1 = 2 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 2 | NULL |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_split(k2, ',') tmp1 as e1 where k1 = 3 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 3 | |
+| 3 | |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_split(k2, ',') tmp1 as e1 where k1 = 4 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 4 | 1 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_split(k2, ',') tmp1 as e1 where k1 = 5 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 5 | 1 |
+| 5 | 2 |
+| 5 | 3 |
++------+------+
+
+mysql> select k1, e1 from example1 lateral view explode_split(k2, ',') tmp1 as e1 where k1 = 6 order by k1, e1;
++------+------+
+| k1 | e1 |
++------+------+
+| 6 | b |
+| 6 | c |
+| 6 | a |
++------+------+
+```
+
+## keyword
+
+ explode_split
\ No newline at end of file
diff --git a/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/lateral-view.md b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/lateral-view.md
new file mode 100644
index 0000000..2b6798e
--- /dev/null
+++ b/docs/zh-CN/sql-reference/sql-statements/Data Manipulation/lateral-view.md
@@ -0,0 +1,94 @@
+---
+{
+ "title": "Lateral View",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Lateral View
+
+## description
+
+Lateral view 语法可以搭配 Table Function，完成将一行数据扩展成多行（列转行）的需求。
+
+语法：
+
+```
+...
+FROM table_name
+lateral_view_ref[ lateral_view_ref ...]
+
+lateral_view_ref:
+
+LATERAL VIEW table_function(...) view_alias as col_name
+```
+
+Lateral view 子句必须跟随在表名或子查询之后。可以包含多个 Lateral view 子句。`view_alias` 是对应 Lateral View 的名称。`col_name` 是表函数 `table_function` 产出的列名。
+
+目前支持的表函数：
+
+1. `explode_split`
+2. `explode_bitmap`
+3. `explode_json_array`
+
+具体函数说明可参阅对应语法帮助文档。
+
+table 中的数据会和各个 Lateral View 产生的结果集做笛卡尔积后返回上层。
+
+## example
+
+这里只给出 Lateral View 的语法示例，具体含义和产出的结果说明，可参阅对应表函数帮助文档。
+
+1.
+
+```
+select k1, e1 from tbl1
+lateral view explode_split(v1, ',') tmp1 as e1 where e1 = "abc";
+```
+
+2.
+
+```
+select k1, e1, e2 from tbl2
+lateral view explode_split(v1, ',') tmp1 as e1
+lateral view explode_bitmap(bitmap1) tmp2 as e2
+where e2 > 3;
+```
+
+3.
+
+```
+select k1, e1, e2 from tbl3
+lateral view explode_json_array_int("[1,2,3]") tmp1 as e1
+lateral view explode_bitmap(bitmap_from_string("4,5,6")) tmp2 as e2;
+```
+
+4.
+
+```
+select k1, e1 from (select k1, bitmap_union(members) as x from tbl1 where k1=10000 group by k1)tmp1
+lateral view explode_bitmap(x) tmp2 as e1;
+```
+
+## keyword
+
+ lateral view
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index 76a81c6..87f8919 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -55,6 +55,7 @@ import java.io.IOException;
import java.text.StringCharacterIterator;
import java.util.Arrays;
import java.util.List;
+
// TODO: for aggregations, we need to unify the code paths for builtins and UDAs.
public class FunctionCallExpr extends Expr {
private static final Logger LOG = LogManager.getLogger(FunctionCallExpr.class);
@@ -84,7 +85,10 @@ public class FunctionCallExpr extends Expr {
private Expr originStmtFnExpr;
private boolean isRewrote = false;
-
+
+ public static final String UNKNOWN_TABLE_FUNCTION_MSG = "Currently only support `explode_split`, `explode_bitmap` " +
+ "and `explode_json_array_xx` table functions";
+
public void setIsAnalyticFnCall(boolean v) {
isAnalyticFnCall = v;
}
@@ -127,7 +131,7 @@ public class FunctionCallExpr extends Expr {
}
private FunctionCallExpr(
- FunctionName fnName, FunctionParams params, boolean isMergeAggFn) {
+ FunctionName fnName, FunctionParams params, boolean isMergeAggFn) {
super();
this.fnName = fnName;
fnParams = params;
@@ -158,7 +162,7 @@ public class FunctionCallExpr extends Expr {
super(other);
fnName = other.fnName;
isAnalyticFnCall = other.isAnalyticFnCall;
- // aggOp = other.aggOp;
+ // aggOp = other.aggOp;
// fnParams = other.fnParams;
// Clone the params in a way that keeps the children_ and the params.exprs()
// in sync. The children have already been cloned in the super c'tor.
@@ -245,12 +249,12 @@ public class FunctionCallExpr extends Expr {
}
if (((FunctionCallExpr) expr).fnParams.isDistinct()) {
sb.append("DISTINCT ");
- }
+ }
boolean isJsonFunction = false;
int len = children.size();
List<String> result = Lists.newArrayList();
if ((fnName.getFunction().equalsIgnoreCase("json_array")) ||
- (fnName.getFunction().equalsIgnoreCase("json_object"))) {
+ (fnName.getFunction().equalsIgnoreCase("json_object"))) {
len = len - 1;
isJsonFunction = true;
}
@@ -277,7 +281,7 @@ public class FunctionCallExpr extends Expr {
public boolean isScalarFunction() {
Preconditions.checkState(fn != null);
- return fn instanceof ScalarFunction ;
+ return fn instanceof ScalarFunction;
}
public boolean isAggregateFunction() {
@@ -289,6 +293,7 @@ public class FunctionCallExpr extends Expr {
Preconditions.checkState(fn != null);
return fn instanceof BuiltinAggregateFunction && !isAnalyticFnCall;
}
+
/**
* Returns true if this is a call to an aggregate function that returns
* non-null on an empty input (e.g. count).
@@ -296,7 +301,7 @@ public class FunctionCallExpr extends Expr {
public boolean returnsNonNullOnEmpty() {
Preconditions.checkNotNull(fn);
return fn instanceof AggregateFunction
- && ((AggregateFunction) fn).returnsNonNullOnEmpty();
+ && ((AggregateFunction) fn).returnsNonNullOnEmpty();
}
public boolean isDistinct() {
@@ -372,8 +377,8 @@ public class FunctionCallExpr extends Expr {
}
return;
}
-
- if(fnName.getFunction().equalsIgnoreCase("json_array")) {
+
+ if (fnName.getFunction().equalsIgnoreCase("json_array")) {
String res = parseJsonDataType(false);
if (children.size() == originChildSize) {
children.add(new StringLiteral(res));
@@ -381,8 +386,8 @@ public class FunctionCallExpr extends Expr {
return;
}
- if(fnName.getFunction().equalsIgnoreCase("json_object")) {
- if ((children.size()&1) == 1 && (originChildSize == children.size())) {
+ if (fnName.getFunction().equalsIgnoreCase("json_object")) {
+ if ((children.size() & 1) == 1 && (originChildSize == children.size())) {
throw new AnalysisException("json_object can't be odd parameters, need even parameters: " + this.toSql());
}
String res = parseJsonDataType(true);
@@ -395,7 +400,7 @@ public class FunctionCallExpr extends Expr {
if (fnName.getFunction().equalsIgnoreCase("group_concat")) {
if (children.size() > 2 || children.isEmpty()) {
throw new AnalysisException(
- "group_concat requires one or two parameters: " + this.toSql());
+ "group_concat requires one or two parameters: " + this.toSql());
}
if (fnParams.isDistinct()) {
@@ -405,7 +410,7 @@ public class FunctionCallExpr extends Expr {
Expr arg0 = getChild(0);
if (!arg0.type.isStringType() && !arg0.type.isNull()) {
throw new AnalysisException(
- "group_concat requires first parameter to be of type STRING: " + this.toSql());
+ "group_concat requires first parameter to be of type STRING: " + this.toSql());
}
if (children.size() == 2) {
@@ -490,7 +495,7 @@ public class FunctionCallExpr extends Expr {
throw new AnalysisException("intersect_count function first argument should be of BITMAP type, but was " + inputType);
}
- for(int i = 2; i < children.size(); i++) {
+ for (int i = 2; i < children.size(); i++) {
if (!getChild(i).isConstant()) {
throw new AnalysisException("intersect_count function filter_values arg must be constant");
}
@@ -667,7 +672,7 @@ public class FunctionCallExpr extends Expr {
type = getChild(0).type.getMaxResolutionType();
}
fn = getBuiltinFunction(analyzer, fnName.getFunction(), new Type[]{type},
- Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+ Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
} else if (fnName.getFunction().equalsIgnoreCase("count_distinct")) {
Type compatibleType = this.children.get(0).getType();
for (int i = 1; i < this.children.size(); ++i) {
@@ -688,36 +693,36 @@ public class FunctionCallExpr extends Expr {
fn = getTableFunction(fnName.getFunction(), childTypes,
Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
if (fn == null) {
- throw new AnalysisException("Doris only support `explode_split(varchar, varchar)` table function");
+ throw new AnalysisException(UNKNOWN_TABLE_FUNCTION_MSG);
}
- return;
- }
- // now first find function in built-in functions
- if (Strings.isNullOrEmpty(fnName.getDb())) {
- Type[] childTypes = collectChildReturnTypes();
- fn = getBuiltinFunction(analyzer, fnName.getFunction(), childTypes,
- Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
- }
-
- // find user defined functions
- if (fn == null) {
- if (!analyzer.isUDFAllowed()) {
- throw new AnalysisException(
- "Does not support non-builtin functions, or function does not exist: " + this.toSqlImpl());
+ } else {
+ // now first find function in built-in functions
+ if (Strings.isNullOrEmpty(fnName.getDb())) {
+ Type[] childTypes = collectChildReturnTypes();
+ fn = getBuiltinFunction(analyzer, fnName.getFunction(), childTypes,
+ Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
}
- String dbName = fnName.analyzeDb(analyzer);
- if (!Strings.isNullOrEmpty(dbName)) {
- // check operation privilege
- if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(
- ConnectContext.get(), dbName, PrivPredicate.SELECT)) {
- ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "SELECT");
+ // find user defined functions
+ if (fn == null) {
+ if (!analyzer.isUDFAllowed()) {
+ throw new AnalysisException(
+ "Does not support non-builtin functions, or function does not exist: " + this.toSqlImpl());
}
- Database db = Catalog.getCurrentCatalog().getDbNullable(dbName);
- if (db != null) {
- Function searchDesc = new Function(
- fnName, Arrays.asList(collectChildReturnTypes()), Type.INVALID, false);
- fn = db.getFunction(searchDesc, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+
+ String dbName = fnName.analyzeDb(analyzer);
+ if (!Strings.isNullOrEmpty(dbName)) {
+ // check operation privilege
+ if (!Catalog.getCurrentCatalog().getAuth().checkDbPriv(
+ ConnectContext.get(), dbName, PrivPredicate.SELECT)) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "SELECT");
+ }
+ Database db = Catalog.getCurrentCatalog().getDbNullable(dbName);
+ if (db != null) {
+ Function searchDesc = new Function(
+ fnName, Arrays.asList(collectChildReturnTypes()), Type.INVALID, false);
+ fn = db.getFunction(searchDesc, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+ }
}
}
}
@@ -729,7 +734,7 @@ public class FunctionCallExpr extends Expr {
}
if (fnName.getFunction().equalsIgnoreCase("from_unixtime")
- || fnName.getFunction().equalsIgnoreCase("date_format")) {
+ || fnName.getFunction().equalsIgnoreCase("date_format")) {
// if has only one child, it has default time format: yyyy-MM-dd HH:mm:ss.SSSSSS
if (children.size() > 1) {
final StringLiteral fmtLiteral = (StringLiteral) children.get(1);
@@ -994,40 +999,34 @@ public class FunctionCallExpr extends Expr {
result = 31 * result + Objects.hashCode(fnParams);
return result;
}
- public String forJSON(String str){
+
+ public String forJSON(String str) {
final StringBuilder result = new StringBuilder();
StringCharacterIterator iterator = new StringCharacterIterator(str);
char character = iterator.current();
- while (character != StringCharacterIterator.DONE){
- if( character == '\"' ){
- result.append("\\\"");
- }
- else if(character == '\\'){
- result.append("\\\\");
- }
- else if(character == '/'){
- result.append("\\/");
- }
- else if(character == '\b'){
- result.append("\\b");
- }
- else if(character == '\f'){
- result.append("\\f");
- }
- else if(character == '\n'){
- result.append("\\n");
- }
- else if(character == '\r'){
- result.append("\\r");
- }
- else if(character == '\t'){
- result.append("\\t");
- }
- else {
- result.append(character);
- }
- character = iterator.next();
+ while (character != StringCharacterIterator.DONE) {
+ if (character == '\"') {
+ result.append("\\\"");
+ } else if (character == '\\') {
+ result.append("\\\\");
+ } else if (character == '/') {
+ result.append("\\/");
+ } else if (character == '\b') {
+ result.append("\\b");
+ } else if (character == '\f') {
+ result.append("\\f");
+ } else if (character == '\n') {
+ result.append("\\n");
+ } else if (character == '\r') {
+ result.append("\\r");
+ } else if (character == '\t') {
+ result.append("\\t");
+ } else {
+ result.append(character);
+ }
+ character = iterator.next();
}
- return result.toString();
- }
+ return result.toString();
+ }
}
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java
index 41baabc..8326854 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java
@@ -18,9 +18,7 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.InlineView;
-import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
@@ -77,32 +75,31 @@ public class LateralViewRef extends TableRef {
if (!(expr instanceof FunctionCallExpr)) {
throw new AnalysisException("Only support function call expr in lateral view");
}
+
+ analyzeFunctionExpr(analyzer);
+
+ // analyze lateral view
+ desc = analyzer.registerTableRef(this);
+ explodeSlotRef = new SlotRef(new TableName(null, viewName), columnName);
+ explodeSlotRef.analyze(analyzer);
+ isAnalyzed = true; // true now that we have assigned desc
+ }
+
+ private void analyzeFunctionExpr(Analyzer analyzer) throws AnalysisException {
fnExpr = (FunctionCallExpr) expr;
fnExpr.setTableFnCall(true);
checkAndSupplyDefaultTableName(fnExpr);
fnExpr.analyze(analyzer);
- if (!fnExpr.getFnName().getFunction().equals(FunctionSet.EXPLODE_SPLIT)) {
- throw new AnalysisException("Only support explode function in lateral view");
- }
checkScalarFunction(fnExpr.getChild(0));
- if (!(fnExpr.getChild(1) instanceof StringLiteral)) {
- throw new AnalysisException("Split separator of explode must be a string const");
- }
fnExpr.getChild(0).collect(SlotRef.class, originSlotRefList);
- // analyze lateral view
- desc = analyzer.registerTableRef(this);
- explodeSlotRef = new SlotRef(new TableName(null, viewName), columnName);
- explodeSlotRef.analyze(analyzer);
- isAnalyzed = true; // true now that we have assigned desc
}
@Override
public TupleDescriptor createTupleDescriptor(Analyzer analyzer) throws AnalysisException {
// Create a fake catalog table for the lateral view
List<Column> columnList = Lists.newArrayList();
- columnList.add(new Column(columnName, Type.VARCHAR,
- false, null, true,
- null, ""));
+ columnList.add(new Column(columnName, fnExpr.getFn().getReturnType(),
+ false, null, true, null, ""));
view = new InlineView(viewName, columnList);
// Create the non-materialized tuple and set the fake table in it.
@@ -180,3 +177,4 @@ public class LateralViewRef extends TableRef {
}
}
}
+
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
index 8c7d169..c7b451d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java
@@ -30,6 +30,7 @@ import org.apache.doris.thrift.TSlotRef;
import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -374,6 +375,19 @@ public class SlotRef extends Expr {
}
}
+ @Override
+ protected Expr substituteImpl(ExprSubstitutionMap smap, Analyzer analyzer) throws AnalysisException {
+ if (isAnalyzed && desc != null && desc.getParent().getTable() == null && desc.getSourceExprs() != null) {
+ List<Expr> newSourceExprs = Lists.newArrayList();
+ for (int i = 0; i < desc.getSourceExprs().size(); ++i) {
+ newSourceExprs.add(desc.getSourceExprs().get(i).substituteImpl(smap, analyzer));
+ }
+ desc.setSourceExprs(newSourceExprs);
+ return this;
+ }
+ return super.substituteImpl(smap, analyzer);
+ }
+
public Table getTable() {
Preconditions.checkState(desc != null);
Table table = desc.getParent().getTable();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
index c74da9f..03327ce 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
@@ -2140,19 +2140,51 @@ public class FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
return builtinFunctions;
}
-
public static final String EXPLODE_SPLIT = "explode_split";
+ public static final String EXPLODE_BITMAP = "explode_bitmap";
+ public static final String EXPLODE_JSON_ARRAY_INT = "explode_json_array_int";
+ public static final String EXPLODE_JSON_ARRAY_DOUBLE = "explode_json_array_double";
+ public static final String EXPLODE_JSON_ARRAY_STRING = "explode_json_array_string";
private void initTableFunction() {
- // init explode_split function
- ArrayList<Type> argsType = Lists.newArrayList();
- argsType.add(Type.VARCHAR);
- argsType.add(Type.VARCHAR);
- Function explodeSplit = ScalarFunction.createBuiltin(
- EXPLODE_SPLIT, Type.VARCHAR, Function.NullableMode.DEPEND_ON_ARGUMENT, argsType, false,
- "", "", "", true);
List<Function> explodeSplits = Lists.newArrayList();
- explodeSplits.add(explodeSplit);
+ explodeSplits.add(ScalarFunction.createBuiltin(
+ EXPLODE_SPLIT, Type.VARCHAR, Function.NullableMode.DEPEND_ON_ARGUMENT,
+ Lists.newArrayList(Type.VARCHAR, Type.VARCHAR), false,
+ "_ZN5doris19DummyTableFunctions13explode_splitEPN9doris_udf15FunctionContextERKNS1_9StringValES6_",
+ null, null, true));
tableFunctions.put(EXPLODE_SPLIT, explodeSplits);
+
+ List<Function> explodeBitmaps = Lists.newArrayList();
+ explodeBitmaps.add(ScalarFunction.createBuiltin(
+ EXPLODE_BITMAP, Type.BIGINT, Function.NullableMode.DEPEND_ON_ARGUMENT,
+ Lists.newArrayList(Type.BITMAP), false,
+ "_ZN5doris19DummyTableFunctions14explode_bitmapEPN9doris_udf15FunctionContextERKNS1_9StringValE",
+ null, null, true));
+ tableFunctions.put(EXPLODE_BITMAP, explodeBitmaps);
+
+ List<Function> explodeJsonArrayInts = Lists.newArrayList();
+ explodeJsonArrayInts.add(ScalarFunction.createBuiltin(
+ EXPLODE_JSON_ARRAY_INT, Type.BIGINT, Function.NullableMode.DEPEND_ON_ARGUMENT,
+ Lists.newArrayList(Type.VARCHAR), false,
+ "_ZN5doris19DummyTableFunctions22explode_json_array_intEPN9doris_udf15FunctionContextERKNS1_9StringValE",
+ null, null, true));
+ tableFunctions.put(EXPLODE_JSON_ARRAY_INT, explodeJsonArrayInts);
+
+ List<Function> explodeJsonArrayDoubles = Lists.newArrayList();
+ explodeJsonArrayDoubles.add(ScalarFunction.createBuiltin(
+ EXPLODE_JSON_ARRAY_DOUBLE, Type.DOUBLE, Function.NullableMode.DEPEND_ON_ARGUMENT,
+ Lists.newArrayList(Type.VARCHAR), false,
+ "_ZN5doris19DummyTableFunctions25explode_json_array_doubleEPN9doris_udf15FunctionContextERKNS1_9StringValE",
+ null, null, true));
+ tableFunctions.put(EXPLODE_JSON_ARRAY_DOUBLE, explodeJsonArrayDoubles);
+
+ List<Function> explodeJsonArrayStrings = Lists.newArrayList();
+ explodeJsonArrayStrings.add(ScalarFunction.createBuiltin(
+ EXPLODE_JSON_ARRAY_STRING, Type.VARCHAR, Function.NullableMode.DEPEND_ON_ARGUMENT,
+ Lists.newArrayList(Type.VARCHAR), false,
+ "_ZN5doris19DummyTableFunctions25explode_json_array_stringEPN9doris_udf15FunctionContextERKNS1_9StringValE",
+ null, null, true));
+ tableFunctions.put(EXPLODE_JSON_ARRAY_STRING, explodeJsonArrayStrings);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
index a77a3ce..4e02aa8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
@@ -293,6 +293,7 @@ public class DistributedPlanner {
private PlanFragment createTableFunctionFragment(PlanNode node, PlanFragment childFragment) {
Preconditions.checkState(node instanceof TableFunctionNode);
node.setChild(0, childFragment.getPlanRoot());
+ node.setNumInstances(childFragment.getPlanRoot().getNumInstances());
childFragment.addPlanRoot(node);
return childFragment;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index 5e72189..a792231 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -573,6 +573,7 @@ public class Coordinator {
LOG.warn("catch a execute exception", e);
exception = e;
code = TStatusCode.THRIFT_RPC_ERROR;
+ BackendServiceProxy.getInstance().removeProxy(pair.first.brpcAddress);
} catch (InterruptedException e) {
LOG.warn("catch a interrupt exception", e);
exception = e;
@@ -2229,3 +2230,4 @@ public class Coordinator {
}
+
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
index 2752467..8617077 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
@@ -19,20 +19,20 @@ package org.apache.doris.planner;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
+import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.UtFrameUtils;
import org.apache.commons.io.FileUtils;
-
-import java.io.File;
-import java.util.UUID;
-
import org.junit.After;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.File;
+import java.util.UUID;
+
public class TableFunctionPlanTest {
private static String runningDir = "fe/mocked/TableFunctionPlanTest/" + UUID.randomUUID().toString() + "/";
private static ConnectContext ctx;
@@ -55,6 +55,11 @@ public class TableFunctionPlanTest {
+ "DUPLICATE KEY(k1) distributed by hash(k1) buckets 3 properties('replication_num' = '1');";
CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTblStmtStr, ctx);
Catalog.getCurrentCatalog().createTable(createTableStmt);
+
+ createTblStmtStr = "create table db1.tbl2(k1 int, k2 varchar, v1 bitmap bitmap_union) "
+ + "distributed by hash(k1) buckets 3 properties('replication_num' = '1');";
+ createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTblStmtStr, ctx);
+ Catalog.getCurrentCatalog().createTable(createTableStmt);
}
// test planner
@@ -172,15 +177,11 @@ public class TableFunctionPlanTest {
public void errorParam() throws Exception {
String sql = "explain select k1, e1 from db1.tbl1 lateral view explode_split(k2) tmp as e1;";
String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql);
- Assert.assertTrue(explainString.contains("Doris only support `explode_split(varchar, varchar)` table function"));
+ Assert.assertTrue(explainString.contains(FunctionCallExpr.UNKNOWN_TABLE_FUNCTION_MSG));
sql = "explain select k1, e1 from db1.tbl1 lateral view explode_split(k1) tmp as e1;";
explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql);
- Assert.assertTrue(explainString.contains("Doris only support `explode_split(varchar, varchar)` table function"));
-
- sql = "explain select k1, e1 from db1.tbl1 lateral view explode_split(k1, k2) tmp as e1;";
- explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql);
- Assert.assertTrue(explainString.contains("Split separator of explode must be a string const"));
+ Assert.assertTrue(explainString.contains(FunctionCallExpr.UNKNOWN_TABLE_FUNCTION_MSG));
}
/* Case2 table function in where stmt
@@ -385,29 +386,59 @@ public class TableFunctionPlanTest {
Assert.assertTrue(explainString.contains("tuple ids: 1 3"));
String formatString = explainString.replaceAll(" ", "");
Assert.assertTrue(formatString.contains(
- "SlotDescriptor{id=0,col=k1,type=INT}\n"
+ "SlotDescriptor{id=0,col=k1,type=INT}\n"
+ "parent=0\n"
+ "materialized=true"
));
Assert.assertTrue(formatString.contains(
- "SlotDescriptor{id=1,col=k2,type=VARCHAR(*)}\n"
+ "SlotDescriptor{id=1,col=k2,type=VARCHAR(*)}\n"
+ "parent=0\n"
+ "materialized=true"
));
Assert.assertTrue(formatString.contains(
- "SlotDescriptor{id=2,col=null,type=INT}\n"
+ "SlotDescriptor{id=2,col=null,type=INT}\n"
+ "parent=1\n"
+ "materialized=true"
));
Assert.assertTrue(formatString.contains(
- "SlotDescriptor{id=3,col=null,type=VARCHAR(*)}\n"
+ "SlotDescriptor{id=3,col=null,type=VARCHAR(*)}\n"
+ "parent=1\n"
+ "materialized=true"
));
Assert.assertTrue(formatString.contains(
- "SlotDescriptor{id=6,col=e1,type=VARCHAR(*)}\n"
+ "SlotDescriptor{id=6,col=e1,type=VARCHAR(*)}\n"
+ "parent=3\n"
+ "materialized=true"
));
}
+
+ @Test
+ public void testExplodeBitmap() throws Exception {
+ String sql = "desc select k1, e1 from db1.tbl2 lateral view explode_bitmap(v1) tmp1 as e1 ";
+ String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql, true);
+ System.out.println(explainString);
+ Assert.assertTrue(explainString.contains("table function: explode_bitmap(`default_cluster:db1`.`tbl2`.`v1`)"));
+ Assert.assertTrue(explainString.contains("output slot id: 1 2"));
+ }
+
+ @Test
+ public void testExplodeJsonArray() throws Exception {
+ String sql = "desc select k1, e1 from db1.tbl2 lateral view explode_json_array_int('[1,2,3]') tmp1 as e1 ";
+ String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql, true);
+ System.out.println(explainString);
+ Assert.assertTrue(explainString.contains("table function: explode_json_array_int('[1,2,3]')"));
+ Assert.assertTrue(explainString.contains("output slot id: 0 1"));
+
+ sql = "desc select k1, e1 from db1.tbl2 lateral view explode_json_array_string('[\"a\",\"b\",\"c\"]') tmp1 as e1 ";
+ explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql, true);
+ System.out.println(explainString);
+ Assert.assertTrue(explainString.contains("table function: explode_json_array_string('[\"a\",\"b\",\"c\"]')"));
+ Assert.assertTrue(explainString.contains("output slot id: 0 1"));
+
+ sql = "desc select k1, e1 from db1.tbl2 lateral view explode_json_array_double('[1.1, 2.2, 3.3]') tmp1 as e1 ";
+ explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql, true);
+ System.out.println(explainString);
+ Assert.assertTrue(explainString.contains("table function: explode_json_array_double('[1.1, 2.2, 3.3]')"));
+ Assert.assertTrue(explainString.contains("output slot id: 0 1"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org