You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2022/06/15 03:41:08 UTC
[GitHub] [incubator-doris] HappenLee commented on a diff in pull request #10103: [feature] support runtime filter on vectorized engine
HappenLee commented on code in PR #10103:
URL: https://github.com/apache/incubator-doris/pull/10103#discussion_r897497731
##########
be/src/util/simd/bits.h:
##########
@@ -57,5 +57,35 @@ inline uint32_t bytes32_mask_to_bits32_mask(const bool* data) {
return bytes32_mask_to_bits32_mask(reinterpret_cast<const uint8_t*>(data));
}
+inline size_t count_zero_num(const int8_t* data, size_t size) {
+ size_t num = 0;
+ const int8_t* end = data + size;
+
+#if defined(__SSE2__) && defined(__POPCNT__)
Review Comment:
rethink we need to do simd by ourself ? the code can auto simd by compiler and have `AVX2`
```
for (; data < end; ++data) {
num += (*data == 0);
}
```
##########
be/src/vec/exprs/vruntimefilter_wrapper.h:
##########
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "vec/exprs/vexpr.h"
+
+namespace doris::vectorized {
+class VRuntimeFilterWrapper final : public VExpr {
+public:
+ VRuntimeFilterWrapper(const TExprNode& node, VExpr* impl);
+ VRuntimeFilterWrapper(const VRuntimeFilterWrapper& vexpr);
+ ~VRuntimeFilterWrapper() = default;
+ virtual doris::Status execute(VExprContext* context, doris::vectorized::Block* block,
+ int* result_column_id) override;
+ virtual doris::Status prepare(doris::RuntimeState* state, const doris::RowDescriptor& desc,
+ VExprContext* context) override;
+ virtual doris::Status open(doris::RuntimeState* state, VExprContext* context,
+ FunctionContext::FunctionStateScope scope) override;
+ virtual std::string debug_string() const override { return _impl->debug_string(); };
+ virtual bool is_constant() const override;
+ virtual void close(doris::RuntimeState* state, VExprContext* context,
+ FunctionContext::FunctionStateScope scope) override;
+ virtual VExpr* clone(doris::ObjectPool* pool) const override {
+ return pool->add(new VRuntimeFilterWrapper(*this));
+ }
+ virtual const std::string& expr_name() const override;
+
+ virtual DataTypePtr& data_type() override { return _impl->data_type(); }
+
+ virtual TypeDescriptor type() override { return _impl->type(); }
+
+ virtual bool is_slot_ref() const override { return _impl->is_slot_ref(); }
+
+ virtual TExprNodeType::type node_type() const override { return _impl->node_type(); }
+
+ virtual void add_child(VExpr* expr) override { _impl->add_child(expr); }
+
+ virtual bool is_nullable() const override { return _impl->is_nullable(); }
+
+ virtual PrimitiveType result_type() const override { return _impl->result_type(); }
+
+ virtual const std::vector<VExpr*>& children() const override { return _impl->children(); }
+ virtual void set_children(std::vector<VExpr*> children) override {
+ _impl->set_children(children);
+ }
+
+ virtual bool is_and_expr() const override { return _impl->is_and_expr(); }
+
+ virtual const TFunction& fn() const override { return _impl->fn(); }
+ virtual ColumnPtrWrapper* get_const_col(VExprContext* context) override {
+ return _impl->get_const_col(context);
+ }
+
+protected:
Review Comment:
`final` change to `private`
##########
be/src/vec/exprs/vruntimefilter_wrapper.h:
##########
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "vec/exprs/vexpr.h"
+
+namespace doris::vectorized {
+class VRuntimeFilterWrapper final : public VExpr {
+public:
+ VRuntimeFilterWrapper(const TExprNode& node, VExpr* impl);
+ VRuntimeFilterWrapper(const VRuntimeFilterWrapper& vexpr);
+ ~VRuntimeFilterWrapper() = default;
+ virtual doris::Status execute(VExprContext* context, doris::vectorized::Block* block,
Review Comment:
`final` not need `virtual`
##########
be/src/vec/exprs/vruntimefilter_wrapper.h:
##########
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "vec/exprs/vexpr.h"
+
+namespace doris::vectorized {
+class VRuntimeFilterWrapper final : public VExpr {
+public:
+ VRuntimeFilterWrapper(const TExprNode& node, VExpr* impl);
+ VRuntimeFilterWrapper(const VRuntimeFilterWrapper& vexpr);
+ ~VRuntimeFilterWrapper() = default;
+ virtual doris::Status execute(VExprContext* context, doris::vectorized::Block* block,
+ int* result_column_id) override;
+ virtual doris::Status prepare(doris::RuntimeState* state, const doris::RowDescriptor& desc,
+ VExprContext* context) override;
+ virtual doris::Status open(doris::RuntimeState* state, VExprContext* context,
+ FunctionContext::FunctionStateScope scope) override;
+ virtual std::string debug_string() const override { return _impl->debug_string(); };
+ virtual bool is_constant() const override;
+ virtual void close(doris::RuntimeState* state, VExprContext* context,
+ FunctionContext::FunctionStateScope scope) override;
+ virtual VExpr* clone(doris::ObjectPool* pool) const override {
+ return pool->add(new VRuntimeFilterWrapper(*this));
+ }
+ virtual const std::string& expr_name() const override;
+
+ virtual DataTypePtr& data_type() override { return _impl->data_type(); }
+
+ virtual TypeDescriptor type() override { return _impl->type(); }
+
+ virtual bool is_slot_ref() const override { return _impl->is_slot_ref(); }
+
+ virtual TExprNodeType::type node_type() const override { return _impl->node_type(); }
+
+ virtual void add_child(VExpr* expr) override { _impl->add_child(expr); }
+
+ virtual bool is_nullable() const override { return _impl->is_nullable(); }
+
+ virtual PrimitiveType result_type() const override { return _impl->result_type(); }
+
+ virtual const std::vector<VExpr*>& children() const override { return _impl->children(); }
+ virtual void set_children(std::vector<VExpr*> children) override {
+ _impl->set_children(children);
+ }
+
+ virtual bool is_and_expr() const override { return _impl->is_and_expr(); }
+
+ virtual const TFunction& fn() const override { return _impl->fn(); }
+ virtual ColumnPtrWrapper* get_const_col(VExprContext* context) override {
+ return _impl->get_const_col(context);
+ }
+
+protected:
+ /// Helper function that calls ctx->register(), sets fn_context_index_, and returns the
+ /// registered FunctionContext
+ virtual void register_function_context(doris::RuntimeState* state,
+ VExprContext* context) override {
+ _impl->register_function_context(state, context);
+ }
+
+ /// Helper function to initialize function context, called in `open` phase of VExpr:
+ /// 1. Set constant columns result of function arguments.
+ /// 2. Call function's prepare() to initialize function state, fragment-local or
+ /// thread-local according the input `FunctionStateScope` argument.
+ virtual Status init_function_context(VExprContext* context,
+ FunctionContext::FunctionStateScope scope,
+ const FunctionBasePtr& function) const override {
+ return _impl->init_function_context(context, scope, function);
+ }
+
+ /// Helper function to close function context, fragment-local or thread-local according
+ /// the input `FunctionStateScope` argument. Called in `close` phase of VExpr.
+ virtual void close_function_context(VExprContext* context,
+ FunctionContext::FunctionStateScope scope,
+ const FunctionBasePtr& function) const override {
+ _impl->close_function_context(context, scope, function);
+ }
+
+private:
+ VExpr* _impl;
+
+ bool _always_true;
+ /// TODO: statistic filter rate in the profile
+ std::atomic<int64_t> _filtered_rows;
+ std::atomic<int64_t> _scan_rows;
+
+ bool _has_calculate_filter = false;
+ // loop size must be power of 2
+ constexpr static int64_t _loop_size = 8192;
Review Comment:
ConstExpr `LOOP_SIZE`
##########
be/src/exec/olap_scan_node.h:
##########
@@ -280,6 +280,8 @@ class OlapScanNode : public ScanNode {
};
std::vector<TRuntimeFilterDesc> _runtime_filter_descs;
std::vector<RuntimeFilterContext> _runtime_filter_ctxs;
+ std::vector<bool> _runtime_filter_ready_flag;
+ std::vector<std::unique_ptr<std::mutex>> _rf_locks;
Review Comment:
why not `std::vector<std::mutex>` ?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org