You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/01/11 04:53:42 UTC
[doris] branch master updated: [fix](nereids) orthogonal_bitmap_intersect's return type should be bitmap (#15784)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 006b3bd61a [fix](nereids) orthogonal_bitmap_intersect's return type should be bitmap (#15784)
006b3bd61a is described below
commit 006b3bd61ad2346116e2ad1d7151b93f8e946307
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Wed Jan 11 12:53:37 2023 +0800
[fix](nereids) orthogonal_bitmap_intersect's return type should be bitmap (#15784)
---
.../functions/agg/OrthogonalBitmapIntersect.java | 12 +
.../test_bitmap_function_nereids.out | 273 +++++++++++++++++++++
.../test_bitmap_function_nereids.groovy | 194 +++++++++++++++
3 files changed, 479 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersect.java
index 7c9b33d2ad..956f585f0a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersect.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/OrthogonalBitmapIntersect.java
@@ -17,13 +17,16 @@
package org.apache.doris.nereids.trees.expressions.functions.agg;
+import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNotNullable;
import org.apache.doris.nereids.trees.expressions.functions.BitmapIntersectFunction;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.BitmapType;
import org.apache.doris.nereids.util.ExpressionUtils;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import java.util.List;
@@ -33,6 +36,10 @@ import java.util.List;
public class OrthogonalBitmapIntersect extends AggregateFunction
implements AlwaysNotNullable, OrthogonalBitmapFunction, BitmapIntersectFunction {
+ static final List<FunctionSignature> FUNCTION_SIGNATURES = SUPPORTED_TYPES.stream()
+ .map(type -> FunctionSignature.ret(BitmapType.INSTANCE).varArgs(BitmapType.INSTANCE, type, type))
+ .collect(ImmutableList.toImmutableList());
+
/**
* constructor with 3 or more arguments.
*/
@@ -49,6 +56,11 @@ public class OrthogonalBitmapIntersect extends AggregateFunction
ExpressionUtils.mergeArguments(arg0, arg1, arg2, varArgs));
}
+ @Override
+ public List<FunctionSignature> getSignatures() {
+ return FUNCTION_SIGNATURES;
+ }
+
/**
* withDistinctAndChildren.
*/
diff --git a/regression-test/data/nereids_syntax_p0/test_bitmap_function_nereids.out b/regression-test/data/nereids_syntax_p0/test_bitmap_function_nereids.out
new file mode 100644
index 0000000000..ed771ea325
--- /dev/null
+++ b/regression-test/data/nereids_syntax_p0/test_bitmap_function_nereids.out
@@ -0,0 +1,273 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+0
+
+-- !sql --
+1
+
+-- !sql --
+1
+
+-- !sql --
+1,2
+
+-- !sql --
+
+
+-- !sql --
+\N
+
+-- !sql --
+false
+
+-- !sql --
+true
+
+-- !sql --
+0
+
+-- !sql --
+
+
+-- !sql --
+0,1,2
+
+-- !sql --
+\N
+
+-- !sql --
+false
+
+-- !sql --
+true
+
+-- !sql --
+true
+
+-- !sql --
+false
+
+-- !sql_bitmap_hash1 --
+1
+
+-- !sql_bitmap_hash2 --
+1
+
+-- !sql_bitmap_hash3 --
+0
+
+-- !sql_bitmap_hash64_1 --
+1
+
+-- !sql_bitmap_hash64_2 --
+1
+
+-- !sql_bitmap_hash64_3 --
+0
+
+-- !sql --
+2
+
+-- !sql --
+1
+
+-- !sql --
+1,2
+
+-- !sql --
+\N
+
+-- !sql --
+0,1,2,10
+
+-- !sql --
+1,2,3,4,5,10
+
+-- !sql --
+0
+
+-- !sql --
+3
+
+-- !sql --
+1
+
+-- !sql --
+2
+
+-- !sql --
+0
+
+-- !sql --
+\N
+
+-- !sql --
+3
+
+-- !sql --
+3
+
+-- !sql --
+5
+
+-- !sql --
+6
+
+-- !sql --
+\N
+
+-- !sql --
+2
+
+-- !sql --
+1,4
+
+-- !sql --
+1,3,5
+
+-- !sql --
+1,3,5
+
+-- !sql --
+\N
+
+-- !sql --
+4
+
+-- !sql --
+0
+
+-- !sql --
+6
+
+-- !sql --
+3
+
+-- !sql --
+3
+
+-- !sql --
+\N
+
+-- !sql --
+0
+
+-- !sql --
+5
+
+-- !sql --
+2
+
+-- !sql --
+2
+
+-- !sql --
+1,2,3,4,5
+
+-- !sql --
+2
+
+-- !sql --
+1,2,3
+
+-- !sql --
+4,5
+
+-- !sql --
+0,1,2
+
+-- !sql --
+2,3
+
+-- !sql --
+2,3,5
+
+-- !sql --
+\N
+
+-- !sql --
+
+
+-- !sql --
+1
+
+-- !sql --
+1,2
+
+-- !sql --
+1 \N
+2 \N
+
+-- !sql --
+1 3
+2 2
+
+-- !sql --
+1 3
+2 2
+
+-- !sql --
+2
+
+-- !sql --
+1,4
+
+-- !sql --
+1,3,5
+
+-- !sql --
+1,3,5
+
+-- !sql --
+\N
+
+-- !sql --
+1
+
+-- !sql --
+
+
+-- !sql --
+\N
+
+-- !sql --
+9999999999
+
+-- !sql --
+4 1,2,3
+3 1,2,3,4,5
+
+-- !sql --
+3
+
+-- !sql --
+\N
+
+-- !sql --
+0
+
+-- !sql --
+0
+
+-- !sql --
+[1, 2, 3]
+[1, 2, 3, 4, 5]
+
+-- !sql --
+[]
+
+-- !sql --
+[3, 4, 100, 200]
+
+-- !sql --
+1,2,3
+
+-- !sql --
+1
+
+-- !sql --
+100
+
+-- !sql --
+20221103
+
diff --git a/regression-test/suites/nereids_syntax_p0/test_bitmap_function_nereids.groovy b/regression-test/suites/nereids_syntax_p0/test_bitmap_function_nereids.groovy
new file mode 100644
index 0000000000..cb00d6cd66
--- /dev/null
+++ b/regression-test/suites/nereids_syntax_p0/test_bitmap_function_nereids.groovy
@@ -0,0 +1,194 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_bitmap_function_nereids") {
+ sql 'set enable_vectorized_engine=true;'
+ sql 'set enable_fallback_to_original_planner=false;'
+ sql 'set enable_nereids_planner=true;'
+ // BITMAP_AND
+ qt_sql """ select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(2))) cnt """
+ qt_sql """ select bitmap_count(bitmap_and(to_bitmap(1), to_bitmap(1))) cnt """
+ qt_sql """ select bitmap_to_string(bitmap_and(to_bitmap(1), to_bitmap(1))) """
+ qt_sql """ select bitmap_to_string(bitmap_and(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'))) """
+ qt_sql """ select bitmap_to_string(bitmap_and(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'),bitmap_empty())) """
+ qt_sql """ select bitmap_to_string(bitmap_and(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'),NULL)) """
+
+ // BITMAP_CONTAINS
+ qt_sql """ select bitmap_contains(to_bitmap(1),2) cnt """
+ qt_sql """ select bitmap_contains(to_bitmap(1),1) cnt """
+
+ // BITMAP_EMPTY
+ qt_sql """ select bitmap_count(bitmap_empty()) """
+
+ // BITMAP_FROM_STRING
+ qt_sql """ select bitmap_to_string(bitmap_empty()) """
+ qt_sql """ select bitmap_to_string(bitmap_from_string("0, 1, 2")) """
+ qt_sql """ select bitmap_from_string("-1, 0, 1, 2") """
+
+ // BITMAP_HAS_ANY
+ qt_sql """ select bitmap_has_any(to_bitmap(1),to_bitmap(2)) cnt """
+ qt_sql """ select bitmap_has_any(to_bitmap(1),to_bitmap(1)) cnt """
+
+ // BITMAP_HAS_ALL
+ qt_sql """ select bitmap_has_all(bitmap_from_string("0, 1, 2"), bitmap_from_string("1, 2")) cnt """
+ qt_sql """ select bitmap_has_all(bitmap_empty(), bitmap_from_string("1, 2")) cnt """
+
+ // BITMAP_HASH
+ qt_sql_bitmap_hash1 """ select bitmap_count(bitmap_hash('hello')) """
+ qt_sql_bitmap_hash2 """ select bitmap_count(bitmap_hash('')) """
+ qt_sql_bitmap_hash3 """ select bitmap_count(bitmap_hash(null)) """
+
+ // BITMAP_HASH64
+ qt_sql_bitmap_hash64_1 """ select bitmap_count(bitmap_hash64('hello')) """
+ qt_sql_bitmap_hash64_2 """ select bitmap_count(bitmap_hash64('')) """
+ qt_sql_bitmap_hash64_3 """ select bitmap_count(bitmap_hash64(null)) """
+
+ // BITMAP_OR
+ qt_sql """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(2))) cnt """
+ qt_sql """ select bitmap_count(bitmap_or(to_bitmap(1), to_bitmap(1))) cnt """
+ qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) """
+ qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), NULL)) """
+ qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2), to_bitmap(10), to_bitmap(0), bitmap_empty())) """
+ qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(10), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'))) """
+
+ // bitmap_and_count
+ qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'),bitmap_empty()) """
+ qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3')) """
+ qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """
+ qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5')) """
+ qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'),bitmap_empty()) """
+ qt_sql """ select bitmap_and_count(bitmap_from_string('1,2,3'), bitmap_from_string('1,2'), bitmap_from_string('1,2,3,4,5'), NULL) """
+
+ // bitmap_or_count
+ qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_empty()) """
+ qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3'))"""
+ qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """
+ qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), bitmap_empty()) """
+ qt_sql """ select bitmap_or_count(bitmap_from_string('1,2,3'), bitmap_from_string('3,4,5'), to_bitmap(100), NULL) """
+
+ // BITMAP_XOR
+ qt_sql """ select bitmap_count(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt """
+ qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) """
+ qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'))) """
+ qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),bitmap_empty())) """
+ qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),NULL)) """
+
+ // BITMAP_XOR_COUNT
+ qt_sql """ select bitmap_xor_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) """
+ qt_sql """ select bitmap_xor_count(bitmap_from_string('1,2,3'),bitmap_from_string('1,2,3')) """
+ qt_sql """ select bitmap_xor_count(bitmap_from_string('1,2,3'),bitmap_from_string('4,5,6')) """
+ qt_sql """ select (bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'))) """
+ qt_sql """ select (bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),bitmap_empty())) """
+ qt_sql """ select (bitmap_xor_count(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),NULL)) """
+
+ // BITMAP_NOT
+ qt_sql """ select bitmap_count(bitmap_not(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt """
+ qt_sql """ select bitmap_to_string(bitmap_not(bitmap_from_string('2,3,5'),bitmap_from_string('1,2,3,4'))) """
+
+ // BITMAP_AND_NOT
+ qt_sql """ select bitmap_count(bitmap_and_not(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5'))) cnt """
+
+ // BITMAP_AND_NOT_COUNT
+ qt_sql """ select bitmap_and_not_count(bitmap_from_string('1,2,3'),bitmap_from_string('3,4,5')) cnt """
+
+ // BITMAP_SUBSET_IN_RANGE
+ qt_sql """ select bitmap_to_string(bitmap_subset_in_range(bitmap_from_string('1,2,3,4,5'), 0, 9)) value """
+ qt_sql """ select bitmap_to_string(bitmap_subset_in_range(bitmap_from_string('1,2,3,4,5'), 2, 3)) value """
+
+ // BITMAP_SUBSET_LIMIT
+ qt_sql """ select bitmap_to_string(bitmap_subset_limit(bitmap_from_string('1,2,3,4,5'), 0, 3)) value """
+ qt_sql """ select bitmap_to_string(bitmap_subset_limit(bitmap_from_string('1,2,3,4,5'), 4, 3)) value """
+
+ // SUB_BITMAP
+ qt_sql """ select bitmap_to_string(sub_bitmap(bitmap_from_string('1,0,1,2,3,1,5'), 0, 3)) value """
+ qt_sql """ select bitmap_to_string(sub_bitmap(bitmap_from_string('1,0,1,2,3,1,5'), -3, 2)) value """
+ qt_sql """ select bitmap_to_string(sub_bitmap(bitmap_from_string('1,0,1,2,3,1,5'), 2, 100)) value """
+
+ // BITMAP_TO_STRING
+ qt_sql """ select bitmap_to_string(null) """
+ qt_sql """ select bitmap_to_string(bitmap_empty()) """
+ qt_sql """ select bitmap_to_string(to_bitmap(1)) """
+ qt_sql """ select bitmap_to_string(bitmap_or(to_bitmap(1), to_bitmap(2))) """
+
+ // BITMAP_UNION
+ def bitmapUnionTable = "test_bitmap_union"
+ sql """ DROP TABLE IF EXISTS ${bitmapUnionTable} """
+ sql """ create table if not exists ${bitmapUnionTable} (page_id int,user_id bitmap bitmap_union) aggregate key (page_id) distributed by hash (page_id) PROPERTIES("replication_num" = "1") """
+
+ sql """ insert into ${bitmapUnionTable} values(1, to_bitmap(1)); """
+ sql """ insert into ${bitmapUnionTable} values(1, to_bitmap(2)); """
+ sql """ insert into ${bitmapUnionTable} values(1, to_bitmap(3)); """
+ sql """ insert into ${bitmapUnionTable} values(2, to_bitmap(1)); """
+ sql """ insert into ${bitmapUnionTable} values(2, to_bitmap(2)); """
+
+ qt_sql """ select page_id, bitmap_union(user_id) from ${bitmapUnionTable} group by page_id order by page_id """
+ qt_sql """ select page_id, bitmap_count(bitmap_union(user_id)) from ${bitmapUnionTable} group by page_id order by page_id """
+ qt_sql """ select page_id, count(distinct user_id) from ${bitmapUnionTable} group by page_id order by page_id """
+
+ sql """ drop table ${bitmapUnionTable} """
+
+ // BITMAP_XOR
+ qt_sql """ select bitmap_count(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))) cnt; """
+ qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'))); """
+ qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'))); """
+ qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),bitmap_empty())); """
+ qt_sql """ select bitmap_to_string(bitmap_xor(bitmap_from_string('2,3'),bitmap_from_string('1,2,3,4'),bitmap_from_string('3,4,5'),NULL)); """
+
+ // TO_BITMAP
+ qt_sql """ select bitmap_count(to_bitmap(10)) """
+ qt_sql """ select bitmap_to_string(to_bitmap(-1)) """
+
+ // BITMAP_MAX
+ qt_sql """ select bitmap_max(bitmap_from_string('')) value; """
+ qt_sql """ select bitmap_max(bitmap_from_string('1,9999999999')) value """
+
+ // INTERSECT_COUNT
+ def intersectCountTable = "test_intersect_count"
+ sql """ DROP TABLE IF EXISTS ${intersectCountTable} """
+ sql """ create table if not exists ${intersectCountTable} (dt int (11),page varchar (10),user_id bitmap BITMAP_UNION ) DISTRIBUTED BY HASH(dt) BUCKETS 2 PROPERTIES("replication_num" = "1") """
+
+
+ sql """ insert into ${intersectCountTable} values(3,"110001", to_bitmap(1)); """
+ sql """ insert into ${intersectCountTable} values(3,"110001", to_bitmap(2)); """
+ sql """ insert into ${intersectCountTable} values(3,"110001", to_bitmap(3)); """
+ sql """ insert into ${intersectCountTable} values(3,"110001", to_bitmap(4)); """
+ sql """ insert into ${intersectCountTable} values(3,"110001", to_bitmap(5)); """
+ sql """ insert into ${intersectCountTable} values(4,"110001", to_bitmap(1)); """
+ sql """ insert into ${intersectCountTable} values(4,"110001", to_bitmap(2)); """
+ sql """ insert into ${intersectCountTable} values(4,"110001", to_bitmap(3)); """
+
+ qt_sql """ select dt,bitmap_to_string(user_id) from ${intersectCountTable} where dt in (3,4) order by dt desc; """
+ qt_sql """ select intersect_count(user_id,dt,3,4) from ${intersectCountTable}; """
+
+ // ARTHOGONAL_BITMAP_****
+ def arthogonalBitmapTable = "test_arthogonal_bitmap"
+ sql """ DROP TABLE IF EXISTS ${arthogonalBitmapTable} """
+ sql """ CREATE TABLE IF NOT EXISTS ${arthogonalBitmapTable} ( tag_group bigint(20) NULL COMMENT "标签组", tag_value_id varchar(64) NULL COMMENT "标签值", tag_range int(11) NOT NULL DEFAULT "0" COMMENT "", partition_sign varchar(32) NOT NULL COMMENT "分区标识", bucket int(11) NOT NULL COMMENT "分桶字段", confidence tinyint(4) NULL DEFAULT "100" COMMENT "置信度", members bitmap BITMAP_UNION NULL COMMENT "人群") ENGINE=OLAP AGGREGATE KEY(tag_group, tag_value_id, tag_range, partition_sign, bucket, confiden [...]
+
+ qt_sql """ select orthogonal_bitmap_intersect(members, tag_group, 1150000, 1150001, 390006) from ${arthogonalBitmapTable} where tag_group in ( 1150000, 1150001, 390006); """
+ qt_sql """ select orthogonal_bitmap_intersect_count(members, tag_group, 1150000, 1150001, 390006) from ${arthogonalBitmapTable} where tag_group in ( 1150000, 1150001, 390006); """
+ qt_sql """ select orthogonal_bitmap_union_count(members) from ${arthogonalBitmapTable} where tag_group in ( 1150000, 1150001, 390006); """
+
+ qt_sql """ select bitmap_to_array(user_id) from ${intersectCountTable} order by dt desc; """
+ qt_sql """ select bitmap_to_array(bitmap_empty()); """
+ qt_sql """ select bitmap_to_array(bitmap_from_string('100,200,3,4')); """
+
+ qt_sql """ select bitmap_to_string(sub_bitmap(bitmap_from_string('1,2,3,4,5'), 0, 3)) value; """
+ qt_sql """ select bitmap_to_string(sub_bitmap(bitmap_from_string('1'), 0, 3)) value; """
+ qt_sql """ select bitmap_to_string(bitmap_subset_limit(bitmap_from_string('100'), 0, 3)) value; """
+ qt_sql """ select bitmap_to_string(bitmap_subset_in_range(bitmap_from_string('20221103'), 0, 20221104)) date_list_bitmap; """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org