You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ya...@apache.org on 2021/11/06 05:31:12 UTC
[incubator-doris] branch master updated: sub_bitmap (#6977)
This is an automated email from the ASF dual-hosted git repository.
yangzhg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new e69249c sub_bitmap (#6977)
e69249c is described below
commit e69249c082796a9cc336305358adb49b301956a0
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Sat Nov 6 13:31:03 2021 +0800
sub_bitmap (#6977)
Starting from the offset position, intercept the specified limit bitmap elements and return a bitmap subset.
Types of chang
---
be/src/exprs/bitmap_function.cpp | 19 +++++++
be/src/exprs/bitmap_function.h | 2 +
be/src/util/bitmap_value.h | 26 +++++++++
be/test/exprs/bitmap_function_test.cpp | 57 ++++++++++++++++++++
docs/.vuepress/sidebar/en.js | 1 +
docs/.vuepress/sidebar/zh-CN.js | 1 +
.../sql-functions/bitmap-functions/sub_bitmap.md | 61 +++++++++++++++++++++
.../sql-functions/bitmap-functions/sub_bitmap.md | 62 ++++++++++++++++++++++
gensrc/script/doris_builtins_functions.py | 4 ++
9 files changed, 233 insertions(+)
diff --git a/be/src/exprs/bitmap_function.cpp b/be/src/exprs/bitmap_function.cpp
index ccab0c5..b469c15 100644
--- a/be/src/exprs/bitmap_function.cpp
+++ b/be/src/exprs/bitmap_function.cpp
@@ -671,6 +671,25 @@ StringVal BitmapFunctions::bitmap_subset_in_range(FunctionContext* ctx, const St
return serialize(ctx, &ret_bitmap);
}
+StringVal BitmapFunctions::sub_bitmap(FunctionContext* ctx, const StringVal& src,
+ const BigIntVal& offset, const BigIntVal& cardinality_limit) {
+ if (src.is_null || offset.is_null || cardinality_limit.is_null || cardinality_limit.val <= 0) {
+ return StringVal::null();
+ }
+
+ BitmapValue ret_bitmap;
+ if (src.len == 0) {
+ ret_bitmap = *reinterpret_cast<BitmapValue*>(src.ptr);
+ } else {
+ BitmapValue bitmap = BitmapValue((char*)src.ptr);
+ if (bitmap.offset_limit(offset.val, cardinality_limit.val, &ret_bitmap) == 0) {
+ return StringVal::null();
+ }
+ }
+
+ return serialize(ctx, &ret_bitmap);
+}
+
StringVal BitmapFunctions::bitmap_subset_limit(FunctionContext* ctx, const StringVal& src,
const BigIntVal& range_start, const BigIntVal& cardinality_limit) {
if (src.is_null || range_start.is_null || cardinality_limit.is_null) {
diff --git a/be/src/exprs/bitmap_function.h b/be/src/exprs/bitmap_function.h
index 5b166a0..363b9b7 100644
--- a/be/src/exprs/bitmap_function.h
+++ b/be/src/exprs/bitmap_function.h
@@ -108,6 +108,8 @@ public:
const BigIntVal& range_start, const BigIntVal& range_end);
static StringVal bitmap_subset_limit(FunctionContext* ctx, const StringVal& src,
const BigIntVal& range_start, const BigIntVal& cardinality_limit);
+ static StringVal sub_bitmap(FunctionContext* ctx, const StringVal& src,
+ const BigIntVal& offset, const BigIntVal& cardinality_limit);
};
} // namespace doris
#endif //DORIS_BE_SRC_QUERY_EXPRS_BITMAP_FUNCTION_H
diff --git a/be/src/util/bitmap_value.h b/be/src/util/bitmap_value.h
index 9bf6f65..6d3710f 100644
--- a/be/src/util/bitmap_value.h
+++ b/be/src/util/bitmap_value.h
@@ -1489,6 +1489,32 @@ public:
break;
}
}
+ return count;
+ }
+
+ /**
+ * Returns the bitmap elements, starting from the offset position.
+ * The number of returned elements is limited by the cardinality_limit parameter.
+ * Analog of the substring string function, but for bitmap.
+ */
+ int64_t offset_limit(const int64_t& offset, const int64_t& limit, BitmapValue* ret_bitmap) {
+ if (std::abs(offset) >= _bitmap.cardinality()) {
+ return 0;
+ }
+ int64_t abs_offset = offset;
+ if (offset < 0) {
+ abs_offset = _bitmap.cardinality() + offset;
+ }
+
+ int64_t count = 0;
+ int64_t offset_count = 0;
+ auto it = _bitmap.begin();
+ for (;it != _bitmap.end() && offset_count < abs_offset; ++it) {
+ ++offset_count;
+ }
+ for (;it != _bitmap.end() && count < limit; ++it, ++count) {
+ ret_bitmap->add(*it);
+ }
return count;
}
diff --git a/be/test/exprs/bitmap_function_test.cpp b/be/test/exprs/bitmap_function_test.cpp
index c753383..8a2a950 100644
--- a/be/test/exprs/bitmap_function_test.cpp
+++ b/be/test/exprs/bitmap_function_test.cpp
@@ -751,7 +751,64 @@ TEST_F(BitmapFunctionsTest, bitmap_subset_in_range) {
res = BitmapFunctions::bitmap_subset_in_range(ctx, bitmap_src, BigIntVal::null(), BigIntVal::null());
ASSERT_TRUE(res.is_null);
+}
+
+TEST_F(BitmapFunctionsTest, sub_bitmap) {
+ // normal
+ BitmapValue bitmap1({0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,100,200,500});
+ StringVal bitmap_src = convert_bitmap_to_string(ctx, bitmap1);
+
+ StringVal res = BitmapFunctions::sub_bitmap(ctx, bitmap_src, BigIntVal(30), BigIntVal(6));
+ BitmapValue bitmap2({30,31,32,33,100,200});
+ ASSERT_EQ(res, convert_bitmap_to_string(ctx, bitmap2));
+
+ res = BitmapFunctions::sub_bitmap(ctx, bitmap_src, BigIntVal(30), BigIntVal(100));
+ BitmapValue bitmap3({30,31,32,33,100,200,500});
+ ASSERT_EQ(res, convert_bitmap_to_string(ctx, bitmap3));
+
+ res = BitmapFunctions::sub_bitmap(ctx, bitmap_src, BigIntVal(30), BigIntVal(INT64_MAX));
+ BitmapValue bitmap4({30,31,32,33,100,200,500});
+ ASSERT_EQ(res, convert_bitmap_to_string(ctx, bitmap4));
+
+ res = BitmapFunctions::sub_bitmap(ctx, bitmap_src, BigIntVal(0), BigIntVal(2));
+ BitmapValue bitmap5({0,1});
+ ASSERT_EQ(res, convert_bitmap_to_string(ctx, bitmap5));
+
+ res = BitmapFunctions::sub_bitmap(ctx, bitmap_src, BigIntVal(-1), BigIntVal(2));
+ BitmapValue bitmap6({500});
+ ASSERT_EQ(res, convert_bitmap_to_string(ctx, bitmap6));
+
+ res = BitmapFunctions::sub_bitmap(ctx, bitmap_src, BigIntVal(-7), BigIntVal(6));
+ BitmapValue bitmap7({30,31,32,33,100,200});
+ ASSERT_EQ(res, convert_bitmap_to_string(ctx, bitmap7));
+
+ // null
+ res = BitmapFunctions::sub_bitmap(ctx, bitmap_src, BigIntVal(0), BigIntVal(0));
+ ASSERT_TRUE(res.is_null);
+
+ res = BitmapFunctions::sub_bitmap(ctx, bitmap_src, BigIntVal(100), BigIntVal(6));
+ ASSERT_TRUE(res.is_null);
+ res = BitmapFunctions::sub_bitmap(ctx, bitmap_src, BigIntVal(-100), BigIntVal(6));
+ ASSERT_TRUE(res.is_null);
+
+ res = BitmapFunctions::sub_bitmap(ctx, bitmap_src, BigIntVal(30), BigIntVal(INT64_MIN));
+ ASSERT_TRUE(res.is_null);
+
+ res = BitmapFunctions::sub_bitmap(ctx, StringVal::null(), BigIntVal(1), BigIntVal(3));
+ ASSERT_TRUE(res.is_null);
+
+ res = BitmapFunctions::sub_bitmap(ctx, bitmap_src, BigIntVal::null(), BigIntVal(20));
+ ASSERT_TRUE(res.is_null);
+
+ res = BitmapFunctions::sub_bitmap(ctx, bitmap_src, BigIntVal(10), BigIntVal::null());
+ ASSERT_TRUE(res.is_null);
+
+ // empty
+ BitmapValue bitmap0;
+ StringVal empty_str = convert_bitmap_to_string(ctx, bitmap0);
+ res = BitmapFunctions::sub_bitmap(ctx, empty_str, BigIntVal(0), BigIntVal(3));
+ ASSERT_TRUE(res.is_null);
}
TEST_F(BitmapFunctionsTest, bitmap_subset_limit) {
diff --git a/docs/.vuepress/sidebar/en.js b/docs/.vuepress/sidebar/en.js
index e585509..89b2d8c 100644
--- a/docs/.vuepress/sidebar/en.js
+++ b/docs/.vuepress/sidebar/en.js
@@ -426,6 +426,7 @@ module.exports = [
"bitmap_and_not_count",
"bitmap_subset_in_range",
"bitmap_subset_limit",
+ "sub_bitmap",
"bitmap_to_string",
"bitmap_union",
"bitmap_xor",
diff --git a/docs/.vuepress/sidebar/zh-CN.js b/docs/.vuepress/sidebar/zh-CN.js
index b393c2f..d0c53a1 100644
--- a/docs/.vuepress/sidebar/zh-CN.js
+++ b/docs/.vuepress/sidebar/zh-CN.js
@@ -430,6 +430,7 @@ module.exports = [
"bitmap_and_not_count",
"bitmap_subset_in_range",
"bitmap_subset_limit",
+ "sub_bitmap",
"bitmap_to_string",
"bitmap_union",
"bitmap_xor",
diff --git a/docs/en/sql-reference/sql-functions/bitmap-functions/sub_bitmap.md b/docs/en/sql-reference/sql-functions/bitmap-functions/sub_bitmap.md
new file mode 100644
index 0000000..eb4aeae
--- /dev/null
+++ b/docs/en/sql-reference/sql-functions/bitmap-functions/sub_bitmap.md
@@ -0,0 +1,61 @@
+---
+{
+ "title": "sub_bitmap",
+ "language": "en"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# sub_bitmap
+
+## description
+### Syntax
+
+`BITMAP SUB_BITMAP(BITMAP src, BIGINT offset, BIGINT cardinality_limit)`
+
+Starting from the position specified by offset, intercept cardinality_limit bitmap elements and return a bitmap subset.
+
+## example
+
+```
+mysql> select bitmap_to_string(sub_bitmap(bitmap_from_string('1,0,1,2,3,1,5'), 0, 3)) value;
++-------+
+| value |
++-------+
+| 0,1,2 |
++-------+
+
+mysql> select bitmap_to_string(sub_bitmap(bitmap_from_string('1,0,1,2,3,1,5'), -3, 2)) value;
++-------+
+| value |
++-------+
+| 2,3 |
++-------+
+
+mysql> select bitmap_to_string(sub_bitmap(bitmap_from_string('1,0,1,2,3,1,5'), 2, 100)) value;
++-------+
+| value |
++-------+
+| 2,3,5 |
++-------+
+```
+
+## keyword
+
+ SUB_BITMAP,BITMAP_SUBSET,BITMAP
diff --git a/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/sub_bitmap.md b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/sub_bitmap.md
new file mode 100644
index 0000000..2118250
--- /dev/null
+++ b/docs/zh-CN/sql-reference/sql-functions/bitmap-functions/sub_bitmap.md
@@ -0,0 +1,62 @@
+---
+{
+ "title": "sub_bitmap",
+ "language": "zh-CN"
+}
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# sub_bitmap
+
+## Description
+
+### Syntax
+
+`BITMAP SUB_BITMAP(BITMAP src, BIGINT offset, BIGINT cardinality_limit)`
+
+从 offset 指定位置开始，截取 cardinality_limit 个 bitmap 元素，返回一个 bitmap 子集。
+
+## example
+
+```
+mysql> select bitmap_to_string(sub_bitmap(bitmap_from_string('1,0,1,2,3,1,5'), 0, 3)) value;
++-------+
+| value |
++-------+
+| 0,1,2 |
++-------+
+
+mysql> select bitmap_to_string(sub_bitmap(bitmap_from_string('1,0,1,2,3,1,5'), -3, 2)) value;
++-------+
+| value |
++-------+
+| 2,3 |
++-------+
+
+mysql> select bitmap_to_string(sub_bitmap(bitmap_from_string('1,0,1,2,3,1,5'), 2, 100)) value;
++-------+
+| value |
++-------+
+| 2,3,5 |
++-------+
+```
+
+## keyword
+
+ SUB_BITMAP,BITMAP_SUBSET,BITMAP
\ No newline at end of file
diff --git a/gensrc/script/doris_builtins_functions.py b/gensrc/script/doris_builtins_functions.py
index 6c990b5..6c9fbbe 100755
--- a/gensrc/script/doris_builtins_functions.py
+++ b/gensrc/script/doris_builtins_functions.py
@@ -1229,6 +1229,10 @@ visible_functions = [
[['bitmap_or_count'], 'BIGINT', ['BITMAP','BITMAP'],
'_ZN5doris15BitmapFunctions15bitmap_or_countEPN9doris_udf15FunctionContextERKNS1_9StringValES6_',
'', '', '', ''],
+ [['sub_bitmap'], 'BITMAP', ['BITMAP', 'BIGINT', 'BIGINT'],
+ '_ZN5doris15BitmapFunctions10sub_bitmapEPN9doris_udf15FunctionContextERKNS1_9StringValERKNS1_9BigIntValES9_',
+ '', '', 'vec', ''],
+
# hash functions
[['murmur_hash3_32'], 'INT', ['VARCHAR', '...'],
'_ZN5doris13HashFunctions15murmur_hash3_32EPN9doris_udf15FunctionContextEiPKNS1_9StringValE',
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org