You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2020/03/05 01:15:19 UTC
[incubator-doris] branch master updated: [Bug] Fix int128 bloom
filter write bug (#2995)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 63051a3 [Bug] Fix int128 bloom filter write bug (#2995)
63051a3 is described below
commit 63051a3b37927776770a607226e00fb1c19d5c24
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Thu Mar 5 09:15:11 2020 +0800
[Bug] Fix int128 bloom filter write bug (#2995)
std::set.insert(int128) core dump because segment fault.
the reason is the __int128 is not aligned.
---
.../rowset/segment_v2/bloom_filter_index_writer.cpp | 19 +++++++++++++++++++
be/src/util/types.h | 7 +++++++
2 files changed, 26 insertions(+)
diff --git a/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp b/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp
index 9c3f8f8..b58f19f 100644
--- a/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/bloom_filter_index_writer.cpp
@@ -46,6 +46,17 @@ struct BloomFilterTraits<Slice> {
using ValueDict = std::set<Slice, Slice::Comparator>;
};
+struct Int128Comparator {
+ bool operator()(const PackedInt128& a, const PackedInt128& b) const {
+ return a.value < b.value;
+ }
+};
+
+template<>
+struct BloomFilterTraits<int128_t> {
+ using ValueDict = std::set<PackedInt128, Int128Comparator>;
+};
+
// Builder for bloom filter. In doris, bloom filter index is used in
// high cardinality key columns and none-agg value columns for high selectivity and storage
// efficiency.
@@ -73,6 +84,10 @@ public:
CppType new_value;
_typeinfo->deep_copy(&new_value, v, &_pool);
_values.insert(new_value);
+ } else if (_is_int128()) {
+ PackedInt128 new_value;
+ new_value.value = *v;
+ _values.insert((*reinterpret_cast<CppType*>(&new_value)));
} else {
_values.insert(*v);
}
@@ -141,6 +156,10 @@ private:
return field_type == OLAP_FIELD_TYPE_VARCHAR || field_type == OLAP_FIELD_TYPE_CHAR;
}
+ bool _is_int128() const {
+ return field_type == OLAP_FIELD_TYPE_LARGEINT;
+ }
+
private:
BloomFilterOptions _bf_options;
const TypeInfo* _typeinfo;
diff --git a/be/src/util/types.h b/be/src/util/types.h
index 76da748..0844164 100644
--- a/be/src/util/types.h
+++ b/be/src/util/types.h
@@ -22,6 +22,13 @@ namespace doris {
// Because __int128 in memory is not aligned, but GCC7 will generate SSE instruction
// for __int128 load/store. This will cause segment fault.
struct PackedInt128 {
+ PackedInt128() : value(0) {
+ }
+
+ PackedInt128(const __int128& value_) {
+ value = value_;
+ }
+
PackedInt128& operator=(const __int128& value_) {
value = value_;
return *this;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org