You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by li...@apache.org on 2020/04/10 04:53:25 UTC
[incubator-doris] branch branch-0.11 updated: Fix bug when use
ZoneMap/BloomFiter on column with REPLACE/REPLACE_IF_NOT_NULL (#3288)
This is an automated email from the ASF dual-hosted git repository.
lichaoyong pushed a commit to branch branch-0.11
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/branch-0.11 by this push:
new fcc2d40 Fix bug when use ZoneMap/BloomFiter on column with REPLACE/REPLACE_IF_NOT_NULL (#3288)
fcc2d40 is described below
commit fcc2d402474539b249e9bf97b60d9e1ed42a44e3
Author: lichaoyong <li...@baidu.com>
AuthorDate: Fri Apr 10 10:22:21 2020 +0800
Fix bug when use ZoneMap/BloomFiter on column with REPLACE/REPLACE_IF_NOT_NULL (#3288)
Now, column with REPLACE/REPLACE_IF_NOT_NULL can be filtered by ZoneMap/BloomFilter
when the rowset is base(version starts with zero). Always we think is an optimization.
But when some case, it will occurs bug.
create table test(
k1 int,
v1 int replace,
v2 int sum
);
If I have two records on different two versions
1 2 2 on version [0-10]
1 3 1 on version 11
If I perform a query
select * from test where k1 = 1 and v1 = 3;
The result will be 1 3 1, this is not right because of the first record is filtered.
The right answer is 1 3 3, the v2 should be summed.
Remove this optimization is necessity to make the result is right.
---
be/src/olap/rowset/beta_rowset_writer.cpp | 2 +-
be/src/olap/rowset/segment_reader.cpp | 8 ++------
.../main/java/org/apache/doris/common/util/PropertyAnalyzer.java | 3 +--
.../test/java/org/apache/doris/common/PropertyAnalyzerTest.java | 4 ++--
4 files changed, 6 insertions(+), 11 deletions(-)
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp
index ccc30be..4281a94 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -184,4 +184,4 @@ OLAPStatus BetaRowsetWriter::_flush_segment_writer() {
return OLAP_SUCCESS;
}
-} // namespace doris
\ No newline at end of file
+} // namespace doris
diff --git a/be/src/olap/rowset/segment_reader.cpp b/be/src/olap/rowset/segment_reader.cpp
index 2ccc4a2..b6dddbf 100644
--- a/be/src/olap/rowset/segment_reader.cpp
+++ b/be/src/olap/rowset/segment_reader.cpp
@@ -491,9 +491,7 @@ OLAPStatus SegmentReader::_pick_row_groups(uint32_t first_block, uint32_t last_b
for (auto& i : _conditions->columns()) {
FieldAggregationMethod aggregation = _get_aggregation_by_index(i.first);
- bool is_continue = (aggregation == OLAP_FIELD_AGGREGATION_NONE
- || (aggregation == OLAP_FIELD_AGGREGATION_REPLACE
- && _segment_group->version().first == 0));
+ bool is_continue = (aggregation == OLAP_FIELD_AGGREGATION_NONE);
if (!is_continue) {
continue;
}
@@ -532,9 +530,7 @@ OLAPStatus SegmentReader::_pick_row_groups(uint32_t first_block, uint32_t last_b
for (uint32_t i : _load_bf_columns) {
FieldAggregationMethod aggregation = _get_aggregation_by_index(i);
- bool is_continue = (aggregation == OLAP_FIELD_AGGREGATION_NONE
- || (aggregation == OLAP_FIELD_AGGREGATION_REPLACE
- && _segment_group->version().first == 0));
+ bool is_continue = (aggregation == OLAP_FIELD_AGGREGATION_NONE);
if (!is_continue) {
continue;
}
diff --git a/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
index 440d03e..9c164d0 100644
--- a/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
+++ b/fe/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java
@@ -275,8 +275,7 @@ public class PropertyAnalyzer {
throw new AnalysisException(type + " is not supported in bloom filter index. "
+ "invalid column: " + bfColumn);
} else if (column.isKey()
- || column.getAggregationType() == AggregateType.NONE
- || column.getAggregationType() == AggregateType.REPLACE) {
+ || column.getAggregationType() == AggregateType.NONE) {
if (!bfColumnSet.add(bfColumn)) {
throw new AnalysisException("Reduplicated bloom filter column: " + bfColumn);
}
diff --git a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
index 45430e4..3aaaf20 100644
--- a/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
+++ b/fe/src/test/java/org/apache/doris/common/PropertyAnalyzerTest.java
@@ -49,10 +49,10 @@ public class PropertyAnalyzerTest {
columns.get(1).setIsKey(true);
Map<String, String> properties = Maps.newHashMap();
- properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k1,v1");
+ properties.put(PropertyAnalyzer.PROPERTIES_BF_COLUMNS, "k1");
Set<String> bfColumns = PropertyAnalyzer.analyzeBloomFilterColumns(properties, columns);
- Assert.assertEquals(Sets.newHashSet("k1", "v1"), bfColumns);
+ Assert.assertEquals(Sets.newHashSet("k1"), bfColumns);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org