You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2026/10/21 15:59:22 UTC
(impala) branch master updated: Revert "IMPALA-14510: Disable ineffective runtime filters more aggressively"
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new aca598ae25 Revert "IMPALA-14510: Disable ineffective runtime filters more aggressively"
aca598ae25 is described below
commit aca598ae253aaa9e098a36eca6cb84f051eef433
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Wed Oct 21 15:18:52 2026 +0000
Revert "IMPALA-14510: Disable ineffective runtime filters more aggressively"
This reverts commit 867c237b5dc62cbfa84be0bdcadec517c1b7bfd8.
Change-Id: I1fa7eb87bcea37a55ce8cf0290163c5128f243f8
Reviewed-on: http://gerrit.cloudera.org:8080/25172
Reviewed-by: Arnab Karmakar <ar...@gmail.com>
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Joe McDonnell <jo...@cloudera.com>
---
be/src/exec/hdfs-scanner.h | 11 -----------
be/src/exec/orc/hdfs-orc-scanner.cc | 12 ------------
be/src/exec/parquet/hdfs-parquet-scanner.cc | 12 ------------
3 files changed, 35 deletions(-)
diff --git a/be/src/exec/hdfs-scanner.h b/be/src/exec/hdfs-scanner.h
index ca53a002e4..c71e940779 100644
--- a/be/src/exec/hdfs-scanner.h
+++ b/be/src/exec/hdfs-scanner.h
@@ -56,17 +56,6 @@ constexpr int BATCHES_PER_FILTER_SELECTIVITY_CHECK = 16;
static_assert(BitUtil::IsPowerOf2(BATCHES_PER_FILTER_SELECTIVITY_CHECK),
"BATCHES_PER_FILTER_SELECTIVITY_CHECK must be a power of two");
-// The number of unfiltered rows between checks to see if a filter is effective and should
-// be disabled. Must be a power of two. Unlike BATCHES_PER_FILTER_SELECTIVITY_CHECK,
-// this is doing the check periodically based on the count of unfiltered rows that
-// the scanner processes. This is useful when runtime filters are being enforced at the
-// row level in the scanner and an ineffective filter is paired with a highly effective
-// filter. The highly effective filter means that very few rows are returned, so
-// the check in HdfsScanner::GetNext() doesn't trigger in a timely manner.
-const int UNFILTERED_ROWS_PER_FILTER_SELECTIVITY_CHECK = 1 << 15;
-static_assert(BitUtil::IsPowerOf2(UNFILTERED_ROWS_PER_FILTER_SELECTIVITY_CHECK),
- "UNFILTERED_ROWS_PER_FILTER_SELECTIVITY_CHECK must be a power of two");
-
/// Intermediate structure used for two pass parsing approach. In the first pass,
/// the FieldLocation structs are filled out and contain where all the fields start and
/// their lengths. In the second pass, the FieldLocation is used to write out the
diff --git a/be/src/exec/orc/hdfs-orc-scanner.cc b/be/src/exec/orc/hdfs-orc-scanner.cc
index 7bb97be591..9f3871407e 100644
--- a/be/src/exec/orc/hdfs-orc-scanner.cc
+++ b/be/src/exec/orc/hdfs-orc-scanner.cc
@@ -1006,19 +1006,7 @@ Status HdfsOrcScanner::AssembleRows(RowBatch* row_batch) {
end_of_stripe_ = true;
return Status::OK();
}
- int64_t rows_before = scan_node_->rows_read_counter()->value();
COUNTER_ADD(scan_node_->rows_read_counter(), orc_root_batch_->numElements);
- int64_t rows_after = rows_before + orc_root_batch_->numElements;
- // Check filter effectiveness every UNFILTERED_ROWS_PER_FILTER_SELECTIVITY_CHECK.
- // This helps disable ineffective filters when they are paired with a highly
- // selective filter. The highly selective filter means that very few rows are
- // returned and the check in HdfsScanner::GetNext() doesn't trigger.
- // NOTE: UNFILTERED_ROWS_PER_FILTER_SELECTIVITY_CHECK is a power of two, so this
- // will use a right shift.
- if (UNLIKELY((rows_after / UNFILTERED_ROWS_PER_FILTER_SELECTIVITY_CHECK) >
- (rows_before / UNFILTERED_ROWS_PER_FILTER_SELECTIVITY_CHECK))) {
- CheckFiltersEffectiveness();
- }
num_rows_read += orc_root_batch_->numElements;
}
diff --git a/be/src/exec/parquet/hdfs-parquet-scanner.cc b/be/src/exec/parquet/hdfs-parquet-scanner.cc
index ba5956e91f..87126da218 100644
--- a/be/src/exec/parquet/hdfs-parquet-scanner.cc
+++ b/be/src/exec/parquet/hdfs-parquet-scanner.cc
@@ -2498,19 +2498,7 @@ Status HdfsParquetScanner::AssembleRows(RowBatch* row_batch, bool* skip_row_grou
}
}
row_group_rows_read_ += num_rows_read;
- int64_t rows_before = scan_node_->rows_read_counter()->value();
COUNTER_ADD(scan_node_->rows_read_counter(), num_rows_read);
- int64_t rows_after = rows_before + num_rows_read;
- // Check filter effectiveness every UNFILTERED_ROWS_PER_FILTER_SELECTIVITY_CHECK.
- // This helps disable ineffective filters when they are paired with a highly
- // selective filter. The highly selective filter means that very few rows are
- // returned and the check in HdfsScanner::GetNext() doesn't trigger.
- // NOTE: UNFILTERED_ROWS_PER_FILTER_SELECTIVITY_CHECK is a power of two, so this
- // will use a right shift.
- if (UNLIKELY((rows_after / UNFILTERED_ROWS_PER_FILTER_SELECTIVITY_CHECK) >
- (rows_before / UNFILTERED_ROWS_PER_FILTER_SELECTIVITY_CHECK))) {
- CheckFiltersEffectiveness();
- }
// Merge Scanner-local counter into HdfsScanNode counter and reset.
COUNTER_ADD(scan_node_->collection_items_read_counter(), coll_items_read_counter_);
coll_items_read_counter_ = 0;