You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2026/07/11 06:12:55 UTC
(impala) 02/02: IMPALA-15157: Avoid decompressing a page that is fully skipped at a page boundary
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 234fb8592bfa7bc8cae415a769789b39f5aba5ca
Author: zhangyifan27 <ch...@163.com>
AuthorDate: Thu Jul 9 17:25:45 2026 +0800
IMPALA-15157: Avoid decompressing a page that is fully skipped at a page boundary
During late materialization / row-level skipping, when the rows to skip
end exactly on a Parquet page boundary (e.g. an entire row group is
filtered out by row-level predicates), BaseScalarColumnReader::
SkipRowsInternal() would fall out of the page-skipping loop and then
decompress that final page via ReadCurrentDataPage() only to then skip
all of its values.
This changes the loop guard from '>' to '>=' so a page whose values are
all skipped is discarded as a whole compressed page (counted in
NumPagesSkippedByLateMaterialization) without decompressing it. When the
skip lands exactly on the page boundary we short-circuit and return
before reading the next page header. Query results are unchanged; this
only removes the unnecessary decompression and in-page value skipping.
Testing:
- Updated and extended parquet-late-materialization-unique-db.test to
cover skipping to a page boundary (whole-page discard) and in-page
value skipping.
Assisted-by: Claude Opus 4.8
Change-Id: I18f0e69c267c9bf041ff9f6a3d677239ae43971a
Reviewed-on: http://gerrit.cloudera.org:8080/24566
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/exec/parquet/parquet-column-readers.cc | 20 ++++++++++-----
be/src/exec/parquet/parquet-column-readers.h | 2 +-
.../parquet-late-materialization-unique-db.test | 29 ++++++++++++++++++----
3 files changed, 39 insertions(+), 12 deletions(-)
diff --git a/be/src/exec/parquet/parquet-column-readers.cc b/be/src/exec/parquet/parquet-column-readers.cc
index a8969c1c2..37019cb5e 100644
--- a/be/src/exec/parquet/parquet-column-readers.cc
+++ b/be/src/exec/parquet/parquet-column-readers.cc
@@ -1765,19 +1765,27 @@ bool BaseScalarColumnReader::SkipRowsInternal(int64_t num_rows, int64_t skip_row
return false;
}
DCHECK_GT(num_buffered_values_, 0);
- // Keep advancing to next page header if rows to be skipped are more than number
- // of values in the page. Note we will just be reading headers and skipping
- // pages without decompressing them as we advance.
+ // Keep advancing to next page header while all rows of the current page are to be
+ // skipped. Note we will just be reading headers and skipping pages without
+ // decompressing them as we advance. This includes the case where the rows to skip
+ // end exactly on a page boundary (e.g. an entire row group is filtered by
+ // row-level predicates): we discard the compressed page without decompressing and
+ // short-circuit before reading a page header that may not exist.
// In this case, the current column is not in any collection. Therefore the number
// of rows in the page is equal to the number of values.
DCHECK_EQ(max_rep_level(), 0);
- while (num_rows > num_buffered_values_) {
+ while (num_rows >= num_buffered_values_) {
COUNTER_ADD(parent_->num_pages_skipped_by_late_materialization_counter_, 1);
num_rows -= num_buffered_values_;
current_row_ += num_buffered_values_;
- if (!col_chunk_reader_.SkipPageData().ok() || !AdvanceNextPageHeader()) {
- return false;
+ if (!col_chunk_reader_.SkipPageData().ok()) return false;
+ if (num_rows == 0) {
+ // Skipped exactly to the end of this page (and possibly the row group). No
+ // need to read the next page header, which may not exist.
+ num_buffered_values_ = 0;
+ return true;
}
+ if (!AdvanceNextPageHeader()) return false;
DCHECK_GT(num_buffered_values_, 0);
}
// Read the data page (includes decompressing them if required).
diff --git a/be/src/exec/parquet/parquet-column-readers.h b/be/src/exec/parquet/parquet-column-readers.h
index 73eb45a0d..149897761 100644
--- a/be/src/exec/parquet/parquet-column-readers.h
+++ b/be/src/exec/parquet/parquet-column-readers.h
@@ -508,7 +508,7 @@ class BaseScalarColumnReader : public ParquetColumnReader {
/// page.
Status ReadNextDataPageHeader();
- /// Reads only the data page coontent and not the header.
+ /// Reads only the data page content and not the header.
/// This should strictly be called only after 'ReadNextDataPageHeader' or
/// 'AdvanceNextPageHeader' has been invoked.
Status ReadCurrentDataPage();
diff --git a/testdata/workloads/functional-query/queries/QueryTest/parquet-late-materialization-unique-db.test b/testdata/workloads/functional-query/queries/QueryTest/parquet-late-materialization-unique-db.test
index 0f459277e..193a0a4f2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/parquet-late-materialization-unique-db.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/parquet-late-materialization-unique-db.test
@@ -16,18 +16,37 @@ INT, INT
aggregation(SUM, NumTopLevelValuesSkipped): 3
====
---- QUERY
-# Test NumTopLevelValuesSkipped counter when there is more than one page in the
-# column.
+# Test that when a column has more than one page, rows that are skipped up to a
+# page boundary are discarded as whole pages (counted in
+# NumPagesSkippedByLateMaterialization) without decompressing them, so no in-page
+# top-level values need to be skipped (NumTopLevelValuesSkipped stays 0).
# Table `decimals_1_10` contains 60 rows, in which 8 rows are selected by the
-# query and 44 + 8 == 52 rows are skipped.
+# query and 52 rows are skipped.
set PARQUET_LATE_MATERIALIZATION_THRESHOLD=1;
set parquet_read_page_index = false;
select count(d_10) from decimals_1_10 where d_1 = 1;
---- RESULTS
8
---- RUNTIME_PROFILE
-aggregation(SUM, NumPagesSkippedByLateMaterialization): 44
-aggregation(SUM, NumTopLevelValuesSkipped): 8
+aggregation(SUM, NumPagesSkippedByLateMaterialization): 52
+aggregation(SUM, NumTopLevelValuesSkipped): 0
+====
+---- QUERY
+# Test the NumTopLevelValuesSkipped counter when the skipped rows end in the middle
+# of a page, so the page must be decompressed and values skipped within it (rather
+# than discarding whole pages). Column `d_1` has 5 values per page, so filtering on
+# `d_10` (1 value per page) and reading `d_1` forces in-page value skipping.
+# `d_10 = 3` selects the third value of each matching page, skipping the values
+# before and after it inside the page.
+# `d_10 = 3` matches 11 rows, 2 of which have a NULL `d_1`, so count(d_1) returns 9.
+set PARQUET_LATE_MATERIALIZATION_THRESHOLD=1;
+set parquet_read_page_index = false;
+select count(d_1) from decimals_1_10 where d_10 = 3;
+---- RESULTS
+9
+---- RUNTIME_PROFILE
+aggregation(SUM, NumPagesSkippedByLateMaterialization): 1
+aggregation(SUM, NumTopLevelValuesSkipped): 44
====
---- QUERY
# Test if PARQUET_LATE_MATERIALIZATION_THRESHOLD is always 1 if there is any