You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2026/07/11 06:12:55 UTC

(impala) 02/02: IMPALA-15157: Avoid decompressing a page that is fully skipped at a page boundary

This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 234fb8592bfa7bc8cae415a769789b39f5aba5ca
Author: zhangyifan27 <ch...@163.com>
AuthorDate: Thu Jul 9 17:25:45 2026 +0800

    IMPALA-15157: Avoid decompressing a page that is fully skipped at a page boundary
    
    During late materialization / row-level skipping, when the rows to skip
    end exactly on a Parquet page boundary (e.g. an entire row group is
    filtered out by row-level predicates), BaseScalarColumnReader::
    SkipRowsInternal() would fall out of the page-skipping loop and then
    decompress that final page via ReadCurrentDataPage() only to then skip
    all of its values.
    
    This changes the loop guard from '>' to '>=' so a page whose values are
    all skipped is discarded as a whole compressed page (counted in
    NumPagesSkippedByLateMaterialization) without decompressing it. When the
    skip lands exactly on the page boundary we short-circuit and return
    before reading the next page header. Query results are unchanged; this
    only removes the unnecessary decompression and in-page value skipping.
    
    Testing:
    - Updated and extended parquet-late-materialization-unique-db.test to
      cover skipping to a page boundary (whole-page discard) and in-page
      value skipping.
    
    Assisted-by: Claude Opus 4.8
    Change-Id: I18f0e69c267c9bf041ff9f6a3d677239ae43971a
    Reviewed-on: http://gerrit.cloudera.org:8080/24566
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 be/src/exec/parquet/parquet-column-readers.cc      | 20 ++++++++++-----
 be/src/exec/parquet/parquet-column-readers.h       |  2 +-
 .../parquet-late-materialization-unique-db.test    | 29 ++++++++++++++++++----
 3 files changed, 39 insertions(+), 12 deletions(-)

diff --git a/be/src/exec/parquet/parquet-column-readers.cc b/be/src/exec/parquet/parquet-column-readers.cc
index a8969c1c2..37019cb5e 100644
--- a/be/src/exec/parquet/parquet-column-readers.cc
+++ b/be/src/exec/parquet/parquet-column-readers.cc
@@ -1765,19 +1765,27 @@ bool BaseScalarColumnReader::SkipRowsInternal(int64_t num_rows, int64_t skip_row
         return false;
       }
       DCHECK_GT(num_buffered_values_, 0);
-      // Keep advancing to next page header if rows to be skipped are more than number
-      // of values in the page. Note we will just be reading headers and skipping
-      // pages without decompressing them as we advance.
+      // Keep advancing to next page header while all rows of the current page are to be
+      // skipped. Note we will just be reading headers and skipping pages without
+      // decompressing them as we advance. This includes the case where the rows to skip
+      // end exactly on a page boundary (e.g. an entire row group is filtered by
+      // row-level predicates): we discard the compressed page without decompressing and
+      // short-circuit before reading a page header that may not exist.
       // In this case, the current column is not in any collection. Therefore the number
       // of rows in the page is equal to the number of values.
       DCHECK_EQ(max_rep_level(), 0);
-      while (num_rows > num_buffered_values_) {
+      while (num_rows >= num_buffered_values_) {
         COUNTER_ADD(parent_->num_pages_skipped_by_late_materialization_counter_, 1);
         num_rows -= num_buffered_values_;
         current_row_ += num_buffered_values_;
-        if (!col_chunk_reader_.SkipPageData().ok() || !AdvanceNextPageHeader()) {
-          return false;
+        if (!col_chunk_reader_.SkipPageData().ok()) return false;
+        if (num_rows == 0) {
+          // Skipped exactly to the end of this page (and possibly the row group). No
+          // need to read the next page header, which may not exist.
+          num_buffered_values_ = 0;
+          return true;
         }
+        if (!AdvanceNextPageHeader()) return false;
         DCHECK_GT(num_buffered_values_, 0);
       }
       // Read the data page (includes decompressing them if required).
diff --git a/be/src/exec/parquet/parquet-column-readers.h b/be/src/exec/parquet/parquet-column-readers.h
index 73eb45a0d..149897761 100644
--- a/be/src/exec/parquet/parquet-column-readers.h
+++ b/be/src/exec/parquet/parquet-column-readers.h
@@ -508,7 +508,7 @@ class BaseScalarColumnReader : public ParquetColumnReader {
   /// page.
   Status ReadNextDataPageHeader();
 
-  /// Reads only the data page coontent and not the header.
+  /// Reads only the data page content and not the header.
   /// This should strictly be called only after 'ReadNextDataPageHeader' or
   /// 'AdvanceNextPageHeader' has been invoked.
   Status ReadCurrentDataPage();
diff --git a/testdata/workloads/functional-query/queries/QueryTest/parquet-late-materialization-unique-db.test b/testdata/workloads/functional-query/queries/QueryTest/parquet-late-materialization-unique-db.test
index 0f459277e..193a0a4f2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/parquet-late-materialization-unique-db.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/parquet-late-materialization-unique-db.test
@@ -16,18 +16,37 @@ INT, INT
 aggregation(SUM, NumTopLevelValuesSkipped): 3
 ====
 ---- QUERY
-# Test NumTopLevelValuesSkipped counter when there is more than one page in the
-# column.
+# Test that when a column has more than one page, rows that are skipped up to a
+# page boundary are discarded as whole pages (counted in
+# NumPagesSkippedByLateMaterialization) without decompressing them, so no in-page
+# top-level values need to be skipped (NumTopLevelValuesSkipped stays 0).
 # Table `decimals_1_10` contains 60 rows, in which 8 rows are selected by the
-# query and 44 + 8 == 52 rows are skipped.
+# query and 52 rows are skipped.
 set PARQUET_LATE_MATERIALIZATION_THRESHOLD=1;
 set parquet_read_page_index = false;
 select count(d_10) from decimals_1_10 where d_1 = 1;
 ---- RESULTS
 8
 ---- RUNTIME_PROFILE
-aggregation(SUM, NumPagesSkippedByLateMaterialization): 44
-aggregation(SUM, NumTopLevelValuesSkipped): 8
+aggregation(SUM, NumPagesSkippedByLateMaterialization): 52
+aggregation(SUM, NumTopLevelValuesSkipped): 0
+====
+---- QUERY
+# Test the NumTopLevelValuesSkipped counter when the skipped rows end in the middle
+# of a page, so the page must be decompressed and values skipped within it (rather
+# than discarding whole pages). Column `d_1` has 5 values per page, so filtering on
+# `d_10` (1 value per page) and reading `d_1` forces in-page value skipping.
+# `d_10 = 3` selects the third value of each matching page, skipping the values
+# before and after it inside the page.
+# `d_10 = 3` matches 11 rows, 2 of which have a NULL `d_1`, so count(d_1) returns 9.
+set PARQUET_LATE_MATERIALIZATION_THRESHOLD=1;
+set parquet_read_page_index = false;
+select count(d_1) from decimals_1_10 where d_10 = 3;
+---- RESULTS
+9
+---- RUNTIME_PROFILE
+aggregation(SUM, NumPagesSkippedByLateMaterialization): 1
+aggregation(SUM, NumTopLevelValuesSkipped): 44
 ====
 ---- QUERY
 # Test if PARQUET_LATE_MATERIALIZATION_THRESHOLD is always 1 if there is any