You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bo...@apache.org on 2026/04/20 13:43:50 UTC

(impala) 02/02: IMPALA-14825: Add syntactic sugar for Iceberg Row Lineage data

This is an automated email from the ASF dual-hosted git repository.

asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git

commit 6128f6b18ecaad8ec7c6f99048d4d28cbf01e6bd
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Thu Apr 9 17:12:04 2026 +0200

    IMPALA-14825: Add syntactic sugar for Iceberg Row Lineage data
    
    Currently Row Lineage fields can only be queried by complex expressions:
    
    * row-id:
      COALESCE(_file_row_id,
               ICEBERG__FIRST__ROW__ID + FILE__POSITION)
    * last-updated-sequence-number
      COALESCE(_file_last_updated_sequence_number,
               ICEBERG__DATA__SEQUENCE__NUMBER)
    
    This patch adds _row_id and _last_updated_sequence_number virtual
    columns to Iceberg V3 tables as syntactic sugars, i.e. they get
    re-written during analysis to the corresponding COALESCE() expr.
    
    _row_id and _last_updated_sequence_number are being used by other
    engines as well, e.g. Spark.
    
    Testing:
    * e2e tests added
    * column-masking tests
    
    Generated-by: Claude Sonnet 4.6
    Generated-by: Claude Opus 4.6
    
    Change-Id: I02aceeb0969243ba6d28e4c0c97d5dd7a5afd765
    Reviewed-on: http://gerrit.cloudera.org:8080/24184
    Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
    Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
 common/thrift/CatalogObjects.thrift                |   4 +-
 .../java/org/apache/impala/analysis/Analyzer.java  |   3 +
 .../org/apache/impala/analysis/SelectList.java     |  18 +-
 .../org/apache/impala/catalog/IcebergTable.java    |   4 +
 .../org/apache/impala/catalog/VirtualColumn.java   |  13 ++
 .../rewrite/IcebergVirtualColumnRewriteRule.java   |  64 ++++++
 .../java/org/apache/impala/util/IcebergUtil.java   |  62 ++++--
 .../queries/QueryTest/iceberg-v3-optimize.test     |  48 +++--
 .../queries/QueryTest/iceberg-v3-row-lineage.test  | 236 +++++++++++++++++++++
 .../ranger_iceberg_v3_column_masking.test          |  27 +++
 tests/authorization/test_ranger.py                 |  46 ++++
 11 files changed, 492 insertions(+), 33 deletions(-)

diff --git a/common/thrift/CatalogObjects.thrift b/common/thrift/CatalogObjects.thrift
index 56a6d645c..b8e51f562 100644
--- a/common/thrift/CatalogObjects.thrift
+++ b/common/thrift/CatalogObjects.thrift
@@ -96,7 +96,9 @@ enum TVirtualColumnType {
   ICEBERG_DATA_SEQUENCE_NUMBER,
   PARTITION_VALUE_SERIALIZED,
   BUCKET_ID,
-  ICEBERG_FIRST_ROW_ID
+  ICEBERG_FIRST_ROW_ID,
+  ICEBERG_ROW_ID,
+  ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER
 }
 
 // TODO: Since compression is also enabled for Kudu columns, we should
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index a8357e972..6f92ce978 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -104,6 +104,7 @@ import org.apache.impala.rewrite.ExprRewriter;
 import org.apache.impala.rewrite.ExtractCommonConjunctRule;
 import org.apache.impala.rewrite.ExtractCompoundVerticalBarExprRule;
 import org.apache.impala.rewrite.FoldConstantsRule;
+import org.apache.impala.rewrite.IcebergVirtualColumnRewriteRule;
 import org.apache.impala.rewrite.NormalizeBinaryPredicatesRule;
 import org.apache.impala.rewrite.NormalizeCountStarRule;
 import org.apache.impala.rewrite.NormalizeExprsRule;
@@ -679,6 +680,8 @@ public class Analyzer {
       // expr rewrites can be disabled via a query option. When rewrites are enabled
       // BetweenPredicates should be rewritten first to help trigger other rules.
       rules.add(BetweenToCompoundRule.INSTANCE);
+      // Iceberg V3 syntactic-sugar virtual columns must be rewritten before planning.
+      rules.add(IcebergVirtualColumnRewriteRule.INSTANCE);
       // Binary predicates must be rewritten to a canonical form for both Kudu predicate
       // pushdown and Parquet row group pruning based on min/max statistics.
       rules.add(NormalizeBinaryPredicatesRule.INSTANCE);
diff --git a/fe/src/main/java/org/apache/impala/analysis/SelectList.java b/fe/src/main/java/org/apache/impala/analysis/SelectList.java
index cb52ce0c8..b3b54cd91 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SelectList.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SelectList.java
@@ -95,10 +95,24 @@ public class SelectList {
   public void rewriteExprs(ExprRewriter rewriter, Analyzer analyzer)
       throws AnalysisException {
     List<Subquery> subqueryExprs = new ArrayList<>();
-    for (SelectListItem item: items_) {
+    for (int i = 0; i < items_.size(); ++i) {
+      SelectListItem item = items_.get(i);
       if (item.isStar()) continue;
       if (item.isSkipExprRewrite()) continue;
-      item.setExpr(rewriter.rewrite(item.getExpr(), analyzer));
+      Expr origExpr = item.getExpr();
+      Expr newExpr = rewriter.rewrite(origExpr, analyzer);
+      if (newExpr != origExpr && item.getAlias() == null
+          && origExpr instanceof SlotRef) {
+        // The expression was rewritten (e.g., an Iceberg virtual-column SlotRef
+        // replaced by its definition). Preserve the original column label as an
+        // explicit alias so that re-analysis does not change the output column
+        // name, which would break outer-query references (e.g., in CTEs).
+        String colLabel = item.toColumnLabel(i, false);
+        items_.set(i, new SelectListItem(newExpr, colLabel));
+        item = items_.get(i);
+      } else {
+        item.setExpr(newExpr);
+      }
       item.getExpr().collect(Subquery.class, subqueryExprs);
     }
     for (Subquery s : subqueryExprs) {
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
index 29357baf1..9d5b565cd 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
@@ -746,6 +746,10 @@ public class IcebergTable extends Table implements FeIcebergTable {
     addVirtualColumn(VirtualColumn.ICEBERG_PARTITION_SERIALIZED);
     addVirtualColumn(VirtualColumn.ICEBERG_DATA_SEQUENCE_NUMBER);
     addVirtualColumn(VirtualColumn.ICEBERG_FIRST_ROW_ID);
+    if (getFormatVersion() >= 3) {
+      addVirtualColumn(VirtualColumn.ICEBERG_ROW_ID);
+      addVirtualColumn(VirtualColumn.ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER);
+    }
   }
 
   private void addHiddenColumns() {
diff --git a/fe/src/main/java/org/apache/impala/catalog/VirtualColumn.java b/fe/src/main/java/org/apache/impala/catalog/VirtualColumn.java
index 37f20bc44..641976e30 100644
--- a/fe/src/main/java/org/apache/impala/catalog/VirtualColumn.java
+++ b/fe/src/main/java/org/apache/impala/catalog/VirtualColumn.java
@@ -54,6 +54,16 @@ public class VirtualColumn extends Column {
       Type.BIGINT,
       TVirtualColumnType.ICEBERG_FIRST_ROW_ID);
 
+  // Iceberg V3 syntactic-sugar virtual columns (rewritten during analysis).
+  public static VirtualColumn ICEBERG_ROW_ID = new VirtualColumn(
+      "_row_id",
+      Type.BIGINT,
+      TVirtualColumnType.ICEBERG_ROW_ID);
+  public static VirtualColumn ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER = new VirtualColumn(
+      "_last_updated_sequence_number",
+      Type.BIGINT,
+      TVirtualColumnType.ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER);
+
   // Paimon-related virtual columns.
   public static VirtualColumn PARTITION_VALUE_SERIALIZED = new
           VirtualColumn("PARTITION__VALUE__SERIALIZED", Type.BINARY,
@@ -72,6 +82,9 @@ public class VirtualColumn extends Column {
       case PARTITION_VALUE_SERIALIZED: return PARTITION_VALUE_SERIALIZED;
       case BUCKET_ID: return BUCKET_ID;
       case ICEBERG_FIRST_ROW_ID: return ICEBERG_FIRST_ROW_ID;
+      case ICEBERG_ROW_ID: return ICEBERG_ROW_ID;
+      case ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER:
+          return ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER;
       default: break;
     }
     return null;
diff --git a/fe/src/main/java/org/apache/impala/rewrite/IcebergVirtualColumnRewriteRule.java b/fe/src/main/java/org/apache/impala/rewrite/IcebergVirtualColumnRewriteRule.java
new file mode 100644
index 000000000..1e7b9b0ca
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/rewrite/IcebergVirtualColumnRewriteRule.java
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.rewrite;
+
+import org.apache.impala.analysis.Analyzer;
+import org.apache.impala.analysis.Expr;
+import org.apache.impala.analysis.SlotDescriptor;
+import org.apache.impala.analysis.SlotRef;
+import org.apache.impala.common.AnalysisException;
+import org.apache.impala.thrift.TVirtualColumnType;
+import org.apache.impala.util.IcebergUtil;
+
+/**
+ * Rewrites references to the syntactic-sugar virtual columns of Iceberg V3 tables
+ * into their equivalent COALESCE expressions:
+ *   _row_id ->
+ *       COALESCE(_file_row_id, ICEBERG__FIRST__ROW__ID + FILE__POSITION)
+ *   _last_updated_sequence_number ->
+ *       COALESCE(_file_last_updated_sequence_number,
+ *           ICEBERG__DATA__SEQUENCE__NUMBER)
+ *
+ * This rule runs as a mandatory rewrite so that these virtual columns are never
+ * visible to the planner or backend.
+ */
+public class IcebergVirtualColumnRewriteRule implements ExprRewriteRule {
+  public static final ExprRewriteRule INSTANCE = new IcebergVirtualColumnRewriteRule();
+
+  @Override
+  public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException {
+    if (!(expr instanceof SlotRef)) return expr;
+    SlotRef slotRef = (SlotRef) expr;
+    if (!slotRef.isAnalyzed()) return expr;
+    SlotDescriptor desc = slotRef.getDesc();
+    if (desc == null) return expr;
+
+    String tableAlias = desc.getParent().getAlias();
+    if (tableAlias == null) return expr;
+
+    TVirtualColumnType colType = desc.getVirtualColumnType();
+    switch (colType) {
+      case ICEBERG_ROW_ID:
+        return IcebergUtil.buildRowIdExpr(tableAlias);
+      case ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER:
+        return IcebergUtil.buildLastUpdatedSeqNoExpr(tableAlias);
+      default:
+        return expr;
+    }
+  }
+}
diff --git a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
index 16d56b08e..ba9ba4c2b 100644
--- a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
@@ -83,6 +83,7 @@ import org.apache.impala.analysis.IcebergPartitionField;
 import org.apache.impala.analysis.IcebergPartitionSpec;
 import org.apache.impala.analysis.IcebergPartitionTransform;
 import org.apache.impala.analysis.NumericLiteral;
+import org.apache.impala.analysis.Path;
 import org.apache.impala.analysis.SlotRef;
 import org.apache.impala.analysis.StatementBase;
 import org.apache.impala.analysis.StringLiteral;
@@ -1592,32 +1593,66 @@ public class IcebergUtil {
       List<Expr> resultExprs) throws AnalysisException {
     FeIcebergTable table = (FeIcebergTable) tableRef.getTable();
     if (table.getFormatVersion() < 3) return;
-    resultExprs.add(IcebergUtil.getRowIdExpr(analyzer, tableRef));
-    resultExprs.add(IcebergUtil.getLastUpdatedSeqNoExpr(analyzer, tableRef));
+    resultExprs.add(getAnalyzedRowIdExpr(analyzer, tableRef));
+    resultExprs.add(getAnalyzedLastUpdatedSeqNoExpr(analyzer, tableRef));
     Preconditions.checkState(resultExprs.size() == table.getColumns().size());
   }
 
+  /**
+   * Builds an unanalyzed expression equivalent to:
+   *   COALESCE(_file_row_id, ICEBERG__FIRST__ROW__ID + FILE__POSITION)
+   */
+  public static Expr buildRowIdExpr(String tableAlias) {
+    SlotRef fileRowId = new SlotRef(
+        Path.createRawPath(tableAlias, "_file_row_id"));
+    SlotRef firstRowId = new SlotRef(
+        Path.createRawPath(tableAlias, "ICEBERG__FIRST__ROW__ID"));
+    SlotRef filePosition = new SlotRef(
+        Path.createRawPath(tableAlias, "FILE__POSITION"));
+    Expr addExpr = new ArithmeticExpr(
+        ArithmeticExpr.Operator.ADD, firstRowId, filePosition);
+    FunctionCallExpr fnCall = new FunctionCallExpr(
+        "coalesce", Arrays.asList(fileRowId, addExpr));
+    fnCall.setIsInternalFnCall(true);
+    return fnCall;
+  }
+
+  /**
+   * Builds an unanalyzed expression equivalent to:
+   *   COALESCE(_file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER)
+   */
+  public static Expr buildLastUpdatedSeqNoExpr(String tableAlias) {
+    SlotRef fileLastUpdatedSeqNo = new SlotRef(
+        Path.createRawPath(tableAlias, "_file_last_updated_sequence_number"));
+    SlotRef dataSeqNo = new SlotRef(
+        Path.createRawPath(tableAlias, "ICEBERG__DATA__SEQUENCE__NUMBER"));
+    FunctionCallExpr fnCall = new FunctionCallExpr(
+        "coalesce", Arrays.asList(fileLastUpdatedSeqNo, dataSeqNo));
+    fnCall.setIsInternalFnCall(true);
+    return fnCall;
+  }
+
   /**
    * Returns COALESCE(_file_row_id, ICEBERG__FIRST__ROW__ID + FILE__POSITION).
    */
-  public static Expr getRowIdExpr(Analyzer analyzer, TableRef tableRef)
+  public static Expr getAnalyzedRowIdExpr(Analyzer analyzer, TableRef tableRef)
       throws AnalysisException {
-    Expr addExpr = new ArithmeticExpr(ArithmeticExpr.Operator.ADD,
-        createSlotRef(analyzer, tableRef, "ICEBERG__FIRST__ROW__ID"),
-        createSlotRef(analyzer, tableRef, "FILE__POSITION"));
-    return createCoalesceFn(analyzer,
-        createSlotRef(analyzer, tableRef, "_file_row_id"), addExpr);
+    Expr expr = buildRowIdExpr(tableRef.getUniqueAlias());
+    expr.analyze(analyzer);
+    analyzer.materializeSlots(expr);
+    return expr;
   }
 
   /**
    * Returns
    * COALESCE(_file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER).
    */
-  public static Expr getLastUpdatedSeqNoExpr(Analyzer analyzer, TableRef tableRef)
+  public static Expr getAnalyzedLastUpdatedSeqNoExpr(Analyzer analyzer, TableRef tableRef)
       throws AnalysisException {
-    return createCoalesceFn(analyzer,
-        createSlotRef(analyzer, tableRef, "_file_last_updated_sequence_number"),
-        createSlotRef(analyzer, tableRef, "ICEBERG__DATA__SEQUENCE__NUMBER"));
+    Expr expr = buildLastUpdatedSeqNoExpr(tableRef.getUniqueAlias());
+    expr.analyze(analyzer);
+    analyzer.materializeSlots(expr);
+    return expr;
   }
 
   /**
@@ -1639,8 +1674,7 @@ public class IcebergUtil {
 
   public static SlotRef createSlotRef(Analyzer analyzer, TableRef tableRef,
       String colName) throws AnalysisException {
-    List<String> path = org.apache.impala.analysis.Path
-        .createRawPath(tableRef.getUniqueAlias(), colName);
+    List<String> path = Path.createRawPath(tableRef.getUniqueAlias(), colName);
     SlotRef ref = new SlotRef(path);
     ref.analyze(analyzer);
     return ref;
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-optimize.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-optimize.test
index ae620692d..e29a392a9 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-optimize.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-optimize.test
@@ -89,34 +89,38 @@ INSERT INTO optimize_part VALUES (1,'aa'),(2,'ab');
 INSERT INTO optimize_part VALUES (3,'ac'),(4,'ad');
 INSERT INTO optimize_part VALUES (5,'ba'),(6,'bb');
 
-SELECT _file_row_id, ICEBERG__FIRST__ROW__ID, _file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER, *
+SELECT _file_row_id, ICEBERG__FIRST__ROW__ID, _row_id, '|',
+       _file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER, _last_updated_sequence_number, '|',
+       *
 FROM optimize_part;
 ---- RESULTS
-NULL,0,NULL,1,1,'aa'
-NULL,0,NULL,1,2,'ab'
-NULL,2,NULL,2,3,'ac'
-NULL,2,NULL,2,4,'ad'
-NULL,4,NULL,3,5,'ba'
-NULL,4,NULL,3,6,'bb'
+NULL,0,0,'|',NULL,1,1,'|',1,'aa'
+NULL,0,1,'|',NULL,1,1,'|',2,'ab'
+NULL,2,2,'|',NULL,2,2,'|',3,'ac'
+NULL,2,3,'|',NULL,2,2,'|',4,'ad'
+NULL,4,4,'|',NULL,3,3,'|',5,'ba'
+NULL,4,5,'|',NULL,3,3,'|',6,'bb'
 ---- TYPES
-BIGINT,BIGINT,BIGINT,BIGINT,INT,STRING
+BIGINT,BIGINT,BIGINT,STRING,BIGINT,BIGINT,BIGINT,STRING,INT,STRING
 ====
 ---- QUERY
 # With FILE_SIZE_THRESHOLD_MB specified, OPTIMIZE only compact partitions with at least
 # two data files.
 OPTIMIZE TABLE optimize_part (FILE_SIZE_THRESHOLD_MB=10);
 
-SELECT _file_row_id, _file_last_updated_sequence_number, *
+SELECT _file_row_id, _row_id, '|',
+       _file_last_updated_sequence_number, _last_updated_sequence_number, '|',
+       *
 FROM optimize_part;
 ---- RESULTS
-0,1,1,'aa'
-1,1,2,'ab'
-2,2,3,'ac'
-3,2,4,'ad'
-NULL,NULL,5,'ba'
-NULL,NULL,6,'bb'
+0,0,'|',1,1,'|',1,'aa'
+1,1,'|',1,1,'|',2,'ab'
+2,2,'|',2,2,'|',3,'ac'
+3,3,'|',2,2,'|',4,'ad'
+NULL,4,'|',NULL,3,'|',5,'ba'
+NULL,5,'|',NULL,3,'|',6,'bb'
 ---- TYPES
-BIGINT,BIGINT,INT,STRING
+BIGINT,BIGINT,STRING,BIGINT,BIGINT,STRING,INT,STRING
 ====
 ---- QUERY
 # Let's do a full compaction on the partially compacted table.
@@ -205,4 +209,16 @@ FROM optimize_upgraded;
 2,8,3,6,3
 ---- TYPES
 BIGINT,BIGINT,BIGINT,BIGINT,INT
+====
+---- QUERY
+SELECT _file_row_id, ICEBERG__FIRST__ROW__ID, _row_id, '|',
+       _file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER, _last_updated_sequence_number, '|',
+       *
+FROM optimize_upgraded;
+---- RESULTS
+0,6,0,'|',1,6,1,'|',1
+1,7,1,'|',2,6,2,'|',2
+2,8,2,'|',3,6,3,'|',3
+---- TYPES
+BIGINT,BIGINT,BIGINT,STRING,BIGINT,BIGINT,BIGINT,STRING,INT
 ====
\ No newline at end of file
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-row-lineage.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-row-lineage.test
index 8c92f88fc..ad557cf65 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-row-lineage.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-row-lineage.test
@@ -77,6 +77,18 @@ from iceberg_v3_row_lineage for system_version as of 2872597867664652808;
 INT,BIGINT,BIGINT
 ====
 ---- QUERY
+# Use syntactic sugars '_row_id' and '_last_updated_sequence_number' in time-travel query.
+# They should provide the same results as the expressions in the previous query.
+select i, _row_id, _last_updated_sequence_number
+from iceberg_v3_row_lineage for system_version as of 2872597867664652808;
+---- RESULTS
+1,0,1
+2,1,2
+3,2,3
+---- TYPES
+INT,BIGINT,BIGINT
+====
+---- QUERY
 # Calculate 'row_id' and 'last_updated_sequence_number'
 select i, coalesce(_file_row_id, ICEBERG__FIRST__ROW__ID + FILE__POSITION),
           coalesce(_file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER)
@@ -89,6 +101,16 @@ from iceberg_v3_row_lineage;
 INT,BIGINT,BIGINT
 ====
 ---- QUERY
+select i, _row_id, _last_updated_sequence_number
+from iceberg_v3_row_lineage;
+---- RESULTS
+1,0,1
+2,1,2
+3,2,3
+---- TYPES
+INT,BIGINT,BIGINT
+====
+---- QUERY
 INSERT INTO iceberg_v3_row_lineage (i) VALUES (4);
 ====
 ---- QUERY
@@ -422,6 +444,17 @@ from iceberg_v3_row_lineage_orc for system_version as of 7033898671372067760;
 INT,BIGINT,BIGINT
 ====
 ---- QUERY
+# Use syntactic sugars '_row_id' and '_last_updated_sequence_number' in time-travel query.
+select i, _row_id, _last_updated_sequence_number
+from iceberg_v3_row_lineage_orc for system_version as of 7033898671372067760;
+---- RESULTS
+1,0,1
+2,1,2
+3,2,3
+---- TYPES
+INT,BIGINT,BIGINT
+====
+---- QUERY
 # Calculate 'row_id' and 'last_updated_sequence_number'
 select i, coalesce(_file_row_id, ICEBERG__FIRST__ROW__ID + FILE__POSITION),
           coalesce(_file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER)
@@ -434,6 +467,16 @@ from iceberg_v3_row_lineage_orc;
 INT,BIGINT,BIGINT
 ====
 ---- QUERY
+select i, _row_id, _last_updated_sequence_number
+from iceberg_v3_row_lineage_orc;
+---- RESULTS
+1,0,1
+2,1,2
+3,2,3
+---- TYPES
+INT,BIGINT,BIGINT
+====
+---- QUERY
 compute stats iceberg_v3_row_lineage_orc;
 show table stats iceberg_v3_row_lineage_orc;
 ---- RESULTS
@@ -500,3 +543,196 @@ alter table iceberg_v3_row_lineage_orc drop column _file_last_updated_sequence_n
 ---- CATCH
 cannot be dropped.
 ====
+---- QUERY
+# Query syntactic sugar columns of row lineage fields.
+select i, _row_id, _last_updated_sequence_number from iceberg_v3_row_lineage;
+---- RESULTS
+1,0,1
+2,1,2
+3,2,3
+---- TYPES
+INT,BIGINT,BIGINT
+---- LABELS
+i,_row_id,_last_updated_sequence_number
+====
+---- QUERY
+# Syntactic sugar column results match the explicit COALESCE expressions.
+select i,
+    _row_id = coalesce(_file_row_id, ICEBERG__FIRST__ROW__ID + FILE__POSITION),
+    _last_updated_sequence_number
+        = coalesce(_file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER)
+from iceberg_v3_row_lineage;
+---- RESULTS
+1,true,true
+2,true,true
+3,true,true
+---- TYPES
+INT,BOOLEAN,BOOLEAN
+====
+---- QUERY
+# WHERE clause: filter using _row_id.
+select i from iceberg_v3_row_lineage where _row_id = 1;
+---- RESULTS
+2
+---- TYPES
+INT
+====
+---- QUERY
+# WHERE clause: filter using _last_updated_sequence_number.
+select i from iceberg_v3_row_lineage where _last_updated_sequence_number > 1;
+---- RESULTS
+2
+3
+---- TYPES
+INT
+====
+---- QUERY
+# ORDER BY using _row_id (descending).
+select i, _row_id from iceberg_v3_row_lineage order by _row_id desc;
+---- RESULTS
+3,2
+2,1
+1,0
+---- TYPES
+INT,BIGINT
+---- LABELS
+i,_row_id
+====
+---- QUERY
+# ORDER BY using _last_updated_sequence_number.
+select i, _last_updated_sequence_number
+from iceberg_v3_row_lineage order by _last_updated_sequence_number desc;
+---- RESULTS
+3,3
+2,2
+1,1
+---- TYPES
+INT,BIGINT
+---- LABELS
+i,_last_updated_sequence_number
+====
+---- QUERY
+# GROUP BY using _row_id.
+select _row_id, count(*) from iceberg_v3_row_lineage group by _row_id order by _row_id;
+---- RESULTS
+0,1
+1,1
+2,1
+---- TYPES
+BIGINT,BIGINT
+====
+---- QUERY
+# GROUP BY using _last_updated_sequence_number.
+select _last_updated_sequence_number, count(*)
+from iceberg_v3_row_lineage
+group by _last_updated_sequence_number
+order by _last_updated_sequence_number;
+---- RESULTS
+1,1
+2,1
+3,1
+---- TYPES
+BIGINT,BIGINT
+====
+---- QUERY
+# HAVING clause using _row_id, grouped by i.
+select i, max(_row_id) row_id from iceberg_v3_row_lineage
+group by i
+having max(_row_id) >= 1
+order by i;
+---- RESULTS
+2,1
+3,2
+---- TYPES
+INT,BIGINT
+====
+---- QUERY
+# Subquery: _row_id used in inner SELECT, referenced by outer WHERE.
+select i from (
+    select i, _row_id from iceberg_v3_row_lineage
+) sub
+where _row_id < 2
+order by i;
+---- RESULTS
+1
+2
+---- TYPES
+INT
+====
+---- QUERY
+# CTE using _row_id and _last_updated_sequence_number.
+with ids as (
+    select i, _row_id, _last_updated_sequence_number
+    from iceberg_v3_row_lineage
+)
+select i, _row_id, _last_updated_sequence_number
+from ids
+order by i;
+---- RESULTS
+1,0,1
+2,1,2
+3,2,3
+---- TYPES
+INT,BIGINT,BIGINT
+====
+---- QUERY
+create table ice_v3_a (x int) stored as iceberg
+tblproperties ('format-version'='3');
+create table ice_v3_b (y int) stored as iceberg
+tblproperties ('format-version'='3');
+insert into ice_v3_a values (10);
+insert into ice_v3_a values (20);
+insert into ice_v3_b values (20);
+insert into ice_v3_b values (10);
+====
+---- QUERY
+# Qualified _row_id and _last_updated_sequence_number across two V3 tables in one query.
+select a.x, a._row_id, a._last_updated_sequence_number,
+       b.y, b._row_id, b._last_updated_sequence_number
+from ice_v3_a a, ice_v3_b b
+where a.x = b.y;
+---- RESULTS
+10,0,1,10,1,2
+20,1,2,20,0,1
+---- TYPES
+INT,BIGINT,BIGINT,INT,BIGINT,BIGINT
+====
+---- QUERY
+# Join: _row_id from both tables in WHERE predicate.
+select a.x, b.y
+from ice_v3_a a cross join ice_v3_b b
+where a._row_id = b._row_id;
+---- RESULTS
+10,20
+20,10
+---- TYPES
+INT,INT
+====
+---- QUERY
+# _row_id on V2 table should fail, column does not exist on non-V3 tables.
+create table ice_v2_no_sugar (i int) stored as iceberg
+tblproperties ('format-version'='2');
+insert into ice_v2_no_sugar values (1);
+select _row_id from ice_v2_no_sugar;
+---- CATCH
+Could not resolve column/field reference: '_row_id'
+====
+---- QUERY
+# _last_updated_sequence_number on V2 table should fail.
+select _last_updated_sequence_number from ice_v2_no_sugar;
+---- CATCH
+Could not resolve column/field reference: '_last_updated_sequence_number'
+====
+---- QUERY
+# Unqualified _row_id is ambiguous when multiple V3 tables are in scope.
+select _row_id from ice_v3_a a, ice_v3_b b;
+---- CATCH
+Column/field reference is ambiguous: '_row_id'
+====
+---- QUERY
+# Unqualified _last_updated_sequence_number is ambiguous when multiple V3 tables
+# are in scope.
+select _last_updated_sequence_number from ice_v3_a a cross join ice_v3_b b;
+---- CATCH
+Column/field reference is ambiguous: '_last_updated_sequence_number'
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/ranger_iceberg_v3_column_masking.test b/testdata/workloads/functional-query/queries/QueryTest/ranger_iceberg_v3_column_masking.test
new file mode 100644
index 000000000..1f43bc2e2
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/ranger_iceberg_v3_column_masking.test
@@ -0,0 +1,27 @@
+====
+---- QUERY
+# _row_id, _file_row_id, _last_updated_sequence_number, _file_last_updated_sequence_number
+# have column masking rules that multiplies their values by -1.
+# _row_id returns _file_row_id if the latter is present, but it doesn't mean we multiply
+# the final result by -1 twice. I.e., the final results are always the original results
+# plus the column masking rule applied on top of it.
+select _row_id, _file_row_id, _last_updated_sequence_number,
+    _file_last_updated_sequence_number, i
+from $UNIQUE_DB.ice_v3 where i < 10 order by _row_id;
+---- RESULTS
+-6001202,-6001202,-3,-3,6
+-6001201,-6001201,-3,-3,5
+-6001200,-6001200,-3,-3,4
+regex:-\d+,NULL,-2,NULL,7
+regex:-\d+,NULL,-2,NULL,7
+regex:-\d+,NULL,-2,NULL,7
+regex:-\d+,NULL,-2,NULL,7
+regex:-\d+,NULL,-2,NULL,7
+regex:-\d+,NULL,-2,NULL,7
+regex:-\d+,NULL,-2,NULL,7
+-2,-2,-1,-1,3
+-1,-1,-1,-1,2
+0,0,-1,-1,1
+---- TYPES
+BIGINT,BIGINT,BIGINT,BIGINT,INT
+====
diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py
index c09d4433d..3a9e84470 100644
--- a/tests/authorization/test_ranger.py
+++ b/tests/authorization/test_ranger.py
@@ -3554,6 +3554,52 @@ class TestRangerLocalCatalog(TestRanger):
     finally:
       admin_client.execute("drop database if exists {0} cascade".format(unique_database))
 
+  @pytest.mark.execute_serially
+  def test_iceberg_v3_column_masking(self, vector, unique_name):
+    """Test column masking on Iceberg V3 virtual columns (_row_id, _file_row_id,
+    _last_updated_sequence_number, _file_last_updated_sequence_number)."""
+    user = getuser()
+    admin_client = self.create_impala_client(user=ADMIN)
+    unique_database = unique_name + "_db"
+    short_table_name = "ice_v3"
+    tbl_name = unique_database + "." + short_table_name
+
+    policy_cnt = 0
+    try:
+      admin_client.execute("drop database if exists {0} cascade"
+                           .format(unique_database))
+      admin_client.execute("create database {0}".format(unique_database))
+      admin_client.execute(
+          "create table {0} (i int) stored by iceberg "
+          "tblproperties ('format-version'='3')".format(tbl_name))
+      admin_client.execute("insert into {0} values (1), (2), (3)".format(tbl_name))
+      admin_client.execute(
+          "insert into {0} select cast(l_orderkey as INT) "
+          "from tpch.lineitem where l_orderkey > 6".format(tbl_name))
+      admin_client.execute("insert into {0} values (4), (5), (6)".format(tbl_name))
+      admin_client.execute(
+          "optimize table {0} (file_size_threshold_mb=1)".format(tbl_name))
+      admin_client.execute("grant select on database {0} to user {1} "
+                           .format(unique_database, user))
+
+      # Add column masking policies that produce the negative values of virtual columns.
+      for col in ["_row_id", "_file_row_id", "_last_updated_sequence_number",
+                  "_file_last_updated_sequence_number"]:
+        TestRanger._add_column_masking_policy(
+            unique_name + str(policy_cnt), user, unique_database, short_table_name,
+            col, "CUSTOM", "-{col}")
+        policy_cnt += 1
+      self.execute_query_expect_success(admin_client, "refresh authorization")
+      self.run_test_case("QueryTest/ranger_iceberg_v3_column_masking", vector,
+                         test_file_vars={'$UNIQUE_DB': unique_database})
+    finally:
+      admin_client.execute("revoke select on database {0} from user {1}"
+                            .format(unique_database, user))
+      admin_client.execute("drop database if exists {0} cascade"
+                           .format(unique_database))
+      for i in range(policy_cnt):
+        TestRanger._remove_policy(unique_name + str(i))
+
   @pytest.mark.execute_serially
   def test_convert_table_to_iceberg(self, unique_name):
     """Test that autorization is taken into account when performing a table migration to