You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bo...@apache.org on 2026/04/20 13:43:50 UTC
(impala) 02/02: IMPALA-14825: Add syntactic sugar for Iceberg Row Lineage data
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 6128f6b18ecaad8ec7c6f99048d4d28cbf01e6bd
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Thu Apr 9 17:12:04 2026 +0200
IMPALA-14825: Add syntactic sugar for Iceberg Row Lineage data
Currently Row Lineage fields can only be queried by complex expressions:
* row-id:
COALESCE(_file_row_id,
ICEBERG__FIRST__ROW__ID + FILE__POSITION)
* last-updated-sequence-number
COALESCE(_file_last_updated_sequence_number,
ICEBERG__DATA__SEQUENCE__NUMBER)
This patch adds _row_id and _last_updated_sequence_number virtual
columns to Iceberg V3 tables as syntactic sugars, i.e. they get
re-written during analysis to the corresponding COALESCE() expr.
_row_id and _last_updated_sequence_number are being used by other
engines as well, e.g. Spark.
Testing:
* e2e tests added
* column-masking tests
Generated-by: Claude Sonnet 4.6
Generated-by: Claude Opus 4.6
Change-Id: I02aceeb0969243ba6d28e4c0c97d5dd7a5afd765
Reviewed-on: http://gerrit.cloudera.org:8080/24184
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
common/thrift/CatalogObjects.thrift | 4 +-
.../java/org/apache/impala/analysis/Analyzer.java | 3 +
.../org/apache/impala/analysis/SelectList.java | 18 +-
.../org/apache/impala/catalog/IcebergTable.java | 4 +
.../org/apache/impala/catalog/VirtualColumn.java | 13 ++
.../rewrite/IcebergVirtualColumnRewriteRule.java | 64 ++++++
.../java/org/apache/impala/util/IcebergUtil.java | 62 ++++--
.../queries/QueryTest/iceberg-v3-optimize.test | 48 +++--
.../queries/QueryTest/iceberg-v3-row-lineage.test | 236 +++++++++++++++++++++
.../ranger_iceberg_v3_column_masking.test | 27 +++
tests/authorization/test_ranger.py | 46 ++++
11 files changed, 492 insertions(+), 33 deletions(-)
diff --git a/common/thrift/CatalogObjects.thrift b/common/thrift/CatalogObjects.thrift
index 56a6d645c..b8e51f562 100644
--- a/common/thrift/CatalogObjects.thrift
+++ b/common/thrift/CatalogObjects.thrift
@@ -96,7 +96,9 @@ enum TVirtualColumnType {
ICEBERG_DATA_SEQUENCE_NUMBER,
PARTITION_VALUE_SERIALIZED,
BUCKET_ID,
- ICEBERG_FIRST_ROW_ID
+ ICEBERG_FIRST_ROW_ID,
+ ICEBERG_ROW_ID,
+ ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER
}
// TODO: Since compression is also enabled for Kudu columns, we should
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index a8357e972..6f92ce978 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -104,6 +104,7 @@ import org.apache.impala.rewrite.ExprRewriter;
import org.apache.impala.rewrite.ExtractCommonConjunctRule;
import org.apache.impala.rewrite.ExtractCompoundVerticalBarExprRule;
import org.apache.impala.rewrite.FoldConstantsRule;
+import org.apache.impala.rewrite.IcebergVirtualColumnRewriteRule;
import org.apache.impala.rewrite.NormalizeBinaryPredicatesRule;
import org.apache.impala.rewrite.NormalizeCountStarRule;
import org.apache.impala.rewrite.NormalizeExprsRule;
@@ -679,6 +680,8 @@ public class Analyzer {
// expr rewrites can be disabled via a query option. When rewrites are enabled
// BetweenPredicates should be rewritten first to help trigger other rules.
rules.add(BetweenToCompoundRule.INSTANCE);
+ // Iceberg V3 syntactic-sugar virtual columns must be rewritten before planning.
+ rules.add(IcebergVirtualColumnRewriteRule.INSTANCE);
// Binary predicates must be rewritten to a canonical form for both Kudu predicate
// pushdown and Parquet row group pruning based on min/max statistics.
rules.add(NormalizeBinaryPredicatesRule.INSTANCE);
diff --git a/fe/src/main/java/org/apache/impala/analysis/SelectList.java b/fe/src/main/java/org/apache/impala/analysis/SelectList.java
index cb52ce0c8..b3b54cd91 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SelectList.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SelectList.java
@@ -95,10 +95,24 @@ public class SelectList {
public void rewriteExprs(ExprRewriter rewriter, Analyzer analyzer)
throws AnalysisException {
List<Subquery> subqueryExprs = new ArrayList<>();
- for (SelectListItem item: items_) {
+ for (int i = 0; i < items_.size(); ++i) {
+ SelectListItem item = items_.get(i);
if (item.isStar()) continue;
if (item.isSkipExprRewrite()) continue;
- item.setExpr(rewriter.rewrite(item.getExpr(), analyzer));
+ Expr origExpr = item.getExpr();
+ Expr newExpr = rewriter.rewrite(origExpr, analyzer);
+ if (newExpr != origExpr && item.getAlias() == null
+ && origExpr instanceof SlotRef) {
+ // The expression was rewritten (e.g., an Iceberg virtual-column SlotRef
+ // replaced by its definition). Preserve the original column label as an
+ // explicit alias so that re-analysis does not change the output column
+ // name, which would break outer-query references (e.g., in CTEs).
+ String colLabel = item.toColumnLabel(i, false);
+ items_.set(i, new SelectListItem(newExpr, colLabel));
+ item = items_.get(i);
+ } else {
+ item.setExpr(newExpr);
+ }
item.getExpr().collect(Subquery.class, subqueryExprs);
}
for (Subquery s : subqueryExprs) {
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
index 29357baf1..9d5b565cd 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
@@ -746,6 +746,10 @@ public class IcebergTable extends Table implements FeIcebergTable {
addVirtualColumn(VirtualColumn.ICEBERG_PARTITION_SERIALIZED);
addVirtualColumn(VirtualColumn.ICEBERG_DATA_SEQUENCE_NUMBER);
addVirtualColumn(VirtualColumn.ICEBERG_FIRST_ROW_ID);
+ if (getFormatVersion() >= 3) {
+ addVirtualColumn(VirtualColumn.ICEBERG_ROW_ID);
+ addVirtualColumn(VirtualColumn.ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER);
+ }
}
private void addHiddenColumns() {
diff --git a/fe/src/main/java/org/apache/impala/catalog/VirtualColumn.java b/fe/src/main/java/org/apache/impala/catalog/VirtualColumn.java
index 37f20bc44..641976e30 100644
--- a/fe/src/main/java/org/apache/impala/catalog/VirtualColumn.java
+++ b/fe/src/main/java/org/apache/impala/catalog/VirtualColumn.java
@@ -54,6 +54,16 @@ public class VirtualColumn extends Column {
Type.BIGINT,
TVirtualColumnType.ICEBERG_FIRST_ROW_ID);
+ // Iceberg V3 syntactic-sugar virtual columns (rewritten during analysis).
+ public static VirtualColumn ICEBERG_ROW_ID = new VirtualColumn(
+ "_row_id",
+ Type.BIGINT,
+ TVirtualColumnType.ICEBERG_ROW_ID);
+ public static VirtualColumn ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER = new VirtualColumn(
+ "_last_updated_sequence_number",
+ Type.BIGINT,
+ TVirtualColumnType.ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER);
+
// Paimon-related virtual columns.
public static VirtualColumn PARTITION_VALUE_SERIALIZED = new
VirtualColumn("PARTITION__VALUE__SERIALIZED", Type.BINARY,
@@ -72,6 +82,9 @@ public class VirtualColumn extends Column {
case PARTITION_VALUE_SERIALIZED: return PARTITION_VALUE_SERIALIZED;
case BUCKET_ID: return BUCKET_ID;
case ICEBERG_FIRST_ROW_ID: return ICEBERG_FIRST_ROW_ID;
+ case ICEBERG_ROW_ID: return ICEBERG_ROW_ID;
+ case ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER:
+ return ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER;
default: break;
}
return null;
diff --git a/fe/src/main/java/org/apache/impala/rewrite/IcebergVirtualColumnRewriteRule.java b/fe/src/main/java/org/apache/impala/rewrite/IcebergVirtualColumnRewriteRule.java
new file mode 100644
index 000000000..1e7b9b0ca
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/rewrite/IcebergVirtualColumnRewriteRule.java
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.rewrite;
+
+import org.apache.impala.analysis.Analyzer;
+import org.apache.impala.analysis.Expr;
+import org.apache.impala.analysis.SlotDescriptor;
+import org.apache.impala.analysis.SlotRef;
+import org.apache.impala.common.AnalysisException;
+import org.apache.impala.thrift.TVirtualColumnType;
+import org.apache.impala.util.IcebergUtil;
+
+/**
+ * Rewrites references to the syntactic-sugar virtual columns of Iceberg V3 tables
+ * into their equivalent COALESCE expressions:
+ * _row_id ->
+ * COALESCE(_file_row_id, ICEBERG__FIRST__ROW__ID + FILE__POSITION)
+ * _last_updated_sequence_number ->
+ * COALESCE(_file_last_updated_sequence_number,
+ * ICEBERG__DATA__SEQUENCE__NUMBER)
+ *
+ * This rule runs as a mandatory rewrite so that these virtual columns are never
+ * visible to the planner or backend.
+ */
+public class IcebergVirtualColumnRewriteRule implements ExprRewriteRule {
+ public static final ExprRewriteRule INSTANCE = new IcebergVirtualColumnRewriteRule();
+
+ @Override
+ public Expr apply(Expr expr, Analyzer analyzer) throws AnalysisException {
+ if (!(expr instanceof SlotRef)) return expr;
+ SlotRef slotRef = (SlotRef) expr;
+ if (!slotRef.isAnalyzed()) return expr;
+ SlotDescriptor desc = slotRef.getDesc();
+ if (desc == null) return expr;
+
+ String tableAlias = desc.getParent().getAlias();
+ if (tableAlias == null) return expr;
+
+ TVirtualColumnType colType = desc.getVirtualColumnType();
+ switch (colType) {
+ case ICEBERG_ROW_ID:
+ return IcebergUtil.buildRowIdExpr(tableAlias);
+ case ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER:
+ return IcebergUtil.buildLastUpdatedSeqNoExpr(tableAlias);
+ default:
+ return expr;
+ }
+ }
+}
diff --git a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
index 16d56b08e..ba9ba4c2b 100644
--- a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
@@ -83,6 +83,7 @@ import org.apache.impala.analysis.IcebergPartitionField;
import org.apache.impala.analysis.IcebergPartitionSpec;
import org.apache.impala.analysis.IcebergPartitionTransform;
import org.apache.impala.analysis.NumericLiteral;
+import org.apache.impala.analysis.Path;
import org.apache.impala.analysis.SlotRef;
import org.apache.impala.analysis.StatementBase;
import org.apache.impala.analysis.StringLiteral;
@@ -1592,32 +1593,66 @@ public class IcebergUtil {
List<Expr> resultExprs) throws AnalysisException {
FeIcebergTable table = (FeIcebergTable) tableRef.getTable();
if (table.getFormatVersion() < 3) return;
- resultExprs.add(IcebergUtil.getRowIdExpr(analyzer, tableRef));
- resultExprs.add(IcebergUtil.getLastUpdatedSeqNoExpr(analyzer, tableRef));
+ resultExprs.add(getAnalyzedRowIdExpr(analyzer, tableRef));
+ resultExprs.add(getAnalyzedLastUpdatedSeqNoExpr(analyzer, tableRef));
Preconditions.checkState(resultExprs.size() == table.getColumns().size());
}
+ /**
+ * Builds an unanalyzed expression equivalent to:
+ * COALESCE(_file_row_id, ICEBERG__FIRST__ROW__ID + FILE__POSITION)
+ */
+ public static Expr buildRowIdExpr(String tableAlias) {
+ SlotRef fileRowId = new SlotRef(
+ Path.createRawPath(tableAlias, "_file_row_id"));
+ SlotRef firstRowId = new SlotRef(
+ Path.createRawPath(tableAlias, "ICEBERG__FIRST__ROW__ID"));
+ SlotRef filePosition = new SlotRef(
+ Path.createRawPath(tableAlias, "FILE__POSITION"));
+ Expr addExpr = new ArithmeticExpr(
+ ArithmeticExpr.Operator.ADD, firstRowId, filePosition);
+ FunctionCallExpr fnCall = new FunctionCallExpr(
+ "coalesce", Arrays.asList(fileRowId, addExpr));
+ fnCall.setIsInternalFnCall(true);
+ return fnCall;
+ }
+
+ /**
+ * Builds an unanalyzed expression equivalent to:
+ * COALESCE(_file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER)
+ */
+ public static Expr buildLastUpdatedSeqNoExpr(String tableAlias) {
+ SlotRef fileLastUpdatedSeqNo = new SlotRef(
+ Path.createRawPath(tableAlias, "_file_last_updated_sequence_number"));
+ SlotRef dataSeqNo = new SlotRef(
+ Path.createRawPath(tableAlias, "ICEBERG__DATA__SEQUENCE__NUMBER"));
+ FunctionCallExpr fnCall = new FunctionCallExpr(
+ "coalesce", Arrays.asList(fileLastUpdatedSeqNo, dataSeqNo));
+ fnCall.setIsInternalFnCall(true);
+ return fnCall;
+ }
+
/**
* Returns COALESCE(_file_row_id, ICEBERG__FIRST__ROW__ID + FILE__POSITION).
*/
- public static Expr getRowIdExpr(Analyzer analyzer, TableRef tableRef)
+ public static Expr getAnalyzedRowIdExpr(Analyzer analyzer, TableRef tableRef)
throws AnalysisException {
- Expr addExpr = new ArithmeticExpr(ArithmeticExpr.Operator.ADD,
- createSlotRef(analyzer, tableRef, "ICEBERG__FIRST__ROW__ID"),
- createSlotRef(analyzer, tableRef, "FILE__POSITION"));
- return createCoalesceFn(analyzer,
- createSlotRef(analyzer, tableRef, "_file_row_id"), addExpr);
+ Expr expr = buildRowIdExpr(tableRef.getUniqueAlias());
+ expr.analyze(analyzer);
+ analyzer.materializeSlots(expr);
+ return expr;
}
/**
* Returns
* COALESCE(_file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER).
*/
- public static Expr getLastUpdatedSeqNoExpr(Analyzer analyzer, TableRef tableRef)
+ public static Expr getAnalyzedLastUpdatedSeqNoExpr(Analyzer analyzer, TableRef tableRef)
throws AnalysisException {
- return createCoalesceFn(analyzer,
- createSlotRef(analyzer, tableRef, "_file_last_updated_sequence_number"),
- createSlotRef(analyzer, tableRef, "ICEBERG__DATA__SEQUENCE__NUMBER"));
+ Expr expr = buildLastUpdatedSeqNoExpr(tableRef.getUniqueAlias());
+ expr.analyze(analyzer);
+ analyzer.materializeSlots(expr);
+ return expr;
}
/**
@@ -1639,8 +1674,7 @@ public class IcebergUtil {
public static SlotRef createSlotRef(Analyzer analyzer, TableRef tableRef,
String colName) throws AnalysisException {
- List<String> path = org.apache.impala.analysis.Path
- .createRawPath(tableRef.getUniqueAlias(), colName);
+ List<String> path = Path.createRawPath(tableRef.getUniqueAlias(), colName);
SlotRef ref = new SlotRef(path);
ref.analyze(analyzer);
return ref;
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-optimize.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-optimize.test
index ae620692d..e29a392a9 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-optimize.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-optimize.test
@@ -89,34 +89,38 @@ INSERT INTO optimize_part VALUES (1,'aa'),(2,'ab');
INSERT INTO optimize_part VALUES (3,'ac'),(4,'ad');
INSERT INTO optimize_part VALUES (5,'ba'),(6,'bb');
-SELECT _file_row_id, ICEBERG__FIRST__ROW__ID, _file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER, *
+SELECT _file_row_id, ICEBERG__FIRST__ROW__ID, _row_id, '|',
+ _file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER, _last_updated_sequence_number, '|',
+ *
FROM optimize_part;
---- RESULTS
-NULL,0,NULL,1,1,'aa'
-NULL,0,NULL,1,2,'ab'
-NULL,2,NULL,2,3,'ac'
-NULL,2,NULL,2,4,'ad'
-NULL,4,NULL,3,5,'ba'
-NULL,4,NULL,3,6,'bb'
+NULL,0,0,'|',NULL,1,1,'|',1,'aa'
+NULL,0,1,'|',NULL,1,1,'|',2,'ab'
+NULL,2,2,'|',NULL,2,2,'|',3,'ac'
+NULL,2,3,'|',NULL,2,2,'|',4,'ad'
+NULL,4,4,'|',NULL,3,3,'|',5,'ba'
+NULL,4,5,'|',NULL,3,3,'|',6,'bb'
---- TYPES
-BIGINT,BIGINT,BIGINT,BIGINT,INT,STRING
+BIGINT,BIGINT,BIGINT,STRING,BIGINT,BIGINT,BIGINT,STRING,INT,STRING
====
---- QUERY
# With FILE_SIZE_THRESHOLD_MB specified, OPTIMIZE only compact partitions with at least
# two data files.
OPTIMIZE TABLE optimize_part (FILE_SIZE_THRESHOLD_MB=10);
-SELECT _file_row_id, _file_last_updated_sequence_number, *
+SELECT _file_row_id, _row_id, '|',
+ _file_last_updated_sequence_number, _last_updated_sequence_number, '|',
+ *
FROM optimize_part;
---- RESULTS
-0,1,1,'aa'
-1,1,2,'ab'
-2,2,3,'ac'
-3,2,4,'ad'
-NULL,NULL,5,'ba'
-NULL,NULL,6,'bb'
+0,0,'|',1,1,'|',1,'aa'
+1,1,'|',1,1,'|',2,'ab'
+2,2,'|',2,2,'|',3,'ac'
+3,3,'|',2,2,'|',4,'ad'
+NULL,4,'|',NULL,3,'|',5,'ba'
+NULL,5,'|',NULL,3,'|',6,'bb'
---- TYPES
-BIGINT,BIGINT,INT,STRING
+BIGINT,BIGINT,STRING,BIGINT,BIGINT,STRING,INT,STRING
====
---- QUERY
# Let's do a full compaction on the partially compacted table.
@@ -205,4 +209,16 @@ FROM optimize_upgraded;
2,8,3,6,3
---- TYPES
BIGINT,BIGINT,BIGINT,BIGINT,INT
+====
+---- QUERY
+SELECT _file_row_id, ICEBERG__FIRST__ROW__ID, _row_id, '|',
+ _file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER, _last_updated_sequence_number, '|',
+ *
+FROM optimize_upgraded;
+---- RESULTS
+0,6,0,'|',1,6,1,'|',1
+1,7,1,'|',2,6,2,'|',2
+2,8,2,'|',3,6,3,'|',3
+---- TYPES
+BIGINT,BIGINT,BIGINT,STRING,BIGINT,BIGINT,BIGINT,STRING,INT
====
\ No newline at end of file
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-row-lineage.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-row-lineage.test
index 8c92f88fc..ad557cf65 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-row-lineage.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-row-lineage.test
@@ -77,6 +77,18 @@ from iceberg_v3_row_lineage for system_version as of 2872597867664652808;
INT,BIGINT,BIGINT
====
---- QUERY
+# Use syntactic sugars '_row_id' and '_last_updated_sequence_number' in time-travel query.
+# They should provide the same results as the expressions in the previous query.
+select i, _row_id, _last_updated_sequence_number
+from iceberg_v3_row_lineage for system_version as of 2872597867664652808;
+---- RESULTS
+1,0,1
+2,1,2
+3,2,3
+---- TYPES
+INT,BIGINT,BIGINT
+====
+---- QUERY
# Calculate 'row_id' and 'last_updated_sequence_number'
select i, coalesce(_file_row_id, ICEBERG__FIRST__ROW__ID + FILE__POSITION),
coalesce(_file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER)
@@ -89,6 +101,16 @@ from iceberg_v3_row_lineage;
INT,BIGINT,BIGINT
====
---- QUERY
+select i, _row_id, _last_updated_sequence_number
+from iceberg_v3_row_lineage;
+---- RESULTS
+1,0,1
+2,1,2
+3,2,3
+---- TYPES
+INT,BIGINT,BIGINT
+====
+---- QUERY
INSERT INTO iceberg_v3_row_lineage (i) VALUES (4);
====
---- QUERY
@@ -422,6 +444,17 @@ from iceberg_v3_row_lineage_orc for system_version as of 7033898671372067760;
INT,BIGINT,BIGINT
====
---- QUERY
+# Use syntactic sugars '_row_id' and '_last_updated_sequence_number' in time-travel query.
+select i, _row_id, _last_updated_sequence_number
+from iceberg_v3_row_lineage_orc for system_version as of 7033898671372067760;
+---- RESULTS
+1,0,1
+2,1,2
+3,2,3
+---- TYPES
+INT,BIGINT,BIGINT
+====
+---- QUERY
# Calculate 'row_id' and 'last_updated_sequence_number'
select i, coalesce(_file_row_id, ICEBERG__FIRST__ROW__ID + FILE__POSITION),
coalesce(_file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER)
@@ -434,6 +467,16 @@ from iceberg_v3_row_lineage_orc;
INT,BIGINT,BIGINT
====
---- QUERY
+select i, _row_id, _last_updated_sequence_number
+from iceberg_v3_row_lineage_orc;
+---- RESULTS
+1,0,1
+2,1,2
+3,2,3
+---- TYPES
+INT,BIGINT,BIGINT
+====
+---- QUERY
compute stats iceberg_v3_row_lineage_orc;
show table stats iceberg_v3_row_lineage_orc;
---- RESULTS
@@ -500,3 +543,196 @@ alter table iceberg_v3_row_lineage_orc drop column _file_last_updated_sequence_n
---- CATCH
cannot be dropped.
====
+---- QUERY
+# Query syntactic sugar columns of row lineage fields.
+select i, _row_id, _last_updated_sequence_number from iceberg_v3_row_lineage;
+---- RESULTS
+1,0,1
+2,1,2
+3,2,3
+---- TYPES
+INT,BIGINT,BIGINT
+---- LABELS
+i,_row_id,_last_updated_sequence_number
+====
+---- QUERY
+# Syntactic sugar column results match the explicit COALESCE expressions.
+select i,
+ _row_id = coalesce(_file_row_id, ICEBERG__FIRST__ROW__ID + FILE__POSITION),
+ _last_updated_sequence_number
+ = coalesce(_file_last_updated_sequence_number, ICEBERG__DATA__SEQUENCE__NUMBER)
+from iceberg_v3_row_lineage;
+---- RESULTS
+1,true,true
+2,true,true
+3,true,true
+---- TYPES
+INT,BOOLEAN,BOOLEAN
+====
+---- QUERY
+# WHERE clause: filter using _row_id.
+select i from iceberg_v3_row_lineage where _row_id = 1;
+---- RESULTS
+2
+---- TYPES
+INT
+====
+---- QUERY
+# WHERE clause: filter using _last_updated_sequence_number.
+select i from iceberg_v3_row_lineage where _last_updated_sequence_number > 1;
+---- RESULTS
+2
+3
+---- TYPES
+INT
+====
+---- QUERY
+# ORDER BY using _row_id (descending).
+select i, _row_id from iceberg_v3_row_lineage order by _row_id desc;
+---- RESULTS
+3,2
+2,1
+1,0
+---- TYPES
+INT,BIGINT
+---- LABELS
+i,_row_id
+====
+---- QUERY
+# ORDER BY using _last_updated_sequence_number.
+select i, _last_updated_sequence_number
+from iceberg_v3_row_lineage order by _last_updated_sequence_number desc;
+---- RESULTS
+3,3
+2,2
+1,1
+---- TYPES
+INT,BIGINT
+---- LABELS
+i,_last_updated_sequence_number
+====
+---- QUERY
+# GROUP BY using _row_id.
+select _row_id, count(*) from iceberg_v3_row_lineage group by _row_id order by _row_id;
+---- RESULTS
+0,1
+1,1
+2,1
+---- TYPES
+BIGINT,BIGINT
+====
+---- QUERY
+# GROUP BY using _last_updated_sequence_number.
+select _last_updated_sequence_number, count(*)
+from iceberg_v3_row_lineage
+group by _last_updated_sequence_number
+order by _last_updated_sequence_number;
+---- RESULTS
+1,1
+2,1
+3,1
+---- TYPES
+BIGINT,BIGINT
+====
+---- QUERY
+# HAVING clause using _row_id, grouped by i.
+select i, max(_row_id) row_id from iceberg_v3_row_lineage
+group by i
+having max(_row_id) >= 1
+order by i;
+---- RESULTS
+2,1
+3,2
+---- TYPES
+INT,BIGINT
+====
+---- QUERY
+# Subquery: _row_id used in inner SELECT, referenced by outer WHERE.
+select i from (
+ select i, _row_id from iceberg_v3_row_lineage
+) sub
+where _row_id < 2
+order by i;
+---- RESULTS
+1
+2
+---- TYPES
+INT
+====
+---- QUERY
+# CTE using _row_id and _last_updated_sequence_number.
+with ids as (
+ select i, _row_id, _last_updated_sequence_number
+ from iceberg_v3_row_lineage
+)
+select i, _row_id, _last_updated_sequence_number
+from ids
+order by i;
+---- RESULTS
+1,0,1
+2,1,2
+3,2,3
+---- TYPES
+INT,BIGINT,BIGINT
+====
+---- QUERY
+create table ice_v3_a (x int) stored as iceberg
+tblproperties ('format-version'='3');
+create table ice_v3_b (y int) stored as iceberg
+tblproperties ('format-version'='3');
+insert into ice_v3_a values (10);
+insert into ice_v3_a values (20);
+insert into ice_v3_b values (20);
+insert into ice_v3_b values (10);
+====
+---- QUERY
+# Qualified _row_id and _last_updated_sequence_number across two V3 tables in one query.
+select a.x, a._row_id, a._last_updated_sequence_number,
+ b.y, b._row_id, b._last_updated_sequence_number
+from ice_v3_a a, ice_v3_b b
+where a.x = b.y;
+---- RESULTS
+10,0,1,10,1,2
+20,1,2,20,0,1
+---- TYPES
+INT,BIGINT,BIGINT,INT,BIGINT,BIGINT
+====
+---- QUERY
+# Join: _row_id from both tables in WHERE predicate.
+select a.x, b.y
+from ice_v3_a a cross join ice_v3_b b
+where a._row_id = b._row_id;
+---- RESULTS
+10,20
+20,10
+---- TYPES
+INT,INT
+====
+---- QUERY
+# _row_id on V2 table should fail, column does not exist on non-V3 tables.
+create table ice_v2_no_sugar (i int) stored as iceberg
+tblproperties ('format-version'='2');
+insert into ice_v2_no_sugar values (1);
+select _row_id from ice_v2_no_sugar;
+---- CATCH
+Could not resolve column/field reference: '_row_id'
+====
+---- QUERY
+# _last_updated_sequence_number on V2 table should fail.
+select _last_updated_sequence_number from ice_v2_no_sugar;
+---- CATCH
+Could not resolve column/field reference: '_last_updated_sequence_number'
+====
+---- QUERY
+# Unqualified _row_id is ambiguous when multiple V3 tables are in scope.
+select _row_id from ice_v3_a a, ice_v3_b b;
+---- CATCH
+Column/field reference is ambiguous: '_row_id'
+====
+---- QUERY
+# Unqualified _last_updated_sequence_number is ambiguous when multiple V3 tables
+# are in scope.
+select _last_updated_sequence_number from ice_v3_a a cross join ice_v3_b b;
+---- CATCH
+Column/field reference is ambiguous: '_last_updated_sequence_number'
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/ranger_iceberg_v3_column_masking.test b/testdata/workloads/functional-query/queries/QueryTest/ranger_iceberg_v3_column_masking.test
new file mode 100644
index 000000000..1f43bc2e2
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/ranger_iceberg_v3_column_masking.test
@@ -0,0 +1,27 @@
+====
+---- QUERY
+# _row_id, _file_row_id, _last_updated_sequence_number, _file_last_updated_sequence_number
+# have column masking rules that multiplies their values by -1.
+# _row_id returns _file_row_id if the latter is present, but it doesn't mean we multiply
+# the final result by -1 twice. I.e., the final results are always the original results
+# plus the column masking rule applied on top of it.
+select _row_id, _file_row_id, _last_updated_sequence_number,
+ _file_last_updated_sequence_number, i
+from $UNIQUE_DB.ice_v3 where i < 10 order by _row_id;
+---- RESULTS
+-6001202,-6001202,-3,-3,6
+-6001201,-6001201,-3,-3,5
+-6001200,-6001200,-3,-3,4
+regex:-\d+,NULL,-2,NULL,7
+regex:-\d+,NULL,-2,NULL,7
+regex:-\d+,NULL,-2,NULL,7
+regex:-\d+,NULL,-2,NULL,7
+regex:-\d+,NULL,-2,NULL,7
+regex:-\d+,NULL,-2,NULL,7
+regex:-\d+,NULL,-2,NULL,7
+-2,-2,-1,-1,3
+-1,-1,-1,-1,2
+0,0,-1,-1,1
+---- TYPES
+BIGINT,BIGINT,BIGINT,BIGINT,INT
+====
diff --git a/tests/authorization/test_ranger.py b/tests/authorization/test_ranger.py
index c09d4433d..3a9e84470 100644
--- a/tests/authorization/test_ranger.py
+++ b/tests/authorization/test_ranger.py
@@ -3554,6 +3554,52 @@ class TestRangerLocalCatalog(TestRanger):
finally:
admin_client.execute("drop database if exists {0} cascade".format(unique_database))
+ @pytest.mark.execute_serially
+ def test_iceberg_v3_column_masking(self, vector, unique_name):
+ """Test column masking on Iceberg V3 virtual columns (_row_id, _file_row_id,
+ _last_updated_sequence_number, _file_last_updated_sequence_number)."""
+ user = getuser()
+ admin_client = self.create_impala_client(user=ADMIN)
+ unique_database = unique_name + "_db"
+ short_table_name = "ice_v3"
+ tbl_name = unique_database + "." + short_table_name
+
+ policy_cnt = 0
+ try:
+ admin_client.execute("drop database if exists {0} cascade"
+ .format(unique_database))
+ admin_client.execute("create database {0}".format(unique_database))
+ admin_client.execute(
+ "create table {0} (i int) stored by iceberg "
+ "tblproperties ('format-version'='3')".format(tbl_name))
+ admin_client.execute("insert into {0} values (1), (2), (3)".format(tbl_name))
+ admin_client.execute(
+ "insert into {0} select cast(l_orderkey as INT) "
+ "from tpch.lineitem where l_orderkey > 6".format(tbl_name))
+ admin_client.execute("insert into {0} values (4), (5), (6)".format(tbl_name))
+ admin_client.execute(
+ "optimize table {0} (file_size_threshold_mb=1)".format(tbl_name))
+ admin_client.execute("grant select on database {0} to user {1} "
+ .format(unique_database, user))
+
+ # Add column masking policies that produce the negative values of virtual columns.
+ for col in ["_row_id", "_file_row_id", "_last_updated_sequence_number",
+ "_file_last_updated_sequence_number"]:
+ TestRanger._add_column_masking_policy(
+ unique_name + str(policy_cnt), user, unique_database, short_table_name,
+ col, "CUSTOM", "-{col}")
+ policy_cnt += 1
+ self.execute_query_expect_success(admin_client, "refresh authorization")
+ self.run_test_case("QueryTest/ranger_iceberg_v3_column_masking", vector,
+ test_file_vars={'$UNIQUE_DB': unique_database})
+ finally:
+ admin_client.execute("revoke select on database {0} from user {1}"
+ .format(unique_database, user))
+ admin_client.execute("drop database if exists {0} cascade"
+ .format(unique_database))
+ for i in range(policy_cnt):
+ TestRanger._remove_policy(unique_name + str(i))
+
@pytest.mark.execute_serially
def test_convert_table_to_iceberg(self, unique_name):
"""Test that autorization is taken into account when performing a table migration to