You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bo...@apache.org on 2026/04/30 19:07:54 UTC
(impala) 03/03: IMPALA-14913: Fix partition-related issues of DELETE and UPDATE on Iceberg V3 tables
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 8d4578e932c7b76b9f41728b69534f38ed3d33d5
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Tue Apr 28 18:46:18 2026 +0200
IMPALA-14913: Fix partition-related issues of DELETE and UPDATE on Iceberg V3 tables
The following cases were problematic in the original patches:
* partition evolution
* updating partition fields
* predicate pushdown
This patch fixes the above issues. It adds a new method to
DmlStatementBase: getShuffleExprs(): this way we can make shuffling
logic and partition writing logic independent of each other. It is
required for Iceberg V3 tables to write Deletion Vectors correctly:
* even non-partitioned table writes require shuffling
* deletes and updates on partitioned tables require shuffling based
on the deleted records' partition fields, this way we can guarantee
that only one DV is created per data file
Prior to this patch the Deletion Vectors written by Impala did not have
partition information, because of this predicate pushdown filtered them
out even if their corresponding data file passed the filter.
Testing
Added tests from the Iceberg V2 DELETE/UPDATE test cases, they include:
* UPDATE FROM statements
* Updating all types
* Partition transforms
* Partition evolution
* Predicate pushdown
* UDF in the target value
* Constrained memory
Change-Id: I8935d2aac0a8e7776ec7e185e361671054a83227
Reviewed-on: http://gerrit.cloudera.org:8080/24250
Reviewed-by: Peter Rozsa <pr...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/runtime/dml-exec-state.cc | 63 ++-
common/fbs/IcebergObjects.fbs | 2 +
.../apache/impala/analysis/DmlStatementBase.java | 12 +
.../apache/impala/analysis/IcebergDeleteImpl.java | 12 +
.../apache/impala/analysis/IcebergModifyImpl.java | 19 +-
.../apache/impala/analysis/IcebergUpdateImpl.java | 15 +
.../org/apache/impala/analysis/ModifyImpl.java | 2 +
.../org/apache/impala/analysis/ModifyStmt.java | 2 +
.../apache/impala/planner/DistributedPlanner.java | 22 +-
.../apache/impala/planner/IcebergScanPlanner.java | 30 +-
.../java/org/apache/impala/planner/Planner.java | 7 +-
.../impala/service/IcebergCatalogOpExecutor.java | 10 +-
.../java/org/apache/impala/util/IcebergUtil.java | 31 +-
.../QueryTest/iceberg-v3-delete-partitioned.test | 499 +++++++++++++++++++++
.../queries/QueryTest/iceberg-v3-delete.test | 116 +++++
.../QueryTest/iceberg-v3-update-partitions.test | 498 ++++++++++++++++++++
.../queries/QueryTest/iceberg-v3-update.test | 488 ++++++++++++++++++++
tests/query_test/test_iceberg.py | 12 +-
18 files changed, 1784 insertions(+), 56 deletions(-)
diff --git a/be/src/runtime/dml-exec-state.cc b/be/src/runtime/dml-exec-state.cc
index 94da22104..c24c4f9fe 100644
--- a/be/src/runtime/dml-exec-state.cc
+++ b/be/src/runtime/dml-exec-state.cc
@@ -577,6 +577,19 @@ createIcebergColumnStats(
return stats_builder.Finish();
}
+vector<flatbuffers::Offset<org::apache::impala::fb::FbIcebergPartitionField>>
+createRawPartitionFields(flatbuffers::FlatBufferBuilder& fbb,
+ const OutputPartition& partition) {
+ using namespace org::apache::impala::fb;
+ vector<flatbuffers::Offset<FbIcebergPartitionField>> raw_partition_fields;
+ for (const string& partition_name : partition.raw_partition_names) {
+ auto data = reinterpret_cast<const uint8_t*>(partition_name.data());
+ auto fb_vector = fbb.CreateVector(data, partition_name.size());
+ raw_partition_fields.push_back(CreateFbIcebergPartitionField(fbb, fb_vector));
+ }
+ return raw_partition_fields;
+}
+
// Helper function to create an Iceberg data file FlatBuffer with deletion vectors.
// Used for puffin files where we need to create one entry per data file with DVs.
// Parameters:
@@ -597,18 +610,12 @@ string createIcebergDataFileWithDeletionVectorString(
using namespace org::apache::impala::fb;
flatbuffers::FlatBufferBuilder fbb;
- // For puffin files with deletion vectors, we don't have column stats
- vector<flatbuffers::Offset<FbIcebergColumnStats>> ice_col_stats_vec;
-
- vector<flatbuffers::Offset<FbIcebergPartitionField>> raw_partition_fields;
- for (const string& partition_name : partition.raw_partition_names) {
- auto data = reinterpret_cast<const uint8_t*>(partition_name.data());
- auto fb_vector = fbb.CreateVector(data, partition_name.size());
- raw_partition_fields.push_back(CreateFbIcebergPartitionField(fbb, fb_vector));
- }
-
THash128 hash = THash128FromFilePath(referenced_data_file);
+ auto raw_partition_fields = createRawPartitionFields(fbb, partition);
+ auto raw_part_fields_for_old_dv = fbb.CreateVector(raw_partition_fields);
+ auto raw_part_fields_for_new_dv = fbb.CreateVector(raw_partition_fields);
+
// Create FlatBuffer deletion vector reference for old DV
flatbuffers::Offset<FbIcebergDeletionVector> old_dv_fb = 0;
if (old_dv != nullptr) {
@@ -618,7 +625,10 @@ string createIcebergDataFileWithDeletionVectorString(
old_dv->content_size_in_bytes,
hash.high,
hash.low,
- old_dv->__isset.record_count ? old_dv->record_count : 0);
+ old_dv->__isset.record_count ? old_dv->record_count : 0,
+ 0,
+ partition.iceberg_spec_id,
+ raw_part_fields_for_old_dv);
}
// Create FlatBuffer deletion vector reference for new DV
@@ -630,12 +640,17 @@ string createIcebergDataFileWithDeletionVectorString(
hash.high,
hash.low,
new_dv->__isset.record_count ? new_dv->record_count : 0,
- puffin_file_size);
+ puffin_file_size,
+ partition.iceberg_spec_id,
+ raw_part_fields_for_new_dv);
// For puffin files, use the deletion vector record count and the total
// Puffin file size.
int64_t num_rows = new_dv->__isset.record_count ? new_dv->record_count : 0;
+ // For puffin files with deletion vectors, we don't have column stats
+ vector<flatbuffers::Offset<FbIcebergColumnStats>> ice_col_stats_vec;
+
flatbuffers::Offset<FbIcebergDataFile> data_file = CreateFbIcebergDataFile(fbb,
fbb.CreateString(puffin_file_path),
FbIcebergDataFileFormat::FbIcebergDataFileFormat_PUFFIN,
@@ -663,13 +678,7 @@ string createIcebergDataFileString(
ice_col_stats_vec.push_back(createIcebergColumnStats(fbb, it->first, it->second));
}
- vector<flatbuffers::Offset<FbIcebergPartitionField>> raw_partition_fields;
-
- for (const string& partition_name : partition.raw_partition_names) {
- auto data = reinterpret_cast<const uint8_t*>(partition_name.data());
- auto fb_vector = fbb.CreateVector(data, partition_name.size());
- raw_partition_fields.push_back(CreateFbIcebergPartitionField(fbb, fb_vector));
- }
+ auto raw_partition_fields = createRawPartitionFields(fbb, partition);
flatbuffers::Offset<FbIcebergDataFile> data_file = CreateFbIcebergDataFile(fbb,
fbb.CreateString(final_path),
@@ -837,6 +846,18 @@ vector<string> DmlExecState::CollectDeletionVectors(GetDV get_dv) {
// Serialize the DV into its own FlatBuffer.
flatbuffers::FlatBufferBuilder builder(256);
auto path_offset = builder.CreateString(dv->path()->c_str());
+
+ vector<flatbuffers::Offset<FbIcebergPartitionField>> raw_part_fields;
+ if (dv->raw_partition_fields()) {
+ for (int j = 0; j < dv->raw_partition_fields()->size(); ++j) {
+ auto* field = dv->raw_partition_fields()->Get(j);
+ auto fv = builder.CreateVector(
+ field->field_value()->data(), field->field_value()->size());
+ raw_part_fields.push_back(CreateFbIcebergPartitionField(builder, fv));
+ }
+ }
+ auto raw_part_fields_offset = builder.CreateVector(raw_part_fields);
+
auto dv_offset = CreateFbIcebergDeletionVector(builder,
path_offset,
dv->content_offset(),
@@ -844,7 +865,9 @@ vector<string> DmlExecState::CollectDeletionVectors(GetDV get_dv) {
dv->referenced_data_file_hash_high(),
dv->referenced_data_file_hash_low(),
dv->record_count(),
- data_file->file_size_in_bytes());
+ data_file->file_size_in_bytes(),
+ dv->spec_id(),
+ raw_part_fields_offset);
builder.Finish(dv_offset);
ret.push_back(string(reinterpret_cast<const char*>(builder.GetBufferPointer()),
diff --git a/common/fbs/IcebergObjects.fbs b/common/fbs/IcebergObjects.fbs
index 931fa624a..0c067738a 100644
--- a/common/fbs/IcebergObjects.fbs
+++ b/common/fbs/IcebergObjects.fbs
@@ -90,5 +90,7 @@ table FbIcebergDeletionVector {
record_count: long;
// Total size of the Puffin file containing this deletion vector, in bytes.
file_size_in_bytes: long;
+ spec_id: ushort;
+ raw_partition_fields: [FbIcebergPartitionField];
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/DmlStatementBase.java b/fe/src/main/java/org/apache/impala/analysis/DmlStatementBase.java
index 10b71be71..836bdc4a2 100644
--- a/fe/src/main/java/org/apache/impala/analysis/DmlStatementBase.java
+++ b/fe/src/main/java/org/apache/impala/analysis/DmlStatementBase.java
@@ -69,7 +69,19 @@ public abstract class DmlStatementBase extends StatementBase {
abstract public DataSink createDataSink();
abstract public void substituteResultExprs(ExprSubstitutionMap smap, Analyzer analyzer);
+ /**
+ * The DML sink writes data into partitions according to the partition key exprs.
+ */
abstract public List<Expr> getPartitionKeyExprs();
+ /**
+ * The DML sink requires shuffling on the returned shuffle exprs. In most cases, it
+ * is the same as partition key exprs, but in specific cases (e.g. Iceberg V3 tables
+ * with Deletion Vectors) we need to shuffle data differently, or we need to shuffle
+ * data even if the target table is not partitioned.
+ * Another use case is to fan out writers when partition key exprs have low
+ * cardinality. In that case, we can add additional shuffle exprs.
+ */
+ public List<Expr> getShuffleExprs() { return getPartitionKeyExprs(); }
abstract public List<Expr> getSortExprs();
abstract public TSortingOrder getSortingOrder();
diff --git a/fe/src/main/java/org/apache/impala/analysis/IcebergDeleteImpl.java b/fe/src/main/java/org/apache/impala/analysis/IcebergDeleteImpl.java
index ccea56cad..f502e779b 100644
--- a/fe/src/main/java/org/apache/impala/analysis/IcebergDeleteImpl.java
+++ b/fe/src/main/java/org/apache/impala/analysis/IcebergDeleteImpl.java
@@ -78,13 +78,25 @@ public class IcebergDeleteImpl extends IcebergModifyImpl {
throws AnalysisException {
deletePartitionKeyExprs_ = getDeletePartitionExprs(analyzer);
deleteResultExprs_ = getDeleteResultExprs(analyzer);
+ shuffleExprs_ = buildShuffleExprs(analyzer);
selectList.addAll(ExprUtil.exprsAsSelectList(deletePartitionKeyExprs_));
selectList.addAll(ExprUtil.exprsAsSelectList(deleteResultExprs_));
+ if (originalTargetTable_.getFormatVersion() >= 3 &&
+ !originalTargetTable_.isPartitioned()) {
+ // For V3 tables, we need to add file name to the select list for shuffle to ensure
+ // that all rows from the same file are shuffled to the same node.
+ selectList.addAll(ExprUtil.exprsAsSelectList(shuffleExprs_));
+ }
}
@Override
public List<Expr> getPartitionKeyExprs() { return deletePartitionKeyExprs_; }
+ @Override
+ protected List<Expr> getPartitionedShuffleExprs() {
+ return getPartitionKeyExprs();
+ }
+
@Override
public void addCastsToAssignmentsInSourceStmt(Analyzer analyzer)
throws AnalysisException {
diff --git a/fe/src/main/java/org/apache/impala/analysis/IcebergModifyImpl.java b/fe/src/main/java/org/apache/impala/analysis/IcebergModifyImpl.java
index 17de31bb5..a047819d2 100644
--- a/fe/src/main/java/org/apache/impala/analysis/IcebergModifyImpl.java
+++ b/fe/src/main/java/org/apache/impala/analysis/IcebergModifyImpl.java
@@ -42,6 +42,10 @@ abstract class IcebergModifyImpl extends ModifyImpl {
// All Iceberg modify statements (DELETE, UPDATE) need delete exprs.
protected List<Expr> deleteResultExprs_ = new ArrayList<>();
protected List<Expr> deletePartitionKeyExprs_ = new ArrayList<>();
+ protected List<Expr> shuffleExprs_ = new ArrayList<>();
+
+ @Override
+ public List<Expr> getShuffleExprs() { return shuffleExprs_; }
// For every column of the target table that is referenced in the optional
// 'sort.columns' table property, this list will contain the corresponding result expr
@@ -96,6 +100,7 @@ abstract class IcebergModifyImpl extends ModifyImpl {
deleteResultExprs_ = Expr.substituteList(deleteResultExprs_, smap, analyzer, true);
deletePartitionKeyExprs_ = Expr.substituteList(
deletePartitionKeyExprs_, smap, analyzer, true);
+ shuffleExprs_ = Expr.substituteList(shuffleExprs_, smap, analyzer, true);
}
public List<Expr> getDeletePartitionExprs(Analyzer analyzer) throws AnalysisException {
@@ -109,7 +114,7 @@ abstract class IcebergModifyImpl extends ModifyImpl {
"INPUT__FILE__NAME", "FILE__POSITION"));
}
- private List<Expr> getSlotRefs(Analyzer analyzer, List<String> cols)
+ protected List<Expr> getSlotRefs(Analyzer analyzer, List<String> cols)
throws AnalysisException {
List<Expr> ret = new ArrayList<>();
for (String col : cols) {
@@ -118,6 +123,18 @@ abstract class IcebergModifyImpl extends ModifyImpl {
return ret;
}
+ protected List<Expr> buildShuffleExprs(Analyzer analyzer) throws AnalysisException {
+ if (!originalTargetTable_.isPartitioned()) {
+ if (originalTargetTable_.getFormatVersion() >= 3) {
+ return getSlotRefs(analyzer, Lists.newArrayList("INPUT__FILE__NAME"));
+ }
+ return Collections.emptyList();
+ }
+ return getPartitionedShuffleExprs();
+ }
+
+ protected abstract List<Expr> getPartitionedShuffleExprs();
+
abstract String getModifyMode();
protected boolean isMergeOnReadAlwaysAllowed() {
diff --git a/fe/src/main/java/org/apache/impala/analysis/IcebergUpdateImpl.java b/fe/src/main/java/org/apache/impala/analysis/IcebergUpdateImpl.java
index ff5d1f2a2..3a048ed9d 100644
--- a/fe/src/main/java/org/apache/impala/analysis/IcebergUpdateImpl.java
+++ b/fe/src/main/java/org/apache/impala/analysis/IcebergUpdateImpl.java
@@ -141,10 +141,17 @@ public class IcebergUpdateImpl extends IcebergModifyImpl {
insertResultExprs_, originalTargetTable_, insertPartitionKeyExprs_, null);
deletePartitionKeyExprs_ = getDeletePartitionExprs(analyzer);
deleteResultExprs_ = getDeleteResultExprs(analyzer);
+ shuffleExprs_ = buildShuffleExprs(analyzer);
selectList.addAll(ExprUtil.exprsAsSelectList(insertResultExprs_));
selectList.addAll(ExprUtil.exprsAsSelectList(insertPartitionKeyExprs_));
selectList.addAll(ExprUtil.exprsAsSelectList(deleteResultExprs_));
selectList.addAll(ExprUtil.exprsAsSelectList(deletePartitionKeyExprs_));
+ if (originalTargetTable_.getFormatVersion() >= 3 &&
+ !originalTargetTable_.isPartitioned()) {
+ // For V3 tables, we need to add file name to the select list for shuffle to ensure
+ // that all rows from the same file are shuffled to the same node.
+ selectList.addAll(ExprUtil.exprsAsSelectList(shuffleExprs_));
+ }
addSortColumns();
}
@@ -191,6 +198,14 @@ public class IcebergUpdateImpl extends IcebergModifyImpl {
return insertPartitionKeyExprs_;
}
+ @Override
+ protected List<Expr> getPartitionedShuffleExprs() {
+ if (originalTargetTable_.getFormatVersion() >= 3) {
+ return deletePartitionKeyExprs_;
+ }
+ return insertPartitionKeyExprs_;
+ }
+
@Override
public TSortingOrder getSortingOrder() { return sortingOrder_; }
diff --git a/fe/src/main/java/org/apache/impala/analysis/ModifyImpl.java b/fe/src/main/java/org/apache/impala/analysis/ModifyImpl.java
index 8cbc7f793..af94700d4 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ModifyImpl.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ModifyImpl.java
@@ -119,6 +119,8 @@ abstract class ModifyImpl {
abstract public List<Expr> getPartitionKeyExprs();
+ public List<Expr> getShuffleExprs() { return getPartitionKeyExprs(); }
+
public List<Expr> getSortExprs() {
return Collections.emptyList();
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java b/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java
index 167350b70..3f943a5cb 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java
@@ -191,6 +191,8 @@ public abstract class ModifyStmt extends DmlStatementBase {
@Override
public List<Expr> getPartitionKeyExprs() { return modifyImpl_.getPartitionKeyExprs(); }
@Override
+ public List<Expr> getShuffleExprs() { return modifyImpl_.getShuffleExprs(); }
+ @Override
public List<Expr> getSortExprs() { return modifyImpl_.getSortExprs(); }
public Expr getWherePredicate() { return wherePredicate_; }
diff --git a/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java b/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
index b8b8e6956..3768faf24 100644
--- a/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/DistributedPlanner.java
@@ -207,22 +207,22 @@ public class DistributedPlanner {
if (dmlStmt.hasNoShuffleHint() && !enforceHdfsWriterLimit) return inputFragment;
- List<Expr> partitionExprs = new ArrayList<>(dmlStmt.getPartitionKeyExprs());
+ List<Expr> shuffleExprs = new ArrayList<>(dmlStmt.getShuffleExprs());
// Ignore constants for the sake of partitioning.
- Expr.removeConstants(partitionExprs);
+ Expr.removeConstants(shuffleExprs);
// Do nothing if the input fragment is already appropriately partitioned. TODO: handle
// Kudu tables here (IMPALA-5254).
DataPartition inputPartition = inputFragment.getDataPartition();
- if (!partitionExprs.isEmpty()
+ if (!shuffleExprs.isEmpty()
&& analyzer.setsHaveValueTransfer(
- inputPartition.getPartitionExprs(), partitionExprs, true)
+ inputPartition.getPartitionExprs(), shuffleExprs, true)
&& !(dmlStmt.getTargetTable() instanceof FeKuduTable)
&& !enforceHdfsWriterLimit) {
return inputFragment;
}
- long numPartitions = getNumDistinctValues(partitionExprs);
+ long numPartitions = getNumDistinctValues(shuffleExprs);
int maxHdfsWriters = analyzer.getQueryOptions().getMax_fs_writers();
// We also consider fragments containing union nodes along with scan fragments
// (leaf fragments) since they are either a part of those scan fragments or are
@@ -244,7 +244,7 @@ public class DistributedPlanner {
inputFragment.getNumNodes(), analyzer.getMaxParallelismPerNode());
int costBasedMaxWriter = minInstances;
- boolean isPartitioned = !partitionExprs.isEmpty();
+ boolean isPartitioned = !shuffleExprs.isEmpty();
PlanNode root = inputFragment.getPlanRoot();
if (root.getCardinality() > -1 && root.getAvgRowSize() > -1) {
// Both cardinality and avg row size is known.
@@ -296,7 +296,7 @@ public class DistributedPlanner {
// don't insert the exchange.
// TODO: make a more sophisticated decision here for partitioned tables and when
// we have info about tablet locations.
- if (partitionExprs.isEmpty()) return inputFragment;
+ if (shuffleExprs.isEmpty()) return inputFragment;
} else if (!enforceHdfsWriterLimit || !hasHdfsScanORUnion
|| (expectedNumInputInstance <= maxHdfsWriters)) {
// Only consider skipping the addition of an exchange node if
@@ -322,10 +322,10 @@ public class DistributedPlanner {
// check if it is distributed across all nodes. If so, don't repartition.
// TODO: If input fragment has ScanNode leave(s), its inputPartitions can be
// empty and numInputPartitions=1. But the scanned table(s) might be a
- // partitioned table(s) that superset of partitionExprs. Repartition (shuffling)
+ // partitioned table(s) that superset of shuffleExprs. Repartition (shuffling)
// might not needed in that case, but skipping repartition can risk producing
// small files.
- if (Expr.isSubset(inputPartition.getPartitionExprs(), partitionExprs)) {
+ if (Expr.isSubset(inputPartition.getPartitionExprs(), shuffleExprs)) {
long numInputPartitions =
getNumDistinctValues(inputPartition.getPartitionExprs());
if (numInputPartitions >= inputInstances) { return inputFragment; }
@@ -350,7 +350,7 @@ public class DistributedPlanner {
exchNode.init(analyzer);
Preconditions.checkState(exchNode.hasValidStats());
DataPartition partition;
- if (partitionExprs.isEmpty()) {
+ if (shuffleExprs.isEmpty()) {
if (enforceHdfsWriterLimit
&& inputFragment.getDataPartition().getType() == TPartitionType.RANDOM) {
// This ensures the parallelism of the writers is maintained while maintaining
@@ -364,7 +364,7 @@ public class DistributedPlanner {
partition = DataPartition.kuduPartitioned(
KuduUtil.createPartitionExpr((InsertStmt)dmlStmt, ctx_.getRootAnalyzer()));
} else {
- partition = DataPartition.hashPartitioned(partitionExprs);
+ partition = DataPartition.hashPartitioned(shuffleExprs);
}
PlanFragment fragment =
new PlanFragment(ctx_.getNextFragmentId(), exchNode, partition);
diff --git a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
index a543ce537..7255e1fad 100644
--- a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
@@ -727,25 +727,29 @@ public class IcebergScanPlanner {
} else {
dataFilesWithDeletes_.add(fileDesc.first);
for (DeleteFile delFile : fileScanTask.deletes()) {
+ if (delFile.format() == FileFormat.PUFFIN) {
+ // A data file can have at most one DV, but may also be covered by legacy
+ // position-delete files simultaneously (e.g. after a partial V3 migration
+ // by an external engine). Assert there is no second DV for this file.
+ Hash128 dataFileHash = IcebergUtil.getFilePathHash(dataFile.location());
+ TIcebergDeletionVector dv = dataFileToDV_.put(dataFileHash,
+ IcebergUtil.createTIcebergDeletionVector(delFile));
+ Preconditions.checkState(dv == null,
+ "More than one deletion vector found for data file: %s",
+ dataFile.location());
+ continue;
+ }
+ // Handle legacy delete files.
Pair<IcebergFileDescriptor, Boolean> delFileDesc =
getFileDescriptor(delFile, fileStore);
if (!delFileDesc.second) ++dataFilesCacheMisses;
if (delFile.content() == FileContent.EQUALITY_DELETES) {
addEqualityDeletesAndIds(delFileDesc.first);
+ } else if (delFile.content() == FileContent.POSITION_DELETES) {
+ positionDeleteFiles_.add(delFileDesc.first);
} else {
- if (delFile.format() == FileFormat.PUFFIN) {
- // A data file can have at most one DV, but may also be covered by legacy
- // position-delete files simultaneously (e.g. after a partial V3 migration
- // by an external engine). Assert there is no second DV for this file.
- Hash128 dataFileHash = IcebergUtil.getFilePathHash(dataFile.location());
- TIcebergDeletionVector dv = dataFileToDV_.put(dataFileHash,
- IcebergUtil.createTIcebergDeletionVector(delFile));
- Preconditions.checkState(dv == null,
- "More than one deletion vector found for data file: %s",
- dataFile.location());
- } else {
- positionDeleteFiles_.add(delFileDesc.first);
- }
+ Preconditions.checkState(false,
+ "Unknown delete file content type: " + delFile.content());
}
}
}
diff --git a/fe/src/main/java/org/apache/impala/planner/Planner.java b/fe/src/main/java/org/apache/impala/planner/Planner.java
index 760767c33..802f21fd7 100644
--- a/fe/src/main/java/org/apache/impala/planner/Planner.java
+++ b/fe/src/main/java/org/apache/impala/planner/Planner.java
@@ -37,7 +37,6 @@ import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.ExprSubstitutionMap;
import org.apache.impala.analysis.InsertStmt;
import org.apache.impala.analysis.JoinOperator;
-import org.apache.impala.analysis.MergeCase;
import org.apache.impala.analysis.MergeStmt;
import org.apache.impala.analysis.ParsedStatement;
import org.apache.impala.analysis.QueryStmt;
@@ -300,7 +299,7 @@ public class Planner {
// IcebergBufferedDeleteSink. UPDATE/MERGE statements will still require to
// sort their data records.
if (!(stmt instanceof DeleteStmt)) {
- createPreDmlSort(stmt, rootFragment, ctx_.getRootAnalyzer());
+ createIcebergPreDmlSort(stmt, rootFragment, ctx_.getRootAnalyzer());
}
return rootFragment;
}
@@ -1029,8 +1028,8 @@ public class Planner {
inputFragment.setPlanRoot(node);
}
- public void createPreDmlSort(DmlStatementBase dmlStmt, PlanFragment inputFragment,
- Analyzer analyzer) throws ImpalaException {
+ public void createIcebergPreDmlSort(DmlStatementBase dmlStmt,
+ PlanFragment inputFragment, Analyzer analyzer) throws ImpalaException {
List<Expr> orderingExprs = new ArrayList<>();
List<Expr> partitionKeyExprs = dmlStmt.getPartitionKeyExprs();
diff --git a/fe/src/main/java/org/apache/impala/service/IcebergCatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/IcebergCatalogOpExecutor.java
index 9e1b38545..72fab0c5c 100644
--- a/fe/src/main/java/org/apache/impala/service/IcebergCatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/IcebergCatalogOpExecutor.java
@@ -498,7 +498,9 @@ public class IcebergCatalogOpExecutor {
String referencedDataFile = dataFileDescriptor
.getAbsolutePath(feIcebergTable.getLocation());
- PartitionSpec partSpec = feIcebergTable.getIcebergApiTable().spec();
+ int specId = dv.specId();
+ PartitionSpec partSpec = feIcebergTable.getIcebergApiTable().specs().get(specId);
+ Preconditions.checkNotNull(partSpec, "Unknown partition spec id: %s", specId);
// Build the delete file metadata
FileMetadata.Builder builder = FileMetadata.deleteFileBuilder(partSpec)
@@ -511,6 +513,12 @@ public class IcebergCatalogOpExecutor {
.withRecordCount(dv.recordCount())
.withFileSizeInBytes(dv.fileSizeInBytes());
+ IcebergPartitionSpec impPartSpec = feIcebergTable.getPartitionSpec(specId);
+ IcebergUtil.PartitionData partitionData =
+ IcebergUtil.partitionDataFromDeletionVector(
+ partSpec.partitionType(), impPartSpec, dv);
+ if (partitionData != null) builder.withPartition(partitionData);
+
return builder.build();
}
diff --git a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
index cc42f7fc3..31462f0fa 100644
--- a/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/IcebergUtil.java
@@ -33,6 +33,7 @@ import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.time.temporal.ChronoUnit;
+import java.util.function.IntFunction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
@@ -111,6 +112,7 @@ import org.apache.impala.common.ImpalaRuntimeException;
import org.apache.impala.common.Pair;
import org.apache.impala.fb.FbFileMetadata;
import org.apache.impala.fb.FbIcebergDataFile;
+import org.apache.impala.fb.FbIcebergPartitionField;
import org.apache.impala.fb.FbIcebergDataFileFormat;
import org.apache.impala.fb.FbIcebergDeletionVector;
import org.apache.impala.fb.FbIcebergMetadata;
@@ -854,7 +856,28 @@ public class IcebergUtil {
public static PartitionData partitionDataFromDataFile(Types.StructType partitionType,
IcebergPartitionSpec spec, FbIcebergDataFile dataFile)
throws ImpalaRuntimeException {
- if (dataFile == null || dataFile.rawPartitionFieldsLength() == 0) return null;
+ if (dataFile == null) return null;
+ return partitionDataFromRawFields(partitionType, spec,
+ dataFile.rawPartitionFieldsLength(), dataFile::rawPartitionFields);
+ }
+
+ /**
+ * Create a PartitionData object using partition information from
+ * FbIcebergDeletionVector.
+ */
+ public static PartitionData partitionDataFromDeletionVector(
+ Types.StructType partitionType, IcebergPartitionSpec spec,
+ FbIcebergDeletionVector dv) throws ImpalaRuntimeException {
+ if (dv == null) return null;
+ return partitionDataFromRawFields(partitionType, spec,
+ dv.rawPartitionFieldsLength(), dv::rawPartitionFields);
+ }
+
+ private static PartitionData partitionDataFromRawFields(
+ Types.StructType partitionType, IcebergPartitionSpec spec,
+ int rawFieldsLength, IntFunction<FbIcebergPartitionField> rawFieldAccessor)
+ throws ImpalaRuntimeException {
+ if (rawFieldsLength == 0) return null;
PartitionData data = new PartitionData(spec.getIcebergPartitionFieldsSize());
int path_i = 0;
@@ -863,9 +886,9 @@ public class IcebergUtil {
TIcebergPartitionTransformType transformType = field.getTransformType();
if (transformType == TIcebergPartitionTransformType.VOID) continue;
- Preconditions.checkState(path_i < dataFile.rawPartitionFieldsLength());
+ Preconditions.checkState(path_i < rawFieldsLength);
ByteBuffer fieldByteBuffer =
- dataFile.rawPartitionFields(path_i).fieldValueAsByteBuffer();
+ rawFieldAccessor.apply(path_i).fieldValueAsByteBuffer();
Charset charset = StandardCharsets.UTF_8;
if (field.getType() == org.apache.impala.catalog.Type.BINARY) {
@@ -1367,7 +1390,7 @@ public class IcebergUtil {
fbb.finish(
FbIcebergDeletionVector.createFbIcebergDeletionVector(fbb, pathOffset,
contentOffset, contentSizeInBytes, referencedDataFileHashHigh,
- referencedDataFileHashLow, recordCount, 0));
+ referencedDataFileHashLow, recordCount, 0, 0, 0));
ByteBuffer bb = fbb.dataBuffer().slice();
ByteBuffer compressedBb = ByteBuffer.allocate(bb.capacity());
compressedBb.put(bb);
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-delete-partitioned.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-delete-partitioned.test
new file mode 100644
index 000000000..34be55a55
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-delete-partitioned.test
@@ -0,0 +1,499 @@
+====
+---- QUERY
+CREATE TABLE id_part (i int, s string)
+PARTITIONED BY SPEC (i)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+====
+---- QUERY
+# Delete from empty table is no-op.
+DELETE FROM id_part where i = 1;
+SELECT * FROM id_part;
+---- RESULTS
+---- TYPES
+INT,STRING
+====
+---- QUERY
+SHOW FILES IN id_part;
+---- RESULTS
+---- TYPES
+STRING, STRING, STRING, STRING
+====
+---- QUERY
+INSERT INTO id_part VALUES(1, 'one'), (2, 'two'), (3, 'three');
+DELETE FROM id_part WHERE i = 2;
+SELECT * FROM id_part;
+---- RESULTS
+1,'one'
+3,'three'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+INSERT INTO id_part VALUES(4, 'four'), (5, 'five'), (6, 'six');
+DELETE FROM id_part WHERE i % 2 = 0;
+SELECT * FROM id_part;
+---- RESULTS
+1,'one'
+3,'three'
+5,'five'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+DELETE FROM id_part WHERE strleft(s, 1) = 'o';
+SELECT * FROM id_part;
+---- RESULTS
+5,'five'
+3,'three'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+DELETE FROM id_part WHERE i > 0;
+SELECT * FROM id_part;
+---- RESULTS
+---- TYPES
+INT,STRING
+====
+---- QUERY
+INSERT INTO id_part VALUES (null, 'null');
+SELECT * FROM id_part;
+---- RESULTS
+NULL,'null'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+DELETE FROM id_part where s = 'null';
+SELECT * FROM id_part;
+---- RESULTS
+---- TYPES
+INT,STRING
+====
+---- QUERY
+CREATE TABLE trunc_part (i int, s string)
+PARTITIONED BY SPEC (truncate(1, s))
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+====
+---- QUERY
+# Delete from empty table is no-op.
+DELETE FROM trunc_part where i = 1;
+SELECT * FROM trunc_part;
+---- RESULTS
+---- TYPES
+INT,STRING
+====
+---- QUERY
+INSERT INTO trunc_part VALUES(1, 'one'), (2, 'two'), (3, 'three');
+DELETE FROM trunc_part WHERE s like 't%';
+SELECT * FROM trunc_part;
+---- RESULTS
+1,'one'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+INSERT INTO trunc_part VALUES(4, 'four'), (5, 'five'), (6, 'six');
+DELETE FROM trunc_part WHERE i % 2 = 0;
+SELECT * FROM trunc_part;
+---- RESULTS
+1,'one'
+5,'five'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+DELETE FROM trunc_part WHERE i > 0;
+SELECT * FROM trunc_part;
+---- RESULTS
+---- TYPES
+INT,STRING
+====
+---- QUERY
+INSERT INTO trunc_part VALUES (0, null);
+SELECT * FROM trunc_part;
+---- RESULTS
+0,'NULL'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+DELETE FROM trunc_part where s is null;
+SELECT * FROM trunc_part;
+---- RESULTS
+---- TYPES
+INT,STRING
+====
+---- QUERY
+---- QUERY
+CREATE TABLE multi_part (i int, s string, f double)
+PARTITIONED BY SPEC (bucket(3, i), truncate(1, s))
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+====
+---- QUERY
+# Delete from empty table is no-op.
+DELETE FROM multi_part where i = 1;
+SELECT * FROM multi_part;
+---- RESULTS
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+INSERT INTO multi_part VALUES(1, 'one', 1.1), (2, 'two', 2.2), (3, 'three', 3.33);
+DELETE FROM multi_part
+WHERE i != (select min(i) from multi_part) and
+ i != (select max(i) from multi_part);
+SELECT * FROM multi_part;
+---- RESULTS
+1,'one',1.1
+3,'three',3.33
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+INSERT INTO multi_part VALUES(4, 'four', 4.4), (5, 'five', 5.5), (6, 'six', 6.6);
+DELETE FROM multi_part WHERE i % 2 = 0;
+SELECT * FROM multi_part;
+---- RESULTS
+1,'one',1.1
+3,'three',3.33
+5,'five',5.5
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+DELETE FROM multi_part WHERE i > 0;
+SELECT * FROM multi_part;
+---- RESULTS
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+INSERT INTO multi_part VALUES (null, 'null',0.0);
+SELECT * FROM multi_part;
+---- RESULTS
+NULL,'null',0.0
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+DELETE FROM multi_part where s = 'null';
+SELECT * FROM multi_part;
+---- RESULTS
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+CREATE TABLE evolve_part (i int, s string, f double)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+====
+---- QUERY
+INSERT INTO evolve_part VALUES(1, 'one', 1.1), (2, 'two', 2.2), (3, 'three', 3.33);
+DELETE FROM evolve_part WHERE i = 2;
+SELECT * FROM evolve_part;
+---- RESULTS
+1,'one',1.1
+3,'three',3.33
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+ALTER TABLE evolve_part SET PARTITION SPEC (i);
+SELECT * FROM evolve_part;
+---- RESULTS
+1,'one',1.1
+3,'three',3.33
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+INSERT INTO evolve_part VALUES (10, 'ten', 10.10), (20, 'twenty', 20.20);
+SELECT * FROM evolve_part;
+---- RESULTS
+1,'one',1.1
+3,'three',3.33
+10,'ten',10.10
+20,'twenty',20.20
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+DELETE FROM evolve_part WHERE i = 20;
+SELECT * FROM evolve_part;
+---- RESULTS
+1,'one',1.1
+3,'three',3.33
+10,'ten',10.10
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+ALTER TABLE evolve_part SET PARTITION SPEC (truncate(1, s));
+SELECT * FROM evolve_part;
+---- RESULTS
+1,'one',1.1
+3,'three',3.33
+10,'ten',10.10
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+INSERT INTO evolve_part VALUES (30, 'thirty', 30.30), (40, 'forty', 40.40), (50, 'fifty', 50.50);
+SELECT * FROM evolve_part;
+---- RESULTS
+1,'one',1.1
+3,'three',3.33
+10,'ten',10.10
+30,'thirty',30.30
+40,'forty',40.40
+50,'fifty',50.50
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+DELETE FROM evolve_part WHERE i = 50;
+SELECT * FROM evolve_part;
+---- RESULTS
+1,'one',1.1
+3,'three',3.33
+10,'ten',10.10
+30,'thirty',30.30
+40,'forty',40.40
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+ALTER TABLE evolve_part SET PARTITION SPEC (void(s));
+SELECT * FROM evolve_part;
+---- RESULTS
+1,'one',1.1
+3,'three',3.33
+10,'ten',10.10
+30,'thirty',30.30
+40,'forty',40.40
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+DELETE FROM evolve_part WHERE length(s) < 4;
+SELECT * FROM evolve_part;
+---- RESULTS
+3,'three',3.33
+30,'thirty',30.3
+40,'forty',40.4
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+CREATE TABLE ice_store_sales PARTITIONED BY SPEC (ss_sold_date_sk)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3')
+AS SELECT * FROM tpcds_parquet.store_sales;
+SELECT count(*) FROM ice_store_sales;
+---- RESULTS
+2880404
+---- TYPES
+BIGINT
+====
+---- QUERY
+SELECT count(*) FROM ice_store_sales where ss_customer_sk % 10 = 0;
+---- RESULTS
+278906
+---- TYPES
+BIGINT
+====
+---- QUERY
+DELETE FROM ice_store_sales where ss_customer_sk % 10 = 0;
+SELECT count(*) FROM ice_store_sales;
+---- RESULTS
+2601498
+---- TYPES
+BIGINT
+====
+---- QUERY
+SELECT * FROM ice_store_sales where ss_customer_sk % 10 = 0;
+---- RESULTS
+---- TYPES
+INT, BIGINT, INT, INT, INT, INT, INT, INT, BIGINT, INT, DECIMAL, DECIMAL, DECIMAL, DECIMAL, DECIMAL, DECIMAL, DECIMAL, DECIMAL, DECIMAL, DECIMAL, DECIMAL, DECIMAL, INT
+====
+---- QUERY
+create table date_year_part (i int, d date)
+partitioned by spec (year(d))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into date_year_part values (1, '1968-01-01 01:02:03'),
+ (2, '1969-12-31 23:59:00'), (3, '1970-01-01 00:00:00'),
+ (4, '2023-11-15 15:31:00');
+====
+---- QUERY
+delete from date_year_part where i > 0;
+select * from date_year_part;
+---- RESULTS
+---- TYPES
+INT, DATE
+====
+---- QUERY
+create table date_month_part (i int, d date)
+partitioned by spec (month(d))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into date_month_part values (1, '1968-01-01'),
+ (2, '1969-12-31'), (3, '1970-01-01'), (4, '2023-11-15');
+delete from date_month_part where i > 0;
+select * from date_month_part;
+---- RESULTS
+---- TYPES
+INT, DATE
+====
+---- QUERY
+create table date_day_part (i int, d date)
+partitioned by spec (day(d))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into date_day_part values (1, '1968-01-01'),
+ (2, '1969-12-31'), (3, '1970-01-01'), (4, '2023-11-15');
+delete from date_day_part where i > 0;
+select * from date_day_part;
+---- RESULTS
+---- TYPES
+INT, DATE
+====
+---- QUERY
+create table ts_year_part (i int, ts timestamp)
+partitioned by spec (year(ts))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into ts_year_part values (1, '1968-01-01'),
+ (2, '1969-12-31'), (3, '1970-01-01'), (4, '2023-11-15');
+delete from ts_year_part where i > 0;
+select * from ts_year_part;
+---- RESULTS
+---- TYPES
+INT, TIMESTAMP
+====
+---- QUERY
+create table ts_month_part (i int, ts timestamp)
+partitioned by spec (month(ts))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into ts_month_part values (1, '1968-01-01 01:02:03'),
+ (2, '1969-12-31 23:59:00'), (3, '1970-01-01 00:00:00'),
+ (4, '2023-11-15 15:31:00');
+delete from ts_month_part where i > 0;
+select * from ts_month_part;
+---- RESULTS
+---- TYPES
+INT, TIMESTAMP
+====
+---- QUERY
+create table ts_day_part (i int, ts timestamp)
+partitioned by spec (day(ts))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into ts_day_part values (1, '1968-01-01 01:02:03'),
+ (2, '1969-12-31 23:59:00'), (3, '1970-01-01 00:00:00'),
+ (4, '2023-11-15 15:31:00');
+delete from ts_day_part where i > 0;
+select * from ts_day_part;
+---- RESULTS
+---- TYPES
+INT, TIMESTAMP
+====
+---- QUERY
+create table ts_hour_part (i int, ts timestamp)
+partitioned by spec (hour(ts))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into ts_hour_part values (1, '1968-01-01 01:02:03'),
+ (2, '1969-12-31 23:59:00'), (3, '1970-01-01 00:00:00'),
+ (4, '2023-11-15 15:31:00');
+delete from ts_hour_part where i > 0;
+select * from ts_hour_part;
+---- RESULTS
+---- TYPES
+INT, TIMESTAMP
+====
+---- QUERY
+create table ts_evolve_part (i int, ts timestamp)
+partitioned by spec (year(ts))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into ts_evolve_part values (1, '1968-01-01 01:02:03');
+alter table ts_evolve_part set partition spec (month(ts));
+insert into ts_evolve_part values (2, '1969-12-31 23:59:00');
+insert into ts_evolve_part values (111, 'invalid');
+alter table ts_evolve_part set partition spec (day(ts));
+insert into ts_evolve_part values (3, '1970-01-01 00:00:00');
+alter table ts_evolve_part set partition spec (hour(ts));
+insert into ts_evolve_part values (4, '2023-11-15 15:31:00');
+delete from ts_evolve_part where i > 0;
+select * from ts_evolve_part;
+---- RESULTS
+---- TYPES
+INT, TIMESTAMP
+====
+---- QUERY
+create table numeric_truncate (id int, int_col int, bigint_col bigint, dec_8_0 decimal(8, 0), dec_10_2 decimal(10, 2))
+partitioned by spec (truncate(10, int_col), truncate(1000, bigint_col), void(id), truncate(20, dec_8_0), truncate(50, dec_10_2))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into numeric_truncate values (1, 12, 1222, 135, 20.75);
+delete from numeric_truncate where id = 1;
+---- DML_RESULTS: numeric_truncate
+---- TYPES
+INT,INT,BIGINT,DECIMAL,DECIMAL
+====
+---- QUERY
+create table ice_alltypes_part_v2 (i INT NULL, p_bool BOOLEAN NULL, p_int INT NULL, p_bigint BIGINT NULL,
+ p_float FLOAT NULL, p_double DOUBLE NULL, p_decimal DECIMAL(6,3) NULL, p_date DATE NULL, p_string STRING NULL)
+PARTITIONED BY SPEC (p_bool, p_int, p_bigint, p_float, p_double, p_decimal, p_date, p_string)
+STORED AS ICEBERG
+TBLPROPERTIES ('format-version'='3');
+insert into ice_alltypes_part_v2 select * from functional_parquet.iceberg_alltypes_part;
+---- DML_RESULTS: ice_alltypes_part_v2
+1,true,1,11,1.100000023841858,2.222,123.321,2022-02-22,'impala'
+2,true,1,11,1.100000023841858,2.222,123.321,2022-02-22,'impala'
+---- TYPES
+INT, BOOLEAN, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, DATE, STRING
+====
+---- QUERY
+delete from ice_alltypes_part_v2 where i=1;
+---- DML_RESULTS: ice_alltypes_part_v2
+2,true,1,11,1.100000023841858,2.222,123.321,2022-02-22,'impala'
+---- TYPES
+INT, BOOLEAN, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, DATE, STRING
+====
+---- QUERY
+# Create table to test IMPALA-13768. It will have multiple partitions, with
+# multiple data files in each partition, where min/max ranges of column 'bi'
+# is disjunct for data files (in same partition).
+create table ice_invalid_deletes (bi bigint, year int)
+partitioned by spec (year)
+stored as iceberg tblproperties ('format-version'='3');
+insert into ice_invalid_deletes
+ select bigint_col, year from functional.alltypes where month = 10;
+with v as (select max(bi) as max_bi from ice_invalid_deletes)
+insert into ice_invalid_deletes select bi + v.max_bi, year from v, ice_invalid_deletes;
+delete from ice_invalid_deletes where bi % 11 = 0;
+====
+---- QUERY
+# Scan single data file (single channel in EXCHANGE sender) with delete file
+# that have references to other data files.
+select count(*) from ice_invalid_deletes where year=2010 and bi = 180;
+---- RESULTS
+31
+---- TYPES
+BIGINT
+====
+---- QUERY
+# Scan single data file per partition with delete files that reference other data files.
+select count(*) from ice_invalid_deletes where year>2000 and bi = 180;
+---- RESULTS
+62
+---- TYPES
+BIGINT
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-delete.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-delete.test
index 12c292c20..fd0bc7780 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-delete.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-delete.test
@@ -421,3 +421,119 @@ DELETE FROM ice_v3_upgraded_from_v2 WHERE id = 1;
---- CATCH
DELETE is not allowed on Iceberg format version 3 table
====
+---- QUERY
+CREATE TABLE ice_delete (i int, s string)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+====
+---- QUERY
+# Delete from empty table is no-op.
+DELETE FROM ice_delete where i = 1;
+SELECT * FROM ice_delete;
+---- RESULTS
+---- TYPES
+INT,STRING
+====
+---- QUERY
+INSERT INTO ice_delete VALUES(1, 'one'), (2, 'two'), (3, 'three');
+DELETE FROM ice_delete WHERE i = 2;
+SELECT * FROM ice_delete;
+---- RESULTS
+1,'one'
+3,'three'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+SELECT count(*) FROM ice_delete;
+---- RESULTS
+2
+---- TYPES
+BIGINT
+====
+---- QUERY
+INSERT INTO ice_delete VALUES (4, 'four'), (5, 'five'), (6, 'six');
+SELECT * FROM ice_delete;
+---- RESULTS
+1,'one'
+3,'three'
+4,'four'
+5,'five'
+6,'six'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+DELETE FROM ice_delete WHERE s like 'f%' and i > 4;
+SELECT * FROM ice_delete;
+---- RESULTS
+1,'one'
+3,'three'
+4,'four'
+6,'six'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+INSERT INTO ice_delete VALUES (7, 'seven'), (8, 'eight');
+DELETE FROM ice_delete WHERE i in (SELECT i FROM ice_delete where s in ('one', 'three'));
+SELECT * FROM ice_delete;
+---- RESULTS
+4,'four'
+6,'six'
+7,'seven'
+8,'eight'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+DELETE FROM ice_delete WHERE FILE__POSITION = 0;
+SELECT * FROM ice_delete;
+---- RESULTS
+6,'six'
+8,'eight'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+INSERT INTO ice_delete VALUES (9, 'nine'), (10, 'ten');
+DELETE FROM ice_delete WHERE s = (SELECT min(s) FROM ice_delete);
+SELECT * FROM ice_delete;
+---- RESULTS
+6,'six'
+9,'nine'
+10,'ten'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+DELETE FROM ice_delete WHERE i < 10;
+SELECT * FROM ice_delete;
+---- RESULTS
+10,'ten'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+DELETE FROM ice_delete WHERE i = 1000;
+SELECT * FROM ice_delete;
+---- RESULTS
+10,'ten'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+CREATE TABLE ice_lineitem STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3')
+AS SELECT * FROM tpch_parquet.lineitem;
+DELETE FROM ice_lineitem WHERE l_orderkey % 5 = 1;
+SELECT count(*) FROM ice_lineitem;
+---- RESULTS
+4799418
+---- TYPES
+BIGINT
+====
+---- QUERY
+SELECT * FROM ice_lineitem WHERE l_orderkey % 5 = 1;
+---- RESULTS
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-update-partitions.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-update-partitions.test
new file mode 100644
index 000000000..e24a54f50
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-update-partitions.test
@@ -0,0 +1,498 @@
+====
+---- QUERY
+CREATE TABLE id_part (i int, s string)
+PARTITIONED BY SPEC (i)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+====
+---- QUERY
+UPDATE id_part SET i = 1;
+---- DML_RESULTS: id_part
+---- TYPES
+INT,STRING
+====
+---- QUERY
+SHOW FILES IN id_part;
+---- RESULTS
+---- TYPES
+STRING, STRING, STRING, STRING
+====
+---- QUERY
+INSERT INTO id_part VALUES(1, 'one'), (2, 'two'), (3, 'three');
+UPDATE id_part SET i = cast(i * 10 as int);
+---- DML_RESULTS: id_part
+10,'one'
+20,'two'
+30,'three'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+INSERT INTO id_part VALUES(4, 'four'), (5, 'five'), (6, 'six');
+UPDATE id_part SET i = cast(i / 2 as int), s = upper(s);
+---- DML_RESULTS: id_part
+5,'ONE'
+10,'TWO'
+15,'THREE'
+2,'FOUR'
+2,'FIVE'
+3,'SIX'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+CREATE TABLE trunc_part (i int, s string)
+PARTITIONED BY SPEC (truncate(1, s))
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+====
+---- QUERY
+# Delete from empty table is no-op.
+UPDATE trunc_part SET i = 1;
+---- DML_RESULTS: trunc_part
+---- TYPES
+INT,STRING
+====
+---- QUERY
+INSERT INTO trunc_part VALUES(1, 'one'), (2, 'two'), (3, 'three');
+UPDATE trunc_part SET i = cast(i + 100 as int) WHERE s like 't%';
+---- DML_RESULTS: trunc_part
+1,'one'
+102,'two'
+103,'three'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+INSERT INTO trunc_part VALUES(4, 'four'), (5, 'five'), (6, 'six');
+UPDATE trunc_part SET s = concat(upper(s), s), i = cast(i + 1000 as int) WHERE i % 2 = 0;
+---- DML_RESULTS: trunc_part
+1,'one'
+1102,'TWOtwo'
+103,'three'
+1004,'FOURfour'
+5,'five'
+1006,'SIXsix'
+---- TYPES
+INT,STRING
+====
+---- QUERY
+CREATE TABLE multi_part (i int, s string, f double)
+PARTITIONED BY SPEC (bucket(7, i), truncate(1, s))
+SORT BY (f, i)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+====
+---- QUERY
+# Delete from empty table is no-op.
+UPDATE multi_part SET i = 1;
+---- DML_RESULTS: multi_part
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+INSERT INTO multi_part VALUES(1, 'one', 1.1), (2, 'two', 2.2), (3, 'three', 3.33);
+UPDATE multi_part
+SET s = concat(s, s), f = 9.9, i = cast(i + 10 as int)
+WHERE i != (select min(i) from multi_part) and
+ i != (select max(i) from multi_part);
+---- DML_RESULTS: multi_part
+1,'one',1.1
+12,'twotwo',9.9
+3,'three', 3.33
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+INSERT INTO multi_part VALUES(4, 'four', 4.4), (5, 'five', 5.5), (6, 'six', 6.6);
+UPDATE multi_part SET i = 0, s = 'void', f = 3.14 WHERE i % 2 = 0;
+---- DML_RESULTS: multi_part
+1,'one',1.1
+0,'void',3.14
+3,'three', 3.33
+0,'void',3.14
+5,'five',5.5
+0,'void',3.14
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+# No matching rows to WHERE condition
+UPDATE multi_part SET i = 0, s = 'void', f = 3.14
+WHERE f > 1000;
+---- DML_RESULTS: multi_part
+1,'one',1.1
+0,'void',3.14
+3,'three',3.33
+0,'void',3.14
+5,'five',5.5
+0,'void',3.14
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+INSERT INTO multi_part VALUES (null, 'null',0.0);
+---- DML_RESULTS: multi_part
+1,'one',1.1
+0,'void',3.14
+3,'three',3.33
+0,'void',3.14
+5,'five',5.5
+0,'void',3.14
+NULL,'null',0.0
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+UPDATE multi_part SET i = 111, s = 'fox', f = 1.1 where i is null;
+---- DML_RESULTS: multi_part
+1,'one',1.1
+0,'void',3.14
+3,'three',3.33
+0,'void',3.14
+5,'five',5.5
+0,'void',3.14
+111,'fox',1.1
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+CREATE TABLE evolve_part (i int, s string, f double)
+SORT BY ZORDER(i,s)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+====
+---- QUERY
+INSERT INTO evolve_part VALUES(1, 'one', 1.1), (2, 'two', 2.2), (3, 'three', 3.33);
+UPDATE evolve_part SET i = 222 WHERE i = 2;
+---- DML_RESULTS: evolve_part
+1,'one',1.1
+222,'two',2.2
+3,'three',3.33
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+ALTER TABLE evolve_part SET PARTITION SPEC (i);
+SELECT * FROM evolve_part;
+---- RESULTS
+1,'one',1.1
+222,'two',2.2
+3,'three',3.33
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+INSERT INTO evolve_part VALUES (10, 'ten', 10.10), (20, 'twenty', 20.20);
+---- DML_RESULTS: evolve_part
+1,'one',1.1
+222,'two',2.2
+3,'three',3.33
+10,'ten',10.10
+20,'twenty',20.20
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+UPDATE evolve_part SET i = cast(i + 1000 as int) where s like 't%';
+---- DML_RESULTS: evolve_part
+1,'one',1.1
+1222,'two',2.2
+1003,'three',3.33
+1010,'ten',10.10
+1020,'twenty',20.20
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+ALTER TABLE evolve_part SET PARTITION SPEC (truncate(1, s));
+---- DML_RESULTS: evolve_part
+1,'one',1.1
+1222,'two',2.2
+1003,'three',3.33
+1010,'ten',10.10
+1020,'twenty',20.20
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+INSERT INTO evolve_part VALUES (30, 'thirty', 30.30), (40, 'forty', 40.40), (50, 'fifty', 50.50);
+---- DML_RESULTS: evolve_part
+1,'one',1.1
+1222,'two',2.2
+1003,'three',3.33
+1010,'ten',10.10
+1020,'twenty',20.20
+30,'thirty',30.30
+40,'forty',40.40
+50,'fifty',50.50
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+UPDATE evolve_part SET i = cast(i + 100 as int), s = concat('+++', s);
+---- DML_RESULTS: evolve_part
+101,'+++one',1.1
+1322,'+++two',2.2
+1103,'+++three',3.33
+1110,'+++ten',10.10
+1120,'+++twenty',20.20
+130,'+++thirty',30.30
+140,'+++forty',40.40
+150,'+++fifty',50.50
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+ALTER TABLE evolve_part SET PARTITION SPEC (void(s));
+---- DML_RESULTS: evolve_part
+101,'+++one',1.1
+1322,'+++two',2.2
+1103,'+++three',3.33
+1110,'+++ten',10.10
+1120,'+++twenty',20.20
+130,'+++thirty',30.30
+140,'+++forty',40.40
+150,'+++fifty',50.50
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+UPDATE evolve_part SET s = substr(s, 4), i = cast(i - 100 as int);
+---- DML_RESULTS: evolve_part
+1,'one',1.1
+1222,'two',2.2
+1003,'three',3.33
+1010,'ten',10.10
+1020,'twenty',20.20
+30,'thirty',30.30
+40,'forty',40.40
+50,'fifty',50.50
+---- TYPES
+INT,STRING,DOUBLE
+====
+---- QUERY
+create table date_year_part (i int, d date)
+partitioned by spec (year(d))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into date_year_part values
+ (1, '1968-01-01'), (2, '1969-12-31'), (3, '1970-01-01'), (4, '2023-11-15');
+UPDATE date_year_part SET d = years_add(d, 10), i = cast(i + 10 as int);
+---- DML_RESULTS: date_year_part
+11,1978-01-01
+12,1979-12-31
+13,1980-01-01
+14,2033-11-15
+---- TYPES
+INT, DATE
+====
+---- QUERY
+create table date_month_part (i int, d date)
+partitioned by spec (month(d))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into date_month_part values
+ (1, '1968-01-01'), (2, '1969-12-31'), (3, '1970-01-01'), (4, '2023-11-15');
+UPDATE date_month_part SET d = years_add(d, 10), i = cast(i + 10 as int);
+---- DML_RESULTS: date_month_part
+11,1978-01-01
+12,1979-12-31
+13,1980-01-01
+14,2033-11-15
+---- TYPES
+INT, DATE
+====
+---- QUERY
+create table date_day_part (i int, d date)
+partitioned by spec (day(d))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into date_day_part values
+ (1, '1968-01-01'), (2, '1969-12-31'), (3, '1970-01-01'), (4, '2023-11-15');
+UPDATE date_day_part SET d = years_add(d, 10), i = cast(i + 10 as int);
+---- DML_RESULTS: date_day_part
+11,1978-01-01
+12,1979-12-31
+13,1980-01-01
+14,2033-11-15
+---- TYPES
+INT, DATE
+====
+---- QUERY
+create table ts_year_part (i int, ts timestamp)
+partitioned by spec (year(ts))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into ts_year_part values (1, '1968-01-01'),
+ (2, '1969-12-31'), (3, '1970-01-01'), (4, '2023-11-15');
+UPDATE ts_year_part SET ts = years_add(ts, -10), i = cast(i + 100 as int);
+---- DML_RESULTS: ts_year_part
+101,1958-01-01 00:00:00
+102,1959-12-31 00:00:00
+103,1960-01-01 00:00:00
+104,2013-11-15 00:00:00
+---- TYPES
+INT, TIMESTAMP
+====
+---- QUERY
+create table ts_month_part (i int, ts timestamp)
+partitioned by spec (month(ts))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into ts_month_part values (1, '1968-01-01 01:02:03'),
+ (2, '1969-12-31 23:59:00'), (3, '1970-01-01 00:00:00'),
+ (4, '2023-11-15 15:31:00');
+UPDATE ts_month_part SET ts = years_add(ts, -10), i = cast(i + 100 as int);
+---- DML_RESULTS: ts_month_part
+101,1958-01-01 01:02:03
+102,1959-12-31 23:59:00
+103,1960-01-01 00:00:00
+104,2013-11-15 15:31:00
+---- TYPES
+INT, TIMESTAMP
+====
+---- QUERY
+create table ts_day_part (i int, ts timestamp)
+partitioned by spec (day(ts))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into ts_day_part values (1, '1968-01-01 01:02:03'),
+ (2, '1969-12-31 23:59:00'), (3, '1970-01-01 00:00:00'),
+ (4, '2023-11-15 15:31:00');
+UPDATE ts_day_part SET ts = years_add(ts, -10), i = cast(i + 100 as int);
+---- DML_RESULTS: ts_day_part
+101,1958-01-01 01:02:03
+102,1959-12-31 23:59:00
+103,1960-01-01 00:00:00
+104,2013-11-15 15:31:00
+---- TYPES
+INT, TIMESTAMP
+====
+---- QUERY
+create table ts_hour_part (i int, ts timestamp)
+partitioned by spec (hour(ts))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into ts_hour_part values (1, '1968-01-01 01:02:03'),
+ (2, '1969-12-31 23:59:00'), (3, '1970-01-01 00:00:00'),
+ (4, '2023-11-15 15:31:00');
+UPDATE ts_hour_part SET ts = years_add(ts, -10), i = cast(i + 100 as int);;
+---- DML_RESULTS: ts_hour_part
+101,1958-01-01 01:02:03
+102,1959-12-31 23:59:00
+103,1960-01-01 00:00:00
+104,2013-11-15 15:31:00
+---- TYPES
+INT, TIMESTAMP
+====
+---- QUERY
+create table ts_evolve_part (i int, ts timestamp)
+partitioned by spec (year(ts))
+sort by (ts, i)
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into ts_evolve_part values (1, '1968-01-01 01:02:03');
+alter table ts_evolve_part set partition spec (month(ts));
+insert into ts_evolve_part values (2, '1969-12-31 23:59:00');
+insert into ts_evolve_part values (111, 'invalid');
+alter table ts_evolve_part set partition spec (day(ts));
+insert into ts_evolve_part values (3, '1970-01-01 00:00:00');
+alter table ts_evolve_part set partition spec (hour(ts));
+insert into ts_evolve_part values (4, '2023-11-15 15:31:00');
+---- DML_RESULTS: ts_evolve_part
+1,1968-01-01 01:02:03
+2,1969-12-31 23:59:00
+3,1970-01-01 00:00:00
+4,2023-11-15 15:31:00
+111,NULL
+---- TYPES
+INT, TIMESTAMP
+====
+---- QUERY
+UPDATE ts_evolve_part set i = cast(i + 1000 as int), ts = days_add(months_add(years_add(ts, 20), 1), 1);
+---- DML_RESULTS: ts_evolve_part
+1001,1988-02-02 01:02:03
+1002,1990-02-01 23:59:00
+1003,1990-02-02 00:00:00
+1004,2043-12-16 15:31:00
+1111,NULL
+---- TYPES
+INT, TIMESTAMP
+====
+---- QUERY
+create table numeric_truncate (id int, int_col int, bigint_col bigint, dec_8_0 decimal(8, 0), dec_10_2 decimal(10, 2))
+partitioned by spec (truncate(10, int_col), truncate(1000, bigint_col), void(id), truncate(20, dec_8_0), truncate(50, dec_10_2))
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into numeric_truncate values (1, 12, 1222, 135, 20.75);
+UPDATE numeric_truncate SET dec_10_2 = 75.20, dec_8_0 = 531, bigint_col = 2111, int_col = 21, id = 11;
+---- DML_RESULTS: numeric_truncate
+11,21,2111,531,75.20
+---- TYPES
+INT,INT,BIGINT,DECIMAL,DECIMAL
+====
+---- QUERY
+create table ice_alltypes_part_v2 (i INT NULL, p_bool BOOLEAN NULL, p_int INT NULL, p_bigint BIGINT NULL,
+ p_float FLOAT NULL, p_double DOUBLE NULL, p_decimal DECIMAL(6,3) NULL, p_date DATE NULL, p_string STRING NULL)
+PARTITIONED BY SPEC (p_bool, p_int, p_bigint, p_float, p_double, p_decimal, p_date, p_string)
+STORED AS ICEBERG
+TBLPROPERTIES ('format-version'='3');
+insert into ice_alltypes_part_v2 select * from functional_parquet.iceberg_alltypes_part;
+---- DML_RESULTS: ice_alltypes_part_v2
+1,true,1,11,1.100000023841858,2.222,123.321,2022-02-22,'impala'
+2,true,1,11,1.100000023841858,2.222,123.321,2022-02-22,'impala'
+---- TYPES
+INT, BOOLEAN, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, DATE, STRING
+====
+---- QUERY
+update ice_alltypes_part_v2 set i = cast(i + 1 as int);
+update ice_alltypes_part_v2 set p_int = i;
+update ice_alltypes_part_v2 set p_date = add_months(p_date, i);
+---- DML_RESULTS: ice_alltypes_part_v2
+2,true,2,11,1.100000023841858,2.222,123.321,2022-04-22,'impala'
+3,true,3,11,1.100000023841858,2.222,123.321,2022-05-22,'impala'
+---- TYPES
+INT, BOOLEAN, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, DATE, STRING
+====
+---- QUERY
+# Regression test for IMPALA-13655.
+# Updating a table with many partitions (with low mem_limit).
+# Then we also check the number of files to verify that
+# the inputs of the writers are actually sorted(/clustered).
+# The table contains 401 files before the update (1 per partition),
+# then the update writes a delete and a data file to each partition.
+CREATE TABLE ice_tpch_many_parts
+PARTITIONED BY SPEC(truncate(500, l_partkey))
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3')
+AS SELECT * FROM tpch_parquet.lineitem
+WHERE l_linenumber=1;
+
+SET MEM_LIMIT=470m;
+UPDATE ice_tpch_many_parts SET l_orderkey=l_orderkey+1;
+SELECT count(*) FROM $DATABASE.ice_tpch_many_parts.`files`;
+---- RESULTS
+1203
+---- TYPES
+BIGINT
+====
+---- QUERY
+# Regression test: UPDATE on V3 table with partition evolution.
+# createDeletionVector() must use the data file's original spec, not the current spec.
+CREATE TABLE v3_evolve_part (i int, s string)
+STORED BY ICEBERG
+TBLPROPERTIES ('format-version'='3');
+INSERT INTO v3_evolve_part VALUES(1, 'one'), (2, 'two'), (3, 'three');
+ALTER TABLE v3_evolve_part SET PARTITION SPEC (i);
+INSERT INTO v3_evolve_part VALUES (10, 'ten'), (20, 'twenty');
+UPDATE v3_evolve_part SET i = cast(i * 10 as int);
+---- DML_RESULTS: v3_evolve_part
+10,'one'
+20,'two'
+30,'three'
+100,'ten'
+200,'twenty'
+---- TYPES
+INT,STRING
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-update.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-update.test
index 4bc53b432..4960fb3e2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-update.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-update.test
@@ -241,3 +241,491 @@ SELECT * FROM update_v3_pos_del ORDER BY i;
---- TYPES
INT,STRING
====
+---- QUERY
+create table single_col (i int)
+stored by iceberg
+tblproperties ('format-version'='3');
+update single_col set i = 4;
+---- DML_RESULTS: single_col
+---- TYPES
+INT
+---- RUNTIME_PROFILE
+NumModifiedRows: 0
+NumDeletedRows: 0
+====
+---- QUERY
+insert into single_col values (1), (2), (3);
+update single_col set i = cast(i + 1 as int);
+---- DML_RESULTS: single_col
+2
+3
+4
+---- TYPES
+INT
+---- RUNTIME_PROFILE
+NumModifiedRows: 3
+NumDeletedRows: 3
+====
+---- QUERY
+update single_col set i = 1 where i = 2
+---- DML_RESULTS: single_col
+1
+3
+4
+---- TYPES
+INT
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+create table ice_alltypes (bool_col boolean, int_col int, bigint_col bigint, float_col float,
+ double_col double, dec_10_0_col decimal(10,0), dec_8_2_col decimal(8,2), date_col date,
+ timestamp_col timestamp, string_col string, binary_col binary)
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into ice_alltypes values (false, 0, 0, 0, 0, 0, 0, '2000-01-01', '2000-01-01 00:00:00',
+ 'zero', cast('zerob' as binary));
+---- DML_RESULTS: ice_alltypes
+false,0,0,0,0,0,0.00,2000-01-01,2000-01-01 00:00:00,'zero','zerob'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 0
+====
+---- QUERY
+update ice_alltypes set bool_col = true;
+---- DML_RESULTS: ice_alltypes
+true,0,0,0,0,0,0.00,2000-01-01,2000-01-01 00:00:00,'zero','zerob'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+update ice_alltypes set int_col = 1;
+---- DML_RESULTS: ice_alltypes
+true,1,0,0,0,0,0.00,2000-01-01,2000-01-01 00:00:00,'zero','zerob'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+update ice_alltypes set bigint_col = 1;
+---- DML_RESULTS: ice_alltypes
+true,1,1,0,0,0,0.00,2000-01-01,2000-01-01 00:00:00,'zero','zerob'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+update ice_alltypes set float_col = 1;
+---- DML_RESULTS: ice_alltypes
+true,1,1,1,0,0,0.00,2000-01-01,2000-01-01 00:00:00,'zero','zerob'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+update ice_alltypes set double_col = 1;
+---- DML_RESULTS: ice_alltypes
+true,1,1,1,1,0,0.00,2000-01-01,2000-01-01 00:00:00,'zero','zerob'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+update ice_alltypes set dec_10_0_col = 1;
+---- DML_RESULTS: ice_alltypes
+true,1,1,1,1,1,0.00,2000-01-01,2000-01-01 00:00:00,'zero','zerob'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+update ice_alltypes set dec_8_2_col = 1;
+---- DML_RESULTS: ice_alltypes
+true,1,1,1,1,1,1.00,2000-01-01,2000-01-01 00:00:00,'zero','zerob'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+update ice_alltypes set date_col = '2001-01-01';
+---- DML_RESULTS: ice_alltypes
+true,1,1,1,1,1,1.00,2001-01-01,2000-01-01 00:00:00,'zero','zerob'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+update ice_alltypes set timestamp_col = '2001-01-01 01:01:01';
+---- DML_RESULTS: ice_alltypes
+true,1,1,1,1,1,1.00,2001-01-01,2001-01-01 01:01:01,'zero','zerob'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+update ice_alltypes set string_col = 'one';
+---- DML_RESULTS: ice_alltypes
+true,1,1,1,1,1,1.00,2001-01-01,2001-01-01 01:01:01,'one','zerob'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+update ice_alltypes set binary_col = cast('oneb' as binary);
+---- DML_RESULTS: ice_alltypes
+true,1,1,1,1,1,1.00,2001-01-01,2001-01-01 01:01:01,'one','oneb'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+update ice_alltypes set bigint_col = 33, int_col = 30, string_col = 'three';
+---- DML_RESULTS: ice_alltypes
+true,30,33,1,1,1,1.00,2001-01-01,2001-01-01 01:01:01,'three','oneb'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+# The row already has the desired values, no need to update. (IMPALA-12588)
+update ice_alltypes set bigint_col = 33, int_col = 30, string_col = 'three';
+---- DML_RESULTS: ice_alltypes
+true,30,33,1,1,1,1.00,2001-01-01,2001-01-01 01:01:01,'three','oneb'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 0
+NumDeletedRows: 0
+====
+---- QUERY
+# Only one field changes.
+update ice_alltypes set bigint_col = 33, int_col = 3, string_col = 'three';
+---- DML_RESULTS: ice_alltypes
+true,3,33,1,1,1,1.00,2001-01-01,2001-01-01 01:01:01,'three','oneb'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+update ice_alltypes set dec_10_0_col = 23, dec_8_2_col = cast(123.123 as decimal(8, 2));
+---- DML_RESULTS: ice_alltypes
+true,3,33,1,1,23,123.12,2001-01-01,2001-01-01 01:01:01,'three','oneb'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+# UPDATE table based on a reference table.
+insert into ice_alltypes values (false, 0, 0, 0, 0, 0, 0, '2000-01-01', '2000-01-01 00:00:00',
+ 'zero', cast('zerob' as binary));
+create table ref_table (i int, bi bigint, s string, d date);
+insert into ref_table values (0, 111, 'IMPALA', '2023-11-07'), (3, 222, 'ICEBERG', '2023-11-08');
+update ice_alltypes set bigint_col=bi, string_col=s, date_col=d from ice_alltypes, ref_table where int_col = i;
+---- DML_RESULTS: ice_alltypes
+false,0,111,0,0,0,0.00,2023-11-07,2000-01-01 00:00:00,'IMPALA','zerob'
+true,3,222,1,1,23,123.12,2023-11-08,2001-01-01 01:01:01,'ICEBERG','oneb'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 2
+NumDeletedRows: 2
+====
+---- QUERY
+# If the JOIN in UPDATE has multiple matches Impala should raise an error.
+# Changing bigint_col to actually have rows to update.
+insert into ref_table values (0, 1111, 'IIMMPPAALLAA', '2023-12-01');
+update ice_alltypes set bigint_col=bi+1, string_col=s, date_col=d from ice_alltypes, ref_table where int_col = i;
+---- CATCH
+Duplicated row in DELETE sink.
+====
+---- QUERY
+# Check that NULL values and expressions evaluating to NULL in assignments are handled correctly.
+update ice_alltypes set dec_10_0_col = NULL, dec_8_2_col = cast(123.123 + NULL as decimal(8, 2));
+---- DML_RESULTS: ice_alltypes
+false,0,111,0,0,NULL,NULL,2023-11-07,2000-01-01 00:00:00,'IMPALA','zerob'
+true,3,222,1,1,NULL,NULL,2023-11-08,2001-01-01 01:01:01,'ICEBERG','oneb'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 2
+NumDeletedRows: 2
+====
+---- QUERY
+# Check that NULL assignments and WHERE condition filter rows correctly.
+update ice_alltypes set dec_10_0_col = cast(123.123 as decimal(10, 0)), dec_8_2_col = NULL where int_col < 2;
+---- DML_RESULTS: ice_alltypes
+false,0,111,0,0,123,NULL,2023-11-07,2000-01-01 00:00:00,'IMPALA','zerob'
+true,3,222,1,1,NULL,NULL,2023-11-08,2001-01-01 01:01:01,'ICEBERG','oneb'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+# Swap two columns
+update ice_alltypes set dec_10_0_col = cast(dec_8_2_col as decimal(10, 0)), dec_8_2_col = cast(dec_10_0_col as decimal(8, 2));
+---- DML_RESULTS: ice_alltypes
+false,0,111,0,0,NULL,123.00,2023-11-07,2000-01-01 00:00:00,'IMPALA','zerob'
+true,3,222,1,1,NULL,NULL,2023-11-08,2001-01-01 01:01:01,'ICEBERG','oneb'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+# Check col_a = col_b + col_c
+update ice_alltypes set dec_10_0_col = cast(bigint_col + dec_8_2_col as decimal(10, 0));
+---- DML_RESULTS: ice_alltypes
+false,0,111,0,0,234,123.00,2023-11-07,2000-01-01 00:00:00,'IMPALA','zerob'
+true,3,222,1,1,NULL,NULL,2023-11-08,2001-01-01 01:01:01,'ICEBERG','oneb'
+---- TYPES
+BOOLEAN,INT,BIGINT,FLOAT,DOUBLE,DECIMAL,DECIMAL,DATE,TIMESTAMP,STRING,BINARY
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+create table ice_id_partitioned (i int, p int, s string)
+partitioned by spec(p)
+stored by iceberg
+tblproperties('format-version'='3');
+insert into ice_id_partitioned values
+(1, 0, 'impala'), (2, 0, 'iceberg'), (3, 0, 'hive'), (4, 1, 'spark'),
+(5, 2, 'kudu');
+update ice_id_partitioned set s='Impala' where i = 1;
+update ice_id_partitioned set s='Kudu' where i = 5;
+---- DML_RESULTS: ice_id_partitioned
+1,0,'Impala'
+2,0,'iceberg'
+3,0,'hive'
+4,1,'spark'
+5,2,'Kudu'
+---- TYPES
+INT,INT,STRING
+====
+---- QUERY
+# Update partitioned Iceberg table based on VIEW
+create table ref_table_1 (ri int, rs string);
+insert into ref_table_1 values (1, 'Apache Impala');
+create table ref_table_2 (ri int, rs string);
+insert into ref_table_2 values (4, 'Apache Spark');
+create view ref_view as select * from ref_table_1 union select * from ref_table_2;
+update ice_id_partitioned SET s = rs
+FROM ice_id_partitioned JOIN ref_view ON (i = ri);
+---- DML_RESULTS: ice_id_partitioned
+1,0,'Apache Impala'
+2,0,'iceberg'
+3,0,'hive'
+4,1,'Apache Spark'
+5,2,'Kudu'
+---- TYPES
+INT,INT,STRING
+====
+---- QUERY
+# Update partitioned Iceberg table based on VIEW
+UPDATE ice_id_partitioned SET s = upper(rs)
+FROM ice_id_partitioned JOIN ref_view ON (i = ri)
+WHERE p < 1 AND rs = (select min(rs) from ref_view);
+---- DML_RESULTS: ice_id_partitioned
+1,0,'APACHE IMPALA'
+2,0,'iceberg'
+3,0,'hive'
+4,1,'Apache Spark'
+5,2,'Kudu'
+---- TYPES
+INT,INT,STRING
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+# Test if changing SKIP_UNNEEDED_UPDATES_COL_LIMIT query option takes effect: the 4th row
+# is updated even though the table content remained the same.
+SET SKIP_UNNEEDED_UPDATES_COL_LIMIT = 1;
+UPDATE ice_id_partitioned SET i=ri, p=1
+FROM ice_id_partitioned JOIN ref_table_2 ON (i = ri);
+---- DML_RESULTS: ice_id_partitioned
+1,0,'APACHE IMPALA'
+2,0,'iceberg'
+3,0,'hive'
+4,1,'Apache Spark'
+5,2,'Kudu'
+---- TYPES
+INT,INT,STRING
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+# Setting SKIP_UNNEEDED_UPDATES_COL_LIMIT back to default. No rows should be updated.
+SET SKIP_UNNEEDED_UPDATES_COL_LIMIT = 10;
+UPDATE ice_id_partitioned SET i=ri, p=1
+FROM ice_id_partitioned JOIN ref_table_2 ON (i = ri);
+---- DML_RESULTS: ice_id_partitioned
+1,0,'APACHE IMPALA'
+2,0,'iceberg'
+3,0,'hive'
+4,1,'Apache Spark'
+5,2,'Kudu'
+---- TYPES
+INT,INT,STRING
+---- RUNTIME_PROFILE
+aggregation(SUM, RowsInserted): 0
+====
+---- QUERY
+# Update partitioned Iceberg table with inline view in FROM clause
+UPDATE ice_id_partitioned SET i=cast(a.max_val*(p+1) as int)
+FROM ice_id_partitioned JOIN ref_view ON (i = ri), (SELECT max(i) max_val FROM ice_id_partitioned) a
+WHERE p = 1;
+---- DML_RESULTS: ice_id_partitioned
+1,0,'APACHE IMPALA'
+2,0,'iceberg'
+3,0,'hive'
+10,1,'Apache Spark'
+5,2,'Kudu'
+---- TYPES
+INT,INT,STRING
+---- RUNTIME_PROFILE
+NumModifiedRows: 1
+NumDeletedRows: 1
+====
+---- QUERY
+# Negative test for UPDATE part 3:
+# updating partition column AND right side is non-constant value AND we have a FROM clause with multiple table refs.
+# For such operations, if there are multiple matches in the JOIN, the duplicated records can
+# get shuffled independently to different sink operators, therefore they cannot check the
+# presence of duplicates. For statements like this we need to raise an error during analysis.
+UPDATE ice_id_partitioned set p = cast(ref_table.bi as int)
+FROM ice_id_partitioned, ref_table
+WHERE ice_id_partitioned.i = ref_table.i;
+---- CATCH
+AnalysisException: Cannot UPDATE partitioning column 'p' via UPDATE FROM statement with multiple table refs, and when right-hand side 'CAST(ref_table.bi AS INT)' is non-constant.
+====
+---- QUERY
+create table ice_bucket_transform(i int, str string, bi bigint, ts timestamp)
+partitioned by spec(bucket(91, str), bucket(199, bi), bucket(403, ts))
+stored by iceberg
+tblproperties('format-version'='3');
+insert into ice_bucket_transform values (1, 'a fairly long string value', 1000, '1999-09-19 12:00:01'),
+ (2, 'bbb', 2030, '2001-01-01 00:00:00'), (3, 'cccccccccccccccccccccccccccccccccccccccc', -123, '2023-11-24 17:44:30');
+update ice_bucket_transform set i = cast(i * 2 as int);
+---- DML_RESULTS: ice_bucket_transform
+2,'a fairly long string value',1000,1999-09-19 12:00:01
+4,'bbb',2030,2001-01-01 00:00:00
+6,'cccccccccccccccccccccccccccccccccccccccc',-123,2023-11-24 17:44:30
+---- TYPES
+INT,STRING,BIGINT,TIMESTAMP
+====
+---- QUERY
+create table ice_time_transforms_timestamp(id decimal(8, 4), ts1 timestamp, ts2 timestamp, ts3 timestamp, ts4 timestamp)
+partitioned by spec(year(ts1), month(ts2), day(ts3), hour(ts4))
+stored by iceberg
+tblproperties('format-version'='3');
+insert into ice_time_transforms_timestamp values
+ (0.75, '2001-01-01 01:01:01', '2001-01-01 01:01:01', '2001-01-01 01:01:01', '2001-01-01 01:01:01'),
+ (1.2345, '2023-11-24 18:02:00', '2023-11-24 18:02:00', '2023-11-24 18:02:00', '2023-11-24 18:02:00'),
+ (999.9999, '2199-12-31 23:59:59', '2199-12-31 23:59:59', '2199-12-31 23:59:59', '2199-12-31 23:59:59');
+update ice_time_transforms_timestamp set id = cast(id * 2 as decimal(8, 4));
+---- DML_RESULTS: ice_time_transforms_timestamp
+1.5000,2001-01-01 01:01:01,2001-01-01 01:01:01,2001-01-01 01:01:01,2001-01-01 01:01:01
+2.4690,2023-11-24 18:02:00,2023-11-24 18:02:00,2023-11-24 18:02:00,2023-11-24 18:02:00
+1999.9998,2199-12-31 23:59:59,2199-12-31 23:59:59,2199-12-31 23:59:59,2199-12-31 23:59:59
+---- TYPES
+DECIMAL,TIMESTAMP,TIMESTAMP,TIMESTAMP,TIMESTAMP
+====
+---- QUERY
+create table ice_time_transforms_date(id decimal(8, 4), ts1 DATE, ts2 DATE, ts3 DATE)
+partitioned by spec(year(ts1), month(ts2), day(ts3))
+stored by iceberg
+tblproperties('format-version'='3');
+insert into ice_time_transforms_date values
+ (0.75, '2001-01-01 01:01:01', '2001-01-01 01:01:01', '2001-01-01 01:01:01'),
+ (1.2345, '2023-11-24 18:02:00', '2023-11-24 18:02:00', '2023-11-24 18:02:00'),
+ (999.9999, '2199-12-31 23:59:59', '2199-12-31 23:59:59', '2199-12-31 23:59:59');
+update ice_time_transforms_date set id = cast(id * 2 as decimal(8, 4));
+---- DML_RESULTS: ice_time_transforms_date
+1.5000,2001-01-01,2001-01-01,2001-01-01
+2.4690,2023-11-24,2023-11-24,2023-11-24
+1999.9998,2199-12-31,2199-12-31,2199-12-31
+---- TYPES
+DECIMAL,DATE,DATE,DATE
+====
+---- QUERY
+create table ice_part_transforms (i int, ts timestamp, s string, p bigint)
+partitioned by spec(day(ts), truncate(1, s), truncate(1000, p))
+stored by iceberg
+tblproperties('format-version'='3');
+insert into ice_part_transforms values (1, '2023-11-13 18:07:05', 'blue', 1234),
+ (2, '2023-11-13 18:07:23', 'gray', 2500), (3, '2023-11-14 19:07:05', 'green', 1700),
+ (4, '2023-11-01 00:11:11', 'black', 722);
+====
+---- QUERY
+update ice_part_transforms set i = cast(i*2 as int) where i % 2 = 0;
+---- DML_RESULTS: ice_part_transforms
+1,2023-11-13 18:07:05,'blue',1234
+4,2023-11-13 18:07:23,'gray',2500
+3,2023-11-14 19:07:05,'green',1700
+8,2023-11-01 00:11:11,'black',722
+---- TYPES
+INT,TIMESTAMP,STRING,BIGINT
+====
+---- QUERY
+# Regression test for IMPALA-13770: using a native UDF in the target value of an UPDATE
+# Note: even with the incorrect behaviour before IMPALA-13770 the test may pass and the
+# values can also be updated successfully. This is because the crash happens after the
+# query is closed, when releasing resources. This test is a valid regression test because
+# even if it passes, there will be a crash: some later queries are likely to fail and
+# there will be a minidump in the build artifacts, so the build will be marked FAILED.
+# Also testing that rows which already have the desired value should not be skipped
+# because there is a UDF in the SET list.
+create function if not exists identity(int) returns int location 'UDF_LOCATION' symbol='Identity';
+create table update_with_udf(int_col INT)
+stored by iceberg
+tblproperties ('format-version'='3');
+insert into update_with_udf values (1), (2);
+====
+---- QUERY
+update update_with_udf set int_col = identity(int_col);
+---- DML_RESULTS: update_with_udf
+1
+2
+---- TYPES
+INT
+---- RUNTIME_PROFILE
+NumModifiedRows: 2
+NumDeletedRows: 2
+====
diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py
index ab3cd7719..22f18a936 100644
--- a/tests/query_test/test_iceberg.py
+++ b/tests/query_test/test_iceberg.py
@@ -1782,7 +1782,6 @@ class TestIcebergV2Table(IcebergTestSuite):
if IS_HDFS and self.should_run_for_hive(vector):
self._delete_partitioned_hive_tests(unique_database)
-
def _delete_partitioned_hive_tests(self, db):
hive_output = self.run_stmt_in_hive("SELECT * FROM {}.{} ORDER BY i".format(
db, "id_part"))
@@ -2350,6 +2349,9 @@ class TestIcebergV3Table(IcebergTestSuite):
self.run_test_case('QueryTest/iceberg-v3-delete-partition-sort',
vector, unique_database)
+ def test_v3_delete_partitioned(self, vector, unique_database):
+ self.run_test_case('QueryTest/iceberg-v3-delete-partitioned', vector, unique_database)
+
def test_v3_delete_on_existing_dv_table(self, vector, unique_database):
"""Test DELETE on a table that already has deletion vectors written by Spark.
Verifies that Impala correctly reads and extends pre-existing Puffin DV files
@@ -2398,7 +2400,13 @@ class TestIcebergV3Table(IcebergTestSuite):
os.remove(local_file)
def test_v3_update(self, vector, unique_database):
- self.run_test_case('QueryTest/iceberg-v3-update', vector, unique_database)
+ udf_location = get_fs_path('/test-warehouse/libTestUdfs.so')
+ self.run_test_case('QueryTest/iceberg-v3-update', vector, unique_database,
+ test_file_vars={'UDF_LOCATION': udf_location})
+
+ def test_v3_update_partitions(self, vector, unique_database):
+ self.run_test_case('QueryTest/iceberg-v3-update-partitions', vector,
+ unique_database)
def test_v3_optimize(self, vector, unique_database):
self.run_test_case('QueryTest/iceberg-v3-optimize', vector, unique_database)