You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ga...@apache.org on 2026/06/21 15:49:34 UTC
(iceberg-cpp) branch main updated: feat: add DeleteFiles update API (#709)
This is an automated email from the ASF dual-hosted git repository.
wgtmac pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 6f503f9a feat: add DeleteFiles update API (#709)
6f503f9a is described below
commit 6f503f9af75978a34b13dfab121a9cbdc9268b6a
Author: Manu Zhang <Ow...@gmail.com>
AuthorDate: Sun Jun 21 23:49:29 2026 +0800
feat: add DeleteFiles update API (#709)
## Summary
- Add a DeleteFiles snapshot update API and wire it through table and
transaction update flows.
- Add DeleteFiles implementation/build integration.
- Add coverage for path matching, case-insensitive row filters, empty
delete commits, and strict-projection partial-match rejection.
## Validation
- `cmake --build build --target table_update_test`
- `./build/src/iceberg/test/table_update_test
'--gtest_filter=DeleteFilesTest.*'`
Co-authored-by: Codex <co...@openai.com>
---
src/iceberg/CMakeLists.txt | 1 +
src/iceberg/meson.build | 1 +
src/iceberg/table.cc | 7 ++
src/iceberg/table.h | 3 +
src/iceberg/test/CMakeLists.txt | 1 +
src/iceberg/test/delete_files_test.cc | 203 ++++++++++++++++++++++++++++++++++
src/iceberg/transaction.cc | 8 ++
src/iceberg/transaction.h | 3 +
src/iceberg/type_fwd.h | 1 +
src/iceberg/update/delete_files.cc | 79 +++++++++++++
src/iceberg/update/delete_files.h | 72 ++++++++++++
src/iceberg/update/meson.build | 1 +
12 files changed, 380 insertions(+)
diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt
index d4224cd6..b3cc8a7a 100644
--- a/src/iceberg/CMakeLists.txt
+++ b/src/iceberg/CMakeLists.txt
@@ -99,6 +99,7 @@ set(ICEBERG_SOURCES
transform.cc
transform_function.cc
type.cc
+ update/delete_files.cc
update/expire_snapshots.cc
update/fast_append.cc
update/merge_append.cc
diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build
index 508d5e0b..0c467605 100644
--- a/src/iceberg/meson.build
+++ b/src/iceberg/meson.build
@@ -124,6 +124,7 @@ iceberg_sources = files(
'transform.cc',
'transform_function.cc',
'type.cc',
+ 'update/delete_files.cc',
'update/expire_snapshots.cc',
'update/fast_append.cc',
'update/merge_append.cc',
diff --git a/src/iceberg/table.cc b/src/iceberg/table.cc
index 9e8a35bd..817e5917 100644
--- a/src/iceberg/table.cc
+++ b/src/iceberg/table.cc
@@ -31,6 +31,7 @@
#include "iceberg/table_properties.h"
#include "iceberg/table_scan.h"
#include "iceberg/transaction.h"
+#include "iceberg/update/delete_files.h"
#include "iceberg/update/expire_snapshots.h"
#include "iceberg/update/fast_append.h"
#include "iceberg/update/merge_append.h"
@@ -224,6 +225,12 @@ Result<std::shared_ptr<MergeAppend>> Table::NewMergeAppend() {
return MergeAppend::Make(name().name, std::move(ctx));
}
+Result<std::shared_ptr<DeleteFiles>> Table::NewDeleteFiles() {
+ ICEBERG_ASSIGN_OR_RAISE(
+ auto ctx, TransactionContext::Make(shared_from_this(), TransactionKind::kUpdate));
+ return DeleteFiles::Make(name().name, std::move(ctx));
+}
+
Result<std::shared_ptr<UpdateStatistics>> Table::NewUpdateStatistics() {
ICEBERG_ASSIGN_OR_RAISE(
auto ctx, TransactionContext::Make(shared_from_this(), TransactionKind::kUpdate));
diff --git a/src/iceberg/table.h b/src/iceberg/table.h
index 45f4bd96..b71a1ddb 100644
--- a/src/iceberg/table.h
+++ b/src/iceberg/table.h
@@ -179,6 +179,9 @@ class ICEBERG_EXPORT Table : public std::enable_shared_from_this<Table> {
/// \brief Create a new MergeAppend to append data files and merge manifests.
virtual Result<std::shared_ptr<MergeAppend>> NewMergeAppend();
+ /// \brief Create a new DeleteFiles to delete data files and commit the changes.
+ virtual Result<std::shared_ptr<DeleteFiles>> NewDeleteFiles();
+
/// \brief Create a new SnapshotManager to manage snapshots and snapshot references.
virtual Result<std::shared_ptr<SnapshotManager>> NewSnapshotManager();
diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt
index 0e738924..1d8ea472 100644
--- a/src/iceberg/test/CMakeLists.txt
+++ b/src/iceberg/test/CMakeLists.txt
@@ -222,6 +222,7 @@ if(ICEBERG_BUILD_BUNDLE)
add_iceberg_test(table_update_test
USE_BUNDLE
SOURCES
+ delete_files_test.cc
expire_snapshots_test.cc
fast_append_test.cc
manifest_filter_manager_test.cc
diff --git a/src/iceberg/test/delete_files_test.cc b/src/iceberg/test/delete_files_test.cc
new file mode 100644
index 00000000..7c547ac4
--- /dev/null
+++ b/src/iceberg/test/delete_files_test.cc
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/update/delete_files.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "iceberg/avro/avro_register.h"
+#include "iceberg/expression/expressions.h"
+#include "iceberg/expression/literal.h"
+#include "iceberg/manifest/manifest_entry.h"
+#include "iceberg/partition_spec.h"
+#include "iceberg/row/partition_values.h"
+#include "iceberg/schema.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table.h"
+#include "iceberg/test/matchers.h"
+#include "iceberg/test/update_test_base.h"
+#include "iceberg/update/fast_append.h"
+
+namespace iceberg {
+
+class DeleteFilesTest : public MinimalUpdateTestBase {
+ protected:
+ static void SetUpTestSuite() { avro::RegisterAll(); }
+
+ void SetUp() override {
+ MinimalUpdateTestBase::SetUp();
+
+ ICEBERG_UNWRAP_OR_FAIL(spec_, table_->spec());
+ ICEBERG_UNWRAP_OR_FAIL(schema_, table_->schema());
+ file_a_ = MakeDataFile("/data/file_a.parquet", /*partition_x=*/1L);
+ file_b_ = MakeDataFile("/data/file_b.parquet", /*partition_x=*/2L);
+ }
+
+ std::shared_ptr<DataFile> MakeDataFile(const std::string& path, int64_t partition_x) {
+ auto file = std::make_shared<DataFile>();
+ file->content = DataFile::Content::kData;
+ file->file_path = table_location_ + path;
+ file->file_format = FileFormatType::kParquet;
+ file->partition = PartitionValues(std::vector<Literal>{Literal::Long(partition_x)});
+ file->file_size_in_bytes = 1024;
+ file->record_count = 100;
+ file->partition_spec_id = spec_->spec_id();
+ return file;
+ }
+
+ void SetLongBounds(const std::shared_ptr<DataFile>& file, int32_t field_id,
+ int64_t lower, int64_t upper) {
+ ASSERT_NE(file, nullptr);
+ ICEBERG_UNWRAP_OR_FAIL(auto lower_bound, Literal::Long(lower).Serialize());
+ ICEBERG_UNWRAP_OR_FAIL(auto upper_bound, Literal::Long(upper).Serialize());
+ file->value_counts[field_id] = file->record_count;
+ file->null_value_counts[field_id] = 0;
+ file->lower_bounds[field_id] = lower_bound;
+ file->upper_bounds[field_id] = upper_bound;
+ }
+
+ void CommitFiles(const std::vector<std::shared_ptr<DataFile>>& files) {
+ ICEBERG_UNWRAP_OR_FAIL(auto append, table_->NewFastAppend());
+ for (const auto& file : files) {
+ append->AppendFile(file);
+ }
+ ASSERT_THAT(append->Commit(), IsOk());
+ ASSERT_THAT(table_->Refresh(), IsOk());
+ }
+
+ void CommitInitialFiles() { CommitFiles({file_a_, file_b_}); }
+
+ void ExpectOneFileDeleted() {
+ ASSERT_THAT(table_->Refresh(), IsOk());
+ ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot());
+ EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kOperation),
+ DataOperation::kDelete);
+ EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kDeletedDataFiles), "1");
+ EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kDeletedRecords), "100");
+ EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kRemovedFileSize), "1024");
+ }
+
+ std::shared_ptr<PartitionSpec> spec_;
+ std::shared_ptr<Schema> schema_;
+ std::shared_ptr<DataFile> file_a_;
+ std::shared_ptr<DataFile> file_b_;
+
+ static constexpr int32_t kYFieldId = 2;
+};
+
+TEST_F(DeleteFilesTest, DeleteFileByPath) {
+ CommitInitialFiles();
+
+ ICEBERG_UNWRAP_OR_FAIL(auto delete_files, table_->NewDeleteFiles());
+ delete_files->DeleteFile(file_a_->file_path);
+
+ EXPECT_THAT(delete_files->Commit(), IsOk());
+ ExpectOneFileDeleted();
+}
+
+TEST_F(DeleteFilesTest, DeleteFileByDataFile) {
+ CommitInitialFiles();
+
+ ICEBERG_UNWRAP_OR_FAIL(auto delete_files, table_->NewDeleteFiles());
+ delete_files->DeleteFile(file_a_);
+
+ EXPECT_THAT(delete_files->Commit(), IsOk());
+ ExpectOneFileDeleted();
+}
+
+TEST_F(DeleteFilesTest, DeleteFromRowFilterCaseInsensitive) {
+ CommitInitialFiles();
+
+ ICEBERG_UNWRAP_OR_FAIL(auto delete_files, table_->NewDeleteFiles());
+ delete_files->CaseSensitive(false).DeleteFromRowFilter(
+ Expressions::Equal("X", Literal::Long(1L)));
+
+ EXPECT_THAT(delete_files->Commit(), IsOk());
+ ExpectOneFileDeleted();
+}
+
+TEST_F(DeleteFilesTest, EmptyDeleteCommit) {
+ CommitInitialFiles();
+ ICEBERG_UNWRAP_OR_FAIL(auto previous_snapshot, table_->current_snapshot());
+
+ ICEBERG_UNWRAP_OR_FAIL(auto delete_files, table_->NewDeleteFiles());
+
+ EXPECT_THAT(delete_files->Commit(), IsOk());
+
+ ASSERT_THAT(table_->Refresh(), IsOk());
+ ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot());
+ ASSERT_TRUE(snapshot->parent_snapshot_id.has_value());
+ EXPECT_EQ(snapshot->parent_snapshot_id.value(), previous_snapshot->snapshot_id);
+ EXPECT_EQ(snapshot->summary.at(SnapshotSummaryFields::kOperation),
+ DataOperation::kDelete);
+ EXPECT_EQ(snapshot->summary.count(SnapshotSummaryFields::kDeletedDataFiles), 0U);
+ EXPECT_EQ(snapshot->summary.count(SnapshotSummaryFields::kDeletedRecords), 0U);
+ EXPECT_EQ(snapshot->summary.count(SnapshotSummaryFields::kRemovedFileSize), 0U);
+}
+
+TEST_F(DeleteFilesTest, DeleteFromRowFilter) {
+ CommitInitialFiles();
+
+ ICEBERG_UNWRAP_OR_FAIL(auto delete_files, table_->NewDeleteFiles());
+ delete_files->DeleteFromRowFilter(Expressions::Equal("x", Literal::Long(1L)));
+
+ EXPECT_THAT(delete_files->Commit(), IsOk());
+ ExpectOneFileDeleted();
+}
+
+TEST_F(DeleteFilesTest, DeleteFromRowFilterRejectsPartialMatchFile) {
+ auto partial_match_file = MakeDataFile("/data/partial_match.parquet",
+ /*partition_x=*/1L);
+ SetLongBounds(partial_match_file, kYFieldId, /*lower=*/0L, /*upper=*/10L);
+ CommitFiles({partial_match_file});
+ ICEBERG_UNWRAP_OR_FAIL(auto previous_snapshot, table_->current_snapshot());
+
+ ICEBERG_UNWRAP_OR_FAIL(auto delete_files, table_->NewDeleteFiles());
+ delete_files->DeleteFromRowFilter(Expressions::Equal("y", Literal::Long(5L)));
+
+ auto status = delete_files->Commit();
+ EXPECT_THAT(status, IsError(ErrorKind::kValidationFailed));
+ EXPECT_THAT(status,
+ HasErrorMessage("Cannot delete file where some, but not all, rows match "
+ "filter"));
+ EXPECT_THAT(status, HasErrorMessage(partial_match_file->file_path));
+
+ ASSERT_THAT(table_->Refresh(), IsOk());
+ ICEBERG_UNWRAP_OR_FAIL(auto snapshot, table_->current_snapshot());
+ EXPECT_EQ(snapshot->snapshot_id, previous_snapshot->snapshot_id);
+}
+
+TEST_F(DeleteFilesTest, ValidateFilesExistRejectsMissingPath) {
+ CommitInitialFiles();
+
+ ICEBERG_UNWRAP_OR_FAIL(auto delete_files, table_->NewDeleteFiles());
+ delete_files->DeleteFile(table_location_ + "/data/missing.parquet")
+ .ValidateFilesExist();
+
+ auto status = delete_files->Commit();
+ EXPECT_THAT(status, IsError(ErrorKind::kValidationFailed));
+ EXPECT_THAT(status, HasErrorMessage("Missing required files to delete"));
+}
+
+} // namespace iceberg
diff --git a/src/iceberg/transaction.cc b/src/iceberg/transaction.cc
index 673f4376..ac1f0824 100644
--- a/src/iceberg/transaction.cc
+++ b/src/iceberg/transaction.cc
@@ -32,6 +32,7 @@
#include "iceberg/table_requirement.h"
#include "iceberg/table_requirements.h"
#include "iceberg/table_update.h"
+#include "iceberg/update/delete_files.h"
#include "iceberg/update/expire_snapshots.h"
#include "iceberg/update/fast_append.h"
#include "iceberg/update/merge_append.h"
@@ -497,6 +498,13 @@ Result<std::shared_ptr<MergeAppend>> Transaction::NewMergeAppend() {
return merge_append;
}
+Result<std::shared_ptr<DeleteFiles>> Transaction::NewDeleteFiles() {
+ ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr<DeleteFiles> delete_files,
+ DeleteFiles::Make(ctx_->table->name().name, ctx_));
+ ICEBERG_RETURN_UNEXPECTED(AddUpdate(delete_files));
+ return delete_files;
+}
+
Result<std::shared_ptr<UpdateStatistics>> Transaction::NewUpdateStatistics() {
ICEBERG_ASSIGN_OR_RAISE(std::shared_ptr<UpdateStatistics> update_statistics,
UpdateStatistics::Make(ctx_));
diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h
index 0cc4cb9a..52a0605c 100644
--- a/src/iceberg/transaction.h
+++ b/src/iceberg/transaction.h
@@ -109,6 +109,9 @@ class ICEBERG_EXPORT Transaction : public std::enable_shared_from_this<Transacti
/// \brief Create a new MergeAppend to append data files and merge manifests.
Result<std::shared_ptr<MergeAppend>> NewMergeAppend();
+ /// \brief Create a new DeleteFiles to delete data files and commit the changes.
+ Result<std::shared_ptr<DeleteFiles>> NewDeleteFiles();
+
/// \brief Create a new SnapshotManager to manage snapshots.
Result<std::shared_ptr<SnapshotManager>> NewSnapshotManager();
diff --git a/src/iceberg/type_fwd.h b/src/iceberg/type_fwd.h
index c82da513..6c34d3a8 100644
--- a/src/iceberg/type_fwd.h
+++ b/src/iceberg/type_fwd.h
@@ -223,6 +223,7 @@ class Transaction;
class TransactionContext;
/// \brief Update family.
+class DeleteFiles;
class ExpireSnapshots;
class FastAppend;
class MergeAppend;
diff --git a/src/iceberg/update/delete_files.cc b/src/iceberg/update/delete_files.cc
new file mode 100644
index 00000000..9759e3eb
--- /dev/null
+++ b/src/iceberg/update/delete_files.cc
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "iceberg/update/delete_files.h"
+
+#include <memory>
+#include <string>
+#include <string_view>
+
+#include "iceberg/snapshot.h"
+#include "iceberg/transaction.h"
+#include "iceberg/util/error_collector.h"
+#include "iceberg/util/macros.h"
+
+namespace iceberg {
+
+Result<std::unique_ptr<DeleteFiles>> DeleteFiles::Make(
+ std::string table_name, std::shared_ptr<TransactionContext> ctx) {
+ ICEBERG_PRECHECK(!table_name.empty(), "Table name cannot be empty");
+ ICEBERG_PRECHECK(ctx != nullptr, "Cannot create DeleteFiles without a context");
+ return std::unique_ptr<DeleteFiles>(
+ new DeleteFiles(std::move(table_name), std::move(ctx)));
+}
+
+DeleteFiles::DeleteFiles(std::string table_name, std::shared_ptr<TransactionContext> ctx)
+ : MergingSnapshotUpdate(std::move(table_name), std::move(ctx)) {}
+
+DeleteFiles& DeleteFiles::DeleteFile(std::string_view path) {
+ ICEBERG_BUILDER_CHECK(!path.empty(), "Cannot delete an empty file path");
+ ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteByPath(path));
+ return *this;
+}
+
+DeleteFiles& DeleteFiles::DeleteFile(const std::shared_ptr<DataFile>& file) {
+ ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteDataFile(file));
+ return *this;
+}
+
+DeleteFiles& DeleteFiles::DeleteFromRowFilter(std::shared_ptr<Expression> expr) {
+ ICEBERG_BUILDER_RETURN_IF_ERROR(DeleteByRowFilter(std::move(expr)));
+ return *this;
+}
+
+DeleteFiles& DeleteFiles::CaseSensitive(bool case_sensitive) {
+ MergingSnapshotUpdate::CaseSensitive(case_sensitive);
+ return *this;
+}
+
+DeleteFiles& DeleteFiles::ValidateFilesExist() {
+ validate_files_to_delete_exist_ = true;
+ return *this;
+}
+
+std::string DeleteFiles::operation() { return DataOperation::kDelete; }
+
+Status DeleteFiles::Validate(const TableMetadata&, const std::shared_ptr<Snapshot>&) {
+ if (validate_files_to_delete_exist_) {
+ FailMissingDeletePaths();
+ }
+ return {};
+}
+
+} // namespace iceberg
diff --git a/src/iceberg/update/delete_files.h b/src/iceberg/update/delete_files.h
new file mode 100644
index 00000000..1be08e35
--- /dev/null
+++ b/src/iceberg/update/delete_files.h
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+/// \file iceberg/update/delete_files.h
+
+#include <memory>
+#include <string>
+#include <string_view>
+
+#include "iceberg/iceberg_export.h"
+#include "iceberg/result.h"
+#include "iceberg/type_fwd.h"
+#include "iceberg/update/merging_snapshot_update.h"
+
+namespace iceberg {
+
+/// \brief API for deleting data files from a table.
+///
+/// This accumulates data-file deletions, produces a new snapshot, and commits that
+/// snapshot as current. File paths are matched exactly against table metadata values;
+/// equivalent but differently-normalized URIs are not considered matches.
+class ICEBERG_EXPORT DeleteFiles : public MergingSnapshotUpdate {
+ public:
+ static Result<std::unique_ptr<DeleteFiles>> Make(
+ std::string table_name, std::shared_ptr<TransactionContext> ctx);
+
+ /// \brief Delete a data-file path from the table.
+ DeleteFiles& DeleteFile(std::string_view path);
+
+ /// \brief Delete a data file tracked by object identity and path.
+ DeleteFiles& DeleteFile(const std::shared_ptr<DataFile>& file);
+
+ /// \brief Delete files whose rows all match the given expression.
+ DeleteFiles& DeleteFromRowFilter(std::shared_ptr<Expression> expr);
+
+ /// \brief Set case sensitivity for expression binding.
+ DeleteFiles& CaseSensitive(bool case_sensitive);
+
+ /// \brief Validate that explicitly requested deleted files still exist.
+ DeleteFiles& ValidateFilesExist();
+
+ std::string operation() override;
+
+ protected:
+ Status Validate(const TableMetadata& current_metadata,
+ const std::shared_ptr<Snapshot>& snapshot) override;
+
+ private:
+ DeleteFiles(std::string table_name, std::shared_ptr<TransactionContext> ctx);
+
+ bool validate_files_to_delete_exist_ = false;
+};
+
+} // namespace iceberg
diff --git a/src/iceberg/update/meson.build b/src/iceberg/update/meson.build
index 031c7945..9f950e8d 100644
--- a/src/iceberg/update/meson.build
+++ b/src/iceberg/update/meson.build
@@ -17,6 +17,7 @@
install_headers(
[
+ 'delete_files.h',
'expire_snapshots.h',
'fast_append.h',
'merge_append.h',