You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ga...@apache.org on 2026/04/17 01:44:17 UTC
(iceberg-cpp) branch main updated: feat: implement incremental changelog scan (#611)
This is an automated email from the ASF dual-hosted git repository.
gangwu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-cpp.git
The following commit(s) were added to refs/heads/main by this push:
new 4876d7c5 feat: implement incremental changelog scan (#611)
4876d7c5 is described below
commit 4876d7c5469628ecea41d4f89776e6ae81521e80
Author: wzhuo <wn...@gmail.com>
AuthorDate: Fri Apr 17 09:44:12 2026 +0800
feat: implement incremental changelog scan (#611)
---
src/iceberg/table_scan.cc | 160 ++++++-
src/iceberg/table_scan.h | 116 ++++-
src/iceberg/test/CMakeLists.txt | 1 +
.../test/incremental_changelog_scan_test.cc | 524 +++++++++++++++++++++
src/iceberg/test/scan_test_base.h | 81 ++++
5 files changed, 872 insertions(+), 10 deletions(-)
diff --git a/src/iceberg/table_scan.cc b/src/iceberg/table_scan.cc
index ed2ede70..2195482b 100644
--- a/src/iceberg/table_scan.cc
+++ b/src/iceberg/table_scan.cc
@@ -19,11 +19,14 @@
#include "iceberg/table_scan.h"
+#include <cstdint>
#include <cstring>
#include <iterator>
+#include <utility>
#include "iceberg/expression/binder.h"
#include "iceberg/expression/expression.h"
+#include "iceberg/expression/residual_evaluator.h"
#include "iceberg/file_reader.h"
#include "iceberg/manifest/manifest_entry.h"
#include "iceberg/manifest/manifest_group.h"
@@ -31,6 +34,7 @@
#include "iceberg/schema.h"
#include "iceberg/snapshot.h"
#include "iceberg/table_metadata.h"
+#include "iceberg/util/content_file_util.h"
#include "iceberg/util/macros.h"
#include "iceberg/util/snapshot_util_internal.h"
#include "iceberg/util/timepoint.h"
@@ -294,6 +298,26 @@ Result<ArrowArrayStream> FileScanTask::ToArrow(
return MakeArrowArrayStream(std::move(reader));
}
+// ChangelogScanTask implementation
+
+int64_t ChangelogScanTask::size_bytes() const {
+ int64_t total_size = data_file_->file_size_in_bytes;
+ for (const auto& delete_file : delete_files_) {
+ ICEBERG_DCHECK(delete_file->content_size_in_bytes.has_value(),
+ "Delete file content size must be available");
+ total_size +=
+ (delete_file->IsDeletionVector() ? delete_file->content_size_in_bytes.value()
+ : delete_file->file_size_in_bytes);
+ }
+ return total_size;
+}
+
+int32_t ChangelogScanTask::files_count() const { return 1 + delete_files_.size(); }
+
+int64_t ChangelogScanTask::estimated_row_count() const {
+ return data_file_->record_count;
+}
+
// Generic template implementation for Make
template <typename ScanType>
Result<std::unique_ptr<TableScanBuilder<ScanType>>> TableScanBuilder<ScanType>::Make(
@@ -747,11 +771,13 @@ Result<std::vector<std::shared_ptr<FileScanTask>>> IncrementalAppendScan::PlanFi
// IncrementalChangelogScan implementation
Result<std::unique_ptr<IncrementalChangelogScan>> IncrementalChangelogScan::Make(
- [[maybe_unused]] std::shared_ptr<TableMetadata> metadata,
- [[maybe_unused]] std::shared_ptr<Schema> schema,
- [[maybe_unused]] std::shared_ptr<FileIO> io,
- [[maybe_unused]] internal::TableScanContext context) {
- return NotImplemented("IncrementalChangelogScan is not implemented");
+ std::shared_ptr<TableMetadata> metadata, std::shared_ptr<Schema> schema,
+ std::shared_ptr<FileIO> io, internal::TableScanContext context) {
+ ICEBERG_PRECHECK(metadata != nullptr, "Table metadata cannot be null");
+ ICEBERG_PRECHECK(schema != nullptr, "Schema cannot be null");
+ ICEBERG_PRECHECK(io != nullptr, "FileIO cannot be null");
+ return std::unique_ptr<IncrementalChangelogScan>(new IncrementalChangelogScan(
+ std::move(metadata), std::move(schema), std::move(io), std::move(context)));
}
Result<std::vector<std::shared_ptr<ChangelogScanTask>>>
@@ -762,7 +788,129 @@ IncrementalChangelogScan::PlanFiles() const {
Result<std::vector<std::shared_ptr<ChangelogScanTask>>>
IncrementalChangelogScan::PlanFiles(std::optional<int64_t> from_snapshot_id_exclusive,
int64_t to_snapshot_id_inclusive) const {
- return NotImplemented("IncrementalChangelogScan::PlanFiles is not implemented");
+ ICEBERG_ASSIGN_OR_RAISE(
+ auto ancestors_snapshots,
+ SnapshotUtil::AncestorsBetween(*metadata_, to_snapshot_id_inclusive,
+ from_snapshot_id_exclusive));
+
+ std::vector<std::pair<std::shared_ptr<Snapshot>, std::unique_ptr<SnapshotCache>>>
+ changelog_snapshots;
+
+ for (const auto& snapshot : std::ranges::reverse_view(ancestors_snapshots)) {
+ auto operation = snapshot->Operation();
+ if (!operation.has_value() || operation.value() != DataOperation::kReplace) {
+ auto snapshot_cache = std::make_unique<SnapshotCache>(snapshot.get());
+ ICEBERG_ASSIGN_OR_RAISE(auto delete_manifests,
+ snapshot_cache->DeleteManifests(io_));
+ if (!delete_manifests.empty()) {
+ return NotSupported(
+ "Delete files are currently not supported in changelog scans");
+ }
+ changelog_snapshots.emplace_back(snapshot, std::move(snapshot_cache));
+ }
+ }
+ if (changelog_snapshots.empty()) {
+ return std::vector<std::shared_ptr<ChangelogScanTask>>{};
+ }
+
+ std::unordered_set<int64_t> snapshot_ids;
+ std::unordered_map<int64_t, int32_t> snapshot_ordinals;
+ for (const auto& snapshot : changelog_snapshots) {
+ ICEBERG_PRECHECK(
+ std::cmp_less_equal(snapshot_ids.size(), std::numeric_limits<int32_t>::max()),
+ "Number of snapshots in changelog scan exceeds maximum supported");
+ snapshot_ids.insert(snapshot.first->snapshot_id);
+ snapshot_ordinals.try_emplace(snapshot.first->snapshot_id,
+ static_cast<int32_t>(snapshot_ordinals.size()));
+ }
+
+ std::vector<ManifestFile> data_manifests;
+ std::unordered_set<std::string> seen_manifest_paths;
+ for (const auto& snapshot : changelog_snapshots) {
+ ICEBERG_ASSIGN_OR_RAISE(auto manifests, snapshot.second->DataManifests(io_));
+ for (auto& manifest : manifests) {
+ if (snapshot_ids.contains(manifest.added_snapshot_id) &&
+ seen_manifest_paths.insert(manifest.manifest_path).second) {
+ data_manifests.push_back(manifest);
+ }
+ }
+ }
+ if (data_manifests.empty()) {
+ return std::vector<std::shared_ptr<ChangelogScanTask>>{};
+ }
+
+ TableMetadataCache metadata_cache(metadata_.get());
+ ICEBERG_ASSIGN_OR_RAISE(auto specs_by_id, metadata_cache.GetPartitionSpecsById());
+
+ ICEBERG_ASSIGN_OR_RAISE(
+ auto manifest_group,
+ ManifestGroup::Make(io_, schema_, specs_by_id, std::move(data_manifests),
+ /*delete_manifests=*/{}));
+
+ manifest_group->CaseSensitive(context_.case_sensitive)
+ .Select(ScanColumns())
+ .FilterData(filter())
+ .FilterManifestEntries([&snapshot_ids](const ManifestEntry& entry) {
+ return entry.snapshot_id.has_value() &&
+ snapshot_ids.contains(entry.snapshot_id.value());
+ })
+ .IgnoreExisting()
+ .ColumnsToKeepStats(context_.columns_to_keep_stats);
+
+ if (context_.ignore_residuals) {
+ manifest_group->IgnoreResiduals();
+ }
+
+ auto create_tasks_func =
+ [&snapshot_ordinals](
+ std::vector<ManifestEntry>&& entries,
+ const TaskContext& ctx) -> Result<std::vector<std::shared_ptr<ScanTask>>> {
+ std::vector<std::shared_ptr<ScanTask>> tasks;
+ tasks.reserve(entries.size());
+
+ for (auto& entry : entries) {
+ ICEBERG_PRECHECK(entry.snapshot_id.has_value() && entry.data_file,
+ "Invalid manifest entry with missing snapshot id or data file");
+
+ int64_t commit_snapshot_id = entry.snapshot_id.value();
+ auto ordinal_it = snapshot_ordinals.find(commit_snapshot_id);
+ ICEBERG_PRECHECK(ordinal_it != snapshot_ordinals.end(),
+ "Invalid manifest entry with missing snapshot ordinal");
+
+ int32_t change_ordinal = ordinal_it->second;
+
+ if (ctx.drop_stats) {
+ ContentFileUtil::DropAllStats(*entry.data_file);
+ } else if (!ctx.columns_to_keep_stats.empty()) {
+ ContentFileUtil::DropUnselectedStats(*entry.data_file, ctx.columns_to_keep_stats);
+ }
+
+ ICEBERG_ASSIGN_OR_RAISE(auto residual,
+ ctx.residuals->ResidualFor(entry.data_file->partition));
+
+ switch (entry.status) {
+ case ManifestStatus::kAdded:
+ tasks.push_back(std::make_shared<AddedRowsScanTask>(
+ change_ordinal, commit_snapshot_id, std::move(entry.data_file),
+ std::vector<std::shared_ptr<DataFile>>{}, std::move(residual)));
+ break;
+ case ManifestStatus::kDeleted:
+ tasks.push_back(std::make_shared<DeletedDataFileScanTask>(
+ change_ordinal, commit_snapshot_id, std::move(entry.data_file),
+ std::vector<std::shared_ptr<DataFile>>{}, std::move(residual)));
+ break;
+ case ManifestStatus::kExisting:
+ return InvalidArgument("Unexpected entry status: EXISTING");
+ }
+ }
+ return tasks;
+ };
+
+ ICEBERG_ASSIGN_OR_RAISE(auto tasks, manifest_group->Plan(create_tasks_func));
+ return tasks | std::views::transform([](const auto& task) {
+ return std::static_pointer_cast<ChangelogScanTask>(task);
+ }) |
+ std::ranges::to<std::vector>();
}
} // namespace iceberg
diff --git a/src/iceberg/table_scan.h b/src/iceberg/table_scan.h
index b21adcca..064825d8 100644
--- a/src/iceberg/table_scan.h
+++ b/src/iceberg/table_scan.h
@@ -102,14 +102,122 @@ class ICEBERG_EXPORT FileScanTask : public ScanTask {
std::shared_ptr<Expression> residual_filter_;
};
+enum class ChangelogOperation : uint8_t {
+ kInsert,
+ kDelete,
+ kUpdateBefore,
+ kUpdateAfter,
+};
+
/// \brief A scan task for reading changelog entries between snapshots.
class ICEBERG_EXPORT ChangelogScanTask : public ScanTask {
public:
+ /// \brief Construct an AddedRowsScanTask.
+ ///
+ /// \param change_ordinal Position in the changelog order (0-based).
+ /// \param commit_snapshot_id The snapshot ID that committed this change.
+ /// \param data_file The data file containing the added rows.
+ /// \param delete_files Delete files that apply to this data file.
+ /// \param residual_filter Optional residual filter to apply after reading.
+ ChangelogScanTask(int32_t change_ordinal, int64_t commit_snapshot_id,
+ std::shared_ptr<DataFile> data_file,
+ std::vector<std::shared_ptr<DataFile>> delete_files = {},
+ std::shared_ptr<Expression> residual_filter = nullptr)
+ : change_ordinal_(change_ordinal),
+ commit_snapshot_id_(commit_snapshot_id),
+ data_file_(std::move(data_file)),
+ delete_files_(std::move(delete_files)),
+ residual_filter_(std::move(residual_filter)) {}
+
Kind kind() const override { return Kind::kChangelogScanTask; }
- // TODO(): Return actual values once member fields are implemented
- int64_t size_bytes() const override { return 0; }
- int32_t files_count() const override { return 0; }
- int64_t estimated_row_count() const override { return 0; }
+
+ int64_t size_bytes() const override;
+ int32_t files_count() const override;
+ int64_t estimated_row_count() const override;
+
+ virtual ChangelogOperation operation() const = 0;
+
+ /// \brief The position of this change in the changelog order (0-based).
+ int32_t change_ordinal() const { return change_ordinal_; }
+
+ /// \brief The snapshot ID that committed this change.
+ int64_t commit_snapshot_id() const { return commit_snapshot_id_; }
+
+ /// \brief Residual filter to apply after reading.
+ const std::shared_ptr<Expression>& residual_filter() const { return residual_filter_; }
+
+ protected:
+ int32_t change_ordinal_;
+ int64_t commit_snapshot_id_;
+ std::shared_ptr<DataFile> data_file_;
+ std::vector<std::shared_ptr<DataFile>> delete_files_;
+ std::shared_ptr<Expression> residual_filter_;
+};
+
+/// \brief A scan task for inserts generated by adding a data file to the table.
+///
+/// This task represents data files that were added to the table, along with any
+/// delete files that should be applied when reading the data.
+///
+/// Added data files may have matching delete files. This may happen if a
+/// matching position delete file is committed in the same snapshot or if changes
+/// for multiple snapshots are squashed together.
+///
+/// Suppose snapshot S1 adds data files F1, F2, F3 and a position delete file,
+/// D1, that marks particular records in F1 as deleted. A scan for changes
+/// generated by S1 should include the following tasks:
+/// - AddedRowsScanTask(file=F1, deletes=[D1], snapshot=S1)
+/// - AddedRowsScanTask(file=F2, deletes=[], snapshot=S1)
+/// - AddedRowsScanTask(file=F3, deletes=[], snapshot=S1)
+///
+/// Readers consuming these tasks should produce added records with metadata
+/// like change ordinal and commit snapshot ID.
+class ICEBERG_EXPORT AddedRowsScanTask : public ChangelogScanTask {
+ public:
+ using ChangelogScanTask::ChangelogScanTask;
+
+ ChangelogOperation operation() const override { return ChangelogOperation::kInsert; }
+
+ /// \brief The data file containing the added rows.
+ const std::shared_ptr<DataFile>& data_file() const { return data_file_; }
+
+ /// \brief A list of delete files to apply when reading the data file in this task.
+ ///
+ /// @return A list of delete files to apply
+ const std::vector<std::shared_ptr<DataFile>>& delete_files() const {
+ return delete_files_;
+ }
+};
+
+/// \brief A scan task for deletes generated by removing a data file from the table.
+///
+/// All historical delete files added earlier must be applied while reading the data file.
+/// This is required to output only those data records that were live when the data file
+/// was removed.
+///
+/// Suppose snapshot S1 contains data files F1, F2, F3. Then snapshot S2 adds a position
+/// delete file, D1, that deletes records from F2 and snapshot S3 removes F2 entirely. A
+/// scan for changes generated by S3 should include the following task:
+/// - DeletedDataFileScanTask(file=F2, existing-deletes=[D1], snapshot=S3)
+///
+/// Readers consuming these tasks should produce deleted records with metadata like
+/// change ordinal and commit snapshot ID.
+class ICEBERG_EXPORT DeletedDataFileScanTask : public ChangelogScanTask {
+ public:
+ using ChangelogScanTask::ChangelogScanTask;
+
+ ChangelogOperation operation() const override { return ChangelogOperation::kDelete; }
+
+ /// \brief The data file that was deleted.
+ const std::shared_ptr<DataFile>& data_file() const { return data_file_; }
+
+ /// \brief A list of previously added delete files to apply when reading the
+ /// data file in this task.
+ ///
+ /// \return A list of delete files to apply
+ const std::vector<std::shared_ptr<DataFile>>& existing_deletes() const {
+ return delete_files_;
+ }
};
namespace internal {
diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt
index eb26b2be..149abb82 100644
--- a/src/iceberg/test/CMakeLists.txt
+++ b/src/iceberg/test/CMakeLists.txt
@@ -182,6 +182,7 @@ if(ICEBERG_BUILD_BUNDLE)
SOURCES
file_scan_task_test.cc
incremental_append_scan_test.cc
+ incremental_changelog_scan_test.cc
table_scan_test.cc)
add_iceberg_test(table_update_test
diff --git a/src/iceberg/test/incremental_changelog_scan_test.cc b/src/iceberg/test/incremental_changelog_scan_test.cc
new file mode 100644
index 00000000..62fed0f2
--- /dev/null
+++ b/src/iceberg/test/incremental_changelog_scan_test.cc
@@ -0,0 +1,524 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include "iceberg/expression/expressions.h"
+#include "iceberg/snapshot.h"
+#include "iceberg/table_scan.h"
+#include "iceberg/test/scan_test_base.h"
+
+namespace iceberg {
+
+namespace {
+
+const std::string& TaskFilePath(const std::shared_ptr<ChangelogScanTask>& task) {
+ if (auto added = std::dynamic_pointer_cast<AddedRowsScanTask>(task)) {
+ return added->data_file()->file_path;
+ }
+ if (auto deleted = std::dynamic_pointer_cast<DeletedDataFileScanTask>(task)) {
+ return deleted->data_file()->file_path;
+ }
+
+ static const std::string empty_path;
+ return empty_path;
+}
+
+/// \brief Sort changelog scan tasks for deterministic ordering.
+/// Sorts by change_ordinal, then by operation type name, then by file path.
+template <typename TaskType>
+void SortTasks(std::vector<std::shared_ptr<TaskType>>& tasks) {
+ std::ranges::sort(tasks, [](const auto& t1, const auto& t2) {
+ if (t1->change_ordinal() != t2->change_ordinal()) {
+ return t1->change_ordinal() < t2->change_ordinal();
+ }
+ if (t1->operation() != t2->operation()) {
+ return static_cast<uint8_t>(t1->operation()) <
+ static_cast<uint8_t>(t2->operation());
+ }
+ return TaskFilePath(std::static_pointer_cast<ChangelogScanTask>(t1)) <
+ TaskFilePath(std::static_pointer_cast<ChangelogScanTask>(t2));
+ });
+}
+
+} // namespace
+
+class IncrementalChangelogScanTest : public ScanTestBase {};
+
+TEST_P(IncrementalChangelogScanTest, DataFilters) {
+ auto version = GetParam();
+
+ // Bucket transform for "data" column: bucket("a", 16) = 8, bucket("k", 16) = 1
+ // By filtering on data="k", we should only match files with bucket=1
+
+ // Create partition value bucket("a") = 8
+ auto partition_a = PartitionValues({Literal::Int(8)});
+ // Create partition value bucket("k") = 1
+ auto partition_b = PartitionValues({Literal::Int(1)});
+
+ // Create snapshot 1 with file_a
+ auto snapshot_a = MakeAppendSnapshotWithPartitionValues(
+ version, 1000L, std::nullopt, 1L, {{"/path/to/file_a.parquet", partition_a}},
+ partitioned_spec_);
+ SnapshotCache cache_a(snapshot_a.get());
+ ICEBERG_UNWRAP_OR_FAIL(auto manifests_a, cache_a.DataManifests(file_io_));
+ ASSERT_EQ(manifests_a.size(), 1);
+ const auto& manifest_a = manifests_a[0];
+
+ // Create snapshot 2 with file_b (separate manifest list, not inheriting from snap1)
+ auto snapshot_b = MakeAppendSnapshotWithPartitionValues(
+ version, 2000L, 1000L, 2L, {{"/path/to/file_b.parquet", partition_b}},
+ partitioned_spec_);
+
+ // Create metadata with partitioned spec as default
+ auto partitioned_metadata = MakeTableMetadata(
+ {snapshot_a, snapshot_b}, 2000L,
+ {{"main", std::make_shared<SnapshotRef>(SnapshotRef{
+ .snapshot_id = 2000L, .retention = SnapshotRef::Branch{}})}},
+ partitioned_spec_);
+
+ // Make the first manifest unavailable. Planning should still succeed because the
+ // partition filter can skip reading file_a's manifest entirely.
+ EXPECT_THAT(file_io_->DeleteFile(manifest_a.manifest_path), IsOk());
+
+ // Filter by data="k" which should match only file_b (bucket("k", 16) = 1)
+ ICEBERG_UNWRAP_OR_FAIL(auto builder, IncrementalChangelogScanBuilder::Make(
+ partitioned_metadata, file_io_));
+ builder->Filter(Expressions::Equal("data", Literal::String("k")));
+ builder->ToSnapshot(2000L);
+ ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build());
+ ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles());
+ ASSERT_EQ(tasks.size(), 1);
+
+ auto t1 = tasks[0];
+ EXPECT_EQ(t1->change_ordinal(), 1);
+ EXPECT_EQ(t1->commit_snapshot_id(), 2000L);
+ EXPECT_EQ(t1->operation(), ChangelogOperation::kInsert);
+ auto insert_t1 = std::dynamic_pointer_cast<AddedRowsScanTask>(t1);
+ ASSERT_NE(insert_t1, nullptr);
+ EXPECT_EQ(insert_t1->data_file()->file_path, "/path/to/file_b.parquet");
+ EXPECT_TRUE(insert_t1->delete_files().empty());
+}
+
+TEST_P(IncrementalChangelogScanTest, Overwrites) {
+ auto version = GetParam();
+
+ // Create initial snapshot with 2 files
+ auto snapshot_a =
+ MakeAppendSnapshot(version, 1000L, std::nullopt, 1L,
+ {"/path/to/file_a.parquet", "/path/to/file_b.parquet"});
+
+ // Overwrite: add file_a2, delete file_a
+ auto snapshot_b =
+ MakeOverwriteSnapshot(version, 2000L, 1000L, 2L, {"/path/to/file_a2.parquet"},
+ {"/path/to/file_a.parquet"});
+
+ auto metadata = MakeTableMetadata(
+ {snapshot_a, snapshot_b}, 2000L,
+ {{"main", std::make_shared<SnapshotRef>(SnapshotRef{
+ .snapshot_id = 2000L, .retention = SnapshotRef::Branch{}})}});
+
+ // from_snapshot_exclusive(snap1).to_snapshot(snap2) should return 2 tasks
+ ICEBERG_UNWRAP_OR_FAIL(auto builder,
+ IncrementalChangelogScanBuilder::Make(metadata, file_io_));
+ builder->FromSnapshot(1000L, /*inclusive=*/false).ToSnapshot(2000L);
+ ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build());
+ ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles());
+ ASSERT_EQ(tasks.size(), 2);
+ SortTasks(tasks);
+
+ // First task: added file (INSERT operation)
+ auto t1 = tasks[0];
+ EXPECT_EQ(t1->change_ordinal(), 0);
+ EXPECT_EQ(t1->commit_snapshot_id(), 2000L);
+ EXPECT_EQ(t1->operation(), ChangelogOperation::kInsert);
+ auto insert_t1 = std::dynamic_pointer_cast<AddedRowsScanTask>(t1);
+ ASSERT_NE(insert_t1, nullptr);
+ EXPECT_EQ(insert_t1->data_file()->file_path, "/path/to/file_a2.parquet");
+ EXPECT_TRUE(insert_t1->delete_files().empty());
+
+ // Second task: deleted file (DELETE operation)
+ auto t2 = tasks[1];
+ EXPECT_EQ(t2->change_ordinal(), 0);
+ EXPECT_EQ(t2->commit_snapshot_id(), 2000L);
+ EXPECT_EQ(t2->operation(), ChangelogOperation::kDelete);
+ auto delete_t2 = std::dynamic_pointer_cast<DeletedDataFileScanTask>(t2);
+ ASSERT_NE(delete_t2, nullptr);
+ EXPECT_EQ(delete_t2->data_file()->file_path, "/path/to/file_a.parquet");
+ EXPECT_TRUE(delete_t2->existing_deletes().empty());
+}
+
+TEST_P(IncrementalChangelogScanTest, DuplicatedManifests) {
+ auto version = GetParam();
+
+ // Create initial snapshot_a with file_a and extract its manifest
+ auto snapshot_a =
+ MakeAppendSnapshot(version, 1000L, std::nullopt, 1L, {"/path/to/file_a.parquet"});
+ SnapshotCache cache_a(snapshot_a.get());
+ ICEBERG_UNWRAP_OR_FAIL(auto manifests_a, cache_a.DataManifests(file_io_));
+ ASSERT_EQ(manifests_a.size(), 1);
+ auto manifest_a = manifests_a[0];
+
+ // Manually construct snapshot_b that includes BOTH a new manifest AND the manifest from
+ // snapshot_a. This simulates what actually happens in Iceberg when appends reuse older
+ // manifests.
+ std::vector<ManifestEntry> entries_b;
+ auto file_b = MakeDataFile("/path/to/file_b.parquet");
+ entries_b.push_back(MakeEntry(ManifestStatus::kAdded, 2000L, 2L, file_b));
+ auto manifest_b = WriteDataManifest(version, 2000L, std::move(entries_b));
+
+ auto manifest_list_b =
+ WriteManifestList(version, 2000L, 1000L, 2L, {manifest_a, manifest_b});
+ TimePointMs timestamp_ms = TimePointMsFromUnixMs(1609459200000L + 2000);
+ auto snapshot_b = std::make_shared<Snapshot>(Snapshot{
+ .snapshot_id = 2000L,
+ .parent_snapshot_id = 1000L,
+ .sequence_number = 2L,
+ .timestamp_ms = timestamp_ms,
+ .manifest_list = manifest_list_b,
+ .summary = {{"operation", "append"}},
+ .schema_id = schema_->schema_id(),
+ });
+
+ auto metadata = MakeTableMetadata(
+ {snapshot_a, snapshot_b}, 2000L,
+ {{"main", std::make_shared<SnapshotRef>(SnapshotRef{
+ .snapshot_id = 2000L, .retention = SnapshotRef::Branch{}})}});
+
+ ICEBERG_UNWRAP_OR_FAIL(auto builder,
+ IncrementalChangelogScanBuilder::Make(metadata, file_io_));
+ builder->ToSnapshot(2000L);
+ ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build());
+ ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles());
+
+ // We expect exactly 2 tasks, one for file_a and one for file_b.
+ // If deduplication is buggy, we might see 3 tasks (file_a twice).
+ ASSERT_EQ(tasks.size(), 2);
+ SortTasks(tasks);
+
+ auto insert_t1 = std::dynamic_pointer_cast<AddedRowsScanTask>(tasks[0]);
+ ASSERT_NE(insert_t1, nullptr);
+ EXPECT_EQ(insert_t1->data_file()->file_path, "/path/to/file_a.parquet");
+ EXPECT_EQ(tasks[0]->commit_snapshot_id(), 1000L);
+
+ auto insert_t2 = std::dynamic_pointer_cast<AddedRowsScanTask>(tasks[1]);
+ ASSERT_NE(insert_t2, nullptr);
+ EXPECT_EQ(insert_t2->data_file()->file_path, "/path/to/file_b.parquet");
+ EXPECT_EQ(tasks[1]->commit_snapshot_id(), 2000L);
+}
+
+TEST_P(IncrementalChangelogScanTest, FileDeletes) {
+ auto version = GetParam();
+
+ // Create initial snapshot with 2 files
+ auto snapshot_a =
+ MakeAppendSnapshot(version, 1000L, std::nullopt, 1L,
+ {"/path/to/file_a.parquet", "/path/to/file_b.parquet"});
+
+ // Delete file_a
+ auto snapshot_b =
+ MakeDeleteSnapshot(version, 2000L, 1000L, 2L, {"/path/to/file_a.parquet"});
+
+ auto metadata = MakeTableMetadata(
+ {snapshot_a, snapshot_b}, 2000L,
+ {{"main", std::make_shared<SnapshotRef>(SnapshotRef{
+ .snapshot_id = 2000L, .retention = SnapshotRef::Branch{}})}});
+
+ ICEBERG_UNWRAP_OR_FAIL(auto builder,
+ IncrementalChangelogScanBuilder::Make(metadata, file_io_));
+ builder->FromSnapshot(1000L, /*inclusive=*/false).ToSnapshot(2000L);
+ ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build());
+ ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles());
+ ASSERT_EQ(tasks.size(), 1);
+
+ auto t1 = tasks[0];
+ EXPECT_EQ(t1->change_ordinal(), 0);
+ EXPECT_EQ(t1->commit_snapshot_id(), 2000L);
+ EXPECT_EQ(t1->operation(), ChangelogOperation::kDelete);
+ auto delete_t1 = std::dynamic_pointer_cast<DeletedDataFileScanTask>(t1);
+ ASSERT_NE(delete_t1, nullptr);
+ EXPECT_EQ(delete_t1->data_file()->file_path, "/path/to/file_a.parquet");
+ EXPECT_TRUE(delete_t1->existing_deletes().empty());
+}
+
+TEST_P(IncrementalChangelogScanTest, ExistingEntriesInNewDataManifestsAreIgnored) {
+ auto version = GetParam();
+
+ auto snapshot_a =
+ MakeAppendSnapshot(version, 1000L, std::nullopt, 1L, {"/path/to/file_a.parquet"});
+ auto snapshot_b =
+ MakeAppendSnapshot(version, 2000L, 1000L, 2L, {"/path/to/file_b.parquet"});
+
+ // Create snapshot C with a manifest that contains existing entries from A and B
+ // plus a new file C. This simulates manifest merging.
+ std::vector<ManifestEntry> merged_entries;
+ auto file_a = MakeDataFile("/path/to/file_a.parquet");
+ merged_entries.push_back(MakeEntry(ManifestStatus::kExisting, 1000L, 1L, file_a));
+ auto file_b = MakeDataFile("/path/to/file_b.parquet");
+ merged_entries.push_back(MakeEntry(ManifestStatus::kExisting, 2000L, 2L, file_b));
+ auto file_c = MakeDataFile("/path/to/file_c.parquet");
+ merged_entries.push_back(MakeEntry(ManifestStatus::kAdded, 3000L, 3L, file_c));
+
+ auto manifest = WriteDataManifest(version, 3000L, std::move(merged_entries));
+ auto manifest_list = WriteManifestList(version, 3000L, 2000L, 3L, {manifest});
+ TimePointMs timestamp_ms = TimePointMsFromUnixMs(1609459200000L + 3000);
+ auto snapshot_c = std::make_shared<Snapshot>(Snapshot{
+ .snapshot_id = 3000L,
+ .parent_snapshot_id = 2000L,
+ .sequence_number = 3L,
+ .timestamp_ms = timestamp_ms,
+ .manifest_list = manifest_list,
+ .summary = {{"operation", "append"}},
+ .schema_id = schema_->schema_id(),
+ });
+
+ auto metadata = MakeTableMetadata(
+ {snapshot_a, snapshot_b, snapshot_c}, 3000L,
+ {{"main", std::make_shared<SnapshotRef>(SnapshotRef{
+ .snapshot_id = 3000L, .retention = SnapshotRef::Branch{}})}});
+
+ // When scanning from_snapshot_inclusive(C).to_snapshot(C), should only return file_c
+ // because file_a and file_b are marked as EXISTING entries
+ ICEBERG_UNWRAP_OR_FAIL(auto builder,
+ IncrementalChangelogScanBuilder::Make(metadata, file_io_));
+ builder->FromSnapshot(3000L, /*inclusive=*/true).ToSnapshot(3000L);
+ ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build());
+ ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles());
+ ASSERT_EQ(tasks.size(), 1);
+
+ auto t1 = tasks[0];
+ EXPECT_EQ(t1->change_ordinal(), 0);
+ EXPECT_EQ(t1->commit_snapshot_id(), 3000L);
+ EXPECT_EQ(t1->operation(), ChangelogOperation::kInsert);
+ auto insert_t1 = std::dynamic_pointer_cast<AddedRowsScanTask>(t1);
+ ASSERT_NE(insert_t1, nullptr);
+ EXPECT_EQ(insert_t1->data_file()->file_path, "/path/to/file_c.parquet");
+ EXPECT_TRUE(insert_t1->delete_files().empty());
+}
+
+TEST_P(IncrementalChangelogScanTest, DataFileRewrites) {
+ auto version = GetParam();
+
+ auto snapshot_a =
+ MakeAppendSnapshot(version, 1000L, std::nullopt, 1L, {"/path/to/file_a.parquet"});
+ auto snapshot_b =
+ MakeAppendSnapshot(version, 2000L, 1000L, 2L, {"/path/to/file_b.parquet"});
+
+ // Create a rewrite/replace snapshot (simulating file compaction)
+ // In a replace operation, the old file is deleted and new file is added,
+ // but this shouldn't appear in the changelog
+ std::vector<ManifestEntry> entries;
+ auto file_a2 = MakeDataFile("/path/to/file_a2.parquet");
+ entries.push_back(MakeEntry(ManifestStatus::kAdded, 3000L, 3L, file_a2));
+ auto file_a = MakeDataFile("/path/to/file_a.parquet");
+ entries.push_back(MakeEntry(ManifestStatus::kDeleted, 3000L, 3L, file_a));
+ auto manifest = WriteDataManifest(version, 3000L, std::move(entries));
+ auto manifest_list = WriteManifestList(version, 3000L, 2000L, 3L, {manifest});
+ TimePointMs timestamp_ms = TimePointMsFromUnixMs(1609459200000L + 3000);
+ auto snapshot_c = std::make_shared<Snapshot>(Snapshot{
+ .snapshot_id = 3000L,
+ .parent_snapshot_id = 2000L,
+ .sequence_number = 3L,
+ .timestamp_ms = timestamp_ms,
+ .manifest_list = manifest_list,
+ .summary = {{"operation", "replace"}}, // Replace operation
+ .schema_id = schema_->schema_id(),
+ });
+
+ auto metadata = MakeTableMetadata(
+ {snapshot_a, snapshot_b, snapshot_c}, 3000L,
+ {{"main", std::make_shared<SnapshotRef>(SnapshotRef{
+ .snapshot_id = 3000L, .retention = SnapshotRef::Branch{}})}});
+
+ // The changelog should only show the original appends (A and B),
+ // not the replace operation
+ ICEBERG_UNWRAP_OR_FAIL(auto builder,
+ IncrementalChangelogScanBuilder::Make(metadata, file_io_));
+ builder->ToSnapshot(3000L);
+ ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build());
+ ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles());
+ ASSERT_EQ(tasks.size(), 2);
+ SortTasks(tasks);
+
+ auto t1 = tasks[0];
+ EXPECT_EQ(t1->change_ordinal(), 0);
+ EXPECT_EQ(t1->commit_snapshot_id(), 1000L);
+ EXPECT_EQ(t1->operation(), ChangelogOperation::kInsert);
+ auto insert_t1 = std::dynamic_pointer_cast<AddedRowsScanTask>(t1);
+ ASSERT_NE(insert_t1, nullptr);
+ EXPECT_EQ(insert_t1->data_file()->file_path, "/path/to/file_a.parquet");
+
+ auto t2 = tasks[1];
+ EXPECT_EQ(t2->change_ordinal(), 1);
+ EXPECT_EQ(t2->commit_snapshot_id(), 2000L);
+ EXPECT_EQ(t2->operation(), ChangelogOperation::kInsert);
+ auto insert_t2 = std::dynamic_pointer_cast<AddedRowsScanTask>(t2);
+ ASSERT_NE(insert_t2, nullptr);
+ EXPECT_EQ(insert_t2->data_file()->file_path, "/path/to/file_b.parquet");
+}
+
+TEST_P(IncrementalChangelogScanTest, ManifestRewritesAreIgnored) {
+ auto version = GetParam();
+
+ auto snapshot_a =
+ MakeAppendSnapshot(version, 1000L, std::nullopt, 1L, {"/path/to/file_a.parquet"});
+ auto snapshot_b =
+ MakeAppendSnapshot(version, 2000L, 1000L, 2L, {"/path/to/file_b.parquet"});
+
+ // Simulate a manifest rewrite (snapshot 3) - creates a new manifest with EXISTING
+ // entries This simulates what happens after table.rewriteManifests()
+ std::vector<ManifestEntry> rewritten_entries;
+ auto file_a = MakeDataFile("/path/to/file_a.parquet");
+ rewritten_entries.push_back(MakeEntry(ManifestStatus::kExisting, 1000L, 1L, file_a));
+ auto file_b = MakeDataFile("/path/to/file_b.parquet");
+ rewritten_entries.push_back(MakeEntry(ManifestStatus::kExisting, 2000L, 2L, file_b));
+
+ auto rewritten_manifest =
+ WriteDataManifest(version, 3000L, std::move(rewritten_entries));
+ auto manifest_list_3 =
+ WriteManifestList(version, 3000L, 2000L, 3L, {rewritten_manifest});
+ TimePointMs timestamp_3 = TimePointMsFromUnixMs(1609459200000L + 3000);
+ auto snapshot_c = std::make_shared<Snapshot>(Snapshot{
+ .snapshot_id = 3000L,
+ .parent_snapshot_id = 2000L,
+ .sequence_number = 3L,
+ .timestamp_ms = timestamp_3,
+ .manifest_list = manifest_list_3,
+ .summary = {{"operation", "replace"}}, // Manifest rewrite uses replace operation
+ .schema_id = schema_->schema_id(),
+ });
+
+ // Add a new append after the rewrite
+ auto snapshot_d =
+ MakeAppendSnapshot(version, 4000L, 3000L, 4L, {"/path/to/file_c.parquet"});
+
+ auto metadata = MakeTableMetadata(
+ {snapshot_a, snapshot_b, snapshot_c, snapshot_d}, 4000L,
+ {{"main", std::make_shared<SnapshotRef>(SnapshotRef{
+ .snapshot_id = 4000L, .retention = SnapshotRef::Branch{}})}});
+
+ // The changelog should show all 3 files from the original appends,
+ // ignoring the manifest rewrite snapshot
+ ICEBERG_UNWRAP_OR_FAIL(auto builder,
+ IncrementalChangelogScanBuilder::Make(metadata, file_io_));
+ builder->ToSnapshot(4000L);
+ ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build());
+ ICEBERG_UNWRAP_OR_FAIL(auto tasks, scan->PlanFiles());
+ ASSERT_EQ(tasks.size(), 3);
+ SortTasks(tasks);
+
+ auto t1 = tasks[0];
+ EXPECT_EQ(t1->change_ordinal(), 0);
+ EXPECT_EQ(t1->commit_snapshot_id(), 1000L);
+ EXPECT_EQ(t1->operation(), ChangelogOperation::kInsert);
+ auto insert_t1 = std::dynamic_pointer_cast<AddedRowsScanTask>(t1);
+ ASSERT_NE(insert_t1, nullptr);
+ EXPECT_EQ(insert_t1->data_file()->file_path, "/path/to/file_a.parquet");
+
+ auto t2 = tasks[1];
+ EXPECT_EQ(t2->change_ordinal(), 1);
+ EXPECT_EQ(t2->commit_snapshot_id(), 2000L);
+ EXPECT_EQ(t2->operation(), ChangelogOperation::kInsert);
+ auto insert_t2 = std::dynamic_pointer_cast<AddedRowsScanTask>(t2);
+ ASSERT_NE(insert_t2, nullptr);
+ EXPECT_EQ(insert_t2->data_file()->file_path, "/path/to/file_b.parquet");
+
+ auto t3 = tasks[2];
+ EXPECT_EQ(t3->change_ordinal(), 2);
+ EXPECT_EQ(t3->commit_snapshot_id(), 4000L);
+ EXPECT_EQ(t3->operation(), ChangelogOperation::kInsert);
+ auto insert_t3 = std::dynamic_pointer_cast<AddedRowsScanTask>(t3);
+ ASSERT_NE(insert_t3, nullptr);
+ EXPECT_EQ(insert_t3->data_file()->file_path, "/path/to/file_c.parquet");
+}
+
+TEST_P(IncrementalChangelogScanTest, DeleteFilesAreNotSupported) {
+ auto version = GetParam();
+ if (version < 2) {
+ GTEST_SKIP() << "Delete files only exist in format version 2+";
+ }
+
+ auto snapshot_a =
+ MakeAppendSnapshot(version, 1000L, std::nullopt, 1L,
+ {"/path/to/file_a.parquet", "/path/to/file_b.parquet"});
+
+ // Create a snapshot with delete files (positional deletes)
+ // This simulates table.newRowDelta().addDeletes(FILE_A_DELETES).commit()
+ std::vector<ManifestEntry> data_entries;
+ auto file_a = MakeDataFile("/path/to/file_a.parquet");
+ data_entries.push_back(MakeEntry(ManifestStatus::kExisting, 1000L, 1L, file_a));
+ auto file_b = MakeDataFile("/path/to/file_b.parquet");
+ data_entries.push_back(MakeEntry(ManifestStatus::kExisting, 1000L, 1L, file_b));
+ auto data_manifest = WriteDataManifest(version, 2000L, std::move(data_entries));
+
+ // Create a delete file entry
+ auto delete_file = std::make_shared<DataFile>(DataFile{
+ .content = DataFile::Content::kPositionDeletes,
+ .file_path = "/path/to/file_a_deletes.parquet",
+ .file_format = FileFormatType::kParquet,
+ .partition = PartitionValues(std::vector<Literal>{}),
+ .record_count = 1,
+ .file_size_in_bytes = 10,
+ .sort_order_id = 0,
+ .partition_spec_id = unpartitioned_spec_->spec_id(),
+ });
+ std::vector<ManifestEntry> delete_entries;
+ delete_entries.push_back(MakeEntry(ManifestStatus::kAdded, 2000L, 2L, delete_file));
+ auto delete_manifest =
+ WriteDeleteManifest(version, 2000L, std::move(delete_entries), unpartitioned_spec_);
+
+ auto manifest_list =
+ WriteManifestList(version, 2000L, 1000L, 2L, {data_manifest, delete_manifest});
+ TimePointMs timestamp_ms = TimePointMsFromUnixMs(1609459200000L + 2000);
+ auto snapshot_b = std::make_shared<Snapshot>(Snapshot{
+ .snapshot_id = 2000L,
+ .parent_snapshot_id = 1000L,
+ .sequence_number = 2L,
+ .timestamp_ms = timestamp_ms,
+ .manifest_list = manifest_list,
+ .summary = {{"operation", "delete"}},
+ .schema_id = schema_->schema_id(),
+ });
+
+ auto metadata = MakeTableMetadata(
+ {snapshot_a, snapshot_b}, 2000L,
+ {{"main", std::make_shared<SnapshotRef>(SnapshotRef{
+ .snapshot_id = 2000L, .retention = SnapshotRef::Branch{}})}});
+
+ ICEBERG_UNWRAP_OR_FAIL(auto builder,
+ IncrementalChangelogScanBuilder::Make(metadata, file_io_));
+ builder->ToSnapshot(2000L);
+ ICEBERG_UNWRAP_OR_FAIL(auto scan, builder->Build());
+ EXPECT_THAT(scan->PlanFiles(),
+ ::testing::AllOf(
+ IsError(ErrorKind::kNotSupported),
+ HasErrorMessage(
+ "Delete files are currently not supported in changelog scans")));
+}
+
+INSTANTIATE_TEST_SUITE_P(IncrementalChangelogScanVersions, IncrementalChangelogScanTest,
+ testing::Values(1, 2, 3));
+
+} // namespace iceberg
diff --git a/src/iceberg/test/scan_test_base.h b/src/iceberg/test/scan_test_base.h
index 65a4e053..1a2f5648 100644
--- a/src/iceberg/test/scan_test_base.h
+++ b/src/iceberg/test/scan_test_base.h
@@ -261,6 +261,87 @@ class ScanTestBase : public testing::TestWithParam<int8_t> {
});
}
+ /// \brief Create a delete snapshot with the given files.
+ std::shared_ptr<Snapshot> MakeDeleteSnapshot(
+ int8_t format_version, int64_t snapshot_id,
+ std::optional<int64_t> parent_snapshot_id, int64_t sequence_number,
+ const std::vector<std::string>& deleted_files) {
+ std::vector<std::pair<std::string, PartitionValues>> files_with_partitions;
+ for (const auto& path : deleted_files) {
+ files_with_partitions.emplace_back(path, PartitionValues(std::vector<Literal>{}));
+ }
+ return MakeDeleteSnapshot(format_version, snapshot_id, parent_snapshot_id,
+ sequence_number, files_with_partitions);
+ }
+
+ /// \brief Create a delete snapshot with partition values for each file.
+ std::shared_ptr<Snapshot> MakeDeleteSnapshot(
+ int8_t format_version, int64_t snapshot_id,
+ std::optional<int64_t> parent_snapshot_id, int64_t sequence_number,
+ const std::vector<std::pair<std::string, PartitionValues>>& deleted_files) {
+ std::vector<ManifestEntry> entries;
+ entries.reserve(deleted_files.size());
+ for (const auto& [path, partition] : deleted_files) {
+ auto file = MakeDataFile(path, partition);
+ entries.push_back(
+ MakeEntry(ManifestStatus::kDeleted, snapshot_id, sequence_number, file));
+ }
+
+ auto manifest = WriteDataManifest(format_version, snapshot_id, std::move(entries));
+ int64_t parent_id = parent_snapshot_id.value_or(0L);
+ auto manifest_list = WriteManifestList(format_version, snapshot_id, parent_id,
+ sequence_number, {manifest});
+ TimePointMs timestamp_ms =
+ TimePointMsFromUnixMs(1609459200000L + sequence_number * 1000);
+ return std::make_shared<Snapshot>(Snapshot{
+ .snapshot_id = snapshot_id,
+ .parent_snapshot_id = parent_snapshot_id,
+ .sequence_number = sequence_number,
+ .timestamp_ms = timestamp_ms,
+ .manifest_list = manifest_list,
+ .summary = {{"operation", "delete"}},
+ .schema_id = schema_->schema_id(),
+ });
+ }
+
+ /// \brief Create an overwrite snapshot with added and deleted files.
+ std::shared_ptr<Snapshot> MakeOverwriteSnapshot(
+ int8_t format_version, int64_t snapshot_id,
+ std::optional<int64_t> parent_snapshot_id, int64_t sequence_number,
+ const std::vector<std::string>& added_file_paths,
+ const std::vector<std::string>& deleted_file_paths) {
+ std::vector<ManifestEntry> entries;
+ entries.reserve(added_file_paths.size() + deleted_file_paths.size());
+
+ for (const auto& path : added_file_paths) {
+ auto file = MakeDataFile(path);
+ entries.push_back(
+ MakeEntry(ManifestStatus::kAdded, snapshot_id, sequence_number, file));
+ }
+
+ for (const auto& path : deleted_file_paths) {
+ auto file = MakeDataFile(path);
+ entries.push_back(
+ MakeEntry(ManifestStatus::kDeleted, snapshot_id, sequence_number, file));
+ }
+
+ auto manifest = WriteDataManifest(format_version, snapshot_id, std::move(entries));
+ int64_t parent_id = parent_snapshot_id.value_or(0L);
+ auto manifest_list = WriteManifestList(format_version, snapshot_id, parent_id,
+ sequence_number, {manifest});
+ TimePointMs timestamp_ms =
+ TimePointMsFromUnixMs(1609459200000L + sequence_number * 1000);
+ return std::make_shared<Snapshot>(Snapshot{
+ .snapshot_id = snapshot_id,
+ .parent_snapshot_id = parent_snapshot_id,
+ .sequence_number = sequence_number,
+ .timestamp_ms = timestamp_ms,
+ .manifest_list = manifest_list,
+ .summary = {{"operation", "overwrite"}},
+ .schema_id = schema_->schema_id(),
+ });
+ }
+
std::shared_ptr<FileIO> file_io_;
std::shared_ptr<Schema> schema_;
std::shared_ptr<PartitionSpec> partitioned_spec_;